From 21eec80d60cb951031ec5fbdaf4537acaf189f0f Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Sun, 31 May 2026 17:42:40 +0800 Subject: [PATCH 1/4] feat: add safe vtl archive pipeline Add a coldstore-vtl crate with an in-memory VTL simulator and safe command-spec/discovery helpers for mhVTL preparation.\n\nBridge the simulator into the tape service behind a TapeBackend abstraction, then wire scheduler archive batching through CacheArchiveClient and TapeArchiveClient adapters. The archive path is covered by in-process metadata/cache/tape gRPC tests and remains free of live mhVTL or host /dev access.\n\nUpdate safe verification targets and docs to describe the current Phase-1 archive pipeline. --- Cargo.toml | 1 + Makefile | 12 +- README_RUST.md | 3 +- crates/scheduler/Cargo.toml | 2 + crates/scheduler/src/service.rs | 899 ++++++++++++++++++++++++- crates/tape/Cargo.toml | 1 + crates/tape/src/service.rs | 616 +++++++++++++++-- crates/tape/tests/simulator_service.rs | 179 +++++ crates/vtl/Cargo.toml | 11 + crates/vtl/README.md | 51 ++ crates/vtl/src/command.rs | 146 ++++ crates/vtl/src/discover.rs | 144 ++++ crates/vtl/src/error.rs | 50 ++ crates/vtl/src/interface.rs | 20 + crates/vtl/src/lib.rs | 22 + crates/vtl/src/mhvtl.rs | 208 ++++++ crates/vtl/src/model.rs | 223 ++++++ crates/vtl/src/simulator.rs | 291 ++++++++ crates/vtl/tests/behavior.rs | 152 +++++ docs/modules/05-scheduler-layer.md | 11 + docs/modules/06-tape-layer.md | 13 + scripts/setup-mhvtl-env.sh | 179 +++++ 22 files changed, 3170 insertions(+), 64 deletions(-) create mode 100644 crates/tape/tests/simulator_service.rs create mode 100644 crates/vtl/Cargo.toml create mode 100644 crates/vtl/README.md create mode 100644 crates/vtl/src/command.rs create mode 100644 crates/vtl/src/discover.rs create mode 100644 crates/vtl/src/error.rs create mode 100644 crates/vtl/src/interface.rs create mode 100644 crates/vtl/src/lib.rs create mode 100644 crates/vtl/src/mhvtl.rs create mode 100644 crates/vtl/src/model.rs create mode 100644 crates/vtl/src/simulator.rs create mode 100644 crates/vtl/tests/behavior.rs create mode 100755 scripts/setup-mhvtl-env.sh diff --git a/Cargo.toml b/Cargo.toml index 6391457..44f3949 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/scheduler", "crates/cache", "crates/tape", + "crates/vtl", ] [workspace.package] diff --git a/Makefile b/Makefile index ddce59d..55c1b50 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build build-debug fmt fmt-check clippy test unit test-unit check check-all check-safe clean help install-hooks install-tools setup lint run-metadata run-gateway run-scheduler run-cache run-tape +.PHONY: build build-debug fmt fmt-check clippy test unit test-unit vtl-unit vtl-check check check-all check-safe clean help install-hooks install-tools setup lint run-metadata run-gateway run-scheduler run-cache run-tape .DEFAULT_GOAL := help @@ -21,11 +21,15 @@ test: @cargo test --workspace --all-features unit test-unit: @cargo test --workspace --lib --bins +vtl-unit: + @cargo test -p coldstore-vtl --lib --tests +vtl-check: + @cargo check -p coldstore-vtl check: @cargo check --all-targets --all-features check-all: fmt-check clippy test -check-safe: fmt-check clippy unit build-debug - @echo "Safe verification passed (fmt/clippy/unit/build only)." +check-safe: fmt-check clippy unit vtl-unit build-debug + @echo "Safe verification passed (fmt/clippy/unit/vtl-unit/build only)." clean: @cargo clean @@ -59,6 +63,6 @@ help: @echo "ColdStore Workspace Makefile" @echo "" @echo "Build: build build-debug clean" - @echo "Quality: fmt fmt-check clippy test unit check check-all check-safe lint" + @echo "Quality: fmt fmt-check clippy test unit vtl-unit vtl-check check check-all check-safe lint" @echo "Run: run-metadata run-gateway run-scheduler run-cache run-tape" @echo "Setup: setup install-tools install-hooks" diff --git a/README_RUST.md b/README_RUST.md index 7d17c91..bb9c418 100644 --- a/README_RUST.md +++ b/README_RUST.md @@ -15,7 +15,7 @@ ColdStore 是**纯冷归档系统**（类似 AWS Glacier Deep Archive）： | 分期 | 状态 | 已落地能力 | |------|------|------------| -| Phase 1 | 已落地 / 可单测 | Gateway/Scheduler/Metadata/Cache 本地闭环；bucket/object CRUD；Put/Head/Get/Delete/Restore/List；HDD Cache staging/restored；Phase-1 archive 标记 Cold 并清理 staging | +| Phase 1 | 已落地 / 可单测 | Gateway/Scheduler/Metadata/Cache 本地闭环；bucket/object CRUD；Put/Head/Get/Delete/Restore/List；HDD Cache staging/restored；Phase-1 archive 标记 Cold 并清理 staging；TapeService + coldstore-vtl 安全模拟驱动/带库/load/unload/filemark 读写闭环；Scheduler archive batch 可注入 cache+tape writer 并写入 ArchiveBundle/filemark 元数据；CacheArchiveClient 已接入真实 CacheService staging list/get/delete 单测；TapeArchiveClient 已接入真实 TapeService WriteBundle/ReadBundle gRPC 单测 | | Phase 2A | 已启动 / 可单测 | Metadata opt-in 二进制 snapshot：`MetadataServiceImpl::new_with_snapshot(config, path)` 支持写入后保存、重启后恢复 bucket/object/task/worker/tape 等状态 | | Phase 2B | 下一步 | 将 Metadata 状态机接入 OpenRaft + RocksDB/openraft-rocksstore，补齐安全的 Raft 状态机单测和多节点一致性测试 | @@ -32,6 +32,7 @@ ColdStore 是**纯冷归档系统**（类似 AWS Glacier Deep Archive）： | coldstore-scheduler | bin | 调度 Worker（业务中枢） | | coldstore-cache | bin | 缓存 Worker（独立进程，HDD/SPDK） | | coldstore-tape | bin | 磁带 Worker（独立物理节点） | +| coldstore-vtl | lib | mhVTL/虚拟磁带库 harness：lsscsi/mtx/mt/sg 命令封装 + 安全内存模拟器 | ## 组件间通信 diff --git a/crates/scheduler/Cargo.toml b/crates/scheduler/Cargo.toml index b68ef4c..27a4a98 100644 --- a/crates/scheduler/Cargo.toml +++ b/crates/scheduler/Cargo.toml @@ -29,4 +29,6 @@ tokio-stream = { workspace = true } prost-types = { workspace = true } [dev-dependencies] +coldstore-cache = { path = "../cache" } coldstore-metadata = { path = "../metadata" } +coldstore-tape = { path = "../tape" } diff --git a/crates/scheduler/src/service.rs b/crates/scheduler/src/service.rs index f53db75..04a9a3f 100644 --- a/crates/scheduler/src/service.rs +++ b/crates/scheduler/src/service.rs @@ -1,7 +1,18 @@ use crate::SchedulerState; +use coldstore_proto::cache::cache_service_client::CacheServiceClient; +use coldstore_proto::cache::get_staging_response::Payload as GetStagingPayload; +use coldstore_proto::cache::{ + DeleteStagingRequest, GetStagingRequest, ListStagingKeysRequest, StagingKeyEntry, + StagingObjectMeta, +}; use coldstore_proto::common; use coldstore_proto::scheduler::scheduler_service_server::SchedulerService; use coldstore_proto::scheduler::*; +use coldstore_proto::tape::tape_service_client::TapeServiceClient as TapeGrpcClient; +use coldstore_proto::tape::write_bundle_request::Payload as TapeWriteRequestPayload; +use coldstore_proto::tape::{ + WriteBundleMeta as TapeWriteBundleMeta, WriteBundleRequest as TapeWriteBundleRequest, +}; use prost_types::Timestamp; use sha2::{Digest, Sha256}; #[cfg(test)] @@ -53,20 +64,347 @@ pub trait Phase1SchedulerBackend: Send + Sync + 'static { ) -> std::result::Result, Status>; } -struct MetadataBackedSchedulerBackend { +#[derive(Debug, Clone)] +pub struct Phase1StagedObject { + pub meta: StagingObjectMeta, + pub data: Vec, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct TapeArchiveWrite { + pub tape_id: String, + pub tape_set: Vec, + pub filemark_start: u32, + pub filemark_end: u32, + pub bytes_written: u64, +} + +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct ArchiveBatchResult { + pub archived_objects: u32, + pub bytes_written: u64, + pub bundle_ids: Vec, +} + +#[tonic::async_trait] +pub trait Phase1ArchiveCache: Send + Sync { + async fn list_staging_keys( + &self, + limit: u32, + ) -> std::result::Result, Status>; + + async fn get_staging( + &self, + bucket: &str, + key: &str, + version_id: Option<&str>, + ) -> std::result::Result; + + async fn delete_staging( + &self, + bucket: &str, + key: &str, + version_id: Option<&str>, + ) -> std::result::Result<(), Status>; +} + +#[derive(Clone)] +pub struct CacheArchiveClient { + client: CacheServiceClient, +} + +impl CacheArchiveClient { + pub fn new(client: CacheServiceClient) -> Self { + Self { client } + } +} + +#[tonic::async_trait] +impl Phase1ArchiveCache for CacheArchiveClient { + async fn list_staging_keys( + &self, + limit: u32, + ) -> std::result::Result, Status> { + let mut client = self.client.clone(); + Ok(client + .list_staging_keys(Request::new(ListStagingKeysRequest { limit, after: None })) + .await? + .into_inner() + .entries) + } + + async fn get_staging( + &self, + bucket: &str, + key: &str, + version_id: Option<&str>, + ) -> std::result::Result { + let mut client = self.client.clone(); + let mut stream = client + .get_staging(Request::new(GetStagingRequest { + bucket: bucket.into(), + key: key.into(), + version_id: version_id.map(str::to_owned), + })) + .await? + .into_inner(); + + let mut meta = None; + let mut data = Vec::new(); + while let Some(message) = stream.message().await? { + match message.payload { + Some(GetStagingPayload::Meta(next_meta)) => { + if meta.replace(next_meta).is_some() { + return Err(Status::invalid_argument( + "cache get_staging returned duplicate metadata", + )); + } + } + Some(GetStagingPayload::Data(chunk)) => data.extend_from_slice(&chunk), + None => return Err(Status::internal("cache get_staging returned empty chunk")), + } + } + + let meta = meta + .ok_or_else(|| Status::internal("cache get_staging stream ended without metadata"))?; + Ok(Phase1StagedObject { meta, data }) + } + + async fn delete_staging( + &self, + bucket: &str, + key: &str, + version_id: Option<&str>, + ) -> std::result::Result<(), Status> { + let mut client = self.client.clone(); + client + .delete_staging(Request::new(DeleteStagingRequest { + bucket: bucket.into(), + key: key.into(), + version_id: version_id.map(str::to_owned), + })) + .await?; + Ok(()) + } +} + +#[tonic::async_trait] +pub trait TapeArchiveWriter: Send + Sync { + async fn write_bundle( + &self, + bundle_id: &str, + object_count: u32, + data: Vec, + ) -> std::result::Result; +} + +#[derive(Clone)] +pub struct TapeArchiveClient { + client: TapeGrpcClient, + drive_id: String, + tape_id: String, + tape_set: Vec, + block_size: u32, +} + +impl TapeArchiveClient { + pub fn new( + client: TapeGrpcClient, + drive_id: impl Into, + tape_id: impl Into, + tape_set: Vec, + block_size: u32, + ) -> Self { + Self { + client, + drive_id: drive_id.into(), + tape_id: tape_id.into(), + tape_set, + block_size, + } + } +} + +#[tonic::async_trait] +impl TapeArchiveWriter for TapeArchiveClient { + async fn write_bundle( + &self, + bundle_id: &str, + object_count: u32, + data: Vec, + ) -> std::result::Result { + let mut client = self.client.clone(); + let response = client + .write_bundle(Request::new(tokio_stream::iter(vec![ + TapeWriteBundleRequest { + payload: Some(TapeWriteRequestPayload::Meta(TapeWriteBundleMeta { + drive_id: self.drive_id.clone(), + bundle_id: bundle_id.into(), + total_size: data.len() as u64, + object_count, + block_size: self.block_size, + })), + }, + TapeWriteBundleRequest { + payload: Some(TapeWriteRequestPayload::Data(data)), + }, + ]))) + .await? + .into_inner(); + + if !response.success { + return Err(Status::internal(format!( + "tape write_bundle failed for {bundle_id}: {}", + response + .error + .unwrap_or_else(|| "unknown tape error".into()) + ))); + } + if response.drive_id != self.drive_id { + return Err(Status::internal(format!( + "tape write_bundle responded for drive {}, expected {}", + response.drive_id, self.drive_id + ))); + } + if response.bundle_id != bundle_id { + return Err(Status::internal(format!( + "tape write_bundle responded for bundle {}, expected {bundle_id}", + response.bundle_id + ))); + } + + Ok(TapeArchiveWrite { + tape_id: self.tape_id.clone(), + tape_set: self.tape_set.clone(), + filemark_start: response.filemark_start, + filemark_end: response.filemark_end, + bytes_written: response.bytes_written, + }) + } +} + +pub struct MetadataBackedSchedulerBackend { metadata: coldstore_proto::metadata::metadata_service_client::MetadataServiceClient< tonic::transport::Channel, >, } impl MetadataBackedSchedulerBackend { - fn new( + pub fn new( metadata: coldstore_proto::metadata::metadata_service_client::MetadataServiceClient< tonic::transport::Channel, >, ) -> Self { Self { metadata } } + + pub async fn archive_staging_batch( + &self, + cache: &C, + tape: &T, + limit: u32, + ) -> std::result::Result + where + C: Phase1ArchiveCache + ?Sized, + T: TapeArchiveWriter + ?Sized, + { + let entries = cache.list_staging_keys(limit).await?; + let mut result = ArchiveBatchResult::default(); + + for entry in entries { + let object = self.head_object(&entry.bucket, &entry.key).await?; + if object.storage_class != common::StorageClass::ColdPending as i32 { + continue; + } + + let staged = cache + .get_staging(&entry.bucket, &entry.key, entry.version_id.as_deref()) + .await?; + if staged.meta.size != staged.data.len() as u64 { + return Err(Status::invalid_argument(format!( + "staging size mismatch for {}/{}: meta={}, bytes={}", + entry.bucket, + entry.key, + staged.meta.size, + staged.data.len() + ))); + } + + let bundle_id = + phase1_bundle_id(&entry.bucket, &entry.key, entry.version_id.as_deref()); + let checksum = staged + .meta + .checksum + .clone() + .unwrap_or_else(|| sha256_hex(&staged.data)); + let write = tape + .write_bundle(&bundle_id, 1, staged.data.clone()) + .await?; + if write.bytes_written != staged.data.len() as u64 { + return Err(Status::internal(format!( + "tape writer reported {} bytes for {bundle_id}, expected {}", + write.bytes_written, + staged.data.len() + ))); + } + + let now = now_timestamp(); + let bundle = common::ArchiveBundle { + id: bundle_id.clone(), + tape_id: write.tape_id.clone(), + tape_set: write.tape_set.clone(), + entries: vec![common::BundleEntry { + bucket: entry.bucket.clone(), + key: entry.key.clone(), + version_id: entry.version_id.clone(), + size: staged.data.len() as u64, + offset_in_bundle: 0, + tape_block_offset: write.filemark_start as u64, + checksum: checksum.clone(), + }], + total_size: write.bytes_written, + filemark_start: write.filemark_start, + filemark_end: write.filemark_end, + checksum: Some(checksum), + status: common::ArchiveBundleStatus::BundleCompleted as i32, + created_at: Some(now), + completed_at: Some(now), + }; + + let mut client = self.metadata.clone(); + client.put_archive_bundle(Request::new(bundle)).await?; + client + .update_archive_location(Request::new( + coldstore_proto::metadata::UpdateArchiveLocationRequest { + bucket: entry.bucket.clone(), + key: entry.key.clone(), + archive_id: bundle_id.clone(), + tape_id: write.tape_id.clone(), + tape_set: write.tape_set.clone(), + tape_block_offset: write.filemark_start as u64, + }, + )) + .await?; + client + .update_storage_class(Request::new( + coldstore_proto::metadata::UpdateStorageClassRequest { + bucket: entry.bucket.clone(), + key: entry.key.clone(), + storage_class: common::StorageClass::Cold as i32, + }, + )) + .await?; + cache + .delete_staging(&entry.bucket, &entry.key, entry.version_id.as_deref()) + .await?; + + result.archived_objects += 1; + result.bytes_written += write.bytes_written; + result.bundle_ids.push(bundle_id); + } + + Ok(result) + } } #[tonic::async_trait] @@ -398,6 +736,13 @@ fn storage_class_label(storage_class: i32) -> &'static str { } } +fn phase1_bundle_id(bucket: &str, key: &str, version_id: Option<&str>) -> String { + match version_id.filter(|version| !version.is_empty()) { + Some(version) => format!("phase1-bundle:{bucket}/{key}#{version}"), + None => format!("phase1-bundle:{bucket}/{key}"), + } +} + #[tonic::async_trait] impl SchedulerService for SchedulerServiceImpl { async fn put_object( @@ -579,13 +924,36 @@ impl SchedulerService for SchedulerServiceImpl { #[cfg(test)] mod tests { use super::*; - use coldstore_common::config::{MetadataConfig, SchedulerConfig}; + use coldstore_cache::service::CacheServiceImpl; + use coldstore_common::config::{ + CacheBackendConfig, CacheConfig, MetadataConfig, SchedulerConfig, TapeConfig, + }; use coldstore_metadata::service::MetadataServiceImpl; + use coldstore_proto::cache::cache_service_client::CacheServiceClient; + use coldstore_proto::cache::cache_service_server::CacheServiceServer; + use coldstore_proto::cache::put_staging_request::Payload as PutStagingPayload; + use coldstore_proto::cache::{ + GetStagingRequest, ListStagingKeysRequest, PutStagingMeta, PutStagingRequest, + StagingKeyEntry, StagingObjectMeta, + }; use coldstore_proto::metadata::metadata_service_server::MetadataServiceServer; + use coldstore_proto::tape::read_bundle_request::Location as TapeReadLocation; + use coldstore_proto::tape::read_bundle_response::Payload as TapeReadPayload; + use coldstore_proto::tape::tape_service_client::TapeServiceClient; + use coldstore_proto::tape::tape_service_server::{ + TapeService as TapeWorkerService, TapeServiceServer, + }; + use coldstore_proto::tape::write_bundle_request::Payload as TapeWritePayload; + use coldstore_proto::tape::{ + LoadTapeRequest, ReadBundleRequest as TapeReadBundleRequest, WriteBundleMeta, + WriteBundleRequest, + }; + use coldstore_tape::service::{SimulatorTapeBackend, TapeServiceImpl as TapeWorkerServiceImpl}; + use std::time::{SystemTime, UNIX_EPOCH}; use tokio::sync::oneshot; use tokio::time::{sleep, Duration}; use tokio_stream::StreamExt; - use tonic::transport::Server; + use tonic::transport::{Channel, Server}; #[derive(Default)] struct InMemoryBackend { @@ -1033,6 +1401,116 @@ mod tests { (SchedulerServiceImpl::new(state.clone()), state, shutdown_tx) } + fn test_cache_config() -> CacheConfig { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time") + .as_nanos(); + CacheConfig { + backend: CacheBackendConfig::Hdd { + path: format!("/tmp/coldstore-scheduler-cache-test-{unique}"), + max_size_gb: 1, + }, + ..CacheConfig::default() + } + } + + async fn cache_backed_service() -> (CacheServiceClient, oneshot::Sender<()>) { + let cache = CacheServiceImpl::new(&test_cache_config()) + .await + .expect("cache service init"); + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind test listener"); + let addr = listener.local_addr().expect("listener addr"); + drop(listener); + + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + tokio::spawn(async move { + Server::builder() + .add_service(CacheServiceServer::new(cache)) + .serve_with_shutdown(addr, async { + let _ = shutdown_rx.await; + }) + .await + .expect("cache server should run"); + }); + + let mut cache_client = None; + for _ in 0..20 { + match CacheServiceClient::connect(format!("http://{addr}")).await { + Ok(client) => { + cache_client = Some(client); + break; + } + Err(_) => sleep(Duration::from_millis(25)).await, + } + } + (cache_client.expect("connect cache client"), shutdown_tx) + } + + async fn tape_backed_service() -> (TapeServiceClient, oneshot::Sender<()>) { + let backend = SimulatorTapeBackend::new(2, 1); + backend.insert_tape("slot-1", "TAPE-GRPC").unwrap(); + let tape = TapeWorkerServiceImpl::new_with_backend(TapeConfig::default(), backend); + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind test listener"); + let addr = listener.local_addr().expect("listener addr"); + drop(listener); + + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + tokio::spawn(async move { + Server::builder() + .add_service(TapeServiceServer::new(tape)) + .serve_with_shutdown(addr, async { + let _ = shutdown_rx.await; + }) + .await + .expect("tape server should run"); + }); + + let mut tape_client = None; + for _ in 0..20 { + match TapeServiceClient::connect(format!("http://{addr}")).await { + Ok(client) => { + tape_client = Some(client); + break; + } + Err(_) => sleep(Duration::from_millis(25)).await, + } + } + let mut tape_client = tape_client.expect("connect tape client"); + tape_client + .load_tape(Request::new(LoadTapeRequest { + tape_id: "TAPE-GRPC".into(), + drive_id: "drive-0".into(), + slot_id: Some("slot-1".into()), + })) + .await + .expect("load tape into in-process simulator"); + (tape_client, shutdown_tx) + } + + async fn read_tape_client_filemark( + client: &mut TapeServiceClient, + filemark: u32, + length: u64, + ) -> Vec { + let mut stream = client + .read_bundle(Request::new(TapeReadBundleRequest { + drive_id: "drive-0".into(), + location: Some(TapeReadLocation::Filemark(filemark)), + length, + })) + .await + .expect("read bundle through tape client") + .into_inner(); + while let Some(message) = stream.next().await { + match message.unwrap().payload.unwrap() { + TapeReadPayload::Meta(_) => {} + TapeReadPayload::Data(data) => return data, + } + } + Vec::new() + } + #[tokio::test] async fn default_service_uses_metadata_for_bucket_ops() { let (svc, _state, shutdown_tx) = metadata_backed_service().await; @@ -1218,6 +1696,419 @@ mod tests { shutdown_tx.send(()).ok(); } + struct TestArchiveCache { + staged: RwLock>, + deleted: RwLock>, + } + + impl TestArchiveCache { + fn with_object(bucket: &str, key: &str, data: Vec) -> Self { + let meta = StagingObjectMeta { + bucket: bucket.into(), + key: key.into(), + version_id: None, + size: data.len() as u64, + checksum: Some(sha256_hex(&data)), + content_type: Some("text/plain".into()), + etag: Some("etag-staged".into()), + staged_at: Some(Timestamp { + seconds: 30, + nanos: 0, + }), + }; + let mut staged = HashMap::new(); + staged.insert(format!("{bucket}/{key}"), Phase1StagedObject { meta, data }); + Self { + staged: RwLock::new(staged), + deleted: RwLock::new(Vec::new()), + } + } + + fn deleted_keys(&self) -> Vec { + self.deleted.read().unwrap().clone() + } + } + + #[tonic::async_trait] + impl Phase1ArchiveCache for TestArchiveCache { + async fn list_staging_keys( + &self, + limit: u32, + ) -> std::result::Result, Status> { + let mut entries: Vec<_> = self + .staged + .read() + .unwrap() + .values() + .map(|object| StagingKeyEntry { + bucket: object.meta.bucket.clone(), + key: object.meta.key.clone(), + version_id: object.meta.version_id.clone(), + size: object.meta.size, + staged_at: object.meta.staged_at, + }) + .collect(); + entries.sort_by(|a, b| (&a.bucket, &a.key).cmp(&(&b.bucket, &b.key))); + entries.truncate(limit as usize); + Ok(entries) + } + + async fn get_staging( + &self, + bucket: &str, + key: &str, + _version_id: Option<&str>, + ) -> std::result::Result { + self.staged + .read() + .unwrap() + .get(&format!("{bucket}/{key}")) + .cloned() + .ok_or_else(|| Status::not_found("staging object missing")) + } + + async fn delete_staging( + &self, + bucket: &str, + key: &str, + _version_id: Option<&str>, + ) -> std::result::Result<(), Status> { + self.staged + .write() + .unwrap() + .remove(&format!("{bucket}/{key}")); + self.deleted + .write() + .unwrap() + .push(format!("{bucket}/{key}")); + Ok(()) + } + } + + struct DirectTapeWriter { + service: TapeWorkerServiceImpl, + } + + impl DirectTapeWriter { + async fn loaded() -> Self { + let backend = SimulatorTapeBackend::new(2, 1); + backend.insert_tape("slot-1", "TAPE-PHASE1").unwrap(); + let service = TapeWorkerServiceImpl::new_with_backend(TapeConfig::default(), backend); + service + .load_tape(Request::new(LoadTapeRequest { + tape_id: "TAPE-PHASE1".into(), + drive_id: "drive-0".into(), + slot_id: Some("slot-1".into()), + })) + .await + .unwrap(); + Self { service } + } + + async fn read_filemark(&self, filemark: u32, length: u64) -> Vec { + let mut stream = self + .service + .read_bundle(Request::new(TapeReadBundleRequest { + drive_id: "drive-0".into(), + location: Some(TapeReadLocation::Filemark(filemark)), + length, + })) + .await + .unwrap() + .into_inner(); + while let Some(message) = stream.next().await { + match message.unwrap().payload.unwrap() { + TapeReadPayload::Meta(_) => {} + TapeReadPayload::Data(data) => return data, + } + } + Vec::new() + } + } + + #[tonic::async_trait] + impl TapeArchiveWriter for DirectTapeWriter { + async fn write_bundle( + &self, + bundle_id: &str, + object_count: u32, + data: Vec, + ) -> std::result::Result { + let response = self + .service + .write_bundle_from_messages(tokio_stream::iter(vec![ + Ok(WriteBundleRequest { + payload: Some(TapeWritePayload::Meta(WriteBundleMeta { + drive_id: "drive-0".into(), + bundle_id: bundle_id.into(), + total_size: data.len() as u64, + object_count, + block_size: 262_144, + })), + }), + Ok(WriteBundleRequest { + payload: Some(TapeWritePayload::Data(data)), + }), + ])) + .await?; + Ok(TapeArchiveWrite { + tape_id: "TAPE-PHASE1".into(), + tape_set: vec!["TAPE-PHASE1".into()], + filemark_start: response.filemark_start, + filemark_end: response.filemark_end, + bytes_written: response.bytes_written, + }) + } + } + + #[tokio::test] + async fn archive_staging_batch_writes_tape_and_updates_metadata() { + let (_svc, state, shutdown_tx) = metadata_backed_service().await; + let backend = MetadataBackedSchedulerBackend::new(state.metadata.clone()); + + backend + .create_bucket("docs") + .await + .expect("create bucket through metadata backend"); + backend + .put_object( + "docs", + "guide.txt", + b"abcdef".to_vec(), + Some("text/plain".into()), + ) + .await + .expect("put object metadata"); + + let cache = TestArchiveCache::with_object("docs", "guide.txt", b"abcdef".to_vec()); + let tape = DirectTapeWriter::loaded().await; + + let archived = backend + .archive_staging_batch(&cache, &tape, 10) + .await + .expect("archive staging batch"); + assert_eq!(archived.archived_objects, 1); + assert_eq!(archived.bytes_written, 6); + assert_eq!(archived.bundle_ids, vec!["phase1-bundle:docs/guide.txt"]); + + let object = backend + .head_object("docs", "guide.txt") + .await + .expect("head archived object"); + assert_eq!(object.storage_class, common::StorageClass::Cold as i32); + assert_eq!( + object.archive_id.as_deref(), + Some("phase1-bundle:docs/guide.txt") + ); + assert_eq!(object.tape_id.as_deref(), Some("TAPE-PHASE1")); + assert_eq!(object.tape_set, vec!["TAPE-PHASE1"]); + assert_eq!(object.tape_block_offset, Some(0)); + + let mut metadata = state.metadata.clone(); + let bundle = metadata + .get_archive_bundle(Request::new( + coldstore_proto::metadata::GetArchiveBundleRequest { + id: "phase1-bundle:docs/guide.txt".into(), + }, + )) + .await + .expect("archive bundle stored") + .into_inner(); + assert_eq!(bundle.tape_id, "TAPE-PHASE1"); + assert_eq!(bundle.filemark_start, 0); + assert_eq!(bundle.filemark_end, 1); + assert_eq!(bundle.total_size, 6); + assert_eq!(bundle.entries.len(), 1); + assert_eq!(bundle.entries[0].bucket, "docs"); + assert_eq!(bundle.entries[0].key, "guide.txt"); + assert_eq!(bundle.entries[0].tape_block_offset, 0); + + assert_eq!(cache.deleted_keys(), vec!["docs/guide.txt"]); + assert_eq!(tape.read_filemark(0, 6).await, b"abcdef"); + + shutdown_tx.send(()).ok(); + } + + #[tokio::test] + async fn archive_staging_batch_consumes_real_cache_service_staging() { + let (_svc, state, metadata_shutdown_tx) = metadata_backed_service().await; + let backend = MetadataBackedSchedulerBackend::new(state.metadata.clone()); + + backend + .create_bucket("docs") + .await + .expect("create bucket through metadata backend"); + backend + .put_object( + "docs", + "from-cache.txt", + b"cache-body".to_vec(), + Some("text/plain".into()), + ) + .await + .expect("put object metadata"); + + let (mut raw_cache_client, cache_shutdown_tx) = cache_backed_service().await; + let body = b"cache-body".to_vec(); + raw_cache_client + .put_staging(Request::new(tokio_stream::iter(vec![ + PutStagingRequest { + payload: Some(PutStagingPayload::Meta(PutStagingMeta { + bucket: "docs".into(), + key: "from-cache.txt".into(), + version_id: None, + size: body.len() as u64, + checksum: Some(sha256_hex(&body)), + content_type: Some("text/plain".into()), + etag: Some("etag-from-cache".into()), + })), + }, + PutStagingRequest { + payload: Some(PutStagingPayload::Data(body.clone())), + }, + ]))) + .await + .expect("put staging through real cache service"); + + let cache = CacheArchiveClient::new(raw_cache_client.clone()); + let tape = DirectTapeWriter::loaded().await; + + let archived = backend + .archive_staging_batch(&cache, &tape, 10) + .await + .expect("archive real cache staging batch"); + assert_eq!(archived.archived_objects, 1); + assert_eq!(archived.bytes_written, body.len() as u64); + assert_eq!( + archived.bundle_ids, + vec!["phase1-bundle:docs/from-cache.txt"] + ); + + let object = backend + .head_object("docs", "from-cache.txt") + .await + .expect("head archived object"); + assert_eq!(object.storage_class, common::StorageClass::Cold as i32); + assert_eq!( + object.archive_id.as_deref(), + Some("phase1-bundle:docs/from-cache.txt") + ); + + let listed = raw_cache_client + .list_staging_keys(Request::new(ListStagingKeysRequest { + limit: 10, + after: None, + })) + .await + .expect("list staging after archive") + .into_inner(); + assert!(listed.entries.is_empty()); + + let missing = raw_cache_client + .get_staging(Request::new(GetStagingRequest { + bucket: "docs".into(), + key: "from-cache.txt".into(), + version_id: None, + })) + .await + .expect_err("staging should be deleted after archive"); + assert_eq!(missing.code(), tonic::Code::NotFound); + assert_eq!(tape.read_filemark(0, body.len() as u64).await, body); + + metadata_shutdown_tx.send(()).ok(); + cache_shutdown_tx.send(()).ok(); + } + + #[tokio::test] + async fn archive_staging_batch_uses_cache_and_tape_grpc_clients() { + let (_svc, state, metadata_shutdown_tx) = metadata_backed_service().await; + let backend = MetadataBackedSchedulerBackend::new(state.metadata.clone()); + + backend + .create_bucket("docs") + .await + .expect("create bucket through metadata backend"); + let body = b"grpc-tape-body".to_vec(); + backend + .put_object( + "docs", + "grpc-tape.txt", + body.clone(), + Some("text/plain".into()), + ) + .await + .expect("put object metadata"); + + let (mut raw_cache_client, cache_shutdown_tx) = cache_backed_service().await; + raw_cache_client + .put_staging(Request::new(tokio_stream::iter(vec![ + PutStagingRequest { + payload: Some(PutStagingPayload::Meta(PutStagingMeta { + bucket: "docs".into(), + key: "grpc-tape.txt".into(), + version_id: None, + size: body.len() as u64, + checksum: Some(sha256_hex(&body)), + content_type: Some("text/plain".into()), + etag: Some("etag-grpc-tape".into()), + })), + }, + PutStagingRequest { + payload: Some(PutStagingPayload::Data(body.clone())), + }, + ]))) + .await + .expect("put staging through real cache service"); + + let (mut raw_tape_client, tape_shutdown_tx) = tape_backed_service().await; + let cache = CacheArchiveClient::new(raw_cache_client.clone()); + let tape = TapeArchiveClient::new( + raw_tape_client.clone(), + "drive-0", + "TAPE-GRPC", + vec!["TAPE-GRPC".into()], + 262_144, + ); + + let archived = backend + .archive_staging_batch(&cache, &tape, 10) + .await + .expect("archive through cache and tape grpc clients"); + assert_eq!(archived.archived_objects, 1); + assert_eq!(archived.bytes_written, body.len() as u64); + assert_eq!( + archived.bundle_ids, + vec!["phase1-bundle:docs/grpc-tape.txt"] + ); + + let object = backend + .head_object("docs", "grpc-tape.txt") + .await + .expect("head archived object"); + assert_eq!(object.storage_class, common::StorageClass::Cold as i32); + assert_eq!(object.tape_id.as_deref(), Some("TAPE-GRPC")); + assert_eq!(object.tape_set, vec!["TAPE-GRPC"]); + assert_eq!(object.tape_block_offset, Some(0)); + + let listed = raw_cache_client + .list_staging_keys(Request::new(ListStagingKeysRequest { + limit: 10, + after: None, + })) + .await + .expect("list staging after archive") + .into_inner(); + assert!(listed.entries.is_empty()); + assert_eq!( + read_tape_client_filemark(&mut raw_tape_client, 0, body.len() as u64).await, + body + ); + + metadata_shutdown_tx.send(()).ok(); + cache_shutdown_tx.send(()).ok(); + tape_shutdown_tx.send(()).ok(); + } + #[test] fn phase1_unimplemented_message_is_stable() { let status = phase1_unimplemented("scheduler.list_buckets"); diff --git a/crates/tape/Cargo.toml b/crates/tape/Cargo.toml index f21fcce..166867b 100644 --- a/crates/tape/Cargo.toml +++ b/crates/tape/Cargo.toml @@ -12,6 +12,7 @@ path = "src/main.rs" [dependencies] coldstore-proto = { workspace = true } coldstore-common = { workspace = true } +coldstore-vtl = { path = "../vtl" } tokio = { workspace = true } tonic = { workspace = true } prost = { workspace = true } diff --git a/crates/tape/src/service.rs b/crates/tape/src/service.rs index 86f6a90..7b9ad1f 100644 --- a/crates/tape/src/service.rs +++ b/crates/tape/src/service.rs @@ -1,113 +1,619 @@ +#![allow(clippy::result_large_err)] + +use std::sync::{Arc, Mutex, MutexGuard}; + use coldstore_common::config::TapeConfig; -use coldstore_proto::common; +use coldstore_proto::common::{self, DriveStatus, TapeStatus}; +use coldstore_proto::tape::read_bundle_request::Location; +use coldstore_proto::tape::read_bundle_response::Payload as ReadPayload; use coldstore_proto::tape::tape_service_server::TapeService; +use coldstore_proto::tape::write_bundle_request::Payload as WritePayload; use coldstore_proto::tape::*; +use coldstore_vtl::model::{ElementAddress, TapeBarcode}; +use coldstore_vtl::simulator::VirtualTapeLibrary; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tokio_stream::{Stream, StreamExt}; use tonic::{Request, Response, Status, Streaming}; +const SIMULATED_TAPE_CAPACITY_BYTES: u64 = 12 * 1024 * 1024 * 1024 * 1024; + +type ServiceResult = std::result::Result; + +pub trait TapeBackend: Send + Sync { + fn list_drives(&self) -> ServiceResult>; + fn get_drive_status(&self, drive_id: &str) -> ServiceResult; + fn acquire_drive( + &self, + preferred_drive_id: Option<&str>, + required_tape_id: Option<&str>, + ) -> ServiceResult; + fn release_drive(&self, drive_id: &str) -> ServiceResult<()>; + fn load_tape(&self, tape_id: &str, drive_id: &str, slot_id: Option<&str>) -> ServiceResult<()>; + fn unload_tape(&self, drive_id: &str, target_slot_id: Option<&str>) -> ServiceResult<()>; + fn rewind(&self, drive_id: &str) -> ServiceResult<()>; + fn seek_to_filemark(&self, drive_id: &str, filemark: u32) -> ServiceResult<()>; + fn get_tape_media_status(&self, drive_id: &str) -> ServiceResult; + fn inventory(&self) -> ServiceResult; + fn write_bundle(&self, drive_id: &str, data: &[u8]) -> ServiceResult<(u32, u32)>; + fn read_bundle(&self, drive_id: &str, filemark: u32, length: u64) -> ServiceResult>; +} + pub struct TapeServiceImpl { _config: TapeConfig, + backend: Arc, } impl TapeServiceImpl { pub fn new(config: &TapeConfig) -> anyhow::Result { - Ok(Self { - _config: config.clone(), + let backend = SimulatorTapeBackend::from_config(config); + Ok(Self::new_with_backend(config.clone(), backend)) + } + + pub fn new_with_backend(config: TapeConfig, backend: B) -> Self + where + B: TapeBackend + 'static, + { + Self { + _config: config, + backend: Arc::new(backend), + } + } + + pub async fn write_bundle_from_messages( + &self, + mut messages: S, + ) -> ServiceResult + where + S: Stream> + Unpin, + { + let first = match messages.next().await { + Some(Ok(request)) => request, + Some(Err(status)) => return Err(status), + None => return Err(Status::invalid_argument("write_bundle stream is empty")), + }; + let meta = match first.payload { + Some(WritePayload::Meta(meta)) => meta, + Some(WritePayload::Data(_)) => { + return Err(Status::invalid_argument( + "first write_bundle message must carry metadata", + )) + } + None => { + return Err(Status::invalid_argument( + "write_bundle message has no payload", + )) + } + }; + + let mut data = Vec::new(); + while let Some(message) = messages.next().await { + match message?.payload { + Some(WritePayload::Data(chunk)) => data.extend_from_slice(&chunk), + Some(WritePayload::Meta(_)) => { + return Err(Status::invalid_argument( + "write_bundle metadata must appear only once as the first message", + )) + } + None => { + return Err(Status::invalid_argument( + "write_bundle message has no payload", + )) + } + } + } + + if meta.total_size != data.len() as u64 { + return Err(Status::invalid_argument(format!( + "write_bundle total_size={} does not match received bytes={}", + meta.total_size, + data.len() + ))); + } + + let (filemark_start, filemark_end) = self.backend.write_bundle(&meta.drive_id, &data)?; + Ok(WriteBundleResponse { + drive_id: meta.drive_id, + bundle_id: meta.bundle_id, + bytes_written: data.len() as u64, + filemark_start, + filemark_end, + checksum: None, + success: true, + error: None, }) } } -fn phase1_unimplemented(op: &str) -> Status { - Status::unimplemented(format!( - "{op} is not implemented in phase-1 safe mode; no tape devices are accessed during unit-test runs" - )) +#[derive(Debug)] +pub struct SimulatorTapeBackend { + state: Mutex, +} + +impl SimulatorTapeBackend { + pub fn new(slot_count: u32, drive_count: u32) -> Self { + Self { + state: Mutex::new(SimulatorState::new(slot_count, drive_count)), + } + } + + pub fn from_config(config: &TapeConfig) -> Self { + let drive_count = config.scsi.devices.len().max(1) as u32; + Self::new(8, drive_count) + } + + pub fn insert_tape(&self, slot_id: &str, tape_id: &str) -> ServiceResult<()> { + let slot = parse_slot_id(slot_id)?; + let mut state = self.lock_state()?; + state + .library + .insert_tape(slot, TapeBarcode::new(tape_id)) + .map_err(vtl_status) + } + + fn lock_state(&self) -> ServiceResult> { + self.state + .lock() + .map_err(|_| Status::internal("simulator tape backend mutex poisoned")) + } +} + +#[derive(Debug)] +struct SimulatorState { + library: VirtualTapeLibrary, + acquired: Vec, + next_filemark: Vec, +} + +impl SimulatorState { + fn new(slot_count: u32, drive_count: u32) -> Self { + Self { + library: VirtualTapeLibrary::new(slot_count, drive_count), + acquired: vec![false; drive_count as usize], + next_filemark: vec![0; drive_count as usize], + } + } + + fn drive_endpoint(&self, drive_index: u32) -> ServiceResult { + let drive = self + .library + .drive(ElementAddress::drive(drive_index)) + .map_err(vtl_status)?; + let acquired = self + .acquired + .get(drive_index as usize) + .copied() + .unwrap_or(false); + Ok(common::DriveEndpoint { + drive_id: drive_id_string(drive_index), + device_path: format!("vtl://drive/{drive_index}"), + drive_type: "virtual-lto".to_string(), + status: if acquired { + DriveStatus::DriveInUse as i32 + } else { + DriveStatus::DriveIdle as i32 + }, + current_tape: drive + .loaded_barcode() + .map(|barcode| barcode.as_str().to_string()), + }) + } + + fn drive_index(&self, drive_id: &str) -> ServiceResult { + let address = parse_drive_id(drive_id)?; + self.library.drive(address).map_err(vtl_status)?; + Ok(address.index()) + } + + fn find_slot_with_tape(&self, tape_id: &str) -> ServiceResult { + self.library + .slots() + .iter() + .find(|slot| slot.barcode().map(|barcode| barcode.as_str()) == Some(tape_id)) + .map(|slot| slot.address()) + .ok_or_else(|| { + Status::not_found(format!("tape {tape_id} not found in simulator slots")) + }) + } + + fn find_empty_slot(&self) -> ServiceResult { + self.library + .slots() + .iter() + .find(|slot| slot.is_empty()) + .map(|slot| slot.address()) + .ok_or_else(|| Status::failed_precondition("no empty simulator slot is available")) + } +} + +impl TapeBackend for SimulatorTapeBackend { + fn list_drives(&self) -> ServiceResult> { + let state = self.lock_state()?; + (0..state.library.drives().len() as u32) + .map(|drive_index| state.drive_endpoint(drive_index)) + .collect() + } + + fn get_drive_status(&self, drive_id: &str) -> ServiceResult { + let state = self.lock_state()?; + let drive_index = state.drive_index(drive_id)?; + state.drive_endpoint(drive_index) + } + + fn acquire_drive( + &self, + preferred_drive_id: Option<&str>, + required_tape_id: Option<&str>, + ) -> ServiceResult { + let mut state = self.lock_state()?; + let drive_index = if let Some(preferred) = preferred_drive_id { + state.drive_index(preferred)? + } else { + state + .library + .drives() + .iter() + .find(|drive| { + !state.acquired[drive.address().index() as usize] + && required_tape_id.is_none_or(|tape_id| { + drive.loaded_barcode().map(|barcode| barcode.as_str()) == Some(tape_id) + }) + }) + .map(|drive| drive.address().index()) + .ok_or_else(|| Status::resource_exhausted("no matching simulator drive is free"))? + }; + + let index = drive_index as usize; + if state.acquired[index] { + return Err(Status::failed_precondition(format!( + "drive {} is already acquired", + drive_id_string(drive_index) + ))); + } + if let Some(tape_id) = required_tape_id { + let drive = state + .library + .drive(ElementAddress::drive(drive_index)) + .map_err(vtl_status)?; + if drive.loaded_barcode().map(|barcode| barcode.as_str()) != Some(tape_id) { + return Err(Status::failed_precondition(format!( + "drive {} does not contain required tape {tape_id}", + drive_id_string(drive_index) + ))); + } + } + + state.acquired[index] = true; + let drive = state + .library + .drive(ElementAddress::drive(drive_index)) + .map_err(vtl_status)?; + Ok(AcquireDriveResponse { + drive_id: drive_id_string(drive_index), + current_tape: drive + .loaded_barcode() + .map(|barcode| barcode.as_str().to_string()), + }) + } + + fn release_drive(&self, drive_id: &str) -> ServiceResult<()> { + let mut state = self.lock_state()?; + let drive_index = state.drive_index(drive_id)?; + state.acquired[drive_index as usize] = false; + Ok(()) + } + + fn load_tape(&self, tape_id: &str, drive_id: &str, slot_id: Option<&str>) -> ServiceResult<()> { + let drive = parse_drive_id(drive_id)?; + let mut state = self.lock_state()?; + state.library.drive(drive).map_err(vtl_status)?; + let slot = if let Some(slot_id) = slot_id { + parse_slot_id(slot_id)? + } else { + state.find_slot_with_tape(tape_id)? + }; + let slot_ref = state.library.slot(slot).map_err(vtl_status)?; + if slot_ref.barcode().map(|barcode| barcode.as_str()) != Some(tape_id) { + return Err(Status::failed_precondition(format!( + "slot {slot} does not contain tape {tape_id}" + ))); + } + state.library.load(slot, drive).map_err(vtl_status) + } + + fn unload_tape(&self, drive_id: &str, target_slot_id: Option<&str>) -> ServiceResult<()> { + let drive = parse_drive_id(drive_id)?; + let mut state = self.lock_state()?; + state.library.drive(drive).map_err(vtl_status)?; + let slot = if let Some(slot_id) = target_slot_id { + parse_slot_id(slot_id)? + } else { + state.find_empty_slot()? + }; + state.library.unload(drive, slot).map_err(vtl_status) + } + + fn rewind(&self, drive_id: &str) -> ServiceResult<()> { + let drive = parse_drive_id(drive_id)?; + let mut state = self.lock_state()?; + state.library.rewind(drive).map_err(vtl_status) + } + + fn seek_to_filemark(&self, drive_id: &str, filemark: u32) -> ServiceResult<()> { + let drive = parse_drive_id(drive_id)?; + let mut state = self.lock_state()?; + state.library.rewind(drive).map_err(vtl_status)?; + if filemark > 0 { + state + .library + .seek_filemark(drive, filemark) + .map_err(vtl_status)?; + } + Ok(()) + } + + fn get_tape_media_status(&self, drive_id: &str) -> ServiceResult { + let state = self.lock_state()?; + let drive_index = state.drive_index(drive_id)?; + let drive = state + .library + .drive(ElementAddress::drive(drive_index)) + .map_err(vtl_status)?; + let Some(tape) = drive.loaded_tape() else { + return Ok(TapeMediaStatus { + drive_id: drive_id_string(drive_index), + tape_id: None, + tape_status: TapeStatus::TapeOffline as i32, + capacity_bytes: SIMULATED_TAPE_CAPACITY_BYTES, + used_bytes: 0, + remaining_bytes: SIMULATED_TAPE_CAPACITY_BYTES, + current_position: 0, + current_filemark: 0, + is_write_protected: false, + }); + }; + + let used_bytes = tape.used_bytes(); + Ok(TapeMediaStatus { + drive_id: drive_id_string(drive_index), + tape_id: Some(tape.barcode().as_str().to_string()), + tape_status: TapeStatus::TapeOnline as i32, + capacity_bytes: SIMULATED_TAPE_CAPACITY_BYTES, + used_bytes, + remaining_bytes: SIMULATED_TAPE_CAPACITY_BYTES.saturating_sub(used_bytes), + current_position: tape.current_position(), + current_filemark: tape.current_filemark(), + is_write_protected: false, + }) + } + + fn inventory(&self) -> ServiceResult { + let state = self.lock_state()?; + let mut slots = Vec::new(); + for slot in state.library.slots() { + slots.push(SlotInfo { + slot_id: slot_id(slot.address().index()), + tape_id: slot.barcode().map(|barcode| barcode.as_str().to_string()), + is_drive: false, + drive_id: None, + is_import_export: false, + }); + } + for drive in state.library.drives() { + slots.push(SlotInfo { + slot_id: drive_id_string(drive.address().index()), + tape_id: drive + .loaded_barcode() + .map(|barcode| barcode.as_str().to_string()), + is_drive: true, + drive_id: Some(drive_id_string(drive.address().index())), + is_import_export: false, + }); + } + Ok(InventoryResponse { slots }) + } + + fn write_bundle(&self, drive_id: &str, data: &[u8]) -> ServiceResult<(u32, u32)> { + let drive = parse_drive_id(drive_id)?; + let mut state = self.lock_state()?; + state.library.drive(drive).map_err(vtl_status)?; + let drive_index = drive.index() as usize; + let filemark_start = state.next_filemark[drive_index]; + state.library.write(drive, data).map_err(vtl_status)?; + state.library.write_filemark(drive).map_err(vtl_status)?; + state.next_filemark[drive_index] += 1; + Ok((filemark_start, state.next_filemark[drive_index])) + } + + fn read_bundle(&self, drive_id: &str, filemark: u32, length: u64) -> ServiceResult> { + let drive = parse_drive_id(drive_id)?; + let mut state = self.lock_state()?; + state.library.rewind(drive).map_err(vtl_status)?; + if filemark > 0 { + state + .library + .seek_filemark(drive, filemark) + .map_err(vtl_status)?; + } + let max_len = if length == 0 { + usize::MAX + } else { + length.min(usize::MAX as u64) as usize + }; + state.library.read(drive, max_len).map_err(vtl_status) + } } #[tonic::async_trait] impl TapeService for TapeServiceImpl { async fn write_bundle( &self, - _req: Request>, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.write_bundle")) + req: Request>, + ) -> ServiceResult> { + self.write_bundle_from_messages(req.into_inner()) + .await + .map(Response::new) } - type ReadBundleStream = - tokio_stream::wrappers::ReceiverStream>; + type ReadBundleStream = ReceiverStream>; async fn read_bundle( &self, - _req: Request, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.read_bundle")) + req: Request, + ) -> ServiceResult> { + let req = req.into_inner(); + let filemark = match req.location { + Some(Location::Filemark(filemark)) => filemark, + Some(Location::BlockOffset(_)) => { + return Err(Status::unimplemented( + "phase-1 simulator supports filemark-based reads; block offset reads require live SCSI/tape integration", + )) + } + None => return Err(Status::invalid_argument("read_bundle location is required")), + }; + let data = self + .backend + .read_bundle(&req.drive_id, filemark, req.length)?; + let (tx, rx) = mpsc::channel(2); + tx.send(Ok(ReadBundleResponse { + payload: Some(ReadPayload::Meta(ReadBundleMeta { + total_size: data.len() as u64, + checksum: None, + })), + })) + .await + .map_err(|_| Status::cancelled("read_bundle receiver dropped before metadata send"))?; + if !data.is_empty() { + tx.send(Ok(ReadBundleResponse { + payload: Some(ReadPayload::Data(data)), + })) + .await + .map_err(|_| Status::cancelled("read_bundle receiver dropped before data send"))?; + } + Ok(Response::new(ReceiverStream::new(rx))) } - async fn list_drives( - &self, - _req: Request<()>, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.list_drives")) + async fn list_drives(&self, _req: Request<()>) -> ServiceResult> { + Ok(Response::new(ListDrivesResponse { + drives: self.backend.list_drives()?, + })) } async fn get_drive_status( &self, - _req: Request, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.get_drive_status")) + req: Request, + ) -> ServiceResult> { + Ok(Response::new( + self.backend.get_drive_status(&req.into_inner().drive_id)?, + )) } async fn acquire_drive( &self, - _req: Request, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.acquire_drive")) + req: Request, + ) -> ServiceResult> { + let req = req.into_inner(); + Ok(Response::new(self.backend.acquire_drive( + req.preferred_drive_id.as_deref(), + req.required_tape_id.as_deref(), + )?)) } async fn release_drive( &self, - _req: Request, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.release_drive")) + req: Request, + ) -> ServiceResult> { + self.backend.release_drive(&req.into_inner().drive_id)?; + Ok(Response::new(())) } - async fn load_tape( - &self, - _req: Request, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.load_tape")) + async fn load_tape(&self, req: Request) -> ServiceResult> { + let req = req.into_inner(); + self.backend + .load_tape(&req.tape_id, &req.drive_id, req.slot_id.as_deref())?; + Ok(Response::new(())) } - async fn unload_tape( - &self, - _req: Request, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.unload_tape")) + async fn unload_tape(&self, req: Request) -> ServiceResult> { + let req = req.into_inner(); + self.backend + .unload_tape(&req.drive_id, req.target_slot_id.as_deref())?; + Ok(Response::new(())) } - async fn rewind( - &self, - _req: Request, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.rewind")) + async fn rewind(&self, req: Request) -> ServiceResult> { + self.backend.rewind(&req.into_inner().drive_id)?; + Ok(Response::new(())) } async fn seek_to_filemark( &self, - _req: Request, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.seek_to_filemark")) + req: Request, + ) -> ServiceResult> { + let req = req.into_inner(); + self.backend.seek_to_filemark(&req.drive_id, req.filemark)?; + Ok(Response::new(())) } async fn get_tape_media_status( &self, - _req: Request, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.get_tape_media_status")) + req: Request, + ) -> ServiceResult> { + Ok(Response::new( + self.backend + .get_tape_media_status(&req.into_inner().drive_id)?, + )) } - async fn inventory( - &self, - _req: Request<()>, - ) -> std::result::Result, Status> { - Err(phase1_unimplemented("tape.inventory")) + async fn inventory(&self, _req: Request<()>) -> ServiceResult> { + Ok(Response::new(self.backend.inventory()?)) + } +} + +fn parse_slot_id(slot_id: &str) -> ServiceResult { + let raw = slot_id.strip_prefix("slot-").unwrap_or(slot_id); + let index = raw + .parse::() + .map_err(|_| Status::invalid_argument(format!("invalid slot id: {slot_id}")))?; + if index == 0 { + return Err(Status::invalid_argument("slot ids are 1-based")); + } + Ok(ElementAddress::slot(index)) +} + +fn parse_drive_id(drive_id: &str) -> ServiceResult { + let raw = drive_id.strip_prefix("drive-").unwrap_or(drive_id); + let index = raw + .parse::() + .map_err(|_| Status::invalid_argument(format!("invalid drive id: {drive_id}")))?; + Ok(ElementAddress::drive(index)) +} + +fn slot_id(index: u32) -> String { + format!("slot-{index}") +} + +fn drive_id_string(index: u32) -> String { + format!("drive-{index}") +} + +fn vtl_status(error: coldstore_vtl::VtlError) -> Status { + match error { + coldstore_vtl::VtlError::SlotEmpty(_) + | coldstore_vtl::VtlError::SlotOccupied(_) + | coldstore_vtl::VtlError::DriveEmpty(_) + | coldstore_vtl::VtlError::DriveOccupied(_) + | coldstore_vtl::VtlError::FilemarkNotFound => { + Status::failed_precondition(error.to_string()) + } + coldstore_vtl::VtlError::SlotOutOfRange(_) + | coldstore_vtl::VtlError::DriveOutOfRange(_) + | coldstore_vtl::VtlError::WrongElement { .. } + | coldstore_vtl::VtlError::InvalidLsscsiLine(_) + | coldstore_vtl::VtlError::InvalidElementAddress(_) => { + Status::invalid_argument(error.to_string()) + } + coldstore_vtl::VtlError::CommandFailed { .. } => Status::unavailable(error.to_string()), + coldstore_vtl::VtlError::Io(_) => Status::unavailable(error.to_string()), } } diff --git a/crates/tape/tests/simulator_service.rs b/crates/tape/tests/simulator_service.rs new file mode 100644 index 0000000..dcedc86 --- /dev/null +++ b/crates/tape/tests/simulator_service.rs @@ -0,0 +1,179 @@ +use coldstore_common::config::TapeConfig; +use coldstore_proto::common::{DriveStatus, TapeStatus}; +use coldstore_proto::tape::read_bundle_request::Location; +use coldstore_proto::tape::read_bundle_response::Payload as ReadPayload; +use coldstore_proto::tape::tape_service_server::TapeService; +use coldstore_proto::tape::write_bundle_request::Payload as WritePayload; +use coldstore_proto::tape::{ + LoadTapeRequest, ReadBundleRequest, RewindRequest, SeekToFilemarkRequest, UnloadTapeRequest, + WriteBundleMeta, WriteBundleRequest, +}; +use coldstore_tape::service::{SimulatorTapeBackend, TapeServiceImpl}; +use tokio_stream::StreamExt; +use tonic::Request; + +#[tokio::test] +async fn simulator_backend_exposes_inventory_and_drive_operations_through_service() { + let backend = SimulatorTapeBackend::new(3, 1); + backend.insert_tape("slot-1", "TAPE0001L9").unwrap(); + let service = TapeServiceImpl::new_with_backend(TapeConfig::default(), backend); + + let drives = service + .list_drives(Request::new(())) + .await + .unwrap() + .into_inner() + .drives; + assert_eq!(drives.len(), 1); + assert_eq!(drives[0].drive_id, "drive-0"); + assert_eq!(drives[0].status, DriveStatus::DriveIdle as i32); + assert_eq!(drives[0].current_tape.as_deref(), None); + + let inventory = service + .inventory(Request::new(())) + .await + .unwrap() + .into_inner(); + let slot_1 = inventory + .slots + .iter() + .find(|slot| slot.slot_id == "slot-1") + .unwrap(); + assert!(!slot_1.is_drive); + assert_eq!(slot_1.tape_id.as_deref(), Some("TAPE0001L9")); + + service + .load_tape(Request::new(LoadTapeRequest { + tape_id: "TAPE0001L9".to_string(), + drive_id: "drive-0".to_string(), + slot_id: None, + })) + .await + .unwrap(); + + let status = service + .get_drive_status(Request::new(coldstore_proto::tape::GetDriveStatusRequest { + drive_id: "drive-0".to_string(), + })) + .await + .unwrap() + .into_inner(); + assert_eq!(status.current_tape.as_deref(), Some("TAPE0001L9")); + + let media = service + .get_tape_media_status(Request::new( + coldstore_proto::tape::GetTapeMediaStatusRequest { + drive_id: "drive-0".to_string(), + }, + )) + .await + .unwrap() + .into_inner(); + assert_eq!(media.tape_id.as_deref(), Some("TAPE0001L9")); + assert_eq!(media.tape_status, TapeStatus::TapeOnline as i32); + + service + .unload_tape(Request::new(UnloadTapeRequest { + drive_id: "drive-0".to_string(), + target_slot_id: Some("slot-2".to_string()), + })) + .await + .unwrap(); + + let inventory = service + .inventory(Request::new(())) + .await + .unwrap() + .into_inner(); + let slot_2 = inventory + .slots + .iter() + .find(|slot| slot.slot_id == "slot-2") + .unwrap(); + assert_eq!(slot_2.tape_id.as_deref(), Some("TAPE0001L9")); +} + +#[tokio::test] +async fn simulator_service_writes_filemark_delimited_bundles_and_reads_them_back() { + let backend = SimulatorTapeBackend::new(2, 1); + backend.insert_tape("slot-1", "TAPE0002L9").unwrap(); + let service = TapeServiceImpl::new_with_backend(TapeConfig::default(), backend); + + service + .load_tape(Request::new(LoadTapeRequest { + tape_id: "TAPE0002L9".to_string(), + drive_id: "drive-0".to_string(), + slot_id: Some("slot-1".to_string()), + })) + .await + .unwrap(); + + let response = service + .write_bundle_from_messages(tokio_stream::iter(vec![ + Ok(WriteBundleRequest { + payload: Some(WritePayload::Meta(WriteBundleMeta { + drive_id: "drive-0".to_string(), + bundle_id: "bundle-a".to_string(), + total_size: 6, + object_count: 1, + block_size: 262_144, + })), + }), + Ok(WriteBundleRequest { + payload: Some(WritePayload::Data(b"abc".to_vec())), + }), + Ok(WriteBundleRequest { + payload: Some(WritePayload::Data(b"def".to_vec())), + }), + ])) + .await + .unwrap(); + assert!(response.success); + assert_eq!(response.bundle_id, "bundle-a"); + assert_eq!(response.bytes_written, 6); + assert_eq!(response.filemark_start, 0); + assert_eq!(response.filemark_end, 1); + + let media = service + .get_tape_media_status(Request::new( + coldstore_proto::tape::GetTapeMediaStatusRequest { + drive_id: "drive-0".to_string(), + }, + )) + .await + .unwrap() + .into_inner(); + assert_eq!(media.used_bytes, 6); + assert_eq!(media.current_filemark, 1); + + service + .rewind(Request::new(RewindRequest { + drive_id: "drive-0".to_string(), + })) + .await + .unwrap(); + + let mut stream = service + .read_bundle(Request::new(ReadBundleRequest { + drive_id: "drive-0".to_string(), + location: Some(Location::Filemark(0)), + length: 6, + })) + .await + .unwrap() + .into_inner(); + + let meta = stream.next().await.unwrap().unwrap().payload.unwrap(); + assert!(matches!(meta, ReadPayload::Meta(_))); + let data = stream.next().await.unwrap().unwrap().payload.unwrap(); + assert_eq!(data, ReadPayload::Data(b"abcdef".to_vec())); + assert!(stream.next().await.is_none()); + + service + .seek_to_filemark(Request::new(SeekToFilemarkRequest { + drive_id: "drive-0".to_string(), + filemark: 1, + })) + .await + .unwrap(); +} diff --git a/crates/vtl/Cargo.toml b/crates/vtl/Cargo.toml new file mode 100644 index 0000000..68ec36b --- /dev/null +++ b/crates/vtl/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "coldstore-vtl" +description = "ColdStore virtual tape library harness and mhVTL command wrappers" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +thiserror = { workspace = true } + +[dev-dependencies] diff --git a/crates/vtl/README.md b/crates/vtl/README.md new file mode 100644 index 0000000..8a009f6 --- /dev/null +++ b/crates/vtl/README.md @@ -0,0 +1,51 @@ +# coldstore-vtl + +ColdStore 的虚拟磁带库 harness。该 crate 的目标不是在开发机上自动安装或启动 mhVTL，而是先把后续 TapeService 闭环开发需要的边界固化下来： + +- 安全单测：不执行系统命令、不加载内核模块、不创建 `/dev/st*` / `/dev/sg*` / `/dev/sch*`。 +- 命令封装：稳定暴露 `lsscsi`、`mtx`、`mt`、`sg3_utils`、`dd`、`vtlcmd` 的命令构造。 +- 真实后端预留：在独立 VM/专用测试机安装 mhVTL 后，用 `SystemCommandRunner` 执行相同命令。 +- 内存模拟器：提供 medium changer + tape drive 的 slot/barcode/load/unload/rewind/filemark/read/write 行为，供 phase-1/phase-2 单测使用。 + +## 模块分层 + +| 模块 | 职责 | 是否触碰宿主 | +|---|---|---| +| `model` | `TapeBarcode`、`ElementAddress`、`VirtualTape`、filemark/cursor 模型 | 否 | +| `interface` | `MediumChanger`、`TapeDrive`、`TapeInventory` trait | 否 | +| `simulator` | 纯内存 VTL，模拟 slot、drive、barcode、load/unload、rewind、filemark、read/write | 否 | +| `discover` | 解析 `lsscsi -g` 输出为 `ScsiInventory` | 否 | +| `command` | `CommandSpec`、`CommandRunner`、`SystemCommandRunner`、`RecordedCommandRunner` | 只有 `SystemCommandRunner` 会执行 | +| `mhvtl` | mhVTL/live 工具链命令构造和可选执行入口 | 构造命令不触碰；执行需显式 runner | + +## 安全验证 + +```bash +cargo test -p coldstore-vtl --lib --tests +``` + +这只运行单测，不安装系统工具，不访问真实设备。 + +## 后续 live mhVTL 使用边界 + +live 环境必须放在专用 VM 或测试机中，不建议直接在当前开发宿主机执行： + +1. 执行前人工阅读 `scripts/setup-mhvtl-env.sh`。 +2. 只在专用 VM 中运行： + ```bash + scripts/setup-mhvtl-env.sh --execute --start-services + ``` +3. 验证设备发现： + ```bash + lsscsi -g + ``` +4. 用 `coldstore-vtl::mhvtl::MhvtlToolchain` 生成命令，再由上层测试 harness 决定是否执行。 + +## 当前已覆盖能力 + +- `lsscsi -g` 样例解析：识别 `mediumx` / `tape`，抽取 HCTL、vendor、product、revision、`/dev/schX`、`/dev/stX`、`/dev/nstX`、`/dev/sgX`。 +- `mtx` 命令封装：status、transfer、load、unload。 +- `mt` 命令封装：status、rewind、offline、weof、fsf。 +- `sg3_utils` 命令封装：sg_inq、sg_turs、sg_logs、sg_modes。 +- `dd` 命令封装：向 tape device 写入/读取数据。 +- 内存 VTL：slot 插带、load/unload、drive 读写、rewind、filemark seek。 diff --git a/crates/vtl/src/command.rs b/crates/vtl/src/command.rs new file mode 100644 index 0000000..e02c652 --- /dev/null +++ b/crates/vtl/src/command.rs @@ -0,0 +1,146 @@ +use crate::error::{Result, VtlError}; +use std::cell::RefCell; +use std::collections::VecDeque; +use std::fmt; +use std::process::Command; + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct CommandSpec { + program: String, + args: Vec, +} + +impl CommandSpec { + pub fn new(program: impl Into) -> Self { + Self { + program: program.into(), + args: Vec::new(), + } + } + + pub fn arg(mut self, arg: impl Into) -> Self { + self.args.push(arg.into()); + self + } + + pub fn args(mut self, args: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.args.extend(args.into_iter().map(Into::into)); + self + } + + pub fn program(&self) -> &str { + &self.program + } + + pub fn arguments(&self) -> &[String] { + &self.args + } + + pub fn argv(&self) -> Vec<&str> { + std::iter::once(self.program.as_str()) + .chain(self.args.iter().map(String::as_str)) + .collect() + } +} + +impl fmt::Display for CommandSpec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", shell_quote(&self.program))?; + for arg in &self.args { + write!(f, " {}", shell_quote(arg))?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Default)] +pub struct CommandOutput { + pub status: i32, + pub stdout: String, + pub stderr: String, +} + +impl CommandOutput { + pub fn success(stdout: impl Into) -> Self { + Self { + status: 0, + stdout: stdout.into(), + stderr: String::new(), + } + } +} + +pub trait CommandRunner { + fn run(&self, command: &CommandSpec) -> Result; + + fn run_checked(&self, command: &CommandSpec) -> Result { + let output = self.run(command)?; + if output.status == 0 { + Ok(output) + } else { + Err(VtlError::CommandFailed { + command: command.clone(), + status: output.status, + stderr: output.stderr, + }) + } + } +} + +#[derive(Debug, Default, Copy, Clone)] +pub struct SystemCommandRunner; + +impl CommandRunner for SystemCommandRunner { + fn run(&self, command: &CommandSpec) -> Result { + let output = Command::new(command.program()) + .args(command.arguments()) + .output()?; + Ok(CommandOutput { + status: output.status.code().unwrap_or(-1), + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + }) + } +} + +#[derive(Debug, Default)] +pub struct RecordedCommandRunner { + commands: RefCell>, + responses: RefCell>, +} + +impl RecordedCommandRunner { + pub fn push_response(&self, response: CommandOutput) { + self.responses.borrow_mut().push_back(response); + } + + pub fn push_stdout(&self, stdout: impl Into) { + self.push_response(CommandOutput::success(stdout)); + } + + pub fn commands(&self) -> Vec { + self.commands.borrow().clone() + } +} + +impl CommandRunner for RecordedCommandRunner { + fn run(&self, command: &CommandSpec) -> Result { + self.commands.borrow_mut().push(command.clone()); + Ok(self.responses.borrow_mut().pop_front().unwrap_or_default()) + } +} + +fn shell_quote(value: &str) -> String { + if value + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '/' | '_' | '-' | '.' | ':' | '=')) + { + value.to_string() + } else { + format!("'{}'", value.replace("'", "'\\''")) + } +} diff --git a/crates/vtl/src/discover.rs b/crates/vtl/src/discover.rs new file mode 100644 index 0000000..7bd8309 --- /dev/null +++ b/crates/vtl/src/discover.rs @@ -0,0 +1,144 @@ +use crate::error::{Result, VtlError}; + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +pub enum ScsiDeviceKind { + TapeDrive, + MediumChanger, + Disk, + CdDvd, + Other, +} + +impl ScsiDeviceKind { + fn from_lsscsi(value: &str) -> Self { + match value { + "tape" => Self::TapeDrive, + "mediumx" => Self::MediumChanger, + "disk" => Self::Disk, + "cd/dvd" => Self::CdDvd, + _ => Self::Other, + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct ScsiDevice { + pub hctl: String, + pub kind: ScsiDeviceKind, + pub vendor: String, + pub product: String, + pub revision: String, + pub primary_device: Option, + pub sg_device: Option, + pub non_rewinding_device: Option, +} + +#[derive(Debug, Clone, Eq, PartialEq, Default)] +pub struct ScsiInventory { + devices: Vec, +} + +impl ScsiInventory { + pub fn new(devices: Vec) -> Self { + Self { devices } + } + + pub fn devices(&self) -> &[ScsiDevice] { + &self.devices + } + + pub fn tape_drives(&self) -> Vec<&ScsiDevice> { + self.devices + .iter() + .filter(|device| device.kind == ScsiDeviceKind::TapeDrive) + .collect() + } + + pub fn medium_changers(&self) -> Vec<&ScsiDevice> { + self.devices + .iter() + .filter(|device| device.kind == ScsiDeviceKind::MediumChanger) + .collect() + } +} + +pub fn parse_lsscsi(output: &str) -> Result { + let mut devices = Vec::new(); + for line in output + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + { + devices.push(parse_lsscsi_line(line)?); + } + Ok(ScsiInventory::new(devices)) +} + +fn parse_lsscsi_line(line: &str) -> Result { + let Some(close) = line.find(']') else { + return Err(VtlError::InvalidLsscsiLine(line.into())); + }; + let hctl = line + .strip_prefix('[') + .and_then(|rest| rest.get(..close - 1)) + .ok_or_else(|| VtlError::InvalidLsscsiLine(line.into()))? + .to_string(); + + let tokens: Vec<&str> = line[close + 1..].split_whitespace().collect(); + if tokens.len() < 4 { + return Err(VtlError::InvalidLsscsiLine(line.into())); + } + + let kind = ScsiDeviceKind::from_lsscsi(tokens[0]); + let vendor = tokens[1].to_string(); + let product = tokens[2].to_string(); + let revision = tokens[3].to_string(); + let primary_device = tokens + .get(4) + .filter(|value| value.starts_with("/dev/")) + .map(|value| (*value).to_string()); + let sg_device = tokens + .get(5) + .filter(|value| value.starts_with("/dev/sg")) + .map(|value| (*value).to_string()) + .or_else(|| { + tokens + .iter() + .find(|value| value.starts_with("/dev/sg")) + .map(|value| (*value).to_string()) + }); + let non_rewinding_device = primary_device + .as_deref() + .and_then(non_rewinding_tape_device); + + Ok(ScsiDevice { + hctl, + kind, + vendor, + product, + revision, + primary_device, + sg_device, + non_rewinding_device, + }) +} + +fn non_rewinding_tape_device(path: &str) -> Option { + let suffix = path.strip_prefix("/dev/st")?; + if suffix.chars().all(|c| c.is_ascii_digit()) { + Some(format!("/dev/nst{suffix}")) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_non_lsscsi_lines() { + let err = parse_lsscsi_line("not a device").expect_err("invalid line"); + assert!(matches!(err, VtlError::InvalidLsscsiLine(_))); + } +} diff --git a/crates/vtl/src/error.rs b/crates/vtl/src/error.rs new file mode 100644 index 0000000..9925c60 --- /dev/null +++ b/crates/vtl/src/error.rs @@ -0,0 +1,50 @@ +use crate::command::CommandSpec; +use thiserror::Error; + +pub type Result = std::result::Result; + +#[derive(Debug, Error)] +pub enum VtlError { + #[error("invalid SCSI/lsscsi line: {0}")] + InvalidLsscsiLine(String), + + #[error("invalid element address: {0}")] + InvalidElementAddress(String), + + #[error("slot {0} does not exist")] + SlotOutOfRange(u32), + + #[error("drive {0} does not exist")] + DriveOutOfRange(u32), + + #[error("slot {0} is empty")] + SlotEmpty(u32), + + #[error("slot {0} is already occupied")] + SlotOccupied(u32), + + #[error("drive {0} is empty")] + DriveEmpty(u32), + + #[error("drive {0} is already occupied")] + DriveOccupied(u32), + + #[error("expected {expected} element, got {actual}")] + WrongElement { + expected: &'static str, + actual: String, + }, + + #[error("filemark not found before end of tape")] + FilemarkNotFound, + + #[error("command failed: {command}; status={status}; stderr={stderr}")] + CommandFailed { + command: CommandSpec, + status: i32, + stderr: String, + }, + + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), +} diff --git a/crates/vtl/src/interface.rs b/crates/vtl/src/interface.rs new file mode 100644 index 0000000..df12a67 --- /dev/null +++ b/crates/vtl/src/interface.rs @@ -0,0 +1,20 @@ +use crate::error::Result; +use crate::model::{ElementAddress, TapeBarcode}; + +pub trait MediumChanger { + fn move_medium(&mut self, from: ElementAddress, to: ElementAddress) -> Result<()>; + fn load(&mut self, slot: ElementAddress, drive: ElementAddress) -> Result<()>; + fn unload(&mut self, drive: ElementAddress, slot: ElementAddress) -> Result<()>; +} + +pub trait TapeDrive { + fn rewind(&mut self, drive: ElementAddress) -> Result<()>; + fn seek_filemark(&mut self, drive: ElementAddress, count: u32) -> Result<()>; + fn write_filemark(&mut self, drive: ElementAddress) -> Result<()>; + fn write(&mut self, drive: ElementAddress, data: &[u8]) -> Result<()>; + fn read(&mut self, drive: ElementAddress, max_len: usize) -> Result>; +} + +pub trait TapeInventory { + fn insert_tape(&mut self, slot: ElementAddress, barcode: TapeBarcode) -> Result<()>; +} diff --git a/crates/vtl/src/lib.rs b/crates/vtl/src/lib.rs new file mode 100644 index 0000000..3051ae4 --- /dev/null +++ b/crates/vtl/src/lib.rs @@ -0,0 +1,22 @@ +//! ColdStore virtual tape library harness and mhVTL command wrappers. +//! +//! The crate is split into safe, unit-testable pieces: +//! - [`model`] contains device/tape/changer value types. +//! - [`discover`] parses `lsscsi -g` output without touching the host. +//! - [`command`] describes and executes external commands behind a small trait. +//! - [`mhvtl`] exposes stable `lsscsi`/`mtx`/`mt`/`sg3_utils` command builders. +//! - [`simulator`] provides an in-memory VTL for unit tests and phase-1 logic. +//! +//! Live mhVTL usage is intentionally explicit: callers must provide a +//! [`command::SystemCommandRunner`] or another runner that actually executes +//! host commands. + +pub mod command; +pub mod discover; +pub mod error; +pub mod interface; +pub mod mhvtl; +pub mod model; +pub mod simulator; + +pub use error::{Result, VtlError}; diff --git a/crates/vtl/src/mhvtl.rs b/crates/vtl/src/mhvtl.rs new file mode 100644 index 0000000..8727220 --- /dev/null +++ b/crates/vtl/src/mhvtl.rs @@ -0,0 +1,208 @@ +use crate::command::{CommandOutput, CommandRunner, CommandSpec}; +use crate::discover::{parse_lsscsi, ScsiInventory}; +use crate::error::Result; +use crate::model::ElementAddress; +use std::path::Path; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct ToolPaths { + pub lsscsi: String, + pub mtx: String, + pub mt: String, + pub dd: String, + pub sg_inq: String, + pub sg_turs: String, + pub sg_logs: String, + pub sg_modes: String, + pub vtlcmd: String, +} + +impl Default for ToolPaths { + fn default() -> Self { + Self { + lsscsi: "lsscsi".into(), + mtx: "mtx".into(), + mt: "mt".into(), + dd: "dd".into(), + sg_inq: "sg_inq".into(), + sg_turs: "sg_turs".into(), + sg_logs: "sg_logs".into(), + sg_modes: "sg_modes".into(), + vtlcmd: "vtlcmd".into(), + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Default)] +pub struct MhvtlToolchain { + paths: ToolPaths, +} + +impl MhvtlToolchain { + pub fn new(paths: ToolPaths) -> Self { + Self { paths } + } + + pub fn paths(&self) -> &ToolPaths { + &self.paths + } + + pub fn discover_command(&self) -> CommandSpec { + CommandSpec::new(self.paths.lsscsi.clone()).arg("-g") + } + + pub fn library_status_command(&self, changer_sg: &str) -> CommandSpec { + CommandSpec::new(self.paths.mtx.clone()).args(["-f", changer_sg, "status"]) + } + + pub fn move_medium_command( + &self, + changer_sg: &str, + from: ElementAddress, + to: ElementAddress, + ) -> CommandSpec { + CommandSpec::new(self.paths.mtx.clone()).args([ + "-f".to_string(), + changer_sg.to_string(), + "transfer".to_string(), + from.mtx_address().to_string(), + to.mtx_address().to_string(), + ]) + } + + pub fn load_command( + &self, + changer_sg: &str, + slot: ElementAddress, + drive: ElementAddress, + ) -> CommandSpec { + CommandSpec::new(self.paths.mtx.clone()).args([ + "-f".to_string(), + changer_sg.to_string(), + "load".to_string(), + slot.mtx_address().to_string(), + drive.mtx_address().to_string(), + ]) + } + + pub fn unload_command( + &self, + changer_sg: &str, + slot: ElementAddress, + drive: ElementAddress, + ) -> CommandSpec { + CommandSpec::new(self.paths.mtx.clone()).args([ + "-f".to_string(), + changer_sg.to_string(), + "unload".to_string(), + slot.mtx_address().to_string(), + drive.mtx_address().to_string(), + ]) + } + + pub fn tape_status_command(&self, tape_device: &str) -> CommandSpec { + CommandSpec::new(self.paths.mt.clone()).args(["-f", tape_device, "status"]) + } + + pub fn rewind_command(&self, tape_device: &str) -> CommandSpec { + CommandSpec::new(self.paths.mt.clone()).args(["-f", tape_device, "rewind"]) + } + + pub fn offline_command(&self, tape_device: &str) -> CommandSpec { + CommandSpec::new(self.paths.mt.clone()).args(["-f", tape_device, "offline"]) + } + + pub fn write_filemark_command(&self, tape_device: &str, count: u32) -> CommandSpec { + CommandSpec::new(self.paths.mt.clone()).args([ + "-f".to_string(), + tape_device.to_string(), + "weof".to_string(), + count.to_string(), + ]) + } + + pub fn seek_filemark_command(&self, tape_device: &str, count: u32) -> CommandSpec { + CommandSpec::new(self.paths.mt.clone()).args([ + "-f".to_string(), + tape_device.to_string(), + "fsf".to_string(), + count.to_string(), + ]) + } + + pub fn write_tape_command( + &self, + input_path: impl AsRef, + tape_device: &str, + block_size: u32, + ) -> CommandSpec { + CommandSpec::new(self.paths.dd.clone()).args([ + format!("if={}", input_path.as_ref().display()), + format!("of={tape_device}"), + format!("bs={block_size}"), + "status=none".to_string(), + ]) + } + + pub fn read_tape_command( + &self, + tape_device: &str, + output_path: impl AsRef, + block_size: u32, + count: u32, + ) -> CommandSpec { + CommandSpec::new(self.paths.dd.clone()).args([ + format!("if={tape_device}"), + format!("of={}", output_path.as_ref().display()), + format!("bs={block_size}"), + format!("count={count}"), + "iflag=fullblock".to_string(), + "status=none".to_string(), + ]) + } + + pub fn inquiry_command(&self, sg_device: &str) -> CommandSpec { + CommandSpec::new(self.paths.sg_inq.clone()).arg(sg_device) + } + + pub fn test_unit_ready_command(&self, sg_device: &str) -> CommandSpec { + CommandSpec::new(self.paths.sg_turs.clone()).arg(sg_device) + } + + pub fn sg_logs_command(&self, sg_device: &str) -> CommandSpec { + CommandSpec::new(self.paths.sg_logs.clone()).arg(sg_device) + } + + pub fn sg_modes_command(&self, sg_device: &str) -> CommandSpec { + CommandSpec::new(self.paths.sg_modes.clone()).arg(sg_device) + } + + pub fn vtlcmd_command(&self, args: I) -> CommandSpec + where + I: IntoIterator, + S: Into, + { + CommandSpec::new(self.paths.vtlcmd.clone()).args(args) + } + + pub fn discover(&self, runner: &dyn CommandRunner) -> Result { + let output = runner.run_checked(&self.discover_command())?; + parse_lsscsi(&output.stdout) + } + + pub fn run_library_status( + &self, + runner: &dyn CommandRunner, + changer_sg: &str, + ) -> Result { + runner.run_checked(&self.library_status_command(changer_sg)) + } + + pub fn run_test_unit_ready( + &self, + runner: &dyn CommandRunner, + sg_device: &str, + ) -> Result { + runner.run_checked(&self.test_unit_ready_command(sg_device)) + } +} diff --git a/crates/vtl/src/model.rs b/crates/vtl/src/model.rs new file mode 100644 index 0000000..1ab3767 --- /dev/null +++ b/crates/vtl/src/model.rs @@ -0,0 +1,223 @@ +use std::fmt; + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub struct TapeBarcode(String); + +impl TapeBarcode { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for TapeBarcode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +pub enum ElementKind { + Slot, + Drive, + ImportExport, +} + +impl fmt::Display for ElementKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ElementKind::Slot => f.write_str("slot"), + ElementKind::Drive => f.write_str("drive"), + ElementKind::ImportExport => f.write_str("import-export"), + } + } +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +pub struct ElementAddress { + kind: ElementKind, + index: u32, +} + +impl ElementAddress { + pub fn slot(index: u32) -> Self { + Self { + kind: ElementKind::Slot, + index, + } + } + + pub fn drive(index: u32) -> Self { + Self { + kind: ElementKind::Drive, + index, + } + } + + pub fn import_export(index: u32) -> Self { + Self { + kind: ElementKind::ImportExport, + index, + } + } + + pub fn kind(self) -> ElementKind { + self.kind + } + + pub fn index(self) -> u32 { + self.index + } + + /// Numeric element value used by `mtx` style commands. + /// + /// `mtx load/unload` uses slot number + drive index; `mtx transfer` accepts + /// numeric element addresses. We keep this value explicit so live wrappers + /// can be reviewed before they touch a host. + pub fn mtx_address(self) -> u32 { + self.index + } +} + +impl fmt::Display for ElementAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}:{}", self.kind, self.index) + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum TapeRecord { + Data(Vec), + Filemark, +} + +#[derive(Debug, Clone, Eq, PartialEq, Default)] +pub struct TapeCursor { + pub record_index: usize, + pub byte_offset: usize, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct VirtualTape { + barcode: TapeBarcode, + records: Vec, + cursor: TapeCursor, +} + +impl VirtualTape { + pub fn new(barcode: TapeBarcode) -> Self { + Self { + barcode, + records: Vec::new(), + cursor: TapeCursor::default(), + } + } + + pub fn barcode(&self) -> &TapeBarcode { + &self.barcode + } + + pub fn records(&self) -> &[TapeRecord] { + &self.records + } + + pub fn cursor(&self) -> &TapeCursor { + &self.cursor + } + + pub fn used_bytes(&self) -> u64 { + self.records + .iter() + .map(|record| match record { + TapeRecord::Data(data) => data.len() as u64, + TapeRecord::Filemark => 0, + }) + .sum() + } + + pub fn current_position(&self) -> u64 { + let completed_bytes: u64 = self.records[..self.cursor.record_index.min(self.records.len())] + .iter() + .map(|record| match record { + TapeRecord::Data(data) => data.len() as u64, + TapeRecord::Filemark => 0, + }) + .sum(); + completed_bytes + self.cursor.byte_offset as u64 + } + + pub fn current_filemark(&self) -> u32 { + self.records[..self.cursor.record_index.min(self.records.len())] + .iter() + .filter(|record| matches!(record, TapeRecord::Filemark)) + .count() as u32 + } + + pub(crate) fn rewind(&mut self) { + self.cursor = TapeCursor::default(); + } + + pub(crate) fn append_data(&mut self, data: &[u8]) { + if self.cursor.record_index < self.records.len() { + self.records.truncate(self.cursor.record_index); + } + self.records.push(TapeRecord::Data(data.to_vec())); + self.cursor.record_index = self.records.len(); + self.cursor.byte_offset = 0; + } + + pub(crate) fn append_filemark(&mut self) { + if self.cursor.record_index < self.records.len() { + self.records.truncate(self.cursor.record_index); + } + self.records.push(TapeRecord::Filemark); + self.cursor.record_index = self.records.len(); + self.cursor.byte_offset = 0; + } + + pub(crate) fn read(&mut self, max_len: usize) -> Vec { + let mut out = Vec::new(); + while out.len() < max_len && self.cursor.record_index < self.records.len() { + match &self.records[self.cursor.record_index] { + TapeRecord::Filemark => { + self.cursor.record_index += 1; + self.cursor.byte_offset = 0; + break; + } + TapeRecord::Data(data) => { + let start = self.cursor.byte_offset.min(data.len()); + let remaining = max_len - out.len(); + let end = (start + remaining).min(data.len()); + out.extend_from_slice(&data[start..end]); + if end == data.len() { + self.cursor.record_index += 1; + self.cursor.byte_offset = 0; + } else { + self.cursor.byte_offset = end; + } + } + } + } + out + } + + pub(crate) fn seek_filemark(&mut self, mut count: u32) -> bool { + while count > 0 && self.cursor.record_index < self.records.len() { + match &self.records[self.cursor.record_index] { + TapeRecord::Filemark => { + count -= 1; + self.cursor.record_index += 1; + self.cursor.byte_offset = 0; + } + TapeRecord::Data(_) => { + self.cursor.record_index += 1; + self.cursor.byte_offset = 0; + } + } + } + count == 0 + } +} diff --git a/crates/vtl/src/simulator.rs b/crates/vtl/src/simulator.rs new file mode 100644 index 0000000..f9a2cc3 --- /dev/null +++ b/crates/vtl/src/simulator.rs @@ -0,0 +1,291 @@ +use crate::error::{Result, VtlError}; +use crate::interface::{MediumChanger, TapeDrive, TapeInventory}; +use crate::model::{ElementAddress, ElementKind, TapeBarcode, VirtualTape}; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Slot { + address: ElementAddress, + tape: Option, +} + +impl Slot { + fn new(address: ElementAddress) -> Self { + Self { + address, + tape: None, + } + } + + pub fn address(&self) -> ElementAddress { + self.address + } + + pub fn is_empty(&self) -> bool { + self.tape.is_none() + } + + pub fn barcode(&self) -> Option<&TapeBarcode> { + self.tape.as_ref().map(VirtualTape::barcode) + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Drive { + address: ElementAddress, + tape: Option, +} + +impl Drive { + fn new(address: ElementAddress) -> Self { + Self { + address, + tape: None, + } + } + + pub fn address(&self) -> ElementAddress { + self.address + } + + pub fn is_empty(&self) -> bool { + self.tape.is_none() + } + + pub fn loaded_barcode(&self) -> Option<&TapeBarcode> { + self.tape.as_ref().map(VirtualTape::barcode) + } + + pub fn loaded_tape(&self) -> Option<&VirtualTape> { + self.tape.as_ref() + } +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct VirtualTapeLibrary { + slots: Vec, + drives: Vec, +} + +impl VirtualTapeLibrary { + pub fn new(slot_count: u32, drive_count: u32) -> Self { + let slots = (1..=slot_count) + .map(|index| Slot::new(ElementAddress::slot(index))) + .collect(); + let drives = (0..drive_count) + .map(|index| Drive::new(ElementAddress::drive(index))) + .collect(); + Self { slots, drives } + } + + pub fn slots(&self) -> &[Slot] { + &self.slots + } + + pub fn drives(&self) -> &[Drive] { + &self.drives + } + + pub fn slot(&self, address: ElementAddress) -> Result<&Slot> { + self.ensure_slot(address)?; + self.slots + .iter() + .find(|slot| slot.address == address) + .ok_or_else(|| VtlError::SlotOutOfRange(address.index())) + } + + pub fn drive(&self, address: ElementAddress) -> Result<&Drive> { + self.ensure_drive(address)?; + self.drives + .iter() + .find(|drive| drive.address == address) + .ok_or_else(|| VtlError::DriveOutOfRange(address.index())) + } + + fn slot_mut(&mut self, address: ElementAddress) -> Result<&mut Slot> { + self.ensure_slot(address)?; + self.slots + .iter_mut() + .find(|slot| slot.address == address) + .ok_or_else(|| VtlError::SlotOutOfRange(address.index())) + } + + fn drive_mut(&mut self, address: ElementAddress) -> Result<&mut Drive> { + self.ensure_drive(address)?; + self.drives + .iter_mut() + .find(|drive| drive.address == address) + .ok_or_else(|| VtlError::DriveOutOfRange(address.index())) + } + + fn ensure_slot(&self, address: ElementAddress) -> Result<()> { + if address.kind() != ElementKind::Slot { + return Err(VtlError::WrongElement { + expected: "slot", + actual: address.to_string(), + }); + } + if address.index() == 0 || address.index() as usize > self.slots.len() { + return Err(VtlError::SlotOutOfRange(address.index())); + } + Ok(()) + } + + fn ensure_drive(&self, address: ElementAddress) -> Result<()> { + if address.kind() != ElementKind::Drive { + return Err(VtlError::WrongElement { + expected: "drive", + actual: address.to_string(), + }); + } + if address.index() as usize >= self.drives.len() { + return Err(VtlError::DriveOutOfRange(address.index())); + } + Ok(()) + } + + fn take_from_slot(&mut self, slot: ElementAddress) -> Result { + self.slot_mut(slot)? + .tape + .take() + .ok_or_else(|| VtlError::SlotEmpty(slot.index())) + } + + fn put_into_slot(&mut self, slot: ElementAddress, tape: VirtualTape) -> Result<()> { + let slot_ref = self.slot_mut(slot)?; + if slot_ref.tape.is_some() { + return Err(VtlError::SlotOccupied(slot.index())); + } + slot_ref.tape = Some(tape); + Ok(()) + } + + fn take_from_drive(&mut self, drive: ElementAddress) -> Result { + self.drive_mut(drive)? + .tape + .take() + .ok_or_else(|| VtlError::DriveEmpty(drive.index())) + } + + fn put_into_drive(&mut self, drive: ElementAddress, tape: VirtualTape) -> Result<()> { + let drive_ref = self.drive_mut(drive)?; + if drive_ref.tape.is_some() { + return Err(VtlError::DriveOccupied(drive.index())); + } + drive_ref.tape = Some(tape); + Ok(()) + } + + fn loaded_tape_mut(&mut self, drive: ElementAddress) -> Result<&mut VirtualTape> { + self.drive_mut(drive)? + .tape + .as_mut() + .ok_or_else(|| VtlError::DriveEmpty(drive.index())) + } +} + +impl TapeInventory for VirtualTapeLibrary { + fn insert_tape(&mut self, slot: ElementAddress, barcode: TapeBarcode) -> Result<()> { + self.put_into_slot(slot, VirtualTape::new(barcode)) + } +} + +impl MediumChanger for VirtualTapeLibrary { + fn move_medium(&mut self, from: ElementAddress, to: ElementAddress) -> Result<()> { + match (from.kind(), to.kind()) { + (ElementKind::Slot, ElementKind::Slot) => { + let tape = self.take_from_slot(from)?; + self.put_into_slot(to, tape) + } + (ElementKind::Slot, ElementKind::Drive) => { + let tape = self.take_from_slot(from)?; + self.put_into_drive(to, tape) + } + (ElementKind::Drive, ElementKind::Slot) => { + let tape = self.take_from_drive(from)?; + self.put_into_slot(to, tape) + } + (ElementKind::Drive, ElementKind::Drive) => { + let tape = self.take_from_drive(from)?; + self.put_into_drive(to, tape) + } + _ => Err(VtlError::InvalidElementAddress(format!( + "unsupported move from {from} to {to}" + ))), + } + } + + fn load(&mut self, slot: ElementAddress, drive: ElementAddress) -> Result<()> { + self.move_medium(slot, drive) + } + + fn unload(&mut self, drive: ElementAddress, slot: ElementAddress) -> Result<()> { + self.move_medium(drive, slot) + } +} + +impl TapeDrive for VirtualTapeLibrary { + fn rewind(&mut self, drive: ElementAddress) -> Result<()> { + self.loaded_tape_mut(drive)?.rewind(); + Ok(()) + } + + fn seek_filemark(&mut self, drive: ElementAddress, count: u32) -> Result<()> { + if self.loaded_tape_mut(drive)?.seek_filemark(count) { + Ok(()) + } else { + Err(VtlError::FilemarkNotFound) + } + } + + fn write_filemark(&mut self, drive: ElementAddress) -> Result<()> { + self.loaded_tape_mut(drive)?.append_filemark(); + Ok(()) + } + + fn write(&mut self, drive: ElementAddress, data: &[u8]) -> Result<()> { + self.loaded_tape_mut(drive)?.append_data(data); + Ok(()) + } + + fn read(&mut self, drive: ElementAddress, max_len: usize) -> Result> { + Ok(self.loaded_tape_mut(drive)?.read(max_len)) + } +} + +impl VirtualTapeLibrary { + pub fn insert_tape(&mut self, slot: ElementAddress, barcode: TapeBarcode) -> Result<()> { + ::insert_tape(self, slot, barcode) + } + + pub fn move_medium(&mut self, from: ElementAddress, to: ElementAddress) -> Result<()> { + ::move_medium(self, from, to) + } + + pub fn load(&mut self, slot: ElementAddress, drive: ElementAddress) -> Result<()> { + ::load(self, slot, drive) + } + + pub fn unload(&mut self, drive: ElementAddress, slot: ElementAddress) -> Result<()> { + ::unload(self, drive, slot) + } + + pub fn rewind(&mut self, drive: ElementAddress) -> Result<()> { + ::rewind(self, drive) + } + + pub fn seek_filemark(&mut self, drive: ElementAddress, count: u32) -> Result<()> { + ::seek_filemark(self, drive, count) + } + + pub fn write_filemark(&mut self, drive: ElementAddress) -> Result<()> { + ::write_filemark(self, drive) + } + + pub fn write(&mut self, drive: ElementAddress, data: &[u8]) -> Result<()> { + ::write(self, drive, data) + } + + pub fn read(&mut self, drive: ElementAddress, max_len: usize) -> Result> { + ::read(self, drive, max_len) + } +} diff --git a/crates/vtl/tests/behavior.rs b/crates/vtl/tests/behavior.rs new file mode 100644 index 0000000..7ff2b73 --- /dev/null +++ b/crates/vtl/tests/behavior.rs @@ -0,0 +1,152 @@ +use coldstore_vtl::command::{CommandSpec, RecordedCommandRunner}; +use coldstore_vtl::discover::{parse_lsscsi, ScsiDeviceKind}; +use coldstore_vtl::mhvtl::MhvtlToolchain; +use coldstore_vtl::model::{ElementAddress, TapeBarcode}; +use coldstore_vtl::simulator::VirtualTapeLibrary; + +const LSSCSI_SAMPLE: &str = r#" +[5:0:0:0] mediumx STK L700 0106 /dev/sch0 /dev/sg8 +[5:0:1:0] tape IBM ULT3580-TD5 0106 /dev/st4 /dev/sg10 +[5:0:2:0] tape IBM ULT3580-TD5 0106 /dev/st1 /dev/sg6 +"#; + +#[test] +fn parses_lsscsi_tape_and_medium_changer_devices() { + let inventory = parse_lsscsi(LSSCSI_SAMPLE).expect("parse lsscsi output"); + + let changers = inventory.medium_changers(); + assert_eq!(changers.len(), 1); + assert_eq!(changers[0].kind, ScsiDeviceKind::MediumChanger); + assert_eq!(changers[0].hctl, "5:0:0:0"); + assert_eq!(changers[0].vendor, "STK"); + assert_eq!(changers[0].product, "L700"); + assert_eq!(changers[0].primary_device.as_deref(), Some("/dev/sch0")); + assert_eq!(changers[0].sg_device.as_deref(), Some("/dev/sg8")); + + let drives = inventory.tape_drives(); + assert_eq!(drives.len(), 2); + assert_eq!(drives[0].kind, ScsiDeviceKind::TapeDrive); + assert_eq!(drives[0].primary_device.as_deref(), Some("/dev/st4")); + assert_eq!(drives[0].non_rewinding_device.as_deref(), Some("/dev/nst4")); + assert_eq!(drives[0].sg_device.as_deref(), Some("/dev/sg10")); +} + +#[test] +fn mhvtl_toolchain_exposes_stable_lsscsi_mtx_mt_and_sg_command_paths() { + let tools = MhvtlToolchain::default(); + + assert_eq!( + tools.discover_command(), + CommandSpec::new("lsscsi").arg("-g") + ); + assert_eq!( + tools.library_status_command("/dev/sg8"), + CommandSpec::new("mtx").args(["-f", "/dev/sg8", "status"]) + ); + assert_eq!( + tools.move_medium_command( + "/dev/sg8", + ElementAddress::slot(1), + ElementAddress::drive(0) + ), + CommandSpec::new("mtx").args(["-f", "/dev/sg8", "transfer", "1", "0"]) + ); + assert_eq!( + tools.load_command( + "/dev/sg8", + ElementAddress::slot(2), + ElementAddress::drive(0) + ), + CommandSpec::new("mtx").args(["-f", "/dev/sg8", "load", "2", "0"]) + ); + assert_eq!( + tools.unload_command( + "/dev/sg8", + ElementAddress::slot(2), + ElementAddress::drive(0) + ), + CommandSpec::new("mtx").args(["-f", "/dev/sg8", "unload", "2", "0"]) + ); + assert_eq!( + tools.rewind_command("/dev/nst0"), + CommandSpec::new("mt").args(["-f", "/dev/nst0", "rewind"]) + ); + assert_eq!( + tools.write_filemark_command("/dev/nst0", 2), + CommandSpec::new("mt").args(["-f", "/dev/nst0", "weof", "2"]) + ); + assert_eq!( + tools.seek_filemark_command("/dev/nst0", 1), + CommandSpec::new("mt").args(["-f", "/dev/nst0", "fsf", "1"]) + ); + assert_eq!( + tools.test_unit_ready_command("/dev/sg10"), + CommandSpec::new("sg_turs").arg("/dev/sg10") + ); + assert_eq!( + tools.inquiry_command("/dev/sg10"), + CommandSpec::new("sg_inq").arg("/dev/sg10") + ); +} + +#[test] +fn mhvtl_discovery_uses_injected_runner_instead_of_host_commands() { + let tools = MhvtlToolchain::default(); + let runner = RecordedCommandRunner::default(); + runner.push_stdout(LSSCSI_SAMPLE); + + let inventory = tools + .discover(&runner) + .expect("discover from recorded output"); + + assert_eq!(inventory.tape_drives().len(), 2); + assert_eq!(runner.commands(), vec![tools.discover_command()]); +} + +#[test] +fn memory_vtl_models_changer_load_unload_and_tape_filemark_flow() { + let mut vtl = VirtualTapeLibrary::new(3, 1); + vtl.insert_tape(ElementAddress::slot(1), TapeBarcode::new("TAPE001")) + .expect("insert tape"); + + vtl.load(ElementAddress::slot(1), ElementAddress::drive(0)) + .expect("load tape"); + assert!(vtl.slot(ElementAddress::slot(1)).unwrap().is_empty()); + assert_eq!( + vtl.drive(ElementAddress::drive(0)) + .unwrap() + .loaded_barcode() + .map(|b| b.as_str()), + Some("TAPE001") + ); + + vtl.write(ElementAddress::drive(0), b"abc") + .expect("write abc"); + vtl.write_filemark(ElementAddress::drive(0)) + .expect("write filemark"); + vtl.write(ElementAddress::drive(0), b"def") + .expect("write def"); + + vtl.rewind(ElementAddress::drive(0)).expect("rewind"); + assert_eq!( + vtl.read(ElementAddress::drive(0), 3).expect("read abc"), + b"abc" + ); + vtl.seek_filemark(ElementAddress::drive(0), 1) + .expect("seek filemark"); + assert_eq!( + vtl.read(ElementAddress::drive(0), 3).expect("read def"), + b"def" + ); + + vtl.unload(ElementAddress::drive(0), ElementAddress::slot(2)) + .expect("unload tape"); + assert!(vtl.drive(ElementAddress::drive(0)).unwrap().is_empty()); + assert_eq!( + vtl.slot(ElementAddress::slot(2)) + .unwrap() + .barcode() + .map(|b| b.as_str()), + Some("TAPE001") + ); +} diff --git a/docs/modules/05-scheduler-layer.md b/docs/modules/05-scheduler-layer.md index f36d702..43044e0 100644 --- a/docs/modules/05-scheduler-layer.md +++ b/docs/modules/05-scheduler-layer.md @@ -7,6 +7,17 @@ 归档取回调度层负责冷数据的归档与取回调度，是 ColdStore 的核心业务逻辑层。设计需充分考虑**磁带的顺序读写特性**，通过聚合与合并策略最大化吞吐、减少换带与 seek。 +### 1.0 当前实现状态（Phase-1 安全基线） + +当前代码已经有一个可单测的安全归档批处理切片： + +- `MetadataBackedSchedulerBackend::archive_staging_batch(cache, tape, limit)` 通过注入的 `Phase1ArchiveCache` 和 `TapeArchiveWriter` 执行归档批处理，不直接访问宿主机设备。 +- `CacheArchiveClient` 已把真实 `CacheService` gRPC client 适配为 `Phase1ArchiveCache`；单测使用 in-process CacheService + 临时 HDD backend 验证 staging list/get/delete，而不是仅靠 fake cache。 +- `TapeArchiveClient` 已把真实 `TapeService` gRPC client 适配为 `TapeArchiveWriter`；单测使用 in-process TapeService + simulator backend 验证 `WriteBundle` filemark 写入与 `ReadBundle` 读回，而不是直接调用 service helper。 +- 该切片会读取 staging 数据、调用 tape writer 写入 bundle、写入 `ArchiveBundle`，再更新对象 `archive_id` / `tape_id` / `tape_set` / `tape_block_offset` 和 `StorageClass::Cold`，最后删除 staging 数据。 +- 单测使用 in-process `coldstore-tape` simulator writer / gRPC client 验证真实 filemark 写入与读回；没有启动 mhVTL、没有访问 `/dev/*`。 +- 当前一条 staging object 对应一个 Phase-1 bundle（`phase1-bundle:/`）；多对象聚合、驱动选择、失败补偿和 live tape worker client 仍是后续阶段。 + > **部署模型**：调度层运行在 **Scheduler Worker** 节点上，是 ColdStore 的**唯一业务中枢**。 > Gateway 的全部 S3 请求都发往 Scheduler Worker 处理。 > Scheduler 通过 gRPC 对接 Cache Worker（同机）和 Tape Worker（远程）， diff --git a/docs/modules/06-tape-layer.md b/docs/modules/06-tape-layer.md index 82440a3..e872e4d 100644 --- a/docs/modules/06-tape-layer.md +++ b/docs/modules/06-tape-layer.md @@ -7,6 +7,19 @@ 磁带管理层提供磁带设备与磁带库的完整管理能力，通过自研 SDK 抽象层实现与底层硬件的解耦，前期对接 Linux SCSI 协议。 +### 1.0 当前实现状态（Phase-1 安全基线） + +当前代码已经包含一个不触碰宿主机设备的 Phase-1 磁带闭环： + +- `crates/vtl` 提供独立的虚拟磁带库 harness，覆盖 `lsscsi -g` 输出解析、`mtx`/`mt`/`sg3_utils`/`dd` 命令规格构造、runner 注入，以及纯内存 `VirtualTapeLibrary`。 +- `crates/tape` 的 `TapeServiceImpl` 通过 `TapeBackend` trait 依赖后端抽象，默认使用 `SimulatorTapeBackend`，不会访问 `/dev/st*`、`/dev/nst*`、`/dev/sg*` 或 `/dev/sch*`。 +- 已通过 gRPC service 层暴露可单测的驱动/带库操作：`ListDrives`、`GetDriveStatus`、`AcquireDrive`、`ReleaseDrive`、`LoadTape`、`UnloadTape`、`Rewind`、`SeekToFilemark`、`GetTapeMediaStatus`、`Inventory`。 +- 已通过 service/helper 和真实 gRPC client 路径暴露 filemark 分隔的 bundle 写入与按 filemark 读取；Scheduler 侧的 `TapeArchiveClient` 可通过 `TapeServiceClient.WriteBundle` 写入 simulator-backed TapeService。 +- `ReadBundle` 的 block offset 定位仍返回显式 `UNIMPLEMENTED`，等待真实 SCSI/磁带定位语义接入。 +- mhVTL 安装脚本位于 `scripts/setup-mhvtl-env.sh`，默认 dry-run；当前安全验证只执行 `bash -n`，不安装系统包、不启动内核模块或服务。 + +真实 mhVTL/SCSI 闭环属于后续专用 VM/live 测试目标，必须显式使用 live runner 或安装脚本 `--execute`，不能作为默认单测路径。 + ### 1.1 完整职责 | 功能域 | 说明 | diff --git a/scripts/setup-mhvtl-env.sh b/scripts/setup-mhvtl-env.sh new file mode 100755 index 0000000..d31629e --- /dev/null +++ b/scripts/setup-mhvtl-env.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Prepare a dedicated Linux VM/test host for mhVTL live tape-library testing. +# +# SAFETY DEFAULT: dry-run only. Nothing is installed unless --execute is passed. +# Do NOT run this on a normal development host unless you intentionally want to +# install kernel modules, systemd services, and virtual SCSI tape/changer devices. + +set -euo pipefail + +EXECUTE=0 +START_SERVICES=0 +MHVTL_REPO="${MHVTL_REPO:-https://github.com/markh794/mhvtl.git}" +MHVTL_SRC="${MHVTL_SRC:-/usr/local/src/mhvtl}" + +usage() { + cat <<'USAGE' +Usage: + scripts/setup-mhvtl-env.sh [--execute] [--start-services] + +Default mode is dry-run: commands are printed but not executed. + +Options: + --execute Actually install packages, clone/build/install mhVTL. + --start-services After install, enable/start mhvtl.target via systemctl. + -h, --help Show this help. + +Environment: + MHVTL_REPO Git repo to clone. Default: https://github.com/markh794/mhvtl.git + MHVTL_SRC Source checkout path. Default: /usr/local/src/mhvtl + +Expected live validation after install: + lsscsi -g + ps -ax | grep '[v]tl' + mtx -f /dev/sgX status # use changer sg path from lsscsi output + mt -f /dev/nstX status # use non-rewinding tape path + sg_inq /dev/sgX + sg_turs /dev/sgX +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --execute) EXECUTE=1 ;; + --start-services) START_SERVICES=1 ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage; exit 2 ;; + esac + shift +done + +if [[ $EXECUTE -eq 0 ]]; then + echo "DRY-RUN mode. Pass --execute to actually modify this host." >&2 +fi + +SUDO="" +if [[ ${EUID:-$(id -u)} -ne 0 ]]; then + SUDO="sudo" +fi + +run() { + printf '+ ' + printf '%q ' "$@" + printf '\n' + if [[ $EXECUTE -eq 1 ]]; then + "$@" + fi +} + +run_sh() { + printf '+ %s\n' "$*" + if [[ $EXECUTE -eq 1 ]]; then + bash -lc "$*" + fi +} + +require_linux() { + if [[ "$(uname -s)" != "Linux" ]]; then + echo "mhVTL requires Linux" >&2 + exit 1 + fi +} + +install_packages() { + if command -v apt-get >/dev/null 2>&1; then + run $SUDO apt-get update + run $SUDO apt-get install -y \ + build-essential git make gcc kmod pkg-config \ + "linux-headers-$(uname -r)" \ + lsscsi sg3-utils mtx mt-st \ + zlib1g-dev liblzo2-dev + elif command -v dnf >/dev/null 2>&1; then + run $SUDO dnf install -y \ + gcc gcc-c++ make git kmod kernel-devel kernel-headers \ + lsscsi sg3_utils mtx mt-st \ + zlib-devel lzo-devel + elif command -v yum >/dev/null 2>&1; then + run $SUDO yum install -y \ + gcc gcc-c++ make git kmod kernel-devel kernel-headers \ + lsscsi sg3_utils mtx mt-st \ + zlib-devel lzo-devel + elif command -v zypper >/dev/null 2>&1; then + run $SUDO zypper --non-interactive install \ + gcc gcc-c++ make git-core kmod kernel-default-devel \ + lsscsi sg3_utils mtx mt_st \ + zlib-devel lzo-devel + elif command -v pacman >/dev/null 2>&1; then + run $SUDO pacman -Sy --needed --noconfirm \ + base-devel git linux-headers kmod \ + lsscsi sg3_utils mtx mt-st \ + zlib lzo + else + echo "unsupported package manager; install build tools, kernel headers, lsscsi, sg3_utils, mtx, mt-st, zlib-dev, lzo-dev manually" >&2 + exit 1 + fi +} + +checkout_mhvtl() { + if [[ -d "$MHVTL_SRC/.git" ]]; then + run git -C "$MHVTL_SRC" fetch --all --tags --prune + else + run $SUDO mkdir -p "$(dirname "$MHVTL_SRC")" + if [[ -n "$SUDO" ]]; then + run $SUDO git clone "$MHVTL_REPO" "$MHVTL_SRC" + else + run git clone "$MHVTL_REPO" "$MHVTL_SRC" + fi + fi +} + +build_and_install_mhvtl() { + # Upstream layouts have changed over time. Build the kernel module directory + # explicitly when present, then run the top-level build/install. + if [[ -d "$MHVTL_SRC/kernel" ]]; then + run make -C "$MHVTL_SRC/kernel" + run $SUDO make -C "$MHVTL_SRC/kernel" install + fi + + run make -C "$MHVTL_SRC" + run $SUDO make -C "$MHVTL_SRC" install + run $SUDO systemctl daemon-reload +} + +start_services() { + if [[ $START_SERVICES -eq 1 ]]; then + run $SUDO systemctl enable mhvtl.target + run $SUDO systemctl start mhvtl.target + run $SUDO systemctl status --no-pager mhvtl.target + else + echo "Skipping service start. Pass --start-services to enable/start mhvtl.target." >&2 + fi +} + +print_next_steps() { + cat <<'NEXT' + +Next manual checks on the dedicated VM/test host: + lsscsi -g + ps -ax | grep '[v]tl' + +Pick paths from lsscsi output: + mediumx ... /dev/sch0 /dev/sgCHANGER + tape ... /dev/stN /dev/sgTAPE + +Then test command paths: + mtx -f /dev/sgCHANGER status + mt -f /dev/nstN status + sg_inq /dev/sgTAPE + sg_turs /dev/sgTAPE + +Do not point ColdStore live tests at production tape devices. +NEXT +} + +require_linux +install_packages +checkout_mhvtl +build_and_install_mhvtl +start_services +print_next_steps From 2755e68e6dbe2eaf4d241b7c4876ad0d94f605ff Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Sun, 5 Jul 2026 10:38:37 +0800 Subject: [PATCH 2/4] feat: wire single-node archive pipeline runtime --- Cargo.toml | 1 + Makefile | 8 +- configs/single-node-no-tape.yaml | 2 + configs/single-node.yaml | 2 + crates/common/src/config.rs | 92 ++- crates/gateway/src/handler.rs | 14 + crates/metadata/src/lib.rs | 44 +- crates/metadata/src/service.rs | 11 + crates/node/Cargo.toml | 24 + crates/node/src/main.rs | 223 ++++++ crates/scheduler/src/lib.rs | 78 +- crates/scheduler/src/service.rs | 1185 ++++++++++++++++++++++++++++-- crates/tape/src/service.rs | 18 +- 13 files changed, 1621 insertions(+), 81 deletions(-) create mode 100644 configs/single-node-no-tape.yaml create mode 100644 configs/single-node.yaml create mode 100644 crates/node/Cargo.toml create mode 100644 crates/node/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 44f3949..374a188 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/scheduler", "crates/cache", "crates/tape", + "crates/node", "crates/vtl", ] diff --git a/Makefile b/Makefile index 55c1b50..8608187 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build build-debug fmt fmt-check clippy test unit test-unit vtl-unit vtl-check check check-all check-safe clean help install-hooks install-tools setup lint run-metadata run-gateway run-scheduler run-cache run-tape +.PHONY: build build-debug fmt fmt-check clippy test unit test-unit vtl-unit vtl-check check check-all check-safe clean help install-hooks install-tools setup lint run-metadata run-gateway run-scheduler run-cache run-tape run-node run-node-no-tape .DEFAULT_GOAL := help @@ -43,6 +43,10 @@ run-cache: @cargo run -p coldstore-cache run-tape: @cargo run -p coldstore-tape +run-node: + @cargo run -p coldstore-node +run-node-no-tape: + @cargo run -p coldstore-node -- --config configs/single-node-no-tape.yaml lint: fmt-check clippy check unit @echo "All lint checks passed (unit-only)." @@ -64,5 +68,5 @@ help: @echo "" @echo "Build: build build-debug clean" @echo "Quality: fmt fmt-check clippy test unit vtl-unit vtl-check check check-all check-safe lint" - @echo "Run: run-metadata run-gateway run-scheduler run-cache run-tape" + @echo "Run: run-node run-node-no-tape run-metadata run-gateway run-scheduler run-cache run-tape" @echo "Setup: setup install-tools install-hooks" diff --git a/configs/single-node-no-tape.yaml b/configs/single-node-no-tape.yaml new file mode 100644 index 0000000..650b4f6 --- /dev/null +++ b/configs/single-node-no-tape.yaml @@ -0,0 +1,2 @@ +data_dir: data/single-node-no-tape +tape_enabled: false diff --git a/configs/single-node.yaml b/configs/single-node.yaml new file mode 100644 index 0000000..9434ccd --- /dev/null +++ b/configs/single-node.yaml @@ -0,0 +1,2 @@ +data_dir: data/single-node +tape_enabled: true diff --git a/crates/common/src/config.rs b/crates/common/src/config.rs index 02dbf41..fd9b7f3 100644 --- a/crates/common/src/config.rs +++ b/crates/common/src/config.rs @@ -63,14 +63,26 @@ impl Default for MetadataConfig { pub struct SchedulerConfig { pub listen: String, pub metadata_addrs: Vec, + #[serde(default = "default_cache_addrs")] + pub cache_addrs: Vec, + #[serde(default = "default_tape_addrs")] + pub tape_addrs: Vec, pub archive: ArchiveSchedulerConfig, pub recall: RecallSchedulerConfig, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ArchiveSchedulerConfig { + #[serde(default = "default_scheduler_enabled")] + pub enabled: bool, pub scan_interval_secs: u64, pub batch_size: usize, + #[serde(default = "default_scheduler_drive_id")] + pub drive_id: String, + #[serde(default = "default_scheduler_tape_id")] + pub tape_id: String, + #[serde(default = "default_scheduler_tape_set")] + pub tape_set: Vec, pub min_archive_size_mb: u64, pub max_archive_size_mb: u64, pub target_throughput_mbps: u64, @@ -81,10 +93,16 @@ pub struct ArchiveSchedulerConfig { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RecallSchedulerConfig { + #[serde(default = "default_scheduler_enabled")] + pub enabled: bool, + #[serde(default = "default_recall_scan_interval_secs")] + pub scan_interval_secs: u64, pub max_concurrent_restores: usize, pub merge_window_secs: u64, pub restore_timeout_secs: u64, pub read_buffer_mb: u64, + #[serde(default = "default_scheduler_drive_id")] + pub drive_id: String, } impl Default for SchedulerConfig { @@ -96,9 +114,15 @@ impl Default for SchedulerConfig { "127.0.0.1:21002".to_string(), "127.0.0.1:21003".to_string(), ], + cache_addrs: default_cache_addrs(), + tape_addrs: default_tape_addrs(), archive: ArchiveSchedulerConfig { + enabled: true, scan_interval_secs: 60, batch_size: 1000, + drive_id: default_scheduler_drive_id(), + tape_id: default_scheduler_tape_id(), + tape_set: default_scheduler_tape_set(), min_archive_size_mb: 100, max_archive_size_mb: 10240, target_throughput_mbps: 300, @@ -107,15 +131,46 @@ impl Default for SchedulerConfig { block_size: 262144, }, recall: RecallSchedulerConfig { + enabled: true, + scan_interval_secs: default_recall_scan_interval_secs(), max_concurrent_restores: 10, merge_window_secs: 60, restore_timeout_secs: 3600, read_buffer_mb: 64, + drive_id: default_scheduler_drive_id(), }, } } } +fn default_cache_addrs() -> Vec { + vec!["127.0.0.1:23001".to_string()] +} + +fn default_tape_addrs() -> Vec { + vec!["127.0.0.1:24001".to_string()] +} + +fn default_scheduler_enabled() -> bool { + true +} + +fn default_recall_scan_interval_secs() -> u64 { + 60 +} + +fn default_scheduler_drive_id() -> String { + "drive-0".to_string() +} + +fn default_scheduler_tape_id() -> String { + "TAPE0001".to_string() +} + +fn default_scheduler_tape_set() -> Vec { + vec![default_scheduler_tape_id()] +} + // --------------------------------------------------------------------------- // Cache Worker 配置 // --------------------------------------------------------------------------- @@ -176,6 +231,8 @@ pub struct TapeConfig { pub metadata_addrs: Vec, pub sdk_backend: String, pub scsi: ScsiConfig, + #[serde(default)] + pub simulator: TapeSimulatorConfig, pub library_device: Option, pub supported_formats: Vec, pub tape_hold_secs: u64, @@ -189,6 +246,26 @@ pub struct ScsiConfig { pub buffer_size_mb: u64, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TapeSimulatorConfig { + #[serde(default = "default_simulator_slot_count")] + pub slot_count: u32, + #[serde(default = "default_simulator_tape_ids")] + pub tape_ids: Vec, + #[serde(default = "default_simulator_autoload_tape_id")] + pub autoload_tape_id: Option, +} + +impl Default for TapeSimulatorConfig { + fn default() -> Self { + Self { + slot_count: default_simulator_slot_count(), + tape_ids: default_simulator_tape_ids(), + autoload_tape_id: default_simulator_autoload_tape_id(), + } + } +} + impl Default for TapeConfig { fn default() -> Self { Self { @@ -198,12 +275,13 @@ impl Default for TapeConfig { "127.0.0.1:21002".to_string(), "127.0.0.1:21003".to_string(), ], - sdk_backend: "scsi".to_string(), + sdk_backend: "simulator".to_string(), scsi: ScsiConfig { devices: vec!["/dev/nst0".to_string()], block_size: 262144, buffer_size_mb: 64, }, + simulator: TapeSimulatorConfig::default(), library_device: None, supported_formats: vec!["LTO-9".to_string(), "LTO-10".to_string()], tape_hold_secs: 300, @@ -211,3 +289,15 @@ impl Default for TapeConfig { } } } + +fn default_simulator_slot_count() -> u32 { + 8 +} + +fn default_simulator_tape_ids() -> Vec { + vec![default_scheduler_tape_id()] +} + +fn default_simulator_autoload_tape_id() -> Option { + Some(default_scheduler_tape_id()) +} diff --git a/crates/gateway/src/handler.rs b/crates/gateway/src/handler.rs index 55698c7..c36364e 100644 --- a/crates/gateway/src/handler.rs +++ b/crates/gateway/src/handler.rs @@ -322,6 +322,7 @@ fn grpc_status_to_s3_response(status: tonic::Status, resource: &str) -> Response }; (code, StatusCode::NOT_FOUND) } + tonic::Code::FailedPrecondition => (S3ErrorCode::InvalidObjectState, StatusCode::FORBIDDEN), tonic::Code::Unimplemented => (S3ErrorCode::NotImplemented, StatusCode::NOT_IMPLEMENTED), _ => (S3ErrorCode::NotImplemented, StatusCode::BAD_GATEWAY), }; @@ -611,6 +612,19 @@ mod tests { assert_eq!(body.as_ref(), b"hello world"); } + #[tokio::test] + async fn failed_precondition_maps_to_invalid_object_state() { + let response = grpc_status_to_s3_response( + tonic::Status::failed_precondition("object must be restored"), + "/docs/readme.txt", + ); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert!(String::from_utf8(body.to_vec()) + .unwrap() + .contains("InvalidObjectState")); + } + #[tokio::test] async fn delete_object_route_uses_backend() { let response = test_router(state()) diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 8e77cc6..c3001ec 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -9,15 +9,32 @@ pub mod raft_storage; use anyhow::Result; use coldstore_common::config::MetadataConfig; +use std::path::PathBuf; use tonic::transport::Server; use tracing::info; pub async fn run(config: MetadataConfig) -> Result<()> { let addr = config.listen.parse()?; + let snapshot_path = default_snapshot_path(&config); - let metadata_service = service::MetadataServiceImpl::new(&config).await?; + #[cfg(feature = "metadata-raft")] + let metadata_service = service::MetadataServiceImpl::new_with_snapshot_and_raft_backend( + &config, + snapshot_path.clone(), + std::sync::Arc::new(crate::raft::RaftMetadataBackend::new()), + ) + .await?; - info!("Metadata 节点 {} 启动在 {}", config.node_id, addr); + #[cfg(not(feature = "metadata-raft"))] + let metadata_service = + service::MetadataServiceImpl::new_with_snapshot(&config, snapshot_path.clone()).await?; + + info!( + "Metadata 节点 {} 启动在 {}，snapshot={}", + config.node_id, + addr, + snapshot_path.display() + ); Server::builder() .add_service( @@ -30,3 +47,26 @@ pub async fn run(config: MetadataConfig) -> Result<()> { Ok(()) } + +fn default_snapshot_path(config: &MetadataConfig) -> PathBuf { + PathBuf::from(&config.data_path).join(format!("node-{}-snapshot.bin", config.node_id)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_snapshot_path_uses_configured_data_path_and_node_id() { + let config = MetadataConfig { + node_id: 7, + data_path: "/tmp/coldstore-meta".into(), + ..MetadataConfig::default() + }; + + assert_eq!( + default_snapshot_path(&config), + PathBuf::from("/tmp/coldstore-meta/node-7-snapshot.bin") + ); + } +} diff --git a/crates/metadata/src/service.rs b/crates/metadata/src/service.rs index afe377d..8ca7aae 100644 --- a/crates/metadata/src/service.rs +++ b/crates/metadata/src/service.rs @@ -64,6 +64,17 @@ impl MetadataServiceImpl { }) } + #[cfg(feature = "metadata-raft")] + pub async fn new_with_snapshot_and_raft_backend( + config: &MetadataConfig, + snapshot_path: PathBuf, + raft_backend: Arc, + ) -> Result { + let mut service = Self::new_with_snapshot(config, snapshot_path).await?; + service.raft_backend = Some(raft_backend); + Ok(service) + } + async fn persist_locked(&self, state: &MetadataState) -> std::result::Result<(), Status> { if let Some(path) = &self.snapshot_path { save_snapshot(path, state) diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml new file mode 100644 index 0000000..c769139 --- /dev/null +++ b/crates/node/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "coldstore-node" +description = "ColdStore single-node runtime for local and small deployments" +version.workspace = true +edition.workspace = true +license.workspace = true + +[[bin]] +name = "coldstore-node" +path = "src/main.rs" + +[dependencies] +coldstore-cache = { path = "../cache" } +coldstore-common = { workspace = true } +coldstore-gateway = { path = "../gateway" } +coldstore-metadata = { path = "../metadata" } +coldstore-scheduler = { path = "../scheduler" } +coldstore-tape = { path = "../tape" } +anyhow = { workspace = true } +config = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs new file mode 100644 index 0000000..def6fab --- /dev/null +++ b/crates/node/src/main.rs @@ -0,0 +1,223 @@ +use anyhow::Result; +use coldstore_common::config::{ + CacheBackendConfig, CacheConfig, GatewayConfig, MetadataConfig, SchedulerConfig, TapeConfig, +}; +use serde::Deserialize; +use std::path::{Path, PathBuf}; +use tokio::task::JoinHandle; +use tokio::time::{sleep, Duration}; +use tracing::{error, info, warn}; + +const METADATA_ADDR: &str = "127.0.0.1:21001"; +const SCHEDULER_ADDR: &str = "127.0.0.1:22001"; +const CACHE_ADDR: &str = "127.0.0.1:23001"; +const TAPE_ADDR: &str = "127.0.0.1:24001"; +const GATEWAY_ADDR: &str = "127.0.0.1:9000"; + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { + "coldstore_node=info,coldstore_metadata=info,coldstore_cache=info,coldstore_tape=info,coldstore_scheduler=info,coldstore_gateway=info".into() + }), + ) + .init(); + + let runtime = load_runtime_config()?; + let data_dir = runtime.data_dir.clone(); + tokio::fs::create_dir_all(&data_dir).await?; + + let configs = single_node_configs(&data_dir, runtime.tape_enabled); + info!( + data_dir = %data_dir.display(), + gateway = GATEWAY_ADDR, + tape_enabled = runtime.tape_enabled, + "starting ColdStore single-node runtime" + ); + + let mut tasks = Vec::new(); + tasks.push(spawn_component( + "metadata", + coldstore_metadata::run(configs.metadata), + )); + tasks.push(spawn_component( + "cache", + coldstore_cache::run(configs.cache), + )); + if let Some(tape) = configs.tape { + tasks.push(spawn_component("tape", coldstore_tape::run(tape))); + } else { + info!("Tape Worker disabled by config; archive and recall loops will not run"); + } + + sleep(Duration::from_millis(300)).await; + tasks.push(spawn_component( + "scheduler", + coldstore_scheduler::run(configs.scheduler), + )); + + sleep(Duration::from_millis(300)).await; + tasks.push(spawn_component( + "gateway", + coldstore_gateway::run(configs.gateway), + )); + + info!("ColdStore single-node runtime is listening on http://{GATEWAY_ADDR}"); + tokio::signal::ctrl_c().await?; + info!("shutting down ColdStore single-node runtime"); + for task in tasks { + task.abort(); + } + Ok(()) +} + +struct SingleNodeConfigs { + metadata: MetadataConfig, + cache: CacheConfig, + tape: Option, + scheduler: SchedulerConfig, + gateway: GatewayConfig, +} + +fn single_node_configs(data_dir: &Path, tape_enabled: bool) -> SingleNodeConfigs { + let metadata = MetadataConfig { + listen: METADATA_ADDR.to_string(), + cluster: format!("1:{METADATA_ADDR}"), + data_path: data_dir.join("metadata").display().to_string(), + ..MetadataConfig::default() + }; + let cache = CacheConfig { + listen: CACHE_ADDR.to_string(), + metadata_addrs: vec![METADATA_ADDR.to_string()], + backend: CacheBackendConfig::Hdd { + path: data_dir.join("cache").display().to_string(), + max_size_gb: 100, + }, + ..CacheConfig::default() + }; + let tape = tape_enabled.then(|| TapeConfig { + listen: TAPE_ADDR.to_string(), + metadata_addrs: vec![METADATA_ADDR.to_string()], + ..TapeConfig::default() + }); + let mut scheduler = SchedulerConfig { + listen: SCHEDULER_ADDR.to_string(), + metadata_addrs: vec![METADATA_ADDR.to_string()], + cache_addrs: vec![CACHE_ADDR.to_string()], + tape_addrs: if tape_enabled { + vec![TAPE_ADDR.to_string()] + } else { + vec![] + }, + ..SchedulerConfig::default() + }; + scheduler.archive.scan_interval_secs = 2; + scheduler.archive.batch_size = 128; + scheduler.recall.scan_interval_secs = 2; + scheduler.recall.max_concurrent_restores = 4; + if !tape_enabled { + scheduler.archive.enabled = false; + scheduler.recall.enabled = false; + } + + let gateway = GatewayConfig { + listen: GATEWAY_ADDR.to_string(), + scheduler_addrs: vec![SCHEDULER_ADDR.to_string()], + }; + + SingleNodeConfigs { + metadata, + cache, + tape, + scheduler, + gateway, + } +} + +#[derive(Debug, Clone, Deserialize)] +struct NodeRuntimeConfig { + #[serde(default = "default_data_dir")] + data_dir: PathBuf, + #[serde(default = "default_tape_enabled")] + tape_enabled: bool, +} + +impl Default for NodeRuntimeConfig { + fn default() -> Self { + Self { + data_dir: default_data_dir(), + tape_enabled: default_tape_enabled(), + } + } +} + +fn load_runtime_config() -> Result { + let mut runtime = if let Some(path) = config_path()? { + let parsed = config::Config::builder() + .add_source(config::File::from(path.clone())) + .build()? + .try_deserialize()?; + info!(config = %path.display(), "loaded ColdStore node config"); + parsed + } else { + NodeRuntimeConfig::default() + }; + + if let Ok(data_dir) = std::env::var("COLDSTORE_DATA_DIR") { + runtime.data_dir = PathBuf::from(data_dir); + } + if no_tape_override() { + warn!("overriding config: Tape Worker disabled by --no-tape or COLDSTORE_NO_TAPE"); + runtime.tape_enabled = false; + } + Ok(runtime) +} + +fn config_path() -> Result> { + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + if arg == "--config" { + return args + .next() + .map(PathBuf::from) + .map(Some) + .ok_or_else(|| anyhow::anyhow!("--config requires a path")); + } + if let Some(path) = arg.strip_prefix("--config=") { + return Ok(Some(PathBuf::from(path))); + } + } + + if let Ok(path) = std::env::var("COLDSTORE_CONFIG") { + return Ok(Some(PathBuf::from(path))); + } + Ok(None) +} + +fn no_tape_override() -> bool { + std::env::args().any(|arg| arg == "--no-tape") + || matches!( + std::env::var("COLDSTORE_NO_TAPE").as_deref(), + Ok("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON") + ) +} + +fn default_data_dir() -> PathBuf { + PathBuf::from("data/single-node") +} + +fn default_tape_enabled() -> bool { + true +} + +fn spawn_component(name: &'static str, future: F) -> JoinHandle<()> +where + F: std::future::Future> + Send + 'static, +{ + tokio::spawn(async move { + if let Err(err) = future.await { + error!(component = name, error = %err, "ColdStore component exited"); + } + }) +} diff --git a/crates/scheduler/src/lib.rs b/crates/scheduler/src/lib.rs index bba2c2b..59df646 100644 --- a/crates/scheduler/src/lib.rs +++ b/crates/scheduler/src/lib.rs @@ -1,12 +1,13 @@ pub mod service; -use anyhow::Result; +use anyhow::{anyhow, Result}; use coldstore_common::config::SchedulerConfig; use coldstore_proto::cache::cache_service_client::CacheServiceClient; use coldstore_proto::metadata::metadata_service_client::MetadataServiceClient; use coldstore_proto::tape::tape_service_client::TapeServiceClient; +use tokio::time::{sleep, Duration}; use tonic::transport::{Channel, Server}; -use tracing::info; +use tracing::{info, warn}; pub struct SchedulerState { pub metadata: MetadataServiceClient, @@ -19,15 +20,31 @@ pub async fn run(config: SchedulerConfig) -> Result<()> { let addr = config.listen.parse()?; let metadata_addr = format!("http://{}", &config.metadata_addrs[0]); - let metadata = MetadataServiceClient::connect(metadata_addr).await?; + let metadata = connect_metadata_with_retry(metadata_addr).await?; + let cache_addr = config + .cache_addrs + .first() + .ok_or_else(|| anyhow!("scheduler requires at least one cache address"))?; + let cache = connect_cache_with_retry(format!("http://{cache_addr}")).await?; + let tape = if config.archive.enabled || config.recall.enabled { + let tape_addr = config + .tape_addrs + .first() + .ok_or_else(|| anyhow!("scheduler requires at least one tape address"))?; + Some(connect_tape_with_retry(format!("http://{tape_addr}")).await?) + } else { + info!("Scheduler tape client disabled because archive and recall loops are disabled"); + None + }; let state = std::sync::Arc::new(SchedulerState { metadata, - cache: None, - tape: None, + cache: Some(cache), + tape, config: config.clone(), }); + service::spawn_background_loops(state.clone()); let scheduler_service = service::SchedulerServiceImpl::new(state); info!("Scheduler Worker 启动在 {}", config.listen); @@ -43,3 +60,54 @@ pub async fn run(config: SchedulerConfig) -> Result<()> { Ok(()) } + +async fn connect_metadata_with_retry(addr: String) -> Result> { + let mut last_error = None; + for attempt in 1..=30 { + match MetadataServiceClient::connect(addr.clone()).await { + Ok(client) => return Ok(client), + Err(err) => { + warn!(attempt, addr, error = %err, "metadata connection not ready"); + last_error = Some(err); + sleep(Duration::from_millis(200)).await; + } + } + } + Err(last_error + .map(anyhow::Error::from) + .unwrap_or_else(|| anyhow!("metadata connection retry loop did not run"))) +} + +async fn connect_cache_with_retry(addr: String) -> Result> { + let mut last_error = None; + for attempt in 1..=30 { + match CacheServiceClient::connect(addr.clone()).await { + Ok(client) => return Ok(client), + Err(err) => { + warn!(attempt, addr, error = %err, "cache connection not ready"); + last_error = Some(err); + sleep(Duration::from_millis(200)).await; + } + } + } + Err(last_error + .map(anyhow::Error::from) + .unwrap_or_else(|| anyhow!("cache connection retry loop did not run"))) +} + +async fn connect_tape_with_retry(addr: String) -> Result> { + let mut last_error = None; + for attempt in 1..=30 { + match TapeServiceClient::connect(addr.clone()).await { + Ok(client) => return Ok(client), + Err(err) => { + warn!(attempt, addr, error = %err, "tape connection not ready"); + last_error = Some(err); + sleep(Duration::from_millis(200)).await; + } + } + } + Err(last_error + .map(anyhow::Error::from) + .unwrap_or_else(|| anyhow!("tape connection retry loop did not run"))) +} diff --git a/crates/scheduler/src/service.rs b/crates/scheduler/src/service.rs index 04a9a3f..30b6e6d 100644 --- a/crates/scheduler/src/service.rs +++ b/crates/scheduler/src/service.rs @@ -1,17 +1,25 @@ use crate::SchedulerState; use coldstore_proto::cache::cache_service_client::CacheServiceClient; +use coldstore_proto::cache::get_response::Payload as CacheGetPayload; use coldstore_proto::cache::get_staging_response::Payload as GetStagingPayload; +use coldstore_proto::cache::put_restored_request::Payload as PutRestoredPayload; +use coldstore_proto::cache::put_staging_request::Payload as PutStagingPayload; use coldstore_proto::cache::{ - DeleteStagingRequest, GetStagingRequest, ListStagingKeysRequest, StagingKeyEntry, + DeleteStagingRequest, GetRequest as CacheGetRequest, GetStagingRequest, ListStagingKeysRequest, + PutRestoredMeta, PutRestoredRequest, PutStagingMeta, PutStagingRequest, StagingKeyEntry, StagingObjectMeta, }; use coldstore_proto::common; use coldstore_proto::scheduler::scheduler_service_server::SchedulerService; use coldstore_proto::scheduler::*; +use coldstore_proto::tape::read_bundle_request::Location as TapeReadLocation; +use coldstore_proto::tape::read_bundle_response::Payload as TapeReadPayload; use coldstore_proto::tape::tape_service_client::TapeServiceClient as TapeGrpcClient; use coldstore_proto::tape::write_bundle_request::Payload as TapeWriteRequestPayload; use coldstore_proto::tape::{ - WriteBundleMeta as TapeWriteBundleMeta, WriteBundleRequest as TapeWriteBundleRequest, + AcquireDriveRequest, LoadTapeRequest, ReadBundleRequest as TapeReadBundleRequest, + ReleaseDriveRequest, WriteBundleMeta as TapeWriteBundleMeta, + WriteBundleRequest as TapeWriteBundleRequest, }; use prost_types::Timestamp; use sha2::{Digest, Sha256}; @@ -21,8 +29,10 @@ use std::sync::Arc; #[cfg(test)] use std::sync::RwLock; use tokio::sync::mpsc; +use tokio::time::{interval, Duration}; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status, Streaming}; +use tracing::{debug, error, info, warn}; #[tonic::async_trait] pub trait Phase1SchedulerBackend: Send + Sync + 'static { @@ -86,6 +96,13 @@ pub struct ArchiveBatchResult { pub bundle_ids: Vec, } +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct RecallBatchResult { + pub restored_objects: u32, + pub bytes_read: u64, + pub task_ids: Vec, +} + #[tonic::async_trait] pub trait Phase1ArchiveCache: Send + Sync { async fn list_staging_keys( @@ -188,6 +205,59 @@ impl Phase1ArchiveCache for CacheArchiveClient { } } +#[tonic::async_trait] +pub trait Phase1RestoreCache: Send + Sync { + async fn put_restored( + &self, + object: &common::ObjectMetadata, + data: Vec, + expire_at: Timestamp, + ) -> std::result::Result<(), Status>; +} + +#[derive(Clone)] +pub struct CacheRestoreClient { + client: CacheServiceClient, +} + +impl CacheRestoreClient { + pub fn new(client: CacheServiceClient) -> Self { + Self { client } + } +} + +#[tonic::async_trait] +impl Phase1RestoreCache for CacheRestoreClient { + async fn put_restored( + &self, + object: &common::ObjectMetadata, + data: Vec, + expire_at: Timestamp, + ) -> std::result::Result<(), Status> { + let mut client = self.client.clone(); + client + .put_restored(Request::new(tokio_stream::iter(vec![ + PutRestoredRequest { + payload: Some(PutRestoredPayload::Meta(PutRestoredMeta { + bucket: object.bucket.clone(), + key: object.key.clone(), + version_id: object.version_id.clone(), + size: data.len() as u64, + checksum: Some(sha256_hex(&data)), + content_type: object.content_type.clone(), + etag: object.etag.clone(), + expire_at: Some(expire_at), + })), + }, + PutRestoredRequest { + payload: Some(PutRestoredPayload::Data(data)), + }, + ]))) + .await?; + Ok(()) + } +} + #[tonic::async_trait] pub trait TapeArchiveWriter: Send + Sync { async fn write_bundle( @@ -225,6 +295,78 @@ impl TapeArchiveClient { } } +#[tonic::async_trait] +pub trait TapeRecallReader: Send + Sync { + async fn read_bundle( + &self, + tape_id: &str, + filemark: u32, + length: u64, + ) -> std::result::Result, Status>; +} + +#[derive(Clone)] +pub struct TapeRecallClient { + client: TapeGrpcClient, + drive_id: String, +} + +impl TapeRecallClient { + pub fn new( + client: TapeGrpcClient, + drive_id: impl Into, + ) -> Self { + Self { + client, + drive_id: drive_id.into(), + } + } +} + +async fn acquire_and_load_tape( + client: &mut TapeGrpcClient, + preferred_drive_id: &str, + tape_id: &str, +) -> std::result::Result { + let acquired = client + .acquire_drive(Request::new(AcquireDriveRequest { + preferred_drive_id: Some(preferred_drive_id.to_string()), + required_tape_id: None, + priority: 0, + timeout_secs: 0, + })) + .await? + .into_inner(); + let drive_id = if acquired.drive_id.is_empty() { + preferred_drive_id.to_string() + } else { + acquired.drive_id + }; + + if acquired.current_tape.as_deref() != Some(tape_id) { + client + .load_tape(Request::new(LoadTapeRequest { + tape_id: tape_id.to_string(), + drive_id: drive_id.clone(), + slot_id: None, + })) + .await?; + } + + Ok(drive_id) +} + +async fn release_drive_best_effort( + client: &mut TapeGrpcClient, + drive_id: &str, +) { + let _ = client + .release_drive(Request::new(ReleaseDriveRequest { + drive_id: drive_id.to_string(), + })) + .await; +} + #[tonic::async_trait] impl TapeArchiveWriter for TapeArchiveClient { async fn write_bundle( @@ -234,23 +376,29 @@ impl TapeArchiveWriter for TapeArchiveClient { data: Vec, ) -> std::result::Result { let mut client = self.client.clone(); - let response = client - .write_bundle(Request::new(tokio_stream::iter(vec![ - TapeWriteBundleRequest { - payload: Some(TapeWriteRequestPayload::Meta(TapeWriteBundleMeta { - drive_id: self.drive_id.clone(), - bundle_id: bundle_id.into(), - total_size: data.len() as u64, - object_count, - block_size: self.block_size, - })), - }, - TapeWriteBundleRequest { - payload: Some(TapeWriteRequestPayload::Data(data)), - }, - ]))) - .await? - .into_inner(); + let drive_id = acquire_and_load_tape(&mut client, &self.drive_id, &self.tape_id).await?; + let response = async { + client + .write_bundle(Request::new(tokio_stream::iter(vec![ + TapeWriteBundleRequest { + payload: Some(TapeWriteRequestPayload::Meta(TapeWriteBundleMeta { + drive_id: drive_id.clone(), + bundle_id: bundle_id.into(), + total_size: data.len() as u64, + object_count, + block_size: self.block_size, + })), + }, + TapeWriteBundleRequest { + payload: Some(TapeWriteRequestPayload::Data(data)), + }, + ]))) + .await + .map(|response| response.into_inner()) + } + .await; + release_drive_best_effort(&mut client, &drive_id).await; + let response = response?; if !response.success { return Err(Status::internal(format!( @@ -260,10 +408,10 @@ impl TapeArchiveWriter for TapeArchiveClient { .unwrap_or_else(|| "unknown tape error".into()) ))); } - if response.drive_id != self.drive_id { + if response.drive_id != drive_id { return Err(Status::internal(format!( "tape write_bundle responded for drive {}, expected {}", - response.drive_id, self.drive_id + response.drive_id, drive_id ))); } if response.bundle_id != bundle_id { @@ -283,10 +431,72 @@ impl TapeArchiveWriter for TapeArchiveClient { } } +#[tonic::async_trait] +impl TapeRecallReader for TapeRecallClient { + async fn read_bundle( + &self, + tape_id: &str, + filemark: u32, + length: u64, + ) -> std::result::Result, Status> { + let mut client = self.client.clone(); + let drive_id = acquire_and_load_tape(&mut client, &self.drive_id, tape_id).await?; + let result = async { + let mut stream = client + .read_bundle(Request::new(TapeReadBundleRequest { + drive_id: drive_id.clone(), + location: Some(TapeReadLocation::Filemark(filemark)), + length, + })) + .await? + .into_inner(); + + let mut expected_size = None; + let mut data = Vec::new(); + while let Some(message) = stream.message().await? { + match message.payload { + Some(TapeReadPayload::Meta(meta)) => { + if expected_size.replace(meta.total_size).is_some() { + return Err(Status::internal("tape read returned duplicate metadata")); + } + } + Some(TapeReadPayload::Data(chunk)) => data.extend_from_slice(&chunk), + None => return Err(Status::internal("tape read returned empty chunk")), + } + } + + if let Some(expected_size) = expected_size { + if expected_size != data.len() as u64 { + return Err(Status::data_loss(format!( + "tape read returned {} bytes, expected {}", + data.len(), + expected_size + ))); + } + } + Ok(data) + } + .await; + release_drive_best_effort(&mut client, &drive_id).await; + result + } +} + pub struct MetadataBackedSchedulerBackend { metadata: coldstore_proto::metadata::metadata_service_client::MetadataServiceClient< tonic::transport::Channel, >, + cache: Option>, +} + +struct StagingPut { + bucket: String, + key: String, + version_id: Option, + data: Vec, + checksum: String, + content_type: Option, + etag: String, } impl MetadataBackedSchedulerBackend { @@ -295,7 +505,146 @@ impl MetadataBackedSchedulerBackend { tonic::transport::Channel, >, ) -> Self { - Self { metadata } + Self { + metadata, + cache: None, + } + } + + pub fn new_with_cache( + metadata: coldstore_proto::metadata::metadata_service_client::MetadataServiceClient< + tonic::transport::Channel, + >, + cache: Option>, + ) -> Self { + Self { metadata, cache } + } + + #[allow(clippy::result_large_err)] + fn cache_client( + &self, + ) -> std::result::Result, Status> { + self.cache + .clone() + .ok_or_else(|| Status::failed_precondition("scheduler cache client is not configured")) + } + + async fn put_staging_object(&self, staging: StagingPut) -> std::result::Result { + let mut client = self.cache_client()?; + let size = staging.data.len() as u64; + Ok(client + .put_staging(Request::new(tokio_stream::iter(vec![ + PutStagingRequest { + payload: Some(PutStagingPayload::Meta(PutStagingMeta { + bucket: staging.bucket, + key: staging.key, + version_id: staging.version_id, + size, + checksum: Some(staging.checksum), + content_type: staging.content_type, + etag: Some(staging.etag), + })), + }, + PutStagingRequest { + payload: Some(PutStagingPayload::Data(staging.data)), + }, + ]))) + .await? + .into_inner() + .staging_id) + } + + async fn delete_staging_best_effort(&self, bucket: &str, key: &str, version_id: Option<&str>) { + if let Some(cache) = &self.cache { + let mut client = cache.clone(); + let _ = client + .delete_staging(Request::new(DeleteStagingRequest { + bucket: bucket.into(), + key: key.into(), + version_id: version_id.map(str::to_owned), + })) + .await; + } + } + + async fn read_restored_object( + &self, + object: &common::ObjectMetadata, + ) -> std::result::Result, Status> { + if object.storage_class != common::StorageClass::Cold as i32 { + return Err(Status::failed_precondition(format!( + "object {}/{} is not archived yet and cannot be read from restored cache", + object.bucket, object.key + ))); + } + + let restore_status = object + .restore_status + .and_then(|status| common::RestoreStatus::try_from(status).ok()); + if restore_status != Some(common::RestoreStatus::RestoreCompleted) { + return Err(Status::failed_precondition(format!( + "object {}/{} must complete restore before GET", + object.bucket, object.key + ))); + } + + let mut client = self.cache_client()?; + let mut stream = client + .get(Request::new(CacheGetRequest { + bucket: object.bucket.clone(), + key: object.key.clone(), + version_id: object.version_id.clone(), + })) + .await + .map_err(|status| { + if status.code() == tonic::Code::NotFound { + Status::failed_precondition(format!( + "restored object {}/{} is not present in cache", + object.bucket, object.key + )) + } else { + status + } + })? + .into_inner(); + + let mut expected_size = None; + let mut data = Vec::new(); + while let Some(message) = stream.message().await? { + match message.payload { + Some(CacheGetPayload::Meta(meta)) => { + if expected_size.replace(meta.size).is_some() { + return Err(Status::invalid_argument( + "cache get returned duplicate metadata", + )); + } + } + Some(CacheGetPayload::Data(chunk)) => data.extend_from_slice(&chunk), + None => return Err(Status::internal("cache get returned empty chunk")), + } + } + + let expected_size = expected_size + .ok_or_else(|| Status::internal("cache get stream ended without metadata"))?; + if expected_size != data.len() as u64 { + return Err(Status::data_loss(format!( + "cache returned {} bytes for {}/{}, expected {}", + data.len(), + object.bucket, + object.key, + expected_size + ))); + } + if object.size != data.len() as u64 { + return Err(Status::data_loss(format!( + "metadata size {} for {}/{} does not match restored cache bytes {}", + object.size, + object.bucket, + object.key, + data.len() + ))); + } + Ok(data) } pub async fn archive_staging_batch( @@ -405,6 +754,166 @@ impl MetadataBackedSchedulerBackend { Ok(result) } + + async fn update_recall_task( + &self, + task: common::RecallTask, + ) -> std::result::Result<(), Status> { + let mut client = self.metadata.clone(); + client.update_recall_task(Request::new(task)).await?; + Ok(()) + } + + async fn update_object_restore_status( + &self, + bucket: &str, + key: &str, + status: common::RestoreStatus, + expire_at: Option, + ) -> std::result::Result<(), Status> { + let mut client = self.metadata.clone(); + client + .update_restore_status(Request::new( + coldstore_proto::metadata::UpdateRestoreStatusRequest { + bucket: bucket.into(), + key: key.into(), + status: status as i32, + expire_at, + }, + )) + .await?; + Ok(()) + } + + async fn mark_recall_failed( + &self, + mut task: common::RecallTask, + error: String, + ) -> std::result::Result<(), Status> { + task.status = common::RestoreStatus::RestoreFailed as i32; + task.retry_count = task.retry_count.saturating_add(1); + task.completed_at = Some(now_timestamp()); + task.error = Some(error); + self.update_recall_task(task.clone()).await?; + self.update_object_restore_status( + &task.bucket, + &task.key, + common::RestoreStatus::RestoreFailed, + None, + ) + .await?; + Ok(()) + } + + pub async fn recall_pending_batch( + &self, + cache: &C, + tape: &T, + limit: usize, + ) -> std::result::Result + where + C: Phase1RestoreCache + ?Sized, + T: TapeRecallReader + ?Sized, + { + let mut client = self.metadata.clone(); + let tasks = client + .list_pending_recall_tasks(Request::new(())) + .await? + .into_inner() + .tasks; + let mut result = RecallBatchResult::default(); + + for task in tasks.into_iter().filter(is_pending_recall_task).take(limit) { + let restore_result = self.process_recall_task(cache, tape, task.clone()).await; + match restore_result { + Ok(bytes_read) => { + result.restored_objects += 1; + result.bytes_read += bytes_read; + result.task_ids.push(task.id); + } + Err(status) => { + self.mark_recall_failed(task, status.message().to_string()) + .await?; + } + } + } + + Ok(result) + } + + async fn process_recall_task( + &self, + cache: &C, + tape: &T, + mut task: common::RecallTask, + ) -> std::result::Result + where + C: Phase1RestoreCache + ?Sized, + T: TapeRecallReader + ?Sized, + { + let object = self.head_object(&task.bucket, &task.key).await?; + if object.storage_class != common::StorageClass::Cold as i32 { + return Err(Status::failed_precondition(format!( + "recall task {} requires a COLD object", + task.id + ))); + } + if object.archive_id.as_deref() != Some(task.archive_id.as_str()) { + return Err(Status::failed_precondition(format!( + "recall task {} archive_id does not match current object metadata", + task.id + ))); + } + + task.status = common::RestoreStatus::RestoreInProgress as i32; + task.started_at = Some(now_timestamp()); + self.update_recall_task(task.clone()).await?; + self.update_object_restore_status( + &task.bucket, + &task.key, + common::RestoreStatus::RestoreInProgress, + Some(task.expire_at.unwrap_or_else(|| days_from_now(1))), + ) + .await?; + + let filemark = u32::try_from(task.tape_block_offset) + .map_err(|_| Status::invalid_argument("recall tape offset does not fit filemark"))?; + let data = tape + .read_bundle(&task.tape_id, filemark, task.object_size) + .await?; + if data.len() as u64 != task.object_size { + return Err(Status::data_loss(format!( + "recall task {} read {} bytes, expected {}", + task.id, + data.len(), + task.object_size + ))); + } + let checksum = sha256_hex(&data); + if checksum != task.checksum { + return Err(Status::data_loss(format!( + "recall task {} checksum mismatch", + task.id + ))); + } + + let expire_at = task.expire_at.unwrap_or_else(|| days_from_now(1)); + cache.put_restored(&object, data, expire_at).await?; + + task.status = common::RestoreStatus::RestoreCompleted as i32; + task.completed_at = Some(now_timestamp()); + task.error = None; + self.update_recall_task(task.clone()).await?; + self.update_object_restore_status( + &task.bucket, + &task.key, + common::RestoreStatus::RestoreCompleted, + Some(expire_at), + ) + .await?; + + Ok(task.object_size) + } } #[tonic::async_trait] @@ -476,10 +985,8 @@ impl Phase1SchedulerBackend for MetadataBackedSchedulerBackend { key: &str, ) -> std::result::Result<(common::ObjectMetadata, Vec), Status> { let object = self.head_object(bucket, key).await?; - Err(Status::failed_precondition(format!( - "scheduler.get_object requires phase-1 cache wiring; metadata is available for {}/{} but body retrieval is not yet connected", - object.bucket, object.key - ))) + let data = self.read_restored_object(&object).await?; + Ok((object, data)) } async fn put_object( @@ -490,12 +997,25 @@ impl Phase1SchedulerBackend for MetadataBackedSchedulerBackend { content_type: Option, ) -> std::result::Result { let checksum = sha256_hex(&body); + let size = body.len() as u64; + let version_id: Option = None; + let _staging_id = self + .put_staging_object(StagingPut { + bucket: bucket.into(), + key: key.into(), + version_id: version_id.clone(), + data: body, + checksum: checksum.clone(), + content_type: content_type.clone(), + etag: checksum.clone(), + }) + .await?; let now = now_timestamp(); let object = common::ObjectMetadata { bucket: bucket.into(), key: key.into(), - version_id: None, - size: body.len() as u64, + version_id: version_id.clone(), + size, checksum: checksum.clone(), content_type, etag: Some(checksum.clone()), @@ -510,7 +1030,11 @@ impl Phase1SchedulerBackend for MetadataBackedSchedulerBackend { updated_at: Some(now), }; let mut client = self.metadata.clone(); - client.put_object(Request::new(object)).await?; + if let Err(status) = client.put_object(Request::new(object)).await { + self.delete_staging_best_effort(bucket, key, version_id.as_deref()) + .await; + return Err(status); + } Ok(PutObjectResponse { etag: checksum, version_id: String::new(), @@ -535,7 +1059,7 @@ impl Phase1SchedulerBackend for MetadataBackedSchedulerBackend { bucket: &str, key: &str, days: u32, - _tier: common::RestoreTier, + tier: common::RestoreTier, ) -> std::result::Result { let mut client = self.metadata.clone(); let object = client @@ -571,16 +1095,59 @@ impl Phase1SchedulerBackend for MetadataBackedSchedulerBackend { )) } Some(common::RestoreStatus::Unspecified) | None => { + let archive_id = object.archive_id.clone().ok_or_else(|| { + Status::failed_precondition("restore_object requires archive_id metadata") + })?; + let tape_id = object.tape_id.clone().ok_or_else(|| { + Status::failed_precondition("restore_object requires tape_id metadata") + })?; + let tape_block_offset = object.tape_block_offset.ok_or_else(|| { + Status::failed_precondition("restore_object requires tape_block_offset metadata") + })?; + let expire_at = days_from_now(days.max(1)); client .update_restore_status(Request::new( coldstore_proto::metadata::UpdateRestoreStatusRequest { bucket: bucket.into(), key: key.into(), status: common::RestoreStatus::RestorePending as i32, - expire_at: Some(days_from_now(days.max(1))), + expire_at: Some(expire_at), }, )) .await?; + let task = common::RecallTask { + id: phase1_recall_task_id(bucket, key, object.version_id.as_deref()), + bucket: bucket.into(), + key: key.into(), + version_id: object.version_id.clone(), + archive_id, + tape_id, + tape_set: object.tape_set.clone(), + tape_block_offset, + object_size: object.size, + checksum: object.checksum.clone(), + tier: tier as i32, + days: days.max(1), + expire_at: Some(expire_at), + status: common::RestoreStatus::RestorePending as i32, + drive_id: None, + retry_count: 0, + created_at: Some(now_timestamp()), + started_at: None, + completed_at: None, + error: None, + }; + if let Err(status) = client.put_recall_task(Request::new(task)).await { + let _ = self + .update_object_restore_status( + bucket, + key, + common::RestoreStatus::RestoreFailed, + None, + ) + .await; + return Err(status); + } Ok(RestoreObjectResponse { status_code: 202 }) } } @@ -616,7 +1183,10 @@ pub struct SchedulerServiceImpl { impl SchedulerServiceImpl { pub fn new(state: Arc) -> Self { - let backend = Arc::new(MetadataBackedSchedulerBackend::new(state.metadata.clone())); + let backend = Arc::new(MetadataBackedSchedulerBackend::new_with_cache( + state.metadata.clone(), + state.cache.clone(), + )); Self { _state: state, backend, @@ -736,6 +1306,10 @@ fn storage_class_label(storage_class: i32) -> &'static str { } } +fn is_pending_recall_task(task: &common::RecallTask) -> bool { + common::RestoreStatus::try_from(task.status) == Ok(common::RestoreStatus::RestorePending) +} + fn phase1_bundle_id(bucket: &str, key: &str, version_id: Option<&str>) -> String { match version_id.filter(|version| !version.is_empty()) { Some(version) => format!("phase1-bundle:{bucket}/{key}#{version}"), @@ -743,6 +1317,143 @@ fn phase1_bundle_id(bucket: &str, key: &str, version_id: Option<&str>) -> String } } +fn phase1_recall_task_id(bucket: &str, key: &str, version_id: Option<&str>) -> String { + match version_id.filter(|version| !version.is_empty()) { + Some(version) => format!( + "phase1-recall:{bucket}/{key}#{version}:{}", + uuid::Uuid::new_v4() + ), + None => format!("phase1-recall:{bucket}/{key}:{}", uuid::Uuid::new_v4()), + } +} + +pub fn spawn_background_loops(state: Arc) { + if state.config.archive.enabled { + tokio::spawn(run_archive_loop(state.clone())); + } else { + info!("scheduler archive background loop disabled"); + } + + if state.config.recall.enabled { + tokio::spawn(run_recall_loop(state)); + } else { + info!("scheduler recall background loop disabled"); + } +} + +async fn run_archive_loop(state: Arc) { + let every = Duration::from_secs(state.config.archive.scan_interval_secs.max(1)); + let mut ticker = interval(every); + info!( + "scheduler archive background loop enabled: interval={}s batch_size={}", + every.as_secs(), + state.config.archive.batch_size + ); + + loop { + ticker.tick().await; + match archive_staging_once(state.clone()).await { + Ok(result) if result.archived_objects > 0 => { + info!( + archived_objects = result.archived_objects, + bytes_written = result.bytes_written, + "scheduler archive loop archived staging objects" + ); + } + Ok(_) => debug!("scheduler archive loop found no staging objects"), + Err(status) => warn!( + code = ?status.code(), + message = status.message(), + "scheduler archive loop failed" + ), + } + } +} + +async fn run_recall_loop(state: Arc) { + let every = Duration::from_secs(state.config.recall.scan_interval_secs.max(1)); + let mut ticker = interval(every); + info!( + "scheduler recall background loop enabled: interval={}s max_concurrent_restores={}", + every.as_secs(), + state.config.recall.max_concurrent_restores + ); + + loop { + ticker.tick().await; + match recall_pending_once(state.clone()).await { + Ok(result) if result.restored_objects > 0 => { + info!( + restored_objects = result.restored_objects, + bytes_read = result.bytes_read, + "scheduler recall loop restored objects" + ); + } + Ok(_) => debug!("scheduler recall loop found no pending recall tasks"), + Err(status) => error!( + code = ?status.code(), + message = status.message(), + "scheduler recall loop failed" + ), + } + } +} + +pub async fn archive_staging_once( + state: Arc, +) -> std::result::Result { + let cache = state + .cache + .clone() + .ok_or_else(|| Status::failed_precondition("scheduler cache client is not configured"))?; + let tape = state + .tape + .clone() + .ok_or_else(|| Status::failed_precondition("scheduler tape client is not configured"))?; + let backend = + MetadataBackedSchedulerBackend::new_with_cache(state.metadata.clone(), Some(cache.clone())); + let cache = CacheArchiveClient::new(cache); + let tape_set = if state.config.archive.tape_set.is_empty() { + vec![state.config.archive.tape_id.clone()] + } else { + state.config.archive.tape_set.clone() + }; + let tape = TapeArchiveClient::new( + tape, + state.config.archive.drive_id.clone(), + state.config.archive.tape_id.clone(), + tape_set, + state.config.archive.block_size, + ); + backend + .archive_staging_batch( + &cache, + &tape, + state.config.archive.batch_size.min(u32::MAX as usize) as u32, + ) + .await +} + +pub async fn recall_pending_once( + state: Arc, +) -> std::result::Result { + let cache = state + .cache + .clone() + .ok_or_else(|| Status::failed_precondition("scheduler cache client is not configured"))?; + let tape = state + .tape + .clone() + .ok_or_else(|| Status::failed_precondition("scheduler tape client is not configured"))?; + let backend = + MetadataBackedSchedulerBackend::new_with_cache(state.metadata.clone(), Some(cache.clone())); + let cache = CacheRestoreClient::new(cache); + let tape = TapeRecallClient::new(tape, state.config.recall.drive_id.clone()); + backend + .recall_pending_batch(&cache, &tape, state.config.recall.max_concurrent_restores) + .await +} + #[tonic::async_trait] impl SchedulerService for SchedulerServiceImpl { async fn put_object( @@ -931,10 +1642,11 @@ mod tests { use coldstore_metadata::service::MetadataServiceImpl; use coldstore_proto::cache::cache_service_client::CacheServiceClient; use coldstore_proto::cache::cache_service_server::CacheServiceServer; + use coldstore_proto::cache::put_restored_request::Payload as PutRestoredPayload; use coldstore_proto::cache::put_staging_request::Payload as PutStagingPayload; use coldstore_proto::cache::{ - GetStagingRequest, ListStagingKeysRequest, PutStagingMeta, PutStagingRequest, - StagingKeyEntry, StagingObjectMeta, + GetStagingRequest, ListStagingKeysRequest, PutRestoredMeta, PutRestoredRequest, + PutStagingMeta, PutStagingRequest, StagingKeyEntry, StagingObjectMeta, }; use coldstore_proto::metadata::metadata_service_server::MetadataServiceServer; use coldstore_proto::tape::read_bundle_request::Location as TapeReadLocation; @@ -1447,6 +2159,98 @@ mod tests { (cache_client.expect("connect cache client"), shutdown_tx) } + async fn seed_object_metadata( + metadata: &mut coldstore_proto::metadata::metadata_service_client::MetadataServiceClient< + Channel, + >, + bucket: &str, + key: &str, + body: &[u8], + storage_class: common::StorageClass, + restore_status: Option, + ) { + let now = Timestamp { + seconds: 10, + nanos: 0, + }; + metadata + .put_object(Request::new(common::ObjectMetadata { + bucket: bucket.into(), + key: key.into(), + version_id: None, + size: body.len() as u64, + checksum: sha256_hex(body), + content_type: Some("text/plain".into()), + etag: Some(sha256_hex(body)), + storage_class: storage_class as i32, + archive_id: None, + tape_id: None, + tape_set: vec![], + tape_block_offset: None, + restore_status: restore_status.map(|status| status as i32), + restore_expire_at: restore_status.map(|_| days_from_now(1)), + created_at: Some(now), + updated_at: Some(now), + })) + .await + .expect("seed object metadata"); + } + + async fn read_staging_object( + cache: &mut CacheServiceClient, + bucket: &str, + key: &str, + ) -> (StagingObjectMeta, Vec) { + let mut stream = cache + .get_staging(Request::new(GetStagingRequest { + bucket: bucket.into(), + key: key.into(), + version_id: None, + })) + .await + .expect("get staging") + .into_inner(); + let mut meta = None; + let mut body = Vec::new(); + while let Some(message) = stream.next().await { + match message.expect("staging message").payload.expect("payload") { + GetStagingPayload::Meta(next_meta) => { + assert!(meta.replace(next_meta).is_none(), "duplicate staging meta"); + } + GetStagingPayload::Data(bytes) => body.extend_from_slice(&bytes), + } + } + (meta.expect("staging meta"), body) + } + + async fn put_restored_cache_object( + cache: &mut CacheServiceClient, + bucket: &str, + key: &str, + body: Vec, + ) { + cache + .put_restored(Request::new(tokio_stream::iter(vec![ + PutRestoredRequest { + payload: Some(PutRestoredPayload::Meta(PutRestoredMeta { + bucket: bucket.into(), + key: key.into(), + version_id: None, + size: body.len() as u64, + checksum: Some(sha256_hex(&body)), + content_type: Some("text/plain".into()), + etag: Some(sha256_hex(&body)), + expire_at: Some(days_from_now(1)), + })), + }, + PutRestoredRequest { + payload: Some(PutRestoredPayload::Data(body)), + }, + ]))) + .await + .expect("put restored cache object"); + } + async fn tape_backed_service() -> (TapeServiceClient, oneshot::Sender<()>) { let backend = SimulatorTapeBackend::new(2, 1); backend.insert_tape("slot-1", "TAPE-GRPC").unwrap(); @@ -1600,6 +2404,19 @@ mod tests { assert_eq!(list.contents[0].key, "guide.txt"); let mut metadata = state.metadata.clone(); + metadata + .update_archive_location(Request::new( + coldstore_proto::metadata::UpdateArchiveLocationRequest { + bucket: "docs".into(), + key: "guide.txt".into(), + archive_id: "archive-guide".into(), + tape_id: "TAPE-PHASE1".into(), + tape_set: vec!["TAPE-PHASE1".into()], + tape_block_offset: 0, + }, + )) + .await + .expect("set archive location"); metadata .update_storage_class(Request::new( coldstore_proto::metadata::UpdateStorageClassRequest { @@ -1659,10 +2476,39 @@ mod tests { } #[tokio::test] - async fn default_metadata_backend_puts_object_and_reports_cache_gap() { + async fn default_metadata_backend_rejects_put_when_cache_is_missing() { let (_svc, state, shutdown_tx) = metadata_backed_service().await; let backend = MetadataBackedSchedulerBackend::new(state.metadata.clone()); + backend + .create_bucket("docs") + .await + .expect("create bucket through metadata backend"); + + let err = backend + .put_object( + "docs", + "guide.txt", + b"hello".to_vec(), + Some("text/plain".into()), + ) + .await + .expect_err("put object requires cache staging"); + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert!(err.message().contains("cache client")); + + shutdown_tx.send(()).ok(); + } + + #[tokio::test] + async fn default_metadata_backend_puts_object_into_cache_staging_and_metadata() { + let (_svc, state, metadata_shutdown_tx) = metadata_backed_service().await; + let (mut raw_cache_client, cache_shutdown_tx) = cache_backed_service().await; + let backend = MetadataBackedSchedulerBackend::new_with_cache( + state.metadata.clone(), + Some(raw_cache_client.clone()), + ); + backend .create_bucket("docs") .await @@ -1676,7 +2522,7 @@ mod tests { Some("text/plain".into()), ) .await - .expect("put object through metadata backend"); + .expect("put object through cache-backed metadata backend"); assert!(!put.etag.is_empty()); let listed = backend @@ -1685,15 +2531,130 @@ mod tests { .expect("list objects through metadata backend"); assert_eq!(listed.len(), 1); assert_eq!(listed[0].etag.as_deref(), Some(put.etag.as_str())); + assert_eq!( + listed[0].storage_class, + common::StorageClass::ColdPending as i32 + ); - let err = backend + let (staging_meta, staging_body) = + read_staging_object(&mut raw_cache_client, "docs", "guide.txt").await; + assert_eq!(staging_meta.size, 5); + assert_eq!(staging_meta.checksum.as_deref(), Some(put.etag.as_str())); + assert_eq!(staging_body, b"hello"); + + metadata_shutdown_tx.send(()).ok(); + cache_shutdown_tx.send(()).ok(); + } + + #[tokio::test] + async fn default_metadata_backend_gets_object_from_restored_cache() { + let (_svc, state, metadata_shutdown_tx) = metadata_backed_service().await; + let (mut raw_cache_client, cache_shutdown_tx) = cache_backed_service().await; + let backend = MetadataBackedSchedulerBackend::new_with_cache( + state.metadata.clone(), + Some(raw_cache_client.clone()), + ); + + backend + .create_bucket("docs") + .await + .expect("create bucket through metadata backend"); + let body = b"restored-body".to_vec(); + let mut metadata = state.metadata.clone(); + seed_object_metadata( + &mut metadata, + "docs", + "guide.txt", + &body, + common::StorageClass::Cold, + Some(common::RestoreStatus::RestoreCompleted), + ) + .await; + put_restored_cache_object(&mut raw_cache_client, "docs", "guide.txt", body.clone()).await; + + let (object, data) = backend .get_object("docs", "guide.txt") .await - .expect_err("get_object should explain that cache is still not wired"); - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!(err.message().contains("cache wiring")); + .expect("get restored object"); + assert_eq!( + object.restore_status, + Some(common::RestoreStatus::RestoreCompleted as i32) + ); + assert_eq!(data, body); - shutdown_tx.send(()).ok(); + metadata_shutdown_tx.send(()).ok(); + cache_shutdown_tx.send(()).ok(); + } + + #[tokio::test] + async fn restore_object_creates_pending_recall_task() { + let (_svc, state, metadata_shutdown_tx) = metadata_backed_service().await; + let backend = MetadataBackedSchedulerBackend::new(state.metadata.clone()); + + backend + .create_bucket("docs") + .await + .expect("create bucket through metadata backend"); + let body = b"archive-me".to_vec(); + let mut metadata = state.metadata.clone(); + seed_object_metadata( + &mut metadata, + "docs", + "restore.txt", + &body, + common::StorageClass::Cold, + None, + ) + .await; + metadata + .update_archive_location(Request::new( + coldstore_proto::metadata::UpdateArchiveLocationRequest { + bucket: "docs".into(), + key: "restore.txt".into(), + archive_id: "archive-restore".into(), + tape_id: "TAPE-RESTORE".into(), + tape_set: vec!["TAPE-RESTORE".into()], + tape_block_offset: 7, + }, + )) + .await + .expect("set archive location"); + + let response = backend + .restore_object("docs", "restore.txt", 5, common::RestoreTier::Standard) + .await + .expect("queue restore"); + assert_eq!(response.status_code, 202); + + let object = backend + .head_object("docs", "restore.txt") + .await + .expect("head restored-pending object"); + assert_eq!( + object.restore_status, + Some(common::RestoreStatus::RestorePending as i32) + ); + + let tasks = metadata + .list_pending_recall_tasks(Request::new(())) + .await + .expect("list pending recall tasks") + .into_inner() + .tasks; + assert_eq!(tasks.len(), 1); + assert_eq!(tasks[0].bucket, "docs"); + assert_eq!(tasks[0].key, "restore.txt"); + assert_eq!(tasks[0].archive_id, "archive-restore"); + assert_eq!(tasks[0].tape_id, "TAPE-RESTORE"); + assert_eq!(tasks[0].tape_block_offset, 7); + assert_eq!(tasks[0].object_size, body.len() as u64); + assert_eq!(tasks[0].checksum, sha256_hex(&body)); + assert_eq!( + tasks[0].status, + common::RestoreStatus::RestorePending as i32 + ); + + metadata_shutdown_tx.send(()).ok(); } struct TestArchiveCache { @@ -1870,15 +2831,16 @@ mod tests { .create_bucket("docs") .await .expect("create bucket through metadata backend"); - backend - .put_object( - "docs", - "guide.txt", - b"abcdef".to_vec(), - Some("text/plain".into()), - ) - .await - .expect("put object metadata"); + let mut metadata = state.metadata.clone(); + seed_object_metadata( + &mut metadata, + "docs", + "guide.txt", + b"abcdef", + common::StorageClass::ColdPending, + None, + ) + .await; let cache = TestArchiveCache::with_object("docs", "guide.txt", b"abcdef".to_vec()); let tape = DirectTapeWriter::loaded().await; @@ -1938,15 +2900,16 @@ mod tests { .create_bucket("docs") .await .expect("create bucket through metadata backend"); - backend - .put_object( - "docs", - "from-cache.txt", - b"cache-body".to_vec(), - Some("text/plain".into()), - ) - .await - .expect("put object metadata"); + let mut metadata = state.metadata.clone(); + seed_object_metadata( + &mut metadata, + "docs", + "from-cache.txt", + b"cache-body", + common::StorageClass::ColdPending, + None, + ) + .await; let (mut raw_cache_client, cache_shutdown_tx) = cache_backed_service().await; let body = b"cache-body".to_vec(); @@ -2029,15 +2992,16 @@ mod tests { .await .expect("create bucket through metadata backend"); let body = b"grpc-tape-body".to_vec(); - backend - .put_object( - "docs", - "grpc-tape.txt", - body.clone(), - Some("text/plain".into()), - ) - .await - .expect("put object metadata"); + let mut metadata = state.metadata.clone(); + seed_object_metadata( + &mut metadata, + "docs", + "grpc-tape.txt", + &body, + common::StorageClass::ColdPending, + None, + ) + .await; let (mut raw_cache_client, cache_shutdown_tx) = cache_backed_service().await; raw_cache_client @@ -2109,6 +3073,91 @@ mod tests { tape_shutdown_tx.send(()).ok(); } + #[tokio::test] + async fn scheduler_background_once_archives_and_restores_via_grpc_clients() { + let (_svc, state, metadata_shutdown_tx) = metadata_backed_service().await; + let (mut raw_cache_client, cache_shutdown_tx) = cache_backed_service().await; + let (raw_tape_client, tape_shutdown_tx) = tape_backed_service().await; + let backend = MetadataBackedSchedulerBackend::new_with_cache( + state.metadata.clone(), + Some(raw_cache_client.clone()), + ); + + backend + .create_bucket("docs") + .await + .expect("create bucket through metadata backend"); + let body = b"background-loop-body".to_vec(); + backend + .put_object("docs", "loop.txt", body.clone(), Some("text/plain".into())) + .await + .expect("put object into staging"); + + let mut config = SchedulerConfig::default(); + config.archive.batch_size = 10; + config.archive.drive_id = "drive-0".into(); + config.archive.tape_id = "TAPE-GRPC".into(); + config.archive.tape_set = vec!["TAPE-GRPC".into()]; + config.recall.max_concurrent_restores = 2; + config.recall.drive_id = "drive-0".into(); + let loop_state = Arc::new(SchedulerState { + metadata: state.metadata.clone(), + cache: Some(raw_cache_client.clone()), + tape: Some(raw_tape_client.clone()), + config, + }); + + let archived = archive_staging_once(loop_state.clone()) + .await + .expect("archive once"); + assert_eq!(archived.archived_objects, 1); + assert_eq!(archived.bytes_written, body.len() as u64); + + let archived_object = backend + .head_object("docs", "loop.txt") + .await + .expect("head archived object"); + assert_eq!( + archived_object.storage_class, + common::StorageClass::Cold as i32 + ); + assert_eq!(archived_object.tape_id.as_deref(), Some("TAPE-GRPC")); + + let restore = backend + .restore_object("docs", "loop.txt", 2, common::RestoreTier::Standard) + .await + .expect("queue restore"); + assert_eq!(restore.status_code, 202); + + let recalled = recall_pending_once(loop_state).await.expect("recall once"); + assert_eq!(recalled.restored_objects, 1); + assert_eq!(recalled.bytes_read, body.len() as u64); + + let (object, restored) = backend + .get_object("docs", "loop.txt") + .await + .expect("get restored object from cache"); + assert_eq!( + object.restore_status, + Some(common::RestoreStatus::RestoreCompleted as i32) + ); + assert_eq!(restored, body); + + let listed = raw_cache_client + .list_staging_keys(Request::new(ListStagingKeysRequest { + limit: 10, + after: None, + })) + .await + .expect("list staging after archive") + .into_inner(); + assert!(listed.entries.is_empty()); + + metadata_shutdown_tx.send(()).ok(); + cache_shutdown_tx.send(()).ok(); + tape_shutdown_tx.send(()).ok(); + } + #[test] fn phase1_unimplemented_message_is_stable() { let status = phase1_unimplemented("scheduler.list_buckets"); diff --git a/crates/tape/src/service.rs b/crates/tape/src/service.rs index 7b9ad1f..46fafca 100644 --- a/crates/tape/src/service.rs +++ b/crates/tape/src/service.rs @@ -46,7 +46,7 @@ pub struct TapeServiceImpl { impl TapeServiceImpl { pub fn new(config: &TapeConfig) -> anyhow::Result { - let backend = SimulatorTapeBackend::from_config(config); + let backend = SimulatorTapeBackend::from_config(config)?; Ok(Self::new_with_backend(config.clone(), backend)) } @@ -137,9 +137,21 @@ impl SimulatorTapeBackend { } } - pub fn from_config(config: &TapeConfig) -> Self { + pub fn from_config(config: &TapeConfig) -> anyhow::Result { let drive_count = config.scsi.devices.len().max(1) as u32; - Self::new(8, drive_count) + let backend = Self::new(config.simulator.slot_count.max(1), drive_count); + for (index, tape_id) in config.simulator.tape_ids.iter().enumerate() { + let slot_id = format!("slot-{}", index + 1); + backend + .insert_tape(&slot_id, tape_id) + .map_err(|status| anyhow::anyhow!(status.message().to_string()))?; + } + if let Some(tape_id) = &config.simulator.autoload_tape_id { + backend + .load_tape(tape_id, "drive-0", None) + .map_err(|status| anyhow::anyhow!(status.message().to_string()))?; + } + Ok(backend) } pub fn insert_tape(&self, slot_id: &str, tape_id: &str) -> ServiceResult<()> { From 6a191a403555451a8c15e3de1941029e7a72085f Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Sun, 5 Jul 2026 19:40:17 +0800 Subject: [PATCH 3/4] Harden cold archive consistency --- crates/cache/src/backend.rs | 2 + crates/cache/src/hdd.rs | 6 + crates/cache/src/lib.rs | 1 + crates/cache/src/service.rs | 1336 +++++++++++++++++++-- crates/cache/src/spdk.rs | 80 ++ crates/common/src/config.rs | 91 +- crates/gateway/Cargo.toml | 1 + crates/gateway/src/handler.rs | 1067 ++++++++++++++--- crates/gateway/src/lib.rs | 63 +- crates/gateway/src/protocol.rs | 183 ++- crates/metadata/Cargo.toml | 7 +- crates/metadata/src/command.rs | 1 + crates/metadata/src/lib.rs | 390 ++++++- crates/metadata/src/raft.rs | 180 ++- crates/metadata/src/raft_storage.rs | 32 +- crates/metadata/src/service.rs | 204 +++- crates/metadata/src/state_machine.rs | 289 ++++- crates/proto/proto/metadata.proto | 14 + crates/scheduler/src/lib.rs | 15 + crates/scheduler/src/service.rs | 1617 +++++++++++++++++++++++--- docs/ARCHITECTURE.md | 278 +++++ docs/DESIGN.md | 2 + docs/README.md | 50 +- docs/modules/README.md | 37 +- 24 files changed, 5410 insertions(+), 536 deletions(-) create mode 100644 crates/cache/src/spdk.rs create mode 100644 docs/ARCHITECTURE.md diff --git a/crates/cache/src/backend.rs b/crates/cache/src/backend.rs index 37e0da5..7ee0672 100644 --- a/crates/cache/src/backend.rs +++ b/crates/cache/src/backend.rs @@ -34,6 +34,8 @@ pub struct CacheXattrs { pub size: u64, pub expire_at: i64, pub cached_at: i64, + pub last_access_at: i64, + pub access_count: u64, pub checksum: Option, pub content_type: Option, pub etag: Option, diff --git a/crates/cache/src/hdd.rs b/crates/cache/src/hdd.rs index 81d74ec..774893c 100644 --- a/crates/cache/src/hdd.rs +++ b/crates/cache/src/hdd.rs @@ -46,6 +46,8 @@ struct XattrsJson { size: u64, expire_at: i64, cached_at: i64, + last_access_at: Option, + access_count: Option, checksum: Option, content_type: Option, etag: Option, @@ -60,6 +62,8 @@ fn to_json(x: &CacheXattrs) -> XattrsJson { size: x.size, expire_at: x.expire_at, cached_at: x.cached_at, + last_access_at: Some(x.last_access_at), + access_count: Some(x.access_count), checksum: x.checksum.clone(), content_type: x.content_type.clone(), etag: x.etag.clone(), @@ -78,6 +82,8 @@ fn from_json(j: &XattrsJson) -> CacheXattrs { size: j.size, expire_at: j.expire_at, cached_at: j.cached_at, + last_access_at: j.last_access_at.unwrap_or(j.cached_at), + access_count: j.access_count.unwrap_or(0), checksum: j.checksum.clone(), content_type: j.content_type.clone(), etag: j.etag.clone(), diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index de0c370..0af8e7d 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -1,6 +1,7 @@ pub mod backend; pub mod hdd; pub mod service; +pub mod spdk; use anyhow::Result; use coldstore_common::config::CacheConfig; diff --git a/crates/cache/src/service.rs b/crates/cache/src/service.rs index 0caa22c..69a6b2d 100644 --- a/crates/cache/src/service.rs +++ b/crates/cache/src/service.rs @@ -1,5 +1,6 @@ use crate::backend::{CacheBackend, CacheCategory, CacheXattrs}; use crate::hdd::HddBackend; +use crate::spdk::SpdkBackend; use anyhow::Result; use coldstore_common::config::{CacheBackendConfig, CacheConfig}; use coldstore_proto::cache::cache_service_server::CacheService; @@ -7,9 +8,10 @@ use coldstore_proto::cache::*; use prost_types::Timestamp; use std::collections::HashMap; use std::sync::Arc; -use tokio::sync::{mpsc, RwLock}; +use tokio::sync::{mpsc, Mutex, RwLock}; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status, Streaming}; +use tracing::warn; const STREAM_CHUNK_SIZE: usize = 64 * 1024; @@ -49,16 +51,103 @@ struct StoredEntry { struct CacheIndex { staging: HashMap, restored: HashMap, + used_capacity: u64, hit_count: u64, miss_count: u64, evict_count: u64, evict_bytes: u64, } +#[derive(Debug, Clone, Copy)] +enum CapacityRejectReason { + ZeroCapacity, + IncomingLargerThanCapacity, + NoEvictionCandidate, + StagingBudgetExceeded, + RestoredBudgetExceeded, + GlobalCapacityExceeded, + LowWatermarkExceeded, +} + +impl CapacityRejectReason { + fn code(self) -> &'static str { + match self { + Self::ZeroCapacity => "zero_capacity", + Self::IncomingLargerThanCapacity => "incoming_larger_than_capacity", + Self::NoEvictionCandidate => "no_eviction_candidate", + Self::StagingBudgetExceeded => "staging_budget_exceeded", + Self::RestoredBudgetExceeded => "restored_budget_exceeded", + Self::GlobalCapacityExceeded => "global_capacity_exceeded", + Self::LowWatermarkExceeded => "low_watermark_exceeded", + } + } +} + +fn capacity_reject_error(reason: CapacityRejectReason, detail: impl Into) -> anyhow::Error { + anyhow::anyhow!("capacity_reject:{}: {}", reason.code(), detail.into()) +} + +impl CacheIndex { + fn recompute_used_capacity(&mut self) { + self.used_capacity = self + .staging + .values() + .chain(self.restored.values()) + .map(|entry| entry.xattrs.size) + .sum(); + } + + fn staging_used(&self) -> u64 { + self.staging.values().map(|entry| entry.xattrs.size).sum() + } + + fn restored_used(&self) -> u64 { + self.restored.values().map(|entry| entry.xattrs.size).sum() + } +} + +fn rebuilt_entry_should_replace(current: &StoredEntry, candidate: &StoredEntry) -> bool { + (candidate.xattrs.cached_at, candidate.storage_id) + > (current.xattrs.cached_at, current.storage_id) +} + +#[derive(Clone, Copy)] +enum EvictionPolicy { + Lru, + Lfu, +} + +impl EvictionPolicy { + fn from_config(value: &str) -> Self { + match value.to_lowercase().as_str() { + "lfu" => Self::Lfu, + _ => Self::Lru, + } + } +} + +#[derive(Clone)] pub struct CacheServiceImpl { backend: Arc, - _config: CacheConfig, + max_size_bytes: u64, + eviction_policy: EvictionPolicy, + eviction_batch_size: usize, + eviction_low_watermark: f64, + staging_capacity_bytes: u64, + restored_capacity_bytes: u64, index: Arc>, + write_lock: Arc>, +} + +#[derive(Clone, Copy)] +struct CapacityPressure { + staging_after: u64, + restored_after: u64, + used_after: u64, + staging_budget: u64, + restored_budget: u64, + low_watermark_target: Option, + total_capacity: u64, } impl CacheServiceImpl { @@ -67,15 +156,33 @@ impl CacheServiceImpl { CacheBackendConfig::Hdd { path, max_size_gb } => { Arc::new(HddBackend::new(path.clone(), *max_size_gb).await?) } - CacheBackendConfig::Spdk { .. } => { - anyhow::bail!("SPDK backend not yet implemented") - } + CacheBackendConfig::Spdk { + config_file, + bdev_name, + max_size_gb, + .. + } => Arc::new( + SpdkBackend::new(config_file.clone(), bdev_name.clone(), *max_size_gb).await?, + ), }; + let max_size_bytes = backend_capacity_bytes(&config.backend); + let eviction_policy = EvictionPolicy::from_config(&config.eviction_policy); + let (staging_capacity_bytes, restored_capacity_bytes) = split_cache_capacity( + max_size_bytes, + config.staging_capacity_ratio, + config.restored_capacity_ratio, + ); let svc = Self { backend, - _config: config.clone(), + max_size_bytes, + eviction_policy, + eviction_batch_size: config.eviction_batch_size.max(1), + eviction_low_watermark: normalize_watermark(config.eviction_low_watermark), + staging_capacity_bytes, + restored_capacity_bytes, index: Arc::new(RwLock::new(CacheIndex::default())), + write_lock: Arc::new(Mutex::new(())), }; svc.rebuild_index().await?; Ok(svc) @@ -83,6 +190,9 @@ impl CacheServiceImpl { async fn rebuild_index(&self) -> Result<()> { let mut index = CacheIndex::default(); + let mut stale_storage_ids = Vec::new(); + let mut duplicate_loser_count = 0u64; + let mut duplicate_loser_bytes = 0u64; for (storage_id, xattrs) in self.backend.list_all().await? { let key = CacheKey::new( xattrs.bucket.clone(), @@ -90,42 +200,54 @@ impl CacheServiceImpl { xattrs.version_id.clone(), ); let entry = StoredEntry { storage_id, xattrs }; - match entry.xattrs.category { - CacheCategory::Staging => { - index.staging.insert(key, entry); - } - CacheCategory::Restored => { - index.restored.insert(key, entry); - } + let replaced = match entry.xattrs.category { + CacheCategory::Staging => match index.staging.get(&key) { + Some(current) if !rebuilt_entry_should_replace(current, &entry) => { + duplicate_loser_count = duplicate_loser_count.saturating_add(1); + duplicate_loser_bytes = + duplicate_loser_bytes.saturating_add(entry.xattrs.size); + stale_storage_ids.push(entry.storage_id); + None + } + _ => index.staging.insert(key, entry), + }, + CacheCategory::Restored => match index.restored.get(&key) { + Some(current) if !rebuilt_entry_should_replace(current, &entry) => { + duplicate_loser_count = duplicate_loser_count.saturating_add(1); + duplicate_loser_bytes = + duplicate_loser_bytes.saturating_add(entry.xattrs.size); + stale_storage_ids.push(entry.storage_id); + None + } + _ => index.restored.insert(key, entry), + }, + }; + if let Some(replaced) = replaced { + duplicate_loser_count = duplicate_loser_count.saturating_add(1); + duplicate_loser_bytes = duplicate_loser_bytes.saturating_add(replaced.xattrs.size); + stale_storage_ids.push(replaced.storage_id); } } + index.recompute_used_capacity(); *self.index.write().await = index; - Ok(()) - } - - async fn remove_existing(&self, key: &CacheKey, category: CacheCategory) -> Result<()> { - let existing = { - let index = self.index.read().await; - match category { - CacheCategory::Staging => index.staging.get(key).cloned(), - CacheCategory::Restored => index.restored.get(key).cloned(), - } - }; - if let Some(existing) = existing { - self.backend.delete(existing.storage_id).await?; - let mut index = self.index.write().await; - match category { - CacheCategory::Staging => { - index.staging.remove(key); - } - CacheCategory::Restored => { - index.restored.remove(key); - } + if duplicate_loser_count > 0 { + warn!( + duplicate_loser_count, + duplicate_loser_bytes, + "cache index rebuild found duplicate cache objects; stale storage objects will be deleted best-effort" + ); + } + for storage_id in stale_storage_ids { + if let Err(err) = self.backend.delete(storage_id).await { + warn!( + storage_id, + error = %err, + "cache index rebuild failed to delete duplicate stale storage object" + ); } - index.evict_count += 1; - index.evict_bytes += existing.xattrs.size; } + Ok(()) } @@ -137,16 +259,14 @@ impl CacheServiceImpl { } } - async fn insert_entry(&self, key: CacheKey, entry: StoredEntry) { + async fn insert_entry(&self, key: CacheKey, entry: StoredEntry) -> Option { let mut index = self.index.write().await; - match entry.xattrs.category { - CacheCategory::Staging => { - index.staging.insert(key, entry); - } - CacheCategory::Restored => { - index.restored.insert(key, entry); - } - } + let previous = match entry.xattrs.category { + CacheCategory::Staging => index.staging.insert(key, entry), + CacheCategory::Restored => index.restored.insert(key, entry), + }; + index.recompute_used_capacity(); + previous } async fn update_hit_state(&self, hit: bool) { @@ -158,17 +278,70 @@ impl CacheServiceImpl { } } - async fn delete_entry(&self, key: &CacheKey, category: CacheCategory) -> Result { - let removed = { - let mut index = self.index.write().await; - match category { - CacheCategory::Staging => index.staging.remove(key), - CacheCategory::Restored => index.restored.remove(key), + async fn touch_entry(&self, key: &CacheKey, category: CacheCategory) { + let now = now_unix(); + let mut index = self.index.write().await; + match category { + CacheCategory::Staging => { + if let Some(entry) = index.staging.get_mut(key) { + entry.xattrs.last_access_at = now; + entry.xattrs.access_count += 1; + } + } + CacheCategory::Restored => { + if let Some(entry) = index.restored.get_mut(key) { + entry.xattrs.last_access_at = now; + entry.xattrs.access_count += 1; + } } + } + } + + async fn delete_entry(&self, key: &CacheKey, category: CacheCategory) -> Result { + let storage_id = match category { + CacheCategory::Staging => self + .index + .read() + .await + .staging + .get(key) + .map(|entry| entry.storage_id), + CacheCategory::Restored => self + .index + .read() + .await + .restored + .get(key) + .map(|entry| entry.storage_id), }; - if let Some(entry) = removed { - self.backend.delete(entry.storage_id).await?; + let Some(storage_id) = storage_id else { + return Ok(false); + }; + + self.backend.delete(storage_id).await?; + + let mut index = self.index.write().await; + let removed = match category { + CacheCategory::Staging => index + .staging + .get(key) + .is_some_and(|entry| entry.storage_id == storage_id), + CacheCategory::Restored => index + .restored + .get(key) + .is_some_and(|entry| entry.storage_id == storage_id), + }; + if removed { + match category { + CacheCategory::Staging => { + index.staging.remove(key); + } + CacheCategory::Restored => { + index.restored.remove(key); + } + } + index.recompute_used_capacity(); Ok(true) } else { Ok(false) @@ -176,13 +349,60 @@ impl CacheServiceImpl { } async fn put_bytes(&self, key: CacheKey, data: Vec, xattrs: CacheXattrs) -> Result { - self.remove_existing(&key, xattrs.category).await?; + let guard = self.write_lock.lock().await; + self.ensure_object_size_within_total_capacity(xattrs.size) + .await?; + let existing = self.find_entry(&key, xattrs.category).await; + let capacity_delta = xattrs + .size + .saturating_sub(existing.as_ref().map_or(0, |entry| entry.xattrs.size)); + let reclaim_skip = if xattrs.category == CacheCategory::Restored { + Some(&key) + } else { + None + }; + self.reclaim_expired_restored_except(reclaim_skip).await?; + self.evict_if_needed(capacity_delta, xattrs.category) + .await?; let storage_id = self.backend.write(&key.as_cursor(), &data, &xattrs).await?; - self.insert_entry(key, StoredEntry { storage_id, xattrs }) + let previous = self + .insert_entry(key, StoredEntry { storage_id, xattrs }) .await; + drop(guard); + + if let Some(previous_storage_id) = previous.map(|entry| entry.storage_id) { + if previous_storage_id != storage_id { + if let Err(err) = self.backend.delete(previous_storage_id).await { + warn!( + previous_storage_id, + new_storage_id = storage_id, + error = %err, + "cache overwrite failed to delete replaced storage object" + ); + } + } + } + Ok(storage_id) } + async fn ensure_object_size_within_total_capacity(&self, object_size: u64) -> Result<()> { + let total_capacity = self.effective_total_capacity().await?; + if total_capacity == 0 { + return Err(capacity_reject_error( + CapacityRejectReason::ZeroCapacity, + "cache capacity is zero", + )); + } + if object_size > total_capacity { + return Err(capacity_reject_error( + CapacityRejectReason::IncomingLargerThanCapacity, + "incoming object exceeds cache capacity", + )); + } + Ok(()) + } + async fn read_restored(&self, key: &CacheKey) -> Result { let Some(entry) = self.find_entry(key, CacheCategory::Restored).await else { self.update_hit_state(false).await; @@ -195,10 +415,376 @@ impl CacheServiceImpl { return Err(Status::not_found("restored object has expired")); } + self.touch_entry(key, CacheCategory::Restored).await; self.update_hit_state(true).await; Ok(entry) } + #[cfg(test)] + async fn reclaim_expired_restored(&self) -> Result<()> { + self.reclaim_expired_restored_except(None).await + } + + async fn reclaim_expired_restored_except(&self, skip: Option<&CacheKey>) -> Result<()> { + let now = now_unix(); + let expired: Vec<(CacheKey, StoredEntry)> = { + let index = self.index.read().await; + index + .restored + .iter() + .filter(|(key, _)| skip != Some(*key)) + .filter(|(_, entry)| is_expired_at(entry.xattrs.expire_at, now)) + .map(|(key, entry)| (key.clone(), entry.clone())) + .collect() + }; + + for (key, _entry) in expired { + let _ = self.delete_entry(&key, CacheCategory::Restored).await; + } + + Ok(()) + } + + async fn effective_total_capacity(&self) -> Result { + let used_capacity = { + let index = self.index.read().await; + index.used_capacity + }; + let available_bytes = self.backend.available_bytes().await?; + let observed_total = used_capacity.saturating_add(available_bytes); + Ok(observed_total.min(self.max_size_bytes)) + } + + fn split_capacity_by_static_ratio(&self, total_capacity: u64) -> (u64, u64) { + let configured_budget_total = self + .staging_capacity_bytes + .saturating_add(self.restored_capacity_bytes); + if configured_budget_total == 0 { + return (0, total_capacity); + } + + let staging_budget = + total_capacity.saturating_mul(self.staging_capacity_bytes) / configured_budget_total; + let restored_budget = total_capacity.saturating_sub(staging_budget); + (staging_budget, restored_budget) + } + + async fn evict_if_needed(&self, incoming_size: u64, category: CacheCategory) -> Result<()> { + if incoming_size == 0 { + return Ok(()); + } + + let total_capacity = self.effective_total_capacity().await?; + if total_capacity == 0 { + return Err(capacity_reject_error( + CapacityRejectReason::ZeroCapacity, + "cache capacity is zero", + )); + } + if incoming_size > total_capacity { + return Err(capacity_reject_error( + CapacityRejectReason::IncomingLargerThanCapacity, + "incoming object exceeds cache capacity", + )); + } + + let (staging_budget, restored_budget) = self.split_capacity_by_static_ratio(total_capacity); + let low_watermark_target = self.effective_low_watermark_used(total_capacity); + + for _ in 0..self.evict_batch_size() { + let (staging_used, restored_used, used_capacity) = { + let index = self.index.read().await; + ( + index.staging_used(), + index.restored_used(), + index.used_capacity, + ) + }; + + let staging_after = + staging_used.saturating_add(if category == CacheCategory::Staging { + incoming_size + } else { + 0 + }); + let restored_after = + restored_used.saturating_add(if category == CacheCategory::Restored { + incoming_size + } else { + 0 + }); + let used_after_incoming = used_capacity.saturating_add(incoming_size); + + let pressure = CapacityPressure { + staging_after, + restored_after, + used_after: used_after_incoming, + staging_budget, + restored_budget, + low_watermark_target, + total_capacity, + }; + + let needs_eviction = self.should_evict(pressure); + + if !needs_eviction { + return Ok(()); + } + + let victim_category = { + let index = self.index.read().await; + self.choose_victim_category(category, pressure, &index) + }; + let Some(victim_category) = victim_category else { + break; + }; + + let victim = { + let index = self.index.read().await; + self.select_eviction_candidate(&index, victim_category) + } + .ok_or_else(|| { + capacity_reject_error( + CapacityRejectReason::NoEvictionCandidate, + "no restored cache victim is available for eviction", + ) + })?; + + { + let mut index = self.index.write().await; + let exists = match victim_category { + CacheCategory::Staging => index + .staging + .get(&victim.0) + .is_some_and(|entry| entry.storage_id == victim.1.storage_id), + CacheCategory::Restored => index + .restored + .get(&victim.0) + .is_some_and(|entry| entry.storage_id == victim.1.storage_id), + }; + if !exists { + continue; + } + + match victim_category { + CacheCategory::Staging => { + index.staging.remove(&victim.0); + } + CacheCategory::Restored => { + index.restored.remove(&victim.0); + } + } + index.evict_count += 1; + index.evict_bytes += victim.1.xattrs.size; + index.recompute_used_capacity(); + } + + match self.backend.delete(victim.1.storage_id).await { + Ok(()) => {} + Err(err) => { + let mut index = self.index.write().await; + match victim_category { + CacheCategory::Staging => { + index.staging.insert( + victim.0, + StoredEntry { + storage_id: victim.1.storage_id, + xattrs: victim.1.xattrs, + }, + ); + } + CacheCategory::Restored => { + index.restored.insert( + victim.0, + StoredEntry { + storage_id: victim.1.storage_id, + xattrs: victim.1.xattrs, + }, + ); + } + } + index.recompute_used_capacity(); + return Err(err); + } + } + } + + let index = self.index.read().await; + let staging_after = + index + .staging_used() + .saturating_add(if category == CacheCategory::Staging { + incoming_size + } else { + 0 + }); + let restored_after = + index + .restored_used() + .saturating_add(if category == CacheCategory::Restored { + incoming_size + } else { + 0 + }); + let used_after = index.used_capacity.saturating_add(incoming_size); + + let pressure = CapacityPressure { + staging_after, + restored_after, + used_after, + staging_budget, + restored_budget, + low_watermark_target, + total_capacity, + }; + let needs_eviction = self.should_evict(pressure); + if !needs_eviction { + return Ok(()); + } + if staging_after > staging_budget { + Err(capacity_reject_error( + CapacityRejectReason::StagingBudgetExceeded, + "not enough staging cache budget for write path; cannot fit after eviction", + )) + } else if restored_after > restored_budget { + Err(capacity_reject_error( + CapacityRejectReason::RestoredBudgetExceeded, + "not enough restored cache budget for write path; cannot fit after eviction", + )) + } else if used_after > total_capacity { + Err(capacity_reject_error( + CapacityRejectReason::GlobalCapacityExceeded, + "not enough global cache capacity", + )) + } else if low_watermark_target.is_some_and(|target| used_after > target) { + Err(capacity_reject_error( + CapacityRejectReason::LowWatermarkExceeded, + "not enough cache capacity to satisfy low-watermark", + )) + } else { + Ok(()) + } + } + + fn should_evict(&self, pressure: CapacityPressure) -> bool { + let staging_pressure = pressure.staging_after > pressure.staging_budget; + let restored_pressure = pressure.restored_after > pressure.restored_budget; + let global_watermark_pressure = match pressure.low_watermark_target { + Some(target) => pressure.used_after > target, + None => false, + }; + staging_pressure + || restored_pressure + || global_watermark_pressure + || pressure.used_after > pressure.total_capacity + } + + fn evict_batch_size(&self) -> usize { + self.eviction_batch_size.max(1) + } + + fn map_capacity_error(err: anyhow::Error) -> Status { + let message = err.to_string(); + let message_lc = message.to_lowercase(); + if message_lc.contains("capacity") + || message_lc.contains("not enough") + || message_lc.contains("not enough staging") + || message_lc.contains("not enough restored") + || message_lc.contains("exceeds") + || message_lc.contains("no available") + || message_lc.contains("insufficient") + { + Status::resource_exhausted(message) + } else { + internal_status(err) + } + } + + fn effective_low_watermark_used(&self, total_bytes: u64) -> Option { + let watermark = self.eviction_low_watermark.clamp(0.0, 1.0); + if !watermark.is_finite() || watermark <= 0.0 { + None + } else { + Some((total_bytes as f64 * watermark) as u64) + } + } + + fn choose_victim_category( + &self, + incoming_category: CacheCategory, + pressure: CapacityPressure, + index: &CacheIndex, + ) -> Option { + let incoming_used_after = pressure.used_after; + let staging_after = pressure.staging_after; + let restored_after = pressure.restored_after; + let staging_budget = pressure.staging_budget; + let restored_budget = pressure.restored_budget; + let total_capacity = pressure.total_capacity; + let low_watermark_target = pressure.low_watermark_target; + let has_restored = !index.restored.is_empty(); + + let staging_pressure = staging_after > staging_budget; + let restored_pressure = restored_after > restored_budget; + let global_watermark_pressure = match low_watermark_target { + Some(target) => incoming_used_after > target, + None => false, + }; + + if staging_pressure { + return None; + } + + if has_restored + && (restored_pressure + || global_watermark_pressure + || incoming_used_after > total_capacity) + { + return Some(CacheCategory::Restored); + } + + let _ = incoming_category; + None + } + + fn select_eviction_candidate( + &self, + index: &CacheIndex, + category: CacheCategory, + ) -> Option<(CacheKey, StoredEntry)> { + let candidate_map = match category { + CacheCategory::Staging => return None, + CacheCategory::Restored => &index.restored, + }; + + match self.eviction_policy { + EvictionPolicy::Lfu => candidate_map + .iter() + .min_by_key(|(key, entry)| { + ( + entry.xattrs.access_count, + entry.xattrs.last_access_at, + entry.xattrs.cached_at, + entry.xattrs.size, + key.as_cursor(), + ) + }) + .map(|(key, entry)| (key.clone(), entry.clone())), + EvictionPolicy::Lru => candidate_map + .iter() + .min_by_key(|(_, entry)| { + ( + entry.xattrs.last_access_at, + entry.xattrs.cached_at, + entry.xattrs.size, + entry.xattrs.access_count, + entry.xattrs.key.as_str(), + ) + }) + .map(|(key, entry)| (key.clone(), entry.clone())), + } + } + async fn build_get_stream( &self, entry: StoredEntry, @@ -304,6 +890,8 @@ impl CacheService for CacheServiceImpl { size: meta.size, expire_at: 0, cached_at: now_unix(), + last_access_at: now_unix(), + access_count: 0, checksum: meta.checksum, content_type: meta.content_type, etag: meta.etag, @@ -312,7 +900,7 @@ impl CacheService for CacheServiceImpl { let storage_id = self .put_bytes(key, data, xattrs) .await - .map_err(internal_status)?; + .map_err(CacheServiceImpl::map_capacity_error)?; Ok(Response::new(PutStagingResponse { staging_id: storage_id.to_string(), @@ -357,6 +945,8 @@ impl CacheService for CacheServiceImpl { size: meta.size, expire_at: expire_at.seconds, cached_at: now_unix(), + last_access_at: now_unix(), + access_count: 0, checksum: meta.checksum, content_type: meta.content_type, etag: meta.etag, @@ -364,7 +954,7 @@ impl CacheService for CacheServiceImpl { }; self.put_bytes(key, data, xattrs) .await - .map_err(internal_status)?; + .map_err(CacheServiceImpl::map_capacity_error)?; Ok(Response::new(())) } @@ -492,29 +1082,61 @@ impl CacheService for CacheServiceImpl { } async fn stats(&self, _req: Request<()>) -> std::result::Result, Status> { - let available = self + let ( + staging_bytes, + restored_bytes, + object_count, + staging_count, + restored_count, + hit_count, + miss_count, + evict_count, + evict_bytes, + ) = { + let index = self.index.read().await; + ( + index + .staging + .values() + .map(|entry| entry.xattrs.size) + .sum::(), + index + .restored + .values() + .map(|entry| entry.xattrs.size) + .sum::(), + (index.staging.len() + index.restored.len()) as u64, + index.staging.len() as u64, + index.restored.len() as u64, + index.hit_count, + index.miss_count, + index.evict_count, + index.evict_bytes, + ) + }; + + let used_capacity = staging_bytes + restored_bytes; + let available_capacity = self .backend .available_bytes() .await .map_err(internal_status)?; - let index = self.index.read().await; - let staging_bytes: u64 = index.staging.values().map(|entry| entry.xattrs.size).sum(); - let restored_bytes: u64 = index.restored.values().map(|entry| entry.xattrs.size).sum(); - let used_capacity = staging_bytes + restored_bytes; - let total_capacity = used_capacity + available; + let total_capacity = used_capacity + .saturating_add(available_capacity) + .min(self.max_size_bytes); Ok(Response::new(CacheStats { total_capacity, used_capacity, - object_count: (index.staging.len() + index.restored.len()) as u64, - staging_count: index.staging.len() as u64, + object_count, + staging_count, staging_bytes, - restored_count: index.restored.len() as u64, + restored_count, restored_bytes, - hit_count: index.hit_count, - miss_count: index.miss_count, - evict_count: index.evict_count, - evict_bytes: index.evict_bytes, + hit_count, + miss_count, + evict_count, + evict_bytes, })) } } @@ -534,6 +1156,49 @@ fn is_expired(expire_at: i64) -> bool { expire_at > 0 && expire_at <= now_unix() } +fn is_expired_at(expire_at: i64, now: i64) -> bool { + expire_at > 0 && expire_at <= now +} + +fn backend_capacity_bytes(config: &CacheBackendConfig) -> u64 { + match config { + CacheBackendConfig::Hdd { max_size_gb, .. } => { + max_size_gb.saturating_mul(1024 * 1024 * 1024) + } + CacheBackendConfig::Spdk { max_size_gb, .. } => { + max_size_gb.saturating_mul(1024 * 1024 * 1024) + } + } +} + +fn normalize_watermark(watermark: f64) -> f64 { + if watermark.is_nan() || !watermark.is_finite() { + return 0.8; + } + watermark.clamp(0.0, 1.0) +} + +fn normalize_cache_ratio(ratio: f64) -> f64 { + if ratio.is_nan() || !ratio.is_finite() { + return 0.0; + } + ratio.clamp(0.0, 1.0) +} + +fn split_cache_capacity(total_bytes: u64, staging_ratio: f64, restored_ratio: f64) -> (u64, u64) { + let staging = normalize_cache_ratio(staging_ratio); + let restored = normalize_cache_ratio(restored_ratio); + let normalized_total = staging + restored; + + if normalized_total == 0.0 { + return (0, total_bytes); + } + + let staging_bytes = ((total_bytes as f64) * (staging / normalized_total)).floor() as u64; + let restored_bytes = total_bytes.saturating_sub(staging_bytes); + (staging_bytes, restored_bytes) +} + fn internal_status(err: anyhow::Error) -> Status { Status::internal(err.to_string()) } @@ -544,6 +1209,7 @@ mod tests { use coldstore_proto::cache::ContainsRequest; use std::time::{SystemTime, UNIX_EPOCH}; use tokio_stream::StreamExt; + use tonic::Code; fn test_config() -> CacheConfig { let unique = SystemTime::now() @@ -591,6 +1257,8 @@ mod tests { size: 11, expire_at: expires_at, cached_at: now_unix(), + last_access_at: now_unix(), + access_count: 0, checksum: Some("sum".into()), content_type: Some("text/plain".into()), etag: Some("etag-1".into()), @@ -650,6 +1318,8 @@ mod tests { size: 5, expire_at: 0, cached_at: now_unix(), + last_access_at: now_unix(), + access_count: 0, checksum: None, content_type: None, etag: Some("etag-2".into()), @@ -672,4 +1342,530 @@ mod tests { assert_eq!(listed.entries[0].bucket, "docs"); assert_eq!(listed.entries[0].key, "draft.txt"); } + + #[tokio::test] + async fn expired_restored_is_cleaned() { + let mut svc = CacheServiceImpl::new(&test_config()) + .await + .expect("service init"); + svc.max_size_bytes = 100; + let now = now_unix(); + let stale_key = CacheKey::new("docs".into(), "expired.txt".into(), None); + svc.put_bytes( + stale_key.clone(), + b"x".to_vec(), + CacheXattrs { + bucket: "docs".into(), + key: "expired.txt".into(), + version_id: None, + size: 1, + expire_at: now - 1, + cached_at: now, + last_access_at: now, + access_count: 0, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Restored, + }, + ) + .await + .expect("put stale restored"); + + svc.reclaim_expired_restored() + .await + .expect("reclaim should run"); + let response = svc + .contains(Request::new(ContainsRequest { + bucket: "docs".into(), + key: "expired.txt".into(), + version_id: None, + })) + .await + .expect("contains after reclaim") + .into_inner(); + assert!(!response.exists); + + let stats = svc + .stats(Request::new(())) + .await + .expect("stats") + .into_inner(); + assert_eq!(stats.used_capacity, 0); + assert!(svc + .find_entry(&stale_key, CacheCategory::Restored) + .await + .is_none()); + } + + #[tokio::test] + async fn restored_eviction_prefers_lru() { + let mut svc = CacheServiceImpl::new(&test_config()) + .await + .expect("service init"); + svc.max_size_bytes = 50; + svc.eviction_low_watermark = 0.9; + svc.eviction_policy = EvictionPolicy::Lru; + svc.eviction_batch_size = 4; + svc.staging_capacity_bytes = 0; + svc.restored_capacity_bytes = 50; + let now = now_unix(); + + svc.put_bytes( + CacheKey::new("docs".into(), "a".into(), None), + vec![0u8; 20], + CacheXattrs { + bucket: "docs".into(), + key: "a".into(), + version_id: None, + size: 20, + expire_at: now + 3600, + cached_at: now - 30, + last_access_at: now - 30, + access_count: 0, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Restored, + }, + ) + .await + .expect("restore a"); + svc.put_bytes( + CacheKey::new("docs".into(), "b".into(), None), + vec![0u8; 20], + CacheXattrs { + bucket: "docs".into(), + key: "b".into(), + version_id: None, + size: 20, + expire_at: now + 3600, + cached_at: now - 20, + last_access_at: now - 20, + access_count: 0, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Restored, + }, + ) + .await + .expect("restore b"); + svc.put_bytes( + CacheKey::new("docs".into(), "c".into(), None), + vec![0u8; 20], + CacheXattrs { + bucket: "docs".into(), + key: "c".into(), + version_id: None, + size: 20, + expire_at: now + 3600, + cached_at: now, + last_access_at: now, + access_count: 0, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Restored, + }, + ) + .await + .expect("restore c"); + + assert!(svc + .find_entry( + &CacheKey::new("docs".into(), "a".into(), None), + CacheCategory::Restored + ) + .await + .is_none()); + assert!(svc + .find_entry( + &CacheKey::new("docs".into(), "b".into(), None), + CacheCategory::Restored + ) + .await + .is_some()); + assert!(svc + .find_entry( + &CacheKey::new("docs".into(), "c".into(), None), + CacheCategory::Restored + ) + .await + .is_some()); + let stats = svc + .stats(Request::new(())) + .await + .expect("stats") + .into_inner(); + assert_eq!(stats.restored_count, 2); + assert_eq!(stats.used_capacity, 40); + } + + #[tokio::test] + async fn restored_eviction_prefers_lfu() { + let mut svc = CacheServiceImpl::new(&test_config()) + .await + .expect("service init"); + svc.max_size_bytes = 50; + svc.eviction_low_watermark = 0.5; + svc.eviction_policy = EvictionPolicy::Lfu; + svc.eviction_batch_size = 4; + svc.staging_capacity_bytes = 0; + svc.restored_capacity_bytes = 50; + let now = now_unix(); + + svc.put_bytes( + CacheKey::new("docs".into(), "a".into(), None), + vec![0u8; 20], + CacheXattrs { + bucket: "docs".into(), + key: "a".into(), + version_id: None, + size: 20, + expire_at: now + 3600, + cached_at: now - 30, + last_access_at: now - 30, + access_count: 1, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Restored, + }, + ) + .await + .expect("restore a"); + svc.put_bytes( + CacheKey::new("docs".into(), "b".into(), None), + vec![0u8; 20], + CacheXattrs { + bucket: "docs".into(), + key: "b".into(), + version_id: None, + size: 20, + expire_at: now + 3600, + cached_at: now - 20, + last_access_at: now - 20, + access_count: 5, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Restored, + }, + ) + .await + .expect("restore b"); + svc.put_bytes( + CacheKey::new("docs".into(), "c".into(), None), + vec![0u8; 20], + CacheXattrs { + bucket: "docs".into(), + key: "c".into(), + version_id: None, + size: 20, + expire_at: now + 3600, + cached_at: now, + last_access_at: now, + access_count: 3, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Restored, + }, + ) + .await + .expect("restore c"); + + assert!(svc + .find_entry( + &CacheKey::new("docs".into(), "a".into(), None), + CacheCategory::Restored + ) + .await + .is_none()); + assert!(svc + .find_entry( + &CacheKey::new("docs".into(), "c".into(), None), + CacheCategory::Restored + ) + .await + .is_some()); + } + + #[tokio::test] + async fn staging_oversubscription_returns_resource_exhausted() { + let mut svc = CacheServiceImpl::new(&test_config()) + .await + .expect("service init"); + svc.max_size_bytes = 4; + let now = now_unix(); + + let result = svc + .put_bytes( + CacheKey::new("docs".into(), "overflow.txt".into(), None), + vec![0u8; 5], + CacheXattrs { + bucket: "docs".into(), + key: "overflow.txt".into(), + version_id: None, + size: 5, + expire_at: 0, + cached_at: now, + last_access_at: now, + access_count: 0, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Staging, + }, + ) + .await + .expect_err("staging oversubscription should fail"); + let mapped = CacheServiceImpl::map_capacity_error(result); + assert_eq!(mapped.code(), Code::ResourceExhausted); + assert!(mapped + .message() + .contains("capacity_reject:incoming_larger_than_capacity")); + } + + #[tokio::test] + async fn staging_budget_pressure_does_not_evict_existing_objects() { + let mut svc = CacheServiceImpl::new(&test_config()) + .await + .expect("service init"); + svc.max_size_bytes = 100; + svc.eviction_low_watermark = 1.0; + svc.eviction_batch_size = 4; + svc.staging_capacity_bytes = 50; + svc.restored_capacity_bytes = 50; + let now = now_unix(); + + let restored_key = CacheKey::new("docs".into(), "restored.bin".into(), None); + svc.put_bytes( + restored_key.clone(), + vec![0u8; 30], + CacheXattrs { + bucket: "docs".into(), + key: "restored.bin".into(), + version_id: None, + size: 30, + expire_at: now + 3600, + cached_at: now, + last_access_at: now, + access_count: 0, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Restored, + }, + ) + .await + .expect("put restored"); + + let staging_key = CacheKey::new("docs".into(), "staging.bin".into(), None); + svc.put_bytes( + staging_key.clone(), + vec![0u8; 40], + CacheXattrs { + bucket: "docs".into(), + key: "staging.bin".into(), + version_id: None, + size: 40, + expire_at: 0, + cached_at: now, + last_access_at: now, + access_count: 0, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Staging, + }, + ) + .await + .expect("put staging"); + + let result = svc + .put_bytes( + CacheKey::new("docs".into(), "overflow-staging.bin".into(), None), + vec![0u8; 20], + CacheXattrs { + bucket: "docs".into(), + key: "overflow-staging.bin".into(), + version_id: None, + size: 20, + expire_at: 0, + cached_at: now, + last_access_at: now, + access_count: 0, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Staging, + }, + ) + .await + .expect_err("staging budget pressure should reject write"); + + let mapped = CacheServiceImpl::map_capacity_error(result); + assert_eq!(mapped.code(), Code::ResourceExhausted); + assert!(mapped + .message() + .contains("capacity_reject:staging_budget_exceeded")); + assert!(svc + .find_entry(&staging_key, CacheCategory::Staging) + .await + .is_some()); + assert!(svc + .find_entry(&restored_key, CacheCategory::Restored) + .await + .is_some()); + let stats = svc + .stats(Request::new(())) + .await + .expect("stats") + .into_inner(); + assert_eq!(stats.evict_count, 0); + } + + #[tokio::test] + async fn failed_staging_overwrite_preserves_existing_object() { + let mut svc = CacheServiceImpl::new(&test_config()) + .await + .expect("service init"); + svc.max_size_bytes = 100; + svc.eviction_low_watermark = 1.0; + svc.staging_capacity_bytes = 80; + svc.restored_capacity_bytes = 20; + let now = now_unix(); + let key = CacheKey::new("docs".into(), "staging.bin".into(), None); + + svc.put_bytes( + key.clone(), + vec![1u8; 80], + CacheXattrs { + bucket: "docs".into(), + key: "staging.bin".into(), + version_id: None, + size: 80, + expire_at: 0, + cached_at: now, + last_access_at: now, + access_count: 0, + checksum: Some("old".into()), + content_type: None, + etag: None, + category: CacheCategory::Staging, + }, + ) + .await + .expect("initial staging write"); + + let result = svc + .put_bytes( + key.clone(), + vec![2u8; 90], + CacheXattrs { + bucket: "docs".into(), + key: "staging.bin".into(), + version_id: None, + size: 90, + expire_at: 0, + cached_at: now + 1, + last_access_at: now + 1, + access_count: 0, + checksum: Some("new".into()), + content_type: None, + etag: None, + category: CacheCategory::Staging, + }, + ) + .await + .expect_err("overwrite should be rejected by staging budget"); + + let mapped = CacheServiceImpl::map_capacity_error(result); + assert_eq!(mapped.code(), Code::ResourceExhausted); + assert!(mapped + .message() + .contains("capacity_reject:staging_budget_exceeded")); + + let entry = svc + .find_entry(&key, CacheCategory::Staging) + .await + .expect("old staging entry must remain"); + assert_eq!(entry.xattrs.size, 80); + assert_eq!(entry.xattrs.checksum.as_deref(), Some("old")); + } + + #[tokio::test] + async fn restored_write_pressure_does_not_evict_staging_objects() { + let mut svc = CacheServiceImpl::new(&test_config()) + .await + .expect("service init"); + svc.max_size_bytes = 100; + svc.eviction_low_watermark = 1.0; + svc.eviction_batch_size = 4; + svc.staging_capacity_bytes = 80; + svc.restored_capacity_bytes = 20; + let now = now_unix(); + + let staging_key = CacheKey::new("docs".into(), "protected-staging.bin".into(), None); + svc.put_bytes( + staging_key.clone(), + vec![0u8; 80], + CacheXattrs { + bucket: "docs".into(), + key: "protected-staging.bin".into(), + version_id: None, + size: 80, + expire_at: 0, + cached_at: now, + last_access_at: now, + access_count: 0, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Staging, + }, + ) + .await + .expect("put staging"); + + let result = svc + .put_bytes( + CacheKey::new("docs".into(), "oversized-restored.bin".into(), None), + vec![0u8; 30], + CacheXattrs { + bucket: "docs".into(), + key: "oversized-restored.bin".into(), + version_id: None, + size: 30, + expire_at: now + 3600, + cached_at: now, + last_access_at: now, + access_count: 0, + checksum: None, + content_type: None, + etag: None, + category: CacheCategory::Restored, + }, + ) + .await + .expect_err("restored pressure should not evict staging"); + + let mapped = CacheServiceImpl::map_capacity_error(result); + assert_eq!(mapped.code(), Code::ResourceExhausted); + assert!(mapped + .message() + .contains("capacity_reject:restored_budget_exceeded")); + assert!(svc + .find_entry(&staging_key, CacheCategory::Staging) + .await + .is_some()); + let stats = svc + .stats(Request::new(())) + .await + .expect("stats") + .into_inner(); + assert_eq!(stats.evict_count, 0); + } } diff --git a/crates/cache/src/spdk.rs b/crates/cache/src/spdk.rs new file mode 100644 index 0000000..097be7a --- /dev/null +++ b/crates/cache/src/spdk.rs @@ -0,0 +1,80 @@ +use crate::backend::{CacheBackend, CacheXattrs}; +use anyhow::Result; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::hdd::HddBackend; + +pub struct SpdkBackend { + _config_file: String, + _bdev_name: String, + inner: Arc, +} + +impl SpdkBackend { + pub async fn new(config_file: String, bdev_name: String, max_size_gb: u64) -> Result { + let root = Path::new(&config_file) + .join("spdk") + .join(sanitize_bdev_name(&bdev_name)); + let inner = + Arc::new(HddBackend::new(root.to_string_lossy().to_string(), max_size_gb).await?); + Ok(Self { + _config_file: config_file, + _bdev_name: bdev_name, + inner, + }) + } +} + +#[tonic::async_trait] +impl CacheBackend for SpdkBackend { + async fn write(&self, cache_key: &str, data: &[u8], xattrs: &CacheXattrs) -> Result { + self.inner.write(cache_key, data, xattrs).await + } + + async fn read(&self, storage_id: u64) -> Result> { + self.inner.read(storage_id).await + } + + async fn delete(&self, storage_id: u64) -> Result<()> { + self.inner.delete(storage_id).await + } + + async fn read_xattrs(&self, storage_id: u64) -> Result { + self.inner.read_xattrs(storage_id).await + } + + async fn list_all(&self) -> Result> { + self.inner.list_all().await + } + + async fn available_bytes(&self) -> Result { + self.inner.available_bytes().await + } +} + +fn sanitize_bdev_name(name: &str) -> PathBuf { + let mut value = PathBuf::new(); + let safe = name + .chars() + .map(|ch| match ch { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' => ch, + _ => '_', + }) + .collect::(); + value.push(safe); + value +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sanitize_bdev_replaces_special_chars() { + assert_eq!( + sanitize_bdev_name("nvme0n1p1@ctrl"), + PathBuf::from("nvme0n1p1_ctrl") + ); + } +} diff --git a/crates/common/src/config.rs b/crates/common/src/config.rs index fd9b7f3..fe1ae23 100644 --- a/crates/common/src/config.rs +++ b/crates/common/src/config.rs @@ -29,9 +29,21 @@ pub struct MetadataConfig { pub listen: String, pub cluster: String, pub data_path: String, + #[serde(default = "default_metadata_consensus_mode")] + pub consensus_mode: MetadataConsensusMode, + #[serde(default)] + pub raft_state_path: Option, pub rocksdb: RocksDbConfig, } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MetadataConsensusMode { + Standalone, + LocalRaft, + PersistentRaft, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RocksDbConfig { pub max_open_files: i32, @@ -44,8 +56,10 @@ impl Default for MetadataConfig { Self { node_id: 1, listen: "0.0.0.0:21001".to_string(), - cluster: "1:127.0.0.1:21001,2:127.0.0.1:21002,3:127.0.0.1:21003".to_string(), + cluster: "1:127.0.0.1:21001".to_string(), data_path: "/var/lib/coldstore/metadata".to_string(), + consensus_mode: default_metadata_consensus_mode(), + raft_state_path: None, rocksdb: RocksDbConfig { max_open_files: 1024, write_buffer_size_mb: 64, @@ -55,12 +69,18 @@ impl Default for MetadataConfig { } } +fn default_metadata_consensus_mode() -> MetadataConsensusMode { + MetadataConsensusMode::PersistentRaft +} + // --------------------------------------------------------------------------- // Scheduler Worker 配置 // --------------------------------------------------------------------------- #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SchedulerConfig { + #[serde(default = "default_scheduler_node_id")] + pub node_id: u64, pub listen: String, pub metadata_addrs: Vec, #[serde(default = "default_cache_addrs")] @@ -77,6 +97,8 @@ pub struct ArchiveSchedulerConfig { pub enabled: bool, pub scan_interval_secs: u64, pub batch_size: usize, + #[serde(default = "default_scheduler_worker_count")] + pub max_workers: usize, #[serde(default = "default_scheduler_drive_id")] pub drive_id: String, #[serde(default = "default_scheduler_tape_id")] @@ -98,6 +120,8 @@ pub struct RecallSchedulerConfig { #[serde(default = "default_recall_scan_interval_secs")] pub scan_interval_secs: u64, pub max_concurrent_restores: usize, + #[serde(default = "default_scheduler_worker_count")] + pub max_workers: usize, pub merge_window_secs: u64, pub restore_timeout_secs: u64, pub read_buffer_mb: u64, @@ -108,18 +132,16 @@ pub struct RecallSchedulerConfig { impl Default for SchedulerConfig { fn default() -> Self { Self { + node_id: 1, listen: "0.0.0.0:22001".to_string(), - metadata_addrs: vec![ - "127.0.0.1:21001".to_string(), - "127.0.0.1:21002".to_string(), - "127.0.0.1:21003".to_string(), - ], + metadata_addrs: vec!["127.0.0.1:21001".to_string()], cache_addrs: default_cache_addrs(), tape_addrs: default_tape_addrs(), archive: ArchiveSchedulerConfig { enabled: true, scan_interval_secs: 60, batch_size: 1000, + max_workers: 1, drive_id: default_scheduler_drive_id(), tape_id: default_scheduler_tape_id(), tape_set: default_scheduler_tape_set(), @@ -134,6 +156,7 @@ impl Default for SchedulerConfig { enabled: true, scan_interval_secs: default_recall_scan_interval_secs(), max_concurrent_restores: 10, + max_workers: 1, merge_window_secs: 60, restore_timeout_secs: 3600, read_buffer_mb: 64, @@ -171,6 +194,14 @@ fn default_scheduler_tape_set() -> Vec { vec![default_scheduler_tape_id()] } +fn default_scheduler_node_id() -> u64 { + 1 +} + +fn default_scheduler_worker_count() -> usize { + 1 +} + // --------------------------------------------------------------------------- // Cache Worker 配置 // --------------------------------------------------------------------------- @@ -184,6 +215,8 @@ pub struct CacheConfig { pub eviction_policy: String, pub eviction_batch_size: usize, pub eviction_low_watermark: f64, + pub staging_capacity_ratio: f64, + pub restored_capacity_ratio: f64, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -204,11 +237,7 @@ impl Default for CacheConfig { fn default() -> Self { Self { listen: "0.0.0.0:23001".to_string(), - metadata_addrs: vec![ - "127.0.0.1:21001".to_string(), - "127.0.0.1:21002".to_string(), - "127.0.0.1:21003".to_string(), - ], + metadata_addrs: vec!["127.0.0.1:21001".to_string()], backend: CacheBackendConfig::Hdd { path: "/var/lib/coldstore/cache".to_string(), max_size_gb: 100, @@ -217,6 +246,8 @@ impl Default for CacheConfig { eviction_policy: "Lru".to_string(), eviction_batch_size: 64, eviction_low_watermark: 0.8, + staging_capacity_ratio: 0.35, + restored_capacity_ratio: 0.65, } } } @@ -270,11 +301,7 @@ impl Default for TapeConfig { fn default() -> Self { Self { listen: "0.0.0.0:24001".to_string(), - metadata_addrs: vec![ - "127.0.0.1:21001".to_string(), - "127.0.0.1:21002".to_string(), - "127.0.0.1:21003".to_string(), - ], + metadata_addrs: vec!["127.0.0.1:21001".to_string()], sdk_backend: "simulator".to_string(), scsi: ScsiConfig { devices: vec!["/dev/nst0".to_string()], @@ -301,3 +328,35 @@ fn default_simulator_tape_ids() -> Vec { fn default_simulator_autoload_tape_id() -> Option { Some(default_scheduler_tape_id()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn metadata_default_consensus_mode_is_persistent_raft() { + assert!(matches!( + MetadataConfig::default().consensus_mode, + MetadataConsensusMode::PersistentRaft + )); + } + + #[test] + fn metadata_default_config_is_single_node_persistent_raft() { + let config = MetadataConfig::default(); + assert_eq!(config.node_id, 1); + assert_eq!(config.cluster, "1:127.0.0.1:21001"); + assert!(matches!( + config.consensus_mode, + MetadataConsensusMode::PersistentRaft + )); + } + + #[test] + fn worker_defaults_use_single_metadata_endpoint() { + let expected = vec!["127.0.0.1:21001".to_string()]; + assert_eq!(SchedulerConfig::default().metadata_addrs, expected); + assert_eq!(CacheConfig::default().metadata_addrs, expected); + assert_eq!(TapeConfig::default().metadata_addrs, expected); + } +} diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index 1f66410..6108f3a 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -15,6 +15,7 @@ coldstore-common = { workspace = true } tokio = { workspace = true } tonic = { workspace = true } prost = { workspace = true } +prost-types = { workspace = true } axum = { workspace = true } tower = { workspace = true, features = ["util"] } tower-http = { workspace = true } diff --git a/crates/gateway/src/handler.rs b/crates/gateway/src/handler.rs index c36364e..dd180de 100644 --- a/crates/gateway/src/handler.rs +++ b/crates/gateway/src/handler.rs @@ -1,13 +1,19 @@ -use crate::protocol::{format_restore_header, is_restore_request, S3ErrorCode, S3ErrorResponse}; -use crate::{DownloadedObject, GatewayState}; +use crate::protocol::{ + format_restore_header, is_restore_request, parse_restore_request, S3ErrorCode, S3ErrorResponse, +}; +use crate::GatewayState; use axum::body::{Body, Bytes}; use axum::extract::{Path, Query, State}; -use axum::http::{header::HeaderName, HeaderMap, HeaderValue, StatusCode}; +use axum::http::{header, HeaderMap, HeaderName, HeaderValue, StatusCode}; use axum::response::Response; use axum::{routing::get, Router}; +use chrono::{DateTime, Utc}; use std::collections::HashMap; use std::sync::Arc; +const DEFAULT_LIST_MAX_KEYS: u32 = 1000; +const MAX_LIST_MAX_KEYS: u32 = 1000; + pub fn router(state: Arc) -> Router { build_router().with_state(state) } @@ -87,10 +93,11 @@ async fn list_objects( let prefix = query.get("prefix").map(String::as_str); let marker = query.get("marker").map(String::as_str); let delimiter = query.get("delimiter").map(String::as_str); - let max_keys = query - .get("max-keys") - .and_then(|value| value.parse::().ok()) - .unwrap_or(1000); + let max_keys = match parse_list_max_keys(query.get("max-keys").map(String::as_str)) { + Ok(max_keys) => max_keys, + Err(response) => return *response, + }; + match state .backend .list_objects(&bucket, prefix, marker, delimiter, max_keys) @@ -101,105 +108,347 @@ async fn list_objects( } } -async fn get_object( - State(state): State>, - Path((bucket, key)): Path<(String, String)>, -) -> Response { - match state.backend.get_object(&bucket, &key).await { - Ok(object) => get_object_success_response(object), - Err(status) => grpc_status_to_s3_response(status, &format!("/{bucket}/{key}")), +fn parse_list_max_keys(value: Option<&str>) -> std::result::Result> { + let raw = value.unwrap_or(""); + let max_keys = if raw.is_empty() { + DEFAULT_LIST_MAX_KEYS + } else { + raw.parse::().map_err(|_| { + Box::new(bad_request_response_with_code( + S3ErrorCode::InvalidArgument, + "max-keys must be an integer between 1 and 1000", + "/", + )) + })? + }; + + if !(1..=MAX_LIST_MAX_KEYS).contains(&max_keys) { + return Err(Box::new(bad_request_response_with_code( + S3ErrorCode::InvalidArgument, + "max-keys must be an integer between 1 and 1000", + "/", + ))); } + + Ok(max_keys) } -async fn put_object( +async fn get_object( State(state): State>, Path((bucket, key)): Path<(String, String)>, headers: HeaderMap, - body: Bytes, ) -> Response { - let content_type = headers - .get(axum::http::header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .map(str::to_string); - match state - .backend - .put_object(&bucket, &key, body.to_vec(), content_type) - .await + let resource = format!("/{bucket}/{key}"); + + if let Some(response) = + match check_read_preconditions(&state, &bucket, &key, &headers, &resource).await { + Ok(response) => response, + Err(status) => return grpc_status_to_s3_response(status, &resource), + } { - Ok(response) => put_object_success_response(response), - Err(status) => grpc_status_to_s3_response(status, &format!("/{bucket}/{key}")), + return response; } -} -async fn delete_object( - State(state): State>, - Path((bucket, key)): Path<(String, String)>, -) -> Response { - match state.backend.delete_object(&bucket, &key).await { - Ok(()) => empty_response(StatusCode::NO_CONTENT), - Err(status) => grpc_status_to_s3_response(status, &format!("/{bucket}/{key}")), + let object = match state.backend.get_object(&bucket, &key).await { + Ok(object) => object, + Err(status) => return grpc_status_to_s3_response(status, &resource), + }; + + let object_size = object.body.len(); + let mut response_body = object.body; + let mut status_code = StatusCode::OK; + + if let Some(range_value) = headers.get(header::RANGE) { + let (start, end) = match range_value.to_str() { + Ok(raw_range) => match parse_range_header(raw_range, object_size) { + Ok(Some(range)) => range, + Ok(None) => { + return bad_request_response_with_code( + S3ErrorCode::InvalidArgument, + "invalid Range header", + &resource, + ) + } + Err(response) => return *response, + }, + Err(_) => { + return bad_request_response_with_code( + S3ErrorCode::InvalidArgument, + "invalid Range header", + &resource, + ) + } + }; + + let range_response_length = end.saturating_sub(start).saturating_add(1); + response_body = response_body[start..=end].to_vec(); + status_code = StatusCode::PARTIAL_CONTENT; + + let mut response = Response::new(Body::from(response_body)); + *response.status_mut() = status_code; + apply_object_headers(response.headers_mut(), &object.head); + response.headers_mut().insert( + header::CONTENT_LENGTH, + HeaderValue::from_str(&range_response_length.to_string()).unwrap(), + ); + response + .headers_mut() + .insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes")); + response.headers_mut().insert( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes {start}-{end}/{object_size}")) + .unwrap_or_else(|_| HeaderValue::from_static("bytes */*")), + ); + return response; } + + let mut response = Response::new(Body::from(response_body)); + *response.status_mut() = status_code; + response + .headers_mut() + .insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes")); + apply_object_headers(response.headers_mut(), &object.head); + response } -async fn head_object( - State(state): State>, - Path((bucket, key)): Path<(String, String)>, -) -> Response { - match state.backend.head_object(&bucket, &key).await { - Ok(head) => head_object_success_response(head), - Err(status) => grpc_status_to_s3_response(status, &format!("/{bucket}/{key}")), - } +fn parse_range_header( + raw_range: &str, + body_length: usize, +) -> std::result::Result, Box> { + parse_byte_range(raw_range, body_length).map_err(|err| match err { + ByteRangeParseError::Malformed => Box::new(bad_request_response_with_code( + S3ErrorCode::InvalidArgument, + "invalid range syntax", + "Range", + )), + ByteRangeParseError::Unsatisfiable => { + Box::new(range_not_satisfiable_response(body_length as u64)) + } + }) } -async fn post_object( - State(state): State>, - Path((bucket, key)): Path<(String, String)>, - Query(query): Query>, -) -> Response { - let raw_query = if query.is_empty() { - None +fn parse_byte_range( + raw_range: &str, + object_size: usize, +) -> std::result::Result, ByteRangeParseError> { + let raw = raw_range.trim(); + let spec = raw + .split_once('=') + .and_then(|(unit, spec)| { + if unit.eq_ignore_ascii_case("bytes") { + Some(spec) + } else { + None + } + }) + .ok_or(ByteRangeParseError::Malformed)?; + if spec.is_empty() { + return Err(ByteRangeParseError::Malformed); + } + + if object_size == 0 { + return Err(ByteRangeParseError::Unsatisfiable); + } + if spec.contains(',') { + return Err(ByteRangeParseError::Malformed); + } + + let (start_raw, end_raw) = spec.split_once('-').ok_or(ByteRangeParseError::Malformed)?; + + if start_raw.is_empty() { + let suffix_len = end_raw + .parse::() + .map_err(|_| ByteRangeParseError::Malformed)?; + if suffix_len == 0 { + return Err(ByteRangeParseError::Malformed); + } + + let start = object_size.saturating_sub(suffix_len); + return Ok(Some((start, object_size.saturating_sub(1)))); + } + + let start = start_raw + .parse::() + .map_err(|_| ByteRangeParseError::Malformed)?; + + if start >= object_size { + return Err(ByteRangeParseError::Unsatisfiable); + } + + let mut end = if end_raw.is_empty() { + object_size.saturating_sub(1) } else { - Some( - query - .iter() - .map(|(k, v)| { - if v.is_empty() { - k.clone() - } else { - format!("{k}={v}") - } - }) - .collect::>() - .join("&"), - ) + end_raw + .parse::() + .map_err(|_| ByteRangeParseError::Malformed)? }; - let resource = format!("/{bucket}/{key}"); - if !is_restore_request(raw_query.as_deref()) { - return bad_request_response("UnsupportedPostAction", &resource); + + if end < start { + return Err(ByteRangeParseError::Unsatisfiable); + } + if end >= object_size { + end = object_size.saturating_sub(1); } - let days = query - .get("days") - .and_then(|value| value.parse::().ok()) - .unwrap_or(1); - let tier = match query.get("tier").map(String::as_str) { - Some("Expedited") => coldstore_proto::common::RestoreTier::Expedited, - Some("Bulk") => coldstore_proto::common::RestoreTier::Bulk, - _ => coldstore_proto::common::RestoreTier::Standard, + Ok(Some((start, end))) +} + +enum ByteRangeParseError { + Malformed, + Unsatisfiable, +} + +async fn check_read_preconditions( + state: &Arc, + bucket: &str, + key: &str, + headers: &HeaderMap, + resource: &str, +) -> std::result::Result, tonic::Status> { + let if_match = headers + .get(header::IF_MATCH) + .and_then(|value| value.to_str().ok()); + let if_none_match = headers + .get(header::IF_NONE_MATCH) + .and_then(|value| value.to_str().ok()); + let if_modified_since = match headers + .get(header::IF_MODIFIED_SINCE) + .and_then(|value| value.to_str().ok()) + { + Some(value) => parse_http_datetime(value) + .map_err(|_| tonic::Status::invalid_argument("invalid If-Modified-Since header"))?, + None => None, + }; + + let if_unmodified_since = match headers + .get(header::IF_UNMODIFIED_SINCE) + .and_then(|value| value.to_str().ok()) + { + Some(value) => parse_http_datetime(value) + .map_err(|_| tonic::Status::invalid_argument("invalid If-Unmodified-Since header"))?, + None => None, }; - match state - .backend - .restore_object(&bucket, &key, days, tier) - .await + if if_match.is_none() + && if_none_match.is_none() + && if_modified_since.is_none() + && if_unmodified_since.is_none() { - Ok(response) => empty_response(if response.status_code == 200 { - StatusCode::OK - } else { - StatusCode::ACCEPTED - }), - Err(status) => grpc_status_to_s3_response(status, &resource), + return Ok(None); + } + + let head = state.backend.head_object(bucket, key).await?; + let object_etag = head.etag.clone(); + let object_modified_at = head + .last_modified + .as_ref() + .map(timestamp_from_prost_timestamp); + + if let Some(value) = if_match { + if !etag_header_matches(value, &object_etag) { + return Ok(Some(precondition_failed_response("If-Match", resource))); + } + } + + if let Some(unmodified_since) = if_unmodified_since { + let Some(object_modified_at) = object_modified_at else { + return Err(tonic::Status::failed_precondition( + "If-Unmodified-Since requires Last-Modified", + )); + }; + + if object_modified_at > unmodified_since { + return Ok(Some(precondition_failed_response( + "If-Unmodified-Since failed", + resource, + ))); + } + } + + if let Some(value) = if_none_match { + if etag_header_matches(value, &object_etag) { + return Ok(Some(not_modified_response(resource, &object_etag))); + } + } + + if let Some(modified_since) = if_modified_since { + let Some(object_modified_at) = object_modified_at else { + return Err(tonic::Status::failed_precondition( + "If-Modified-Since requires Last-Modified", + )); + }; + + if object_modified_at <= modified_since { + return Ok(Some(not_modified_response(resource, &object_etag))); + } } + + Ok(None) +} + +fn parse_http_datetime(value: &str) -> std::result::Result, String> { + let value = value.trim(); + if value.is_empty() { + return Ok(None); + } + + const RFC_1123_HTTP_DATE: &str = "%a, %d %b %Y %H:%M:%S GMT"; + const RFC_850_HTTP_DATE: &str = "%A, %d-%b-%y %H:%M:%S GMT"; + const ANSI_C_DATE: &str = "%a %b %e %H:%M:%S %Y"; + + for format in [RFC_1123_HTTP_DATE, RFC_850_HTTP_DATE, ANSI_C_DATE] { + if let Ok(time) = DateTime::parse_from_str(value, format) { + return Ok(Some(time.timestamp())); + } + } + + DateTime::parse_from_rfc2822(value) + .map(|time| Some(time.timestamp())) + .map_err(|_| "unable to parse HTTP date".into()) +} + +fn timestamp_from_prost_timestamp(timestamp: &prost_types::Timestamp) -> i64 { + timestamp.seconds +} + +fn etag_header_matches(header_value: &str, etag: &str) -> bool { + header_value + .split(',') + .map(str::trim) + .map(|token| token.trim_matches('"')) + .any(|token| token == "*" || token == etag || token == format!("W/\"{etag}\"")) +} + +fn not_modified_response(resource: &str, etag: &str) -> Response { + let mut response = empty_response(StatusCode::NOT_MODIFIED); + response.headers_mut().insert( + header::ETAG, + HeaderValue::from_str(etag).unwrap_or_else(|_| HeaderValue::from_static("\"\"")), + ); + response.headers_mut().insert( + HeaderName::from_static("x-amz-request-id"), + HeaderValue::from_str(resource).unwrap_or_else(|_| HeaderValue::from_static("-")), + ); + response +} + +fn precondition_failed_response(message: &str, resource: &str) -> Response { + let body = S3ErrorResponse { + code: S3ErrorCode::PreconditionFailed, + message, + resource, + } + .to_xml(); + s3_xml_response(StatusCode::PRECONDITION_FAILED, body) +} + +fn range_not_satisfiable_response(body_size: u64) -> Response { + let mut response = empty_response(StatusCode::RANGE_NOT_SATISFIABLE); + response.headers_mut().insert( + header::CONTENT_RANGE, + HeaderValue::from_str(&format!("bytes */{body_size}")).unwrap(), + ); + response } fn list_buckets_xml_response( @@ -208,7 +457,7 @@ fn list_buckets_xml_response( let buckets_xml = response .buckets .iter() - .map(|bucket| format!("{}", bucket.name)) + .map(|bucket| format!("{}", escape_xml(&bucket.name))) .collect::>() .join(""); let body = format!( @@ -221,39 +470,56 @@ fn list_buckets_xml_response( fn list_objects_xml_response( response: &coldstore_proto::scheduler::ListObjectsResponse, ) -> Response { - let contents = response - .contents - .iter() - .map(|entry| { - format!( - "{}{}{}{}", - entry.key, entry.etag, entry.size, entry.storage_class - ) - }) - .collect::>() - .join(""); + let mut contents = String::new(); + for entry in &response.contents { + contents.push_str(&format!( + "{}{}{}{}", + escape_xml(&entry.key), + escape_xml(&entry.etag), + entry.size, + escape_xml(&entry.storage_class) + )); + } + + let mut common_prefixes = String::new(); + for common_prefix in &response.common_prefixes { + common_prefixes.push_str(&format!( + "{}", + escape_xml(&common_prefix.prefix) + )); + } + let body = format!( - "{}{}", - response.bucket, contents + "{}{}{}{}{}{}{}{}", + escape_xml(&response.bucket), + escape_xml(&response.prefix.clone().unwrap_or_default()), + escape_xml(&response.marker.clone().unwrap_or_default()), + escape_xml(&response.next_marker.clone().unwrap_or_default()), + response.max_keys, + response.is_truncated, + contents, + common_prefixes, ); xml_response(StatusCode::OK, body) } +fn escape_xml(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + fn put_object_success_response( - response: coldstore_proto::scheduler::PutObjectResponse, + put_response: coldstore_proto::scheduler::PutObjectResponse, ) -> Response { - let mut http = empty_response(StatusCode::OK); - http.headers_mut().insert( + let mut response = empty_response(StatusCode::OK); + response.headers_mut().insert( axum::http::header::ETAG, - HeaderValue::from_str(&response.etag).unwrap(), + HeaderValue::from_str(&put_response.etag).unwrap(), ); - http -} - -fn get_object_success_response(object: DownloadedObject) -> Response { - let mut response = Response::new(Body::from(object.body)); - *response.status_mut() = StatusCode::OK; - apply_object_headers(response.headers_mut(), &object.head); response } @@ -270,7 +536,8 @@ fn apply_object_headers( ) { headers.insert( axum::http::header::CONTENT_LENGTH, - HeaderValue::from_str(&head.content_length.to_string()).unwrap(), + HeaderValue::from_str(&head.content_length.to_string()) + .unwrap_or_else(|_| HeaderValue::from_static("0")), ); if let Some(content_type) = &head.content_type { headers.insert( @@ -288,12 +555,33 @@ fn apply_object_headers( HeaderValue::from_str(&normalize_restore_info(restore_info)).unwrap(), ); } + + if let Some(last_modified) = &head.last_modified { + if let Some(last_modified) = + DateTime::from_timestamp(last_modified.seconds, 0).map(|time| time.with_timezone(&Utc)) + { + headers.insert( + header::LAST_MODIFIED, + HeaderValue::from_str(&format_http_date(last_modified)) + .unwrap_or_else(|_| HeaderValue::from_static("")), + ); + } + } +} + +fn format_http_date(time: DateTime) -> String { + time.format("%a, %d %b %Y %H:%M:%S GMT").to_string() } fn normalize_restore_info(restore_info: &str) -> String { - if let Some(expiry) = restore_info.strip_prefix("ongoing-request=\"false\", expiry-ts=\"") { + let restore_info = restore_info.trim(); + if let Some(expiry) = restore_info.strip_prefix("ongoing-request=\"false\", expiry-date=\"") { format_restore_header(false, Some(expiry.trim_end_matches('"'))) - } else if restore_info == "ongoing-request=\"false\"" { + } else if let Some(expiry) = + restore_info.strip_prefix("ongoing-request=\"false\", expiry-ts=\"") + { + format_restore_header(false, Some(expiry.trim_end_matches('"'))) + } else if restore_info.starts_with("ongoing-request=\"false\"") { format_restore_header(false, None) } else { format_restore_header(true, None) @@ -301,10 +589,17 @@ fn normalize_restore_info(restore_info: &str) -> String { } fn bad_request_response(operation: &str, resource: &str) -> Response { - let message = format!("unsupported POST action for {operation}"); + bad_request_response_with_code( + S3ErrorCode::NotImplemented, + &format!("unsupported POST action for {operation}"), + resource, + ) +} + +fn bad_request_response_with_code(code: S3ErrorCode, message: &str, resource: &str) -> Response { let body = S3ErrorResponse { - code: S3ErrorCode::NotImplemented, - message: &message, + code, + message, resource, } .to_xml(); @@ -312,27 +607,119 @@ fn bad_request_response(operation: &str, resource: &str) -> Response { } fn grpc_status_to_s3_response(status: tonic::Status, resource: &str) -> Response { - let (code, http_status) = match status.code() { - tonic::Code::AlreadyExists => (S3ErrorCode::NotImplemented, StatusCode::CONFLICT), + let status_message = status.message(); + let (code, http_status, retry_after) = match status.code() { + tonic::Code::AlreadyExists => ( + S3ErrorCode::RestoreAlreadyInProgress, + StatusCode::CONFLICT, + None, + ), tonic::Code::NotFound => { let code = if resource.matches('/').count() > 1 { S3ErrorCode::NoSuchKey } else { S3ErrorCode::NoSuchBucket }; - (code, StatusCode::NOT_FOUND) + (code, StatusCode::NOT_FOUND, None) + } + tonic::Code::InvalidArgument => { + (S3ErrorCode::InvalidArgument, StatusCode::BAD_REQUEST, None) + } + tonic::Code::FailedPrecondition => { + (S3ErrorCode::InvalidObjectState, StatusCode::FORBIDDEN, None) + } + tonic::Code::Unavailable + if status_message + .to_lowercase() + .contains("glacier expedited retrieval is not available") => + { + ( + S3ErrorCode::GlacierExpeditedRetrievalNotAvailable, + StatusCode::SERVICE_UNAVAILABLE, + Some(1), + ) + } + tonic::Code::Unavailable => ( + S3ErrorCode::ServiceUnavailable, + StatusCode::SERVICE_UNAVAILABLE, + Some(1), + ), + tonic::Code::ResourceExhausted => match cache_reject_status_hint(status_message) { + Some(CacheRejectStatus::Backpressure) => ( + S3ErrorCode::SlowDown, + StatusCode::SERVICE_UNAVAILABLE, + Some(1), + ), + Some(CacheRejectStatus::RateLimited) => ( + S3ErrorCode::SlowDown, + StatusCode::SERVICE_UNAVAILABLE, + Some(1), + ), + Some(CacheRejectStatus::InvalidRequest) => { + (S3ErrorCode::InvalidArgument, StatusCode::BAD_REQUEST, None) + } + None => ( + S3ErrorCode::SlowDown, + StatusCode::SERVICE_UNAVAILABLE, + Some(1), + ), + }, + tonic::Code::Unimplemented => ( + S3ErrorCode::NotImplemented, + StatusCode::NOT_IMPLEMENTED, + None, + ), + _ => (S3ErrorCode::NotImplemented, StatusCode::BAD_GATEWAY, None), + }; + let mut response = { + let body = S3ErrorResponse { + code, + message: status.message(), + resource, } - tonic::Code::FailedPrecondition => (S3ErrorCode::InvalidObjectState, StatusCode::FORBIDDEN), - tonic::Code::Unimplemented => (S3ErrorCode::NotImplemented, StatusCode::NOT_IMPLEMENTED), - _ => (S3ErrorCode::NotImplemented, StatusCode::BAD_GATEWAY), + .to_xml(); + s3_xml_response(http_status, body) }; - let body = S3ErrorResponse { - code, - message: status.message(), - resource, + + if let Some(seconds) = retry_after { + response.headers_mut().insert( + header::RETRY_AFTER, + HeaderValue::from_str(&seconds.to_string()) + .unwrap_or_else(|_| HeaderValue::from_static("1")), + ); + } + + response +} + +enum CacheRejectStatus { + Backpressure, + RateLimited, + InvalidRequest, +} + +fn cache_reject_status_hint(message: &str) -> Option { + let message = message.trim(); + if !message.starts_with("capacity_reject:") { + return None; + } + let reason = message["capacity_reject:".len()..] + .split(':') + .next() + .unwrap_or_default(); + match reason { + "incoming_larger_than_capacity" => Some(CacheRejectStatus::InvalidRequest), + "client_rate_limited" | "concurrency_limit_exceeded" | "request_rate_exceeded" => { + Some(CacheRejectStatus::RateLimited) + } + "staging_budget_exceeded" + | "restored_budget_exceeded" + | "global_capacity_exceeded" + | "low_watermark_exceeded" + | "no_eviction_candidate" + | "zero_capacity" => Some(CacheRejectStatus::Backpressure), + _ => None, } - .to_xml(); - s3_xml_response(http_status, body) } fn s3_xml_response(status: StatusCode, body: String) -> Response { @@ -355,10 +742,119 @@ fn empty_response(status: StatusCode) -> Response { response } +async fn put_object( + State(state): State>, + Path((bucket, key)): Path<(String, String)>, + headers: HeaderMap, + body: Bytes, +) -> Response { + let content_type = headers + .get(axum::http::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + match state + .backend + .put_object(&bucket, &key, body.to_vec(), content_type) + .await + { + Ok(response) => put_object_success_response(response), + Err(status) => grpc_status_to_s3_response(status, &format!("/{bucket}/{key}")), + } +} + +async fn delete_object( + State(state): State>, + Path((bucket, key)): Path<(String, String)>, +) -> Response { + match state.backend.delete_object(&bucket, &key).await { + Ok(()) => empty_response(StatusCode::NO_CONTENT), + Err(status) => grpc_status_to_s3_response(status, &format!("/{bucket}/{key}")), + } +} + +async fn head_object( + State(state): State>, + Path((bucket, key)): Path<(String, String)>, + headers: HeaderMap, +) -> Response { + let resource = format!("/{bucket}/{key}"); + + if let Some(response) = + match check_read_preconditions(&state, &bucket, &key, &headers, &resource).await { + Ok(response) => response, + Err(status) => return grpc_status_to_s3_response(status, &resource), + } + { + return response; + } + + match state.backend.head_object(&bucket, &key).await { + Ok(head) => head_object_success_response(head), + Err(status) => grpc_status_to_s3_response(status, &resource), + } +} + +async fn post_object( + State(state): State>, + Path((bucket, key)): Path<(String, String)>, + Query(query): Query>, + body: Bytes, +) -> Response { + let raw_query = if query.is_empty() { + None + } else { + Some( + query + .iter() + .map(|(k, v)| { + if v.is_empty() { + k.clone() + } else { + format!("{k}={v}") + } + }) + .collect::>() + .join("&"), + ) + }; + let resource = format!("/{bucket}/{key}"); + if !is_restore_request(raw_query.as_deref()) { + return bad_request_response("UnsupportedPostAction", &resource); + } + + let restore_request = match parse_restore_request(&query, body.as_ref()) { + Ok(request) => request, + Err(_) => { + return bad_request_response_with_code( + S3ErrorCode::InvalidArgument, + "invalid restore request payload", + &resource, + ); + } + }; + + match state + .backend + .restore_object(&bucket, &key, restore_request.days, restore_request.tier) + .await + { + Ok(response) => { + let status = match response.status_code { + 200 => StatusCode::OK, + 202 => StatusCode::ACCEPTED, + 409 => StatusCode::CONFLICT, + _ => StatusCode::ACCEPTED, + }; + empty_response(status) + } + Err(status) => grpc_status_to_s3_response(status, &resource), + } +} + #[cfg(test)] mod tests { use super::*; - use crate::GatewayBackend; + use crate::{DownloadedObject, GatewayBackend}; use axum::body::to_bytes; use axum::http::Request; use coldstore_proto::scheduler::{ @@ -453,7 +949,11 @@ mod tests { ) -> std::result::Result { Ok(DownloadedObject { head: self.head_object(bucket, key).await?, - body: b"hello world".to_vec(), + body: if key == "empty.txt" { + Vec::new() + } else { + b"hello world".to_vec() + }, }) } @@ -483,6 +983,15 @@ mod tests { restore_info: Some("ongoing-request=\"false\", expiry-ts=\"123\"".into()), last_modified: None, }) + } else if bucket == "docs" && key == "empty.txt" { + Ok(HeadObjectResponse { + content_length: 0, + content_type: Some("text/plain".into()), + etag: "etag-empty".into(), + storage_class: 2, + restore_info: None, + last_modified: None, + }) } else { Err(tonic::Status::not_found("object missing")) } @@ -497,6 +1006,8 @@ mod tests { ) -> std::result::Result { if bucket == "docs" && key == "readme.txt" { Ok(RestoreObjectResponse { status_code: 202 }) + } else if bucket == "docs" && key == "pending.txt" { + Ok(RestoreObjectResponse { status_code: 409 }) } else { Err(tonic::Status::not_found("object missing")) } @@ -549,6 +1060,38 @@ mod tests { assert!(text.contains("readme.txt")); } + #[tokio::test] + async fn list_objects_route_rejects_invalid_max_keys() { + let response = test_router(state()) + .oneshot( + Request::builder() + .uri("/docs?max-keys=abc") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert!(String::from_utf8(body.to_vec()) + .unwrap() + .contains("InvalidArgument")); + } + + #[tokio::test] + async fn list_objects_route_rejects_invalid_max_keys_range() { + let response = test_router(state()) + .oneshot( + Request::builder() + .uri("/docs?max-keys=0") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + #[tokio::test] async fn create_bucket_route_uses_backend() { let response = test_router(state()) @@ -613,16 +1156,80 @@ mod tests { } #[tokio::test] - async fn failed_precondition_maps_to_invalid_object_state() { - let response = grpc_status_to_s3_response( - tonic::Status::failed_precondition("object must be restored"), - "/docs/readme.txt", - ); - assert_eq!(response.status(), StatusCode::FORBIDDEN); + async fn get_object_route_supports_single_range() { + let response = test_router(state()) + .oneshot( + Request::builder() + .uri("/docs/readme.txt") + .header("range", "bytes=1-3") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); - assert!(String::from_utf8(body.to_vec()) - .unwrap() - .contains("InvalidObjectState")); + assert_eq!(body.as_ref(), b"ell"); + } + + #[tokio::test] + async fn get_object_route_range_on_large_offsets() { + let response = test_router(state()) + .oneshot( + Request::builder() + .uri("/docs/readme.txt") + .header("range", "bytes=-3") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!(body.as_ref(), b"rld"); + } + + #[tokio::test] + async fn get_object_route_range_on_empty_object_is_not_satisfiable() { + let response = test_router(state()) + .oneshot( + Request::builder() + .uri("/docs/empty.txt") + .header("range", "bytes=0-0") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + assert_eq!( + response + .headers() + .get(axum::http::header::CONTENT_RANGE) + .and_then(|value| value.to_str().ok()) + .unwrap(), + "bytes */0" + ); + } + + #[tokio::test] + async fn head_object_route_sets_restore_header() { + let response = test_router(state()) + .oneshot( + Request::builder() + .method("HEAD") + .uri("/docs/readme.txt") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers()["etag"], "etag-1"); + assert_eq!( + response.headers()["x-amz-restore"], + "ongoing-request=\"false\", expiry-date=\"123\"" + ); } #[tokio::test] @@ -646,8 +1253,10 @@ mod tests { .oneshot( Request::builder() .method("POST") - .uri("/docs/readme.txt?restore=true&days=2&tier=Bulk") - .body(Body::empty()) + .uri("/docs/readme.txt?restore=true") + .body(Body::from( + "2Bulk", + )) .unwrap(), ) .await @@ -655,6 +1264,42 @@ mod tests { assert_eq!(response.status(), StatusCode::ACCEPTED); } + #[tokio::test] + async fn restore_post_route_rejects_invalid_body() { + let response = test_router(state()) + .oneshot( + Request::builder() + .method("POST") + .uri("/docs/readme.txt?restore=true") + .body(Body::from( + "0Bulk", + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert!(String::from_utf8(body.to_vec()) + .unwrap() + .contains("InvalidArgument")); + } + + #[tokio::test] + async fn restore_post_route_returns_conflict_for_in_progress_restore() { + let response = test_router(state()) + .oneshot( + Request::builder() + .method("POST") + .uri("/docs/pending.txt?restore=true") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CONFLICT); + } + #[tokio::test] async fn head_bucket_route_uses_backend() { let response = test_router(state()) @@ -671,22 +1316,150 @@ mod tests { } #[tokio::test] - async fn head_object_route_sets_restore_header() { + async fn failed_precondition_maps_to_invalid_object_state() { + let response = grpc_status_to_s3_response( + tonic::Status::failed_precondition("object must be restored"), + "/docs/readme.txt", + ); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert!(String::from_utf8(body.to_vec()) + .unwrap() + .contains("InvalidObjectState")); + } + + #[tokio::test] + async fn resource_exhausted_maps_to_s3_slow_down_with_retry_after() { + let response = grpc_status_to_s3_response( + tonic::Status::resource_exhausted("cache quota exceeded"), + "/docs/readme.txt", + ); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + response + .headers() + .get(axum::http::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()), + Some("1") + ); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert!(String::from_utf8(body.to_vec()) + .unwrap() + .contains("SlowDown")); + } + + #[tokio::test] + async fn resource_exhausted_maps_staging_budget_pressure_to_slow_down() { + let response = grpc_status_to_s3_response( + tonic::Status::resource_exhausted( + "capacity_reject:staging_budget_exceeded: not enough staging cache budget", + ), + "/docs/readme.txt", + ); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + response + .headers() + .get(axum::http::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()), + Some("1") + ); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert!(String::from_utf8(body.to_vec()) + .unwrap() + .contains("SlowDown")); + } + + #[tokio::test] + async fn unavailable_maps_to_glacier_expedited_retrieval_not_available() { + let response = grpc_status_to_s3_response( + tonic::Status::unavailable( + "glacier expedited retrieval is not available in this environment", + ), + "/docs/readme.txt", + ); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert!(String::from_utf8(body.to_vec()) + .unwrap() + .contains("GlacierExpeditedRetrievalNotAvailable")); + } + + #[tokio::test] + async fn unavailable_maps_to_service_unavailable_with_retry_after() { + let response = grpc_status_to_s3_response( + tonic::Status::unavailable("scheduler down"), + "/docs/readme.txt", + ); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert!(String::from_utf8(body.to_vec()) + .unwrap() + .contains("ServiceUnavailable")); + } + + #[tokio::test] + async fn rate_limited_resource_exhausted_maps_to_s3_slow_down_with_retry_after() { + let response = grpc_status_to_s3_response( + tonic::Status::resource_exhausted( + "capacity_reject:concurrency_limit_exceeded: client throttled", + ), + "/docs/readme.txt", + ); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + response + .headers() + .get(axum::http::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()), + Some("1") + ); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert!(String::from_utf8(body.to_vec()) + .unwrap() + .contains("SlowDown")); + } + + #[tokio::test] + async fn s3_error_response_escapes_backend_message_and_resource() { + let response = grpc_status_to_s3_response( + tonic::Status::invalid_argument("bad & \"quote\""), + "/docs/a&b.txt", + ); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let body = String::from_utf8(body.to_vec()).unwrap(); + assert!(body.contains("bad <tag> & "quote"")); + assert!(body.contains("/docs/a&b<raw>.txt")); + } + + #[tokio::test] + async fn get_with_if_none_match_returns_not_modified() { let response = test_router(state()) .oneshot( Request::builder() - .method("HEAD") .uri("/docs/readme.txt") + .header("if-none-match", "etag-1") .body(Body::empty()) .unwrap(), ) .await .unwrap(); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response.headers()["etag"], "etag-1"); - assert_eq!( - response.headers()["x-amz-restore"], - "ongoing-request=\"false\", expiry-date=\"123\"" - ); + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + } + + #[tokio::test] + async fn get_with_if_match_returns_not_precondition_failed() { + let response = test_router(state()) + .oneshot( + Request::builder() + .uri("/docs/readme.txt") + .header("if-match", "etag-other") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::PRECONDITION_FAILED); } } diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index e3f265b..05f3a9d 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -11,6 +11,7 @@ use coldstore_proto::scheduler::{ ListObjectsResponse, PutObjectMeta, PutObjectRequest, PutObjectResponse, RestoreObjectRequest, RestoreObjectResponse, }; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use tonic::transport::Channel; use tracing::info; @@ -66,18 +67,49 @@ pub trait GatewayBackend: Send + Sync + 'static { } pub struct GrpcGatewayBackend { - scheduler_addr: String, + scheduler_addrs: Vec, + next_addr: Arc, } impl GrpcGatewayBackend { - pub fn new(scheduler_addr: String) -> Self { - Self { scheduler_addr } + pub fn new(scheduler_addrs: Vec) -> Self { + Self { + scheduler_addrs, + next_addr: Arc::new(AtomicUsize::new(0)), + } } async fn connect(&self) -> std::result::Result, tonic::Status> { - SchedulerServiceClient::connect(self.scheduler_addr.clone()) - .await - .map_err(|err| tonic::Status::unavailable(err.to_string())) + if self.scheduler_addrs.is_empty() { + return Err(tonic::Status::unavailable( + "no scheduler addresses configured", + )); + } + + let mut last_error: Option = None; + let base = self.next_addr.fetch_add(1, Ordering::Relaxed); + for offset in 0..self.scheduler_addrs.len() { + let index = (base + offset) % self.scheduler_addrs.len(); + let addr = normalize_scheduler_addr(&self.scheduler_addrs[index]); + match SchedulerServiceClient::connect(addr).await { + Ok(client) => return Ok(client), + Err(err) => { + last_error = Some(tonic::Status::unavailable(err.to_string())); + } + } + } + + Err(last_error.unwrap_or_else(|| { + tonic::Status::unavailable("all scheduler addresses are unavailable") + })) + } +} + +fn normalize_scheduler_addr(addr: &str) -> String { + if addr.starts_with("http://") || addr.starts_with("https://") { + addr.to_string() + } else { + format!("http://{addr}") } } @@ -273,9 +305,24 @@ pub struct GatewayState { } pub async fn run(config: GatewayConfig) -> Result<()> { - let scheduler_addr = format!("http://{}", &config.scheduler_addrs[0]); + if config.scheduler_addrs.is_empty() { + anyhow::bail!("gateway requires at least one scheduler address"); + } + + let scheduler_addrs = config + .scheduler_addrs + .into_iter() + .map(|addr| normalize_scheduler_addr(&addr)) + .collect::>(); + + info!( + scheduler_count = scheduler_addrs.len(), + first_scheduler_addr = &scheduler_addrs[0], + "Gateway routing to scheduler fleet" + ); + let state = Arc::new(GatewayState { - backend: Arc::new(GrpcGatewayBackend::new(scheduler_addr)), + backend: Arc::new(GrpcGatewayBackend::new(scheduler_addrs)), }); let app = handler::router(state); diff --git a/crates/gateway/src/protocol.rs b/crates/gateway/src/protocol.rs index 209a5fa..4f5b32d 100644 --- a/crates/gateway/src/protocol.rs +++ b/crates/gateway/src/protocol.rs @@ -6,15 +6,25 @@ //! - x-amz-restore 响应头生成 //! - 错误码映射 (InvalidObjectState, RestoreAlreadyInProgress 等) //! - GET 行为控制 (冷对象需先 Restore) +use coldstore_proto::common; +use std::collections::HashMap; + +const MAX_RESTORE_DAYS: u32 = 365; /// S3 错误码 #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum S3ErrorCode { InvalidObjectState, + InvalidArgument, RestoreAlreadyInProgress, GlacierExpeditedRetrievalNotAvailable, + SlowDown, + ServiceUnavailable, + TooManyRequests, NoSuchKey, NoSuchBucket, + PreconditionFailed, + NotModified, NotImplemented, } @@ -22,12 +32,18 @@ impl S3ErrorCode { pub fn as_str(&self) -> &'static str { match self { S3ErrorCode::InvalidObjectState => "InvalidObjectState", + S3ErrorCode::InvalidArgument => "InvalidArgument", S3ErrorCode::RestoreAlreadyInProgress => "RestoreAlreadyInProgress", S3ErrorCode::GlacierExpeditedRetrievalNotAvailable => { "GlacierExpeditedRetrievalNotAvailable" } + S3ErrorCode::SlowDown => "SlowDown", + S3ErrorCode::ServiceUnavailable => "ServiceUnavailable", + S3ErrorCode::TooManyRequests => "TooManyRequests", S3ErrorCode::NoSuchKey => "NoSuchKey", S3ErrorCode::NoSuchBucket => "NoSuchBucket", + S3ErrorCode::PreconditionFailed => "PreconditionFailed", + S3ErrorCode::NotModified => "NotModified", S3ErrorCode::NotImplemented => "NotImplemented", } } @@ -35,10 +51,16 @@ impl S3ErrorCode { pub fn http_status(&self) -> u16 { match self { S3ErrorCode::InvalidObjectState => 403, + S3ErrorCode::InvalidArgument => 400, S3ErrorCode::RestoreAlreadyInProgress => 409, S3ErrorCode::GlacierExpeditedRetrievalNotAvailable => 503, + S3ErrorCode::SlowDown => 503, + S3ErrorCode::ServiceUnavailable => 503, + S3ErrorCode::TooManyRequests => 429, S3ErrorCode::NoSuchKey => 404, S3ErrorCode::NoSuchBucket => 404, + S3ErrorCode::PreconditionFailed => 412, + S3ErrorCode::NotModified => 304, S3ErrorCode::NotImplemented => 501, } } @@ -53,15 +75,26 @@ pub struct S3ErrorResponse<'a> { impl<'a> S3ErrorResponse<'a> { pub fn to_xml(&self) -> String { + let message = escape_xml(self.message); + let resource = escape_xml(self.resource); format!( "{}{}{}", self.code.as_str(), - self.message, - self.resource, + message, + resource, ) } } +fn escape_xml(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + /// 生成 x-amz-restore 响应头 pub fn format_restore_header(ongoing: bool, expiry_date: Option<&str>) -> String { if ongoing { @@ -80,9 +113,97 @@ pub fn is_restore_request(query: Option<&str>) -> bool { .any(|item| item == "restore" || item.starts_with("restore=")) } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RestoreRequest { + pub days: u32, + pub tier: common::RestoreTier, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RestoreParseError { + InvalidBody, + InvalidDays, + InvalidTier, +} + +fn find_xml_value(source: &str, tag: &str) -> Option { + let open = format!("<{tag}>"); + let close = format!(""); + let open_pos = source.find(&open)?; + let value_start = open_pos + open.len(); + let value_end_rel = source[value_start..].find(&close)?; + let value_end = value_start + value_end_rel; + Some(source[value_start..value_end].trim().to_string()) +} + +fn parse_tier(raw: &str) -> Option { + match raw.trim().to_lowercase().as_str() { + "expedited" => Some(common::RestoreTier::Expedited), + "standard" => Some(common::RestoreTier::Standard), + "bulk" => Some(common::RestoreTier::Bulk), + _ => None, + } +} + +pub fn parse_restore_request( + query: &HashMap, + body: &[u8], +) -> Result { + let body_xml = if body.is_empty() { + None + } else { + Some( + std::str::from_utf8(body) + .map_err(|_| RestoreParseError::InvalidBody)? + .to_string(), + ) + }; + + let days = match body_xml + .as_deref() + .and_then(|body| find_xml_value(body, "Days")) + { + Some(days_raw) => { + let days = days_raw + .parse::() + .map_err(|_| RestoreParseError::InvalidDays)?; + if days == 0 || days > MAX_RESTORE_DAYS { + return Err(RestoreParseError::InvalidDays); + } + days + } + None => { + let days = query + .get("days") + .map(|days| days.parse::()) + .transpose() + .map_err(|_| RestoreParseError::InvalidDays)? + .unwrap_or(1); + if days == 0 || days > MAX_RESTORE_DAYS { + return Err(RestoreParseError::InvalidDays); + } + days + } + }; + + let tier = match body_xml + .as_deref() + .and_then(|body| find_xml_value(body, "Tier")) + { + Some(raw_tier) => parse_tier(&raw_tier).ok_or(RestoreParseError::InvalidTier)?, + None => { + let raw_tier = query.get("tier").map(String::as_str).unwrap_or("Standard"); + parse_tier(raw_tier).ok_or(RestoreParseError::InvalidTier)? + } + }; + + Ok(RestoreRequest { days, tier }) +} + #[cfg(test)] mod tests { use super::*; + use std::collections::HashMap; #[test] fn restore_header_formats_completed_state() { @@ -98,6 +219,64 @@ mod tests { assert!(!is_restore_request(Some("foo=bar"))); } + #[test] + fn restore_request_parse_xml_body() { + let mut query = HashMap::new(); + query.insert("restore".into(), "true".into()); + let request = parse_restore_request( + &query, + b"10Expedited", + ) + .expect("parse restore request"); + assert_eq!(request.days, 10); + assert_eq!(request.tier, common::RestoreTier::Expedited); + } + + #[test] + fn restore_request_parse_xml_body_case_insensitive_tier() { + let mut query = HashMap::new(); + query.insert("restore".into(), "true".into()); + let request = parse_restore_request( + &query, + b"10bulk", + ) + .expect("parse restore request"); + assert_eq!(request.tier, common::RestoreTier::Bulk); + } + + #[test] + fn restore_request_rejects_too_many_days() { + let mut query = HashMap::new(); + query.insert("restore".into(), "true".into()); + assert_eq!( + parse_restore_request(&query, b"366") + .unwrap_err(), + RestoreParseError::InvalidDays + ); + } + + #[test] + fn restore_request_fallback_to_query_parameters() { + let mut query = HashMap::new(); + query.insert("restore".into(), "true".into()); + query.insert("days".into(), "3".into()); + query.insert("tier".into(), "Bulk".into()); + let request = parse_restore_request(&query, b"").expect("parse restore request"); + assert_eq!(request.days, 3); + assert_eq!(request.tier, common::RestoreTier::Bulk); + } + + #[test] + fn restore_request_treats_invalid_tier_as_error() { + let mut query = HashMap::new(); + query.insert("restore".into(), "true".into()); + query.insert("tier".into(), "Turbo".into()); + assert_eq!( + parse_restore_request(&query, b"").unwrap_err(), + RestoreParseError::InvalidTier + ); + } + #[test] fn s3_error_xml_contains_code_and_resource() { let xml = S3ErrorResponse { diff --git a/crates/metadata/Cargo.toml b/crates/metadata/Cargo.toml index 93c7688..bce79c4 100644 --- a/crates/metadata/Cargo.toml +++ b/crates/metadata/Cargo.toml @@ -10,7 +10,7 @@ name = "coldstore-metadata" path = "src/main.rs" [features] -default = [] +default = ["metadata-raft-rocksdb"] metadata-raft = ["dep:openraft"] metadata-raft-rocksdb = ["metadata-raft", "dep:rocksdb"] @@ -30,8 +30,7 @@ chrono = { workspace = true } uuid = { workspace = true } config = { workspace = true } -# Phase 2A starts with opt-in local binary snapshot persistence in service.rs. -# Phase 2B keeps OpenRaft + RocksDB behind the `metadata-raft` feature until the -# command/state-machine boundary is stable enough to become the default backend. +# Default metadata execution path is now persistent Raft-local mode. +# Disable default features only when explicitly embedding with a custom runtime. openraft = { version = "0.10.0-alpha.18", optional = true } rocksdb = { version = "0.24", default-features = false, optional = true } diff --git a/crates/metadata/src/command.rs b/crates/metadata/src/command.rs index 750f293..ede23dc 100644 --- a/crates/metadata/src/command.rs +++ b/crates/metadata/src/command.rs @@ -7,6 +7,7 @@ pub enum MetadataCommand { DeleteObject(DeleteObjectRequest), UpdateStorageClass(UpdateStorageClassRequest), UpdateArchiveLocation(UpdateArchiveLocationRequest), + CompleteArchiveObject(CompleteArchiveObjectRequest), UpdateRestoreStatus(UpdateRestoreStatusRequest), CreateBucket(common::BucketInfo), DeleteBucket(DeleteBucketRequest), diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index c3001ec..bb28b74 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -9,25 +9,88 @@ pub mod raft_storage; use anyhow::Result; use coldstore_common::config::MetadataConfig; -use std::path::PathBuf; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; use tonic::transport::Server; use tracing::info; pub async fn run(config: MetadataConfig) -> Result<()> { let addr = config.listen.parse()?; + let cluster_nodes = parse_cluster_nodes(&config.cluster)?; + let consensus_mode = config.consensus_mode.clone(); + + let effective_config = MetadataConfig { + consensus_mode: consensus_mode.clone(), + ..config.clone() + }; + enforce_consensus_mode(&effective_config, &cluster_nodes)?; + let snapshot_path = default_snapshot_path(&config); + let metadata_service = match consensus_mode { + coldstore_common::config::MetadataConsensusMode::Standalone => { + #[cfg(feature = "metadata-raft")] + { + let raft_backend = std::sync::Arc::new(crate::raft::RaftMetadataBackend::new()); + service::MetadataServiceImpl::new_with_snapshot_and_raft_backend( + &effective_config, + snapshot_path.clone(), + raft_backend, + ) + .await? + } - #[cfg(feature = "metadata-raft")] - let metadata_service = service::MetadataServiceImpl::new_with_snapshot_and_raft_backend( - &config, - snapshot_path.clone(), - std::sync::Arc::new(crate::raft::RaftMetadataBackend::new()), - ) - .await?; + #[cfg(not(feature = "metadata-raft"))] + { + anyhow::bail!( + "consensus_mode=standalone requires `metadata-raft` feature (enable via metadata crate default feature)", + ) + } + } + coldstore_common::config::MetadataConsensusMode::LocalRaft => { + #[cfg(feature = "metadata-raft")] + { + let raft_backend = std::sync::Arc::new(crate::raft::RaftMetadataBackend::new()); + service::MetadataServiceImpl::new_with_snapshot_and_raft_backend( + &effective_config, + snapshot_path.clone(), + raft_backend, + ) + .await? + } - #[cfg(not(feature = "metadata-raft"))] - let metadata_service = - service::MetadataServiceImpl::new_with_snapshot(&config, snapshot_path.clone()).await?; + #[cfg(not(feature = "metadata-raft"))] + { + anyhow::bail!( + "consensus_mode=local_raft requires `metadata-raft` feature (enable via metadata crate default feature)", + ) + } + } + coldstore_common::config::MetadataConsensusMode::PersistentRaft => { + #[cfg(feature = "metadata-raft-rocksdb")] + { + let raft_storage_path = effective_config + .raft_state_path + .clone() + .unwrap_or_else(|| default_raft_path(&effective_config)); + let raft_backend = + std::sync::Arc::new(crate::raft::RaftMetadataBackend::new_with_storage( + Path::new(&raft_storage_path), + )?); + service::MetadataServiceImpl::new_with_snapshot_and_raft_backend( + &effective_config, + snapshot_path.clone(), + raft_backend, + ) + .await? + } + #[cfg(not(feature = "metadata-raft-rocksdb"))] + { + anyhow::bail!( + "metadata consensus_mode=persistent_raft requires metadata-raft-rocksdb feature" + ) + } + } + }; info!( "Metadata 节点 {} 启动在 {}，snapshot={}", @@ -35,6 +98,12 @@ pub async fn run(config: MetadataConfig) -> Result<()> { addr, snapshot_path.display() ); + info!( + "Metadata raft consensus mode: {:?}, cluster_size={}, cluster_nodes={:?}", + effective_config.consensus_mode, + cluster_nodes.len(), + cluster_nodes + ); Server::builder() .add_service( @@ -52,6 +121,151 @@ fn default_snapshot_path(config: &MetadataConfig) -> PathBuf { PathBuf::from(&config.data_path).join(format!("node-{}-snapshot.bin", config.node_id)) } +fn default_raft_path(config: &MetadataConfig) -> String { + Path::new(&config.data_path) + .join(format!("node-{}/raft", config.node_id)) + .to_string_lossy() + .to_string() +} + +#[derive(Debug)] +struct MetadataClusterNode { + node_id: u64, + listen: String, +} + +fn parse_cluster_nodes(cluster: &str) -> Result> { + let mut parsed = Vec::new(); + let mut node_ids = HashSet::new(); + let mut listen_addrs = HashSet::new(); + + for raw_entry in cluster + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + { + let (node_id, listen) = raw_entry.split_once(':').ok_or_else(|| { + anyhow::anyhow!("invalid cluster entry '{raw_entry}', expected node_id:listen_addr") + })?; + let node_id = node_id + .trim() + .parse::() + .map_err(|_| anyhow::anyhow!("invalid node id in cluster entry '{raw_entry}'"))?; + let listen = normalize_listen_addr(listen); + if listen.is_empty() { + anyhow::bail!("invalid cluster entry '{raw_entry}', listen address is empty"); + } + + if !node_ids.insert(node_id) { + anyhow::bail!("duplicated metadata node_id '{node_id}' in cluster config"); + } + if !listen_addrs.insert(listen.clone()) { + anyhow::bail!("duplicated metadata listen address '{listen}' in cluster config"); + } + + parsed.push(MetadataClusterNode { node_id, listen }); + } + + if parsed.is_empty() { + anyhow::bail!("metadata cluster configuration is invalid or empty; expected node_id:listen_addr entries"); + } + + Ok(parsed) +} + +fn normalize_listen_addr(listen: &str) -> String { + strip_listen_scheme(listen) + .trim() + .trim_end_matches('/') + .trim() + .to_string() +} + +fn strip_listen_scheme(entry: &str) -> &str { + entry + .trim() + .trim_start_matches("http://") + .trim_start_matches("https://") + .trim_start_matches("grpc://") +} + +fn enforce_consensus_mode( + config: &MetadataConfig, + cluster_nodes: &[MetadataClusterNode], +) -> Result<()> { + let cluster_size = cluster_nodes.len(); + let local_node = cluster_nodes + .iter() + .find(|node| node.node_id == config.node_id) + .ok_or_else(|| { + anyhow::anyhow!( + "metadata node_id {} not present in cluster definition", + config.node_id + ) + })?; + + let local_listen = normalize_listen_addr(&config.listen); + if !listen_addrs_match(&local_listen, &local_node.listen) { + anyhow::bail!( + "metadata listen mismatch: node_id {} uses {} but cluster entry expects {}", + config.node_id, + config.listen, + local_node.listen, + ); + } + + let is_clustered = cluster_size > 1; + if cluster_size == 0 { + anyhow::bail!("metadata cluster_size must be greater than 0"); + } + match (config.consensus_mode.clone(), is_clustered) { + (coldstore_common::config::MetadataConsensusMode::Standalone, true) => { + anyhow::bail!( + "standalone mode only supports a single metadata node; distributed metadata clusters require a real Raft runtime" + ) + } + (coldstore_common::config::MetadataConsensusMode::LocalRaft, true) => { + anyhow::bail!( + "local_raft mode is not valid for multi-node metadata clusters; distributed metadata clusters require a real Raft runtime" + ) + } + (coldstore_common::config::MetadataConsensusMode::PersistentRaft, true) => { + anyhow::bail!( + "persistent_raft multi-node metadata clusters require the distributed Raft runtime; this build only supports single-node persistent_raft safely" + ) + } + _ => Ok(()), + } +} + +fn listen_addrs_match(local_listen: &str, cluster_listen: &str) -> bool { + let cluster_listen = normalize_listen_addr(cluster_listen); + if local_listen == cluster_listen { + return true; + } + + let Some((local_host, local_port)) = split_host_port(local_listen) else { + return false; + }; + let Some((_cluster_host, cluster_port)) = split_host_port(&cluster_listen) else { + return false; + }; + + local_port == cluster_port && is_wildcard_host(local_host) +} + +fn split_host_port(addr: &str) -> Option<(&str, &str)> { + let (host, port) = addr.rsplit_once(':')?; + if host.is_empty() || port.is_empty() { + return None; + } + Some((host.trim_matches(['[', ']']), port)) +} + +fn is_wildcard_host(host: &str) -> bool { + matches!(host, "0.0.0.0" | "::" | "*") +} + #[cfg(test)] mod tests { use super::*; @@ -69,4 +283,158 @@ mod tests { PathBuf::from("/tmp/coldstore-meta/node-7-snapshot.bin") ); } + + #[test] + fn default_raft_path_is_scoped_by_node_id() { + let config = MetadataConfig { + node_id: 7, + data_path: "/tmp/coldstore-meta".into(), + ..MetadataConfig::default() + }; + + assert_eq!( + default_raft_path(&config), + "/tmp/coldstore-meta/node-7/raft" + ); + } + + #[test] + fn parse_cluster_nodes_rejects_invalid_entry() { + let err = parse_cluster_nodes("bad-entry,2:").expect_err("invalid node must fail"); + assert!(err.to_string().contains("invalid cluster entry"),); + } + + #[test] + fn parse_cluster_nodes_rejects_duplicate_node_id() { + assert!(parse_cluster_nodes("1:127.0.0.1:21001,1:127.0.0.1:21002") + .expect_err("duplicate node id must fail") + .to_string() + .contains("duplicated metadata node_id"),); + } + + #[test] + fn parse_cluster_nodes_rejects_duplicate_normalized_listen_addr() { + assert!( + parse_cluster_nodes("1:grpc://127.0.0.1:21001,2:http://127.0.0.1:21001") + .expect_err("duplicate normalized listen addr must fail") + .to_string() + .contains("duplicated metadata listen address"), + ); + } + + #[test] + fn enforce_consensus_mode_requires_local_node_in_cluster() { + let config = MetadataConfig { + node_id: 9, + consensus_mode: coldstore_common::config::MetadataConsensusMode::LocalRaft, + cluster: "1:127.0.0.1:21001".into(), + ..MetadataConfig::default() + }; + let cluster_nodes = parse_cluster_nodes("1:127.0.0.1:21001").expect("cluster parse"); + let err = + enforce_consensus_mode(&config, &cluster_nodes).expect_err("local node not in cluster"); + assert!(err.to_string().contains("metadata node_id 9 not present")); + } + + #[test] + fn enforce_consensus_mode_rejects_cluster_with_localraft() { + let config = MetadataConfig { + node_id: 1, + listen: "127.0.0.1:21001".into(), + consensus_mode: coldstore_common::config::MetadataConsensusMode::LocalRaft, + cluster: "1:127.0.0.1:21001,2:127.0.0.1:21002".into(), + ..MetadataConfig::default() + }; + let cluster_nodes = + parse_cluster_nodes("1:127.0.0.1:21001,2:127.0.0.1:21002").expect("cluster parse"); + let err = enforce_consensus_mode(&config, &cluster_nodes) + .expect_err("local raft in cluster should reject"); + assert!(err.to_string().contains("local_raft mode")); + } + + #[test] + fn enforce_consensus_mode_rejects_two_node_persistent_raft_cluster_until_runtime_exists() { + let config = MetadataConfig { + node_id: 1, + listen: "127.0.0.1:21001".into(), + consensus_mode: coldstore_common::config::MetadataConsensusMode::PersistentRaft, + cluster: "1:127.0.0.1:21001,2:127.0.0.1:21002".into(), + ..MetadataConfig::default() + }; + let cluster_nodes = parse_cluster_nodes(&config.cluster).expect("cluster parse"); + let err = enforce_consensus_mode(&config, &cluster_nodes) + .expect_err("two-node persistent raft should reject"); + assert!(err.to_string().contains("distributed Raft runtime")); + } + + #[test] + fn enforce_consensus_mode_rejects_even_sized_persistent_raft_cluster_until_runtime_exists() { + let config = MetadataConfig { + node_id: 1, + listen: "127.0.0.1:21001".into(), + consensus_mode: coldstore_common::config::MetadataConsensusMode::PersistentRaft, + cluster: "1:127.0.0.1:21001,2:127.0.0.1:21002,3:127.0.0.1:21003,4:127.0.0.1:21004" + .into(), + ..MetadataConfig::default() + }; + let cluster_nodes = parse_cluster_nodes(&config.cluster).expect("cluster parse"); + let err = enforce_consensus_mode(&config, &cluster_nodes) + .expect_err("even-sized persistent raft should reject"); + assert!(err.to_string().contains("distributed Raft runtime")); + } + + #[test] + fn enforce_consensus_mode_rejects_persistent_raft_cluster_until_distributed_runtime_exists() { + let config = MetadataConfig { + node_id: 1, + listen: "127.0.0.1:21001".into(), + consensus_mode: coldstore_common::config::MetadataConsensusMode::PersistentRaft, + cluster: "1:127.0.0.1:21001,2:127.0.0.1:21002,3:127.0.0.1:21003".into(), + ..MetadataConfig::default() + }; + let cluster_nodes = parse_cluster_nodes(&config.cluster).expect("cluster parse"); + + let err = enforce_consensus_mode(&config, &cluster_nodes) + .expect_err("multi-node persistent raft must fail until distributed runtime exists"); + assert!(err.to_string().contains("distributed Raft runtime")); + } + + #[test] + fn enforce_consensus_mode_rejects_local_listen_mismatch() { + let config = MetadataConfig { + node_id: 1, + listen: "127.0.0.1:21001".into(), + consensus_mode: coldstore_common::config::MetadataConsensusMode::PersistentRaft, + ..MetadataConfig::default() + }; + let cluster_nodes = + parse_cluster_nodes("1:127.0.0.1:31001,2:127.0.0.1:31002").expect("cluster parse"); + let err = enforce_consensus_mode(&config, &cluster_nodes) + .expect_err("listen address mismatch must fail"); + assert!(err.to_string().contains("metadata listen mismatch")); + } + + #[test] + fn enforce_consensus_mode_normalizes_cluster_listen_prefix() { + let config = MetadataConfig { + node_id: 1, + listen: "http://127.0.0.1:21001".into(), + consensus_mode: coldstore_common::config::MetadataConsensusMode::PersistentRaft, + ..MetadataConfig::default() + }; + let cluster_nodes = parse_cluster_nodes("1:grpc://127.0.0.1:21001").expect("cluster parse"); + enforce_consensus_mode(&config, &cluster_nodes).expect("consistency accepted"); + } + + #[test] + fn enforce_consensus_mode_allows_wildcard_bind_with_cluster_advertise_addr() { + let config = MetadataConfig { + cluster: "1:127.0.0.1:21001".into(), + ..MetadataConfig::default() + }; + let cluster_nodes = parse_cluster_nodes(&config.cluster).expect("cluster parse"); + + enforce_consensus_mode(&config, &cluster_nodes) + .expect("wildcard listen should match local advertised cluster address"); + } } diff --git a/crates/metadata/src/raft.rs b/crates/metadata/src/raft.rs index a334d39..ea358f0 100644 --- a/crates/metadata/src/raft.rs +++ b/crates/metadata/src/raft.rs @@ -1,9 +1,12 @@ -//! Feature-gated local propose backend for metadata Raft integration. +//! Feature-gated Raft-like metadata backend abstraction. //! -//! This is not a complete OpenRaft runtime. It provides an explicit, opt-in -//! proposal boundary so `MetadataServiceImpl` can route writes through the same -//! command/state-machine path that a real Raft state machine will use later. +//! In this phase, this module provides strict proposal semantics and a pluggable +//! backend model: +//! - local in-memory apply path for deterministic single-process behavior +//! - optional persistent local path (rocksdb-backed storage module) for production +//! readiness preparation and recovery tests +use std::path::Path; use std::sync::Arc; use tokio::sync::RwLock; use tonic::Status; @@ -11,33 +14,158 @@ use tonic::Status; use crate::command::MetadataCommand; use crate::state_machine::{apply_command, MetadataState}; +#[cfg(feature = "metadata-raft-rocksdb")] +use crate::raft_storage::LocalSingleNodeMetadataRaft; + pub type ColdStoreNodeId = u64; pub type ColdStoreNode = openraft::BasicNode; -#[derive(Debug, Default)] +enum RaftBackendMode { + Memory, + #[cfg(feature = "metadata-raft-rocksdb")] + Persistent { + storage: Arc, + }, +} + pub struct RaftMetadataBackend { + inner: RaftBackendMode, proposed_commands: RwLock, } +impl Default for RaftMetadataBackend { + fn default() -> Self { + Self::new() + } +} + impl RaftMetadataBackend { pub fn new() -> Self { - Self::default() + Self { + inner: RaftBackendMode::Memory, + proposed_commands: RwLock::new(0), + } + } + + #[cfg(feature = "metadata-raft-rocksdb")] + pub fn new_with_storage(storage_path: impl AsRef) -> anyhow::Result { + let storage = Arc::new(LocalSingleNodeMetadataRaft::open(storage_path)?); + let committed = storage.applied_log_count()?; + Ok(Self { + inner: RaftBackendMode::Persistent { storage }, + proposed_commands: RwLock::new(committed), + }) } pub async fn proposed_commands(&self) -> u64 { *self.proposed_commands.read().await } + pub async fn cluster_term_and_leader(&self) -> (u64, Option) { + match &self.inner { + RaftBackendMode::Memory => (1, None), + #[cfg(feature = "metadata-raft-rocksdb")] + RaftBackendMode::Persistent { storage } => storage + .load_vote() + .unwrap_or(None) + .map(|(term, node_id)| (term, Some(node_id))) + .unwrap_or((0, None)), + } + } + + pub fn mode_label(&self) -> &'static str { + self.mode() + } + + #[cfg(feature = "metadata-raft-rocksdb")] + pub fn mode(&self) -> &'static str { + match self.inner { + RaftBackendMode::Memory => "local_memory", + RaftBackendMode::Persistent { .. } => "persistent_local", + } + } + + #[cfg(not(feature = "metadata-raft-rocksdb"))] + pub fn mode(&self) -> &'static str { + "local_memory" + } + + pub async fn bootstrapped_state(&self) -> anyhow::Result> { + match &self.inner { + RaftBackendMode::Memory => Ok(None), + #[cfg(feature = "metadata-raft-rocksdb")] + RaftBackendMode::Persistent { storage } => Ok(Some(storage.snapshot()?)), + } + } + + /// Propose a metadata command for the active backend and apply it through the + /// local state machine view. pub async fn propose_local_apply( &self, state: &Arc>, command: MetadataCommand, ) -> std::result::Result<(), Status> { - let mut guard = state.write().await; - apply_command(&mut guard, command)?; - *self.proposed_commands.write().await += 1; + // Always apply command locally first for deterministic state progression. + { + let mut guard = state.write().await; + apply_command(&mut guard, command.clone())?; + } + + if let Some(next_committed_index) = self.persist_if_needed(&command).await? { + let mut counter = self.proposed_commands.write().await; + *counter = next_committed_index; + return Ok(()); + } + + { + let mut counter = self.proposed_commands.write().await; + *counter = counter.saturating_add(1); + } + Ok(()) } + + pub async fn bootstrap_state_if_empty( + &self, + state: &MetadataState, + ) -> std::result::Result<(), Status> { + if state.is_empty() { + return Ok(()); + } + + match &self.inner { + RaftBackendMode::Memory => Ok(()), + #[cfg(feature = "metadata-raft-rocksdb")] + RaftBackendMode::Persistent { storage } => { + let existing = storage + .load_state_machine_snapshot() + .map_err(|err| Status::internal(err.to_string()))?; + if existing.is_some() { + return Ok(()); + } + + storage + .save_state_machine_snapshot(state) + .map_err(|err| Status::internal(err.to_string())) + } + } + } + + async fn persist_if_needed( + &self, + command: &MetadataCommand, + ) -> std::result::Result, Status> { + match &self.inner { + RaftBackendMode::Memory => Ok(None), + #[cfg(feature = "metadata-raft-rocksdb")] + RaftBackendMode::Persistent { storage } => { + let next = storage + .propose(command.clone()) + .map_err(|err| Status::internal(err.to_string()))?; + Ok(Some(next)) + } + } + } } #[cfg(test)] @@ -57,7 +185,7 @@ mod tests { } #[tokio::test] - async fn raft_backend_proposes_commands_through_state_machine_apply_path() { + async fn raft_backend_proposes_commands_through_state_machine_apply() { let state = Arc::new(RwLock::new(MetadataState::default())); let backend = RaftMetadataBackend::new(); @@ -68,5 +196,37 @@ mod tests { assert_eq!(backend.proposed_commands().await, 1); assert!(state.read().await.buckets.contains_key("docs")); + assert_eq!(backend.mode(), "local_memory"); + } + + #[cfg(feature = "metadata-raft-rocksdb")] + #[tokio::test] + async fn raft_backend_bootstraps_from_persistent_storage() { + use std::path::PathBuf; + + let dir = std::env::temp_dir().join(format!( + "coldstore-metadata-raft-backend-{}", + uuid::Uuid::new_v4() + )); + let storage = crate::raft_storage::LocalSingleNodeMetadataRaft::open(&dir) + .expect("open local single node raft storage"); + + storage + .propose(MetadataCommand::CreateBucket(test_bucket("docs"))) + .expect("persist bootstrap snapshot command"); + let _ = storage.applied_log_count().expect("applied count"); + drop(storage); + + let backend = RaftMetadataBackend::new_with_storage(PathBuf::from(&dir)) + .expect("backend from storage"); + let restored = backend + .bootstrapped_state() + .await + .expect("read bootstrap state") + .expect("bootstrap exists"); + + assert!(restored.bucket("docs").is_some()); + assert_eq!(backend.mode(), "persistent_local"); + let _ = std::fs::remove_dir_all(dir); } } diff --git a/crates/metadata/src/raft_storage.rs b/crates/metadata/src/raft_storage.rs index 52d46c4..6331200 100644 --- a/crates/metadata/src/raft_storage.rs +++ b/crates/metadata/src/raft_storage.rs @@ -107,7 +107,7 @@ impl LocalSingleNodeMetadataRaft { }) } - pub fn propose(&self, command: MetadataCommand) -> Result<()> { + pub fn propose(&self, command: MetadataCommand) -> Result { let index = self.storage.log_entry_count()? + 1; self.storage.append_log_entry(index, 1, &command)?; let mut state_machine = self @@ -119,13 +119,25 @@ impl LocalSingleNodeMetadataRaft { .map_err(|status| anyhow::anyhow!(status.to_string()))?; self.storage .save_state_machine_snapshot(state_machine.state())?; - Ok(()) + Ok(index) } pub fn applied_log_count(&self) -> Result { self.storage.log_entry_count() } + pub fn load_vote(&self) -> Result> { + self.storage.load_vote() + } + + pub fn load_state_machine_snapshot(&self) -> Result> { + self.storage.load_state_machine_snapshot() + } + + pub fn save_state_machine_snapshot(&self, state: &MetadataState) -> Result<()> { + self.storage.save_state_machine_snapshot(state) + } + pub fn snapshot(&self) -> Result { Ok(self .storage @@ -222,13 +234,17 @@ mod tests { uuid::Uuid::new_v4() )); let node = LocalSingleNodeMetadataRaft::open(&dir).expect("open single node raft"); - node.propose(MetadataCommand::CreateBucket(test_bucket("docs"))) + let first_index = node + .propose(MetadataCommand::CreateBucket(test_bucket("docs"))) .expect("propose create bucket"); - node.propose(MetadataCommand::PutObject(test_object( - "docs", - "readme.txt", - ))) - .expect("propose put object"); + let second_index = node + .propose(MetadataCommand::PutObject(test_object( + "docs", + "readme.txt", + ))) + .expect("propose put object"); + assert_eq!(first_index, 1); + assert_eq!(second_index, 2); assert_eq!(node.applied_log_count().expect("applied count"), 2); drop(node); diff --git a/crates/metadata/src/service.rs b/crates/metadata/src/service.rs index 8ca7aae..42a9322 100644 --- a/crates/metadata/src/service.rs +++ b/crates/metadata/src/service.rs @@ -1,7 +1,7 @@ use crate::command::MetadataCommand; use crate::state_machine::{ find_object, is_active_restore_status, is_pending_restore_status, load_snapshot, now_timestamp, - save_snapshot, MetadataState, MetadataStateMachine, + save_snapshot, MetadataState, }; use anyhow::Result; use coldstore_common::config::MetadataConfig; @@ -10,15 +10,15 @@ use coldstore_proto::metadata::metadata_service_server::MetadataService; use coldstore_proto::metadata::*; use std::path::PathBuf; use std::sync::Arc; -use tokio::sync::RwLock; +use tokio::sync::{Mutex, RwLock}; use tonic::{Request, Response, Status}; pub struct MetadataServiceImpl { config: MetadataConfig, state: Arc>, snapshot_path: Option, - #[cfg(feature = "metadata-raft")] raft_backend: Option>, + write_lock: Arc>, } impl MetadataServiceImpl { @@ -27,8 +27,8 @@ impl MetadataServiceImpl { config: config.clone(), state: Arc::new(RwLock::new(MetadataState::default())), snapshot_path: None, - #[cfg(feature = "metadata-raft")] - raft_backend: None, + raft_backend: Some(std::sync::Arc::new(crate::raft::RaftMetadataBackend::new())), + write_lock: Arc::new(Mutex::new(())), }) } @@ -46,33 +46,57 @@ impl MetadataServiceImpl { config: config.clone(), state: Arc::new(RwLock::new(state)), snapshot_path: Some(snapshot_path), - #[cfg(feature = "metadata-raft")] - raft_backend: None, + raft_backend: Some(std::sync::Arc::new(crate::raft::RaftMetadataBackend::new())), + write_lock: Arc::new(Mutex::new(())), }) } - #[cfg(feature = "metadata-raft")] pub async fn new_with_raft_backend( config: &MetadataConfig, raft_backend: Arc, ) -> Result { + let state = raft_backend.bootstrapped_state().await?.unwrap_or_default(); Ok(Self { config: config.clone(), - state: Arc::new(RwLock::new(MetadataState::default())), + state: Arc::new(RwLock::new(state)), snapshot_path: None, raft_backend: Some(raft_backend), + write_lock: Arc::new(Mutex::new(())), }) } - #[cfg(feature = "metadata-raft")] pub async fn new_with_snapshot_and_raft_backend( config: &MetadataConfig, snapshot_path: PathBuf, raft_backend: Arc, ) -> Result { - let mut service = Self::new_with_snapshot(config, snapshot_path).await?; - service.raft_backend = Some(raft_backend); - Ok(service) + let raft_state = raft_backend.bootstrapped_state().await?.unwrap_or_default(); + let state = if raft_state.is_empty() { + match tokio::fs::try_exists(&snapshot_path).await? { + true => { + let snapshot_state = load_snapshot(&snapshot_path).await?; + if snapshot_state.is_empty() { + MetadataState::default() + } else { + raft_backend + .bootstrap_state_if_empty(&snapshot_state) + .await?; + snapshot_state + } + } + false => MetadataState::default(), + } + } else { + raft_state + }; + + Ok(Self { + config: config.clone(), + state: Arc::new(RwLock::new(state)), + snapshot_path: Some(snapshot_path), + raft_backend: Some(raft_backend), + write_lock: Arc::new(Mutex::new(())), + }) } async fn persist_locked(&self, state: &MetadataState) -> std::result::Result<(), Status> { @@ -88,23 +112,19 @@ impl MetadataServiceImpl { &self, command: MetadataCommand, ) -> std::result::Result<(), Status> { - #[cfg(feature = "metadata-raft")] - if let Some(raft_backend) = &self.raft_backend { - raft_backend - .propose_local_apply(&self.state, command) - .await?; - let state = self.state.read().await; - return self.persist_locked(&state).await; - } - - let mut state = self.state.write().await; - let mut state_machine = MetadataStateMachine::new(state.clone()); - state_machine.apply(command)?; - *state = state_machine.into_state(); + let _write_guard = self.write_lock.lock().await; + let raft_backend = self + .raft_backend + .as_ref() + .ok_or_else(|| Status::internal("metadata raft backend is not initialized"))?; + raft_backend + .propose_local_apply(&self.state, command) + .await?; + let state = self.state.read().await; self.persist_locked(&state).await } - fn metadata_nodes(&self) -> Vec { + fn metadata_nodes(&self, leader_hint: Option) -> Vec { self.config .cluster .split(',') @@ -114,8 +134,8 @@ impl MetadataServiceImpl { Some(common::MetadataNodeInfo { node_id, addr: addr.to_string(), - raft_role: if node_id == self.config.node_id { - "LeaderCandidate".into() + raft_role: if Some(node_id) == leader_hint { + "Leader".into() } else { "Follower".into() }, @@ -249,6 +269,15 @@ impl MetadataService for MetadataServiceImpl { Ok(Response::new(())) } + async fn complete_archive_object( + &self, + request: Request, + ) -> std::result::Result, Status> { + self.apply_and_persist(MetadataCommand::CompleteArchiveObject(request.into_inner())) + .await?; + Ok(Response::new(())) + } + async fn update_restore_status( &self, request: Request, @@ -564,15 +593,27 @@ impl MetadataService for MetadataServiceImpl { _request: Request<()>, ) -> std::result::Result, Status> { let state = self.state.read().await; + #[cfg(feature = "metadata-raft")] + let (leader_id, term, committed_index) = if let Some(backend) = &self.raft_backend { + let (term, leader_id) = backend.cluster_term_and_leader().await; + (leader_id, term, backend.proposed_commands().await) + } else { + (None, 1, state.objects.len() as u64) + }; + + #[cfg(not(feature = "metadata-raft"))] + let (leader_id, term, committed_index) = (None, 0_u64, state.objects.len() as u64); + + let metadata_nodes = self.metadata_nodes(leader_id); Ok(Response::new(common::ClusterInfo { cluster_id: "coldstore-phase1".into(), - metadata_nodes: self.metadata_nodes(), + metadata_nodes, scheduler_workers: state.scheduler_workers.values().cloned().collect(), cache_workers: state.cache_workers.values().cloned().collect(), tape_workers: state.tape_workers.values().cloned().collect(), - leader_id: Some(self.config.node_id), - term: 1, - committed_index: state.objects.len() as u64, + leader_id, + term, + committed_index, })) } @@ -876,6 +917,103 @@ mod tests { assert_eq!(bucket.total_size, 5); } + #[tokio::test] + async fn complete_archive_object_is_atomic_and_checks_generation() { + let svc = MetadataServiceImpl::new(&MetadataConfig::default()) + .await + .expect("service init"); + svc.create_bucket(Request::new(test_bucket("docs"))) + .await + .expect("create bucket"); + svc.put_object(Request::new(test_object("docs", "readme.txt"))) + .await + .expect("put object"); + + let before = svc + .head_object(Request::new(HeadObjectRequest { + bucket: "docs".into(), + key: "readme.txt".into(), + })) + .await + .expect("head object") + .into_inner(); + let mut stale_updated_at = before.updated_at; + if let Some(ts) = stale_updated_at.as_mut() { + ts.seconds = ts.seconds.saturating_sub(1); + } + + let bundle = common::ArchiveBundle { + id: "bundle-1".into(), + tape_id: "TAPE0001".into(), + tape_set: vec!["TAPE0001".into()], + entries: vec![common::BundleEntry { + bucket: "docs".into(), + key: "readme.txt".into(), + version_id: None, + size: before.size, + offset_in_bundle: 0, + tape_block_offset: 7, + checksum: before.checksum.clone(), + }], + total_size: before.size, + filemark_start: 7, + filemark_end: 8, + checksum: Some(before.checksum.clone()), + status: common::ArchiveBundleStatus::BundleCompleted as i32, + created_at: before.updated_at, + completed_at: before.updated_at, + }; + + let err = svc + .complete_archive_object(Request::new(CompleteArchiveObjectRequest { + bucket: "docs".into(), + key: "readme.txt".into(), + version_id: before.version_id.clone(), + expected_size: Some(before.size), + expected_checksum: Some(before.checksum.clone()), + expected_storage_class: Some(before.storage_class), + expected_updated_at: stale_updated_at, + bundle: Some(bundle.clone()), + tape_block_offset: 7, + storage_class: common::StorageClass::Cold as i32, + })) + .await + .expect_err("stale generation must reject archive completion"); + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert!(svc + .get_archive_bundle(Request::new(GetArchiveBundleRequest { + id: bundle.id.clone(), + })) + .await + .is_err()); + + svc.complete_archive_object(Request::new(CompleteArchiveObjectRequest { + bucket: "docs".into(), + key: "readme.txt".into(), + version_id: before.version_id.clone(), + expected_size: Some(before.size), + expected_checksum: Some(before.checksum.clone()), + expected_storage_class: Some(before.storage_class), + expected_updated_at: before.updated_at, + bundle: Some(bundle.clone()), + tape_block_offset: 7, + storage_class: common::StorageClass::Cold as i32, + })) + .await + .expect("archive completion should succeed"); + + let archived = svc + .head_object(Request::new(HeadObjectRequest { + bucket: "docs".into(), + key: "readme.txt".into(), + })) + .await + .expect("head archived object") + .into_inner(); + assert_eq!(archived.storage_class, common::StorageClass::Cold as i32); + assert_eq!(archived.archive_id.as_deref(), Some("bundle-1")); + } + #[tokio::test] async fn persistent_snapshot_survives_service_restart() { let snapshot_path = std::env::temp_dir().join(format!( diff --git a/crates/metadata/src/state_machine.rs b/crates/metadata/src/state_machine.rs index 365145e..0d7c5ff 100644 --- a/crates/metadata/src/state_machine.rs +++ b/crates/metadata/src/state_machine.rs @@ -55,6 +55,18 @@ impl MetadataState { pub fn object_count(&self) -> usize { self.objects.len() } + + pub fn is_empty(&self) -> bool { + self.objects.is_empty() + && self.buckets.is_empty() + && self.archive_bundles.is_empty() + && self.archive_tasks.is_empty() + && self.recall_tasks.is_empty() + && self.tapes.is_empty() + && self.scheduler_workers.is_empty() + && self.cache_workers.is_empty() + && self.tape_workers.is_empty() + } } #[derive(Default, Clone, Debug)] @@ -139,6 +151,47 @@ pub(crate) fn apply_command( object.tape_block_offset = Some(request.tape_block_offset); object.updated_at = Some(now_timestamp()); } + MetadataCommand::CompleteArchiveObject(mut request) => { + let bundle = request + .bundle + .take() + .ok_or_else(|| Status::invalid_argument("missing archive bundle"))?; + if bundle.id.is_empty() { + return Err(Status::invalid_argument("archive bundle id is empty")); + } + let object = find_object( + state, + &request.bucket, + &request.key, + request.version_id.as_deref(), + )?; + validate_complete_archive_preconditions(object, &request)?; + + if let Some(existing) = state.archive_bundles.get(&bundle.id) { + if existing != &bundle { + return Err(Status::already_exists( + "archive bundle already exists with different payload", + )); + } + } else { + state + .archive_bundles + .insert(bundle.id.clone(), bundle.clone()); + } + + let object = find_object_mut( + state, + &request.bucket, + &request.key, + request.version_id.as_deref(), + )?; + object.archive_id = Some(bundle.id); + object.tape_id = Some(bundle.tape_id); + object.tape_set = bundle.tape_set; + object.tape_block_offset = Some(request.tape_block_offset); + object.storage_class = request.storage_class; + object.updated_at = Some(now_timestamp()); + } MetadataCommand::UpdateRestoreStatus(request) => { let object = find_object_mut(state, &request.bucket, &request.key, None)?; validate_restore_transition(object.restore_status, request.status)?; @@ -192,7 +245,16 @@ pub(crate) fn apply_command( if task.created_at.is_none() { task.created_at = Some(now_timestamp()); } - state.archive_tasks.insert(task.id.clone(), task); + match state.archive_tasks.get(&task.id) { + Some(existing) if is_same_archive_task(existing, &task) => Ok(()), + Some(_) => Err(Status::already_exists( + "archive task already exists with different payload", + )), + None => { + state.archive_tasks.insert(task.id.clone(), task); + Ok(()) + } + }?; } MetadataCommand::UpdateArchiveTask(task) => { let current = state @@ -206,7 +268,16 @@ pub(crate) fn apply_command( if task.created_at.is_none() { task.created_at = Some(now_timestamp()); } - state.recall_tasks.insert(task.id.clone(), task); + match state.recall_tasks.get(&task.id) { + Some(existing) if is_same_recall_task(existing, &task) => Ok(()), + Some(_) => Err(Status::already_exists( + "recall task already exists with different payload", + )), + None => { + state.recall_tasks.insert(task.id.clone(), task); + Ok(()) + } + }?; } MetadataCommand::UpdateRecallTask(task) => { let current = state @@ -558,6 +629,55 @@ fn timestamp_sort_key(ts: &Option) -> (i64, i32) { .unwrap_or_default() } +#[allow(clippy::result_large_err)] +fn validate_complete_archive_preconditions( + object: common::ObjectMetadata, + request: &CompleteArchiveObjectRequest, +) -> Result<(), Status> { + if object.version_id != request.version_id { + return Err(Status::failed_precondition( + "object version changed before archive completion", + )); + } + let expected_size = request.expected_size.ok_or_else(|| { + Status::failed_precondition("archive completion requires expected_size precondition") + })?; + if object.size != expected_size { + return Err(Status::failed_precondition( + "object size changed before archive completion", + )); + } + let expected_checksum = request.expected_checksum.as_ref().ok_or_else(|| { + Status::failed_precondition("archive completion requires expected_checksum precondition") + })?; + if object.checksum != *expected_checksum { + return Err(Status::failed_precondition( + "object checksum changed before archive completion", + )); + } + let expected_storage_class = request.expected_storage_class.ok_or_else(|| { + Status::failed_precondition( + "archive completion requires expected_storage_class precondition", + ) + })?; + if object.storage_class != expected_storage_class { + return Err(Status::failed_precondition( + "object storage class changed before archive completion", + )); + } + if request.expected_updated_at.is_none() { + return Err(Status::failed_precondition( + "archive completion requires expected_updated_at precondition", + )); + } + if object.updated_at != request.expected_updated_at { + return Err(Status::failed_precondition( + "object generation changed before archive completion", + )); + } + Ok(()) +} + #[allow(clippy::result_large_err)] fn validate_restore_transition(current: Option, next: i32) -> Result<(), Status> { let current = current.and_then(|value| common::RestoreStatus::try_from(value).ok()); @@ -573,7 +693,9 @@ fn validate_restore_transition(current: Option, next: i32) -> Result<(), St ), Some(common::RestoreStatus::RestoreWaitingForMedia) => matches!( next, - common::RestoreStatus::RestorePending | common::RestoreStatus::RestoreFailed + common::RestoreStatus::RestoreInProgress + | common::RestoreStatus::RestorePending + | common::RestoreStatus::RestoreFailed ), Some(common::RestoreStatus::RestoreInProgress) => { matches!( @@ -662,10 +784,40 @@ pub(crate) fn is_pending_restore_status(status: i32) -> bool { common::RestoreStatus::try_from(status), Ok(common::RestoreStatus::RestorePending) | Ok(common::RestoreStatus::RestoreWaitingForMedia) - | Ok(common::RestoreStatus::RestoreInProgress) ) } +fn is_same_archive_task(existing: &common::ArchiveTask, incoming: &common::ArchiveTask) -> bool { + existing.id == incoming.id + && existing.bundle_id == incoming.bundle_id + && existing.tape_id == incoming.tape_id + && existing.drive_id == incoming.drive_id + && existing.object_count == incoming.object_count + && existing.total_size == incoming.total_size + && existing.bytes_written == incoming.bytes_written + && existing.status == incoming.status + && existing.retry_count == incoming.retry_count + && existing.error == incoming.error +} + +fn is_same_recall_task(existing: &common::RecallTask, incoming: &common::RecallTask) -> bool { + existing.id == incoming.id + && existing.bucket == incoming.bucket + && existing.key == incoming.key + && existing.version_id == incoming.version_id + && existing.archive_id == incoming.archive_id + && existing.tape_id == incoming.tape_id + && existing.tape_set == incoming.tape_set + && existing.tape_block_offset == incoming.tape_block_offset + && existing.object_size == incoming.object_size + && existing.checksum == incoming.checksum + && existing.tier == incoming.tier + && existing.days == incoming.days + && existing.status == incoming.status + && existing.drive_id == incoming.drive_id + && existing.retry_count == incoming.retry_count +} + pub(crate) fn is_active_restore_status(status: i32) -> bool { matches!( common::RestoreStatus::try_from(status), @@ -680,6 +832,52 @@ pub(crate) fn is_active_restore_status(status: i32) -> bool { mod tests { use super::*; + fn archive_task(id: &str) -> common::ArchiveTask { + common::ArchiveTask { + id: id.into(), + bundle_id: "bundle-1".into(), + tape_id: "TAPE-1".into(), + drive_id: Some("drive-0".into()), + object_count: 1, + total_size: 1024, + bytes_written: 0, + status: common::ArchiveTaskStatus::ArchiveTaskPending as i32, + retry_count: 0, + created_at: None, + started_at: None, + completed_at: None, + error: None, + } + } + + fn recall_task(id: &str, days: u32) -> common::RecallTask { + common::RecallTask { + id: id.into(), + bucket: "docs".into(), + key: "readme.txt".into(), + version_id: None, + archive_id: "bundle-1".into(), + tape_id: "TAPE-1".into(), + tape_set: vec!["TAPE-1".into()], + tape_block_offset: 0, + object_size: 1024, + checksum: "sha256".into(), + tier: common::RestoreTier::Standard as i32, + days, + expire_at: Some(Timestamp { + seconds: 1000, + nanos: 0, + }), + status: common::RestoreStatus::RestorePending as i32, + drive_id: None, + retry_count: 0, + created_at: None, + started_at: None, + completed_at: None, + error: None, + } + } + #[test] fn decode_snapshot_rejects_excessive_section_count_before_allocating() { let mut bytes = SNAPSHOT_MAGIC.to_vec(); @@ -703,4 +901,87 @@ mod tests { .to_string() .contains("metadata snapshot message is too large")); } + + #[test] + fn put_archive_task_is_idempotent_for_identical_payload() { + let mut state = MetadataState::default(); + apply_command( + &mut state, + MetadataCommand::PutArchiveTask(archive_task("archive-task-1")), + ) + .expect("first archive task insert"); + + apply_command( + &mut state, + MetadataCommand::PutArchiveTask(archive_task("archive-task-1")), + ) + .expect("second identical insert should be idempotent"); + + assert_eq!(state.archive_tasks.len(), 1); + } + + #[test] + fn put_archive_task_requires_same_payload_for_idempotent_write() { + let mut state = MetadataState::default(); + apply_command( + &mut state, + MetadataCommand::PutArchiveTask(archive_task("archive-task-2")), + ) + .expect("first archive task insert"); + + let mut changed = archive_task("archive-task-2"); + changed.tape_id = "TAPE-2".into(); + + let err = apply_command(&mut state, MetadataCommand::PutArchiveTask(changed)) + .expect_err("different payload should be rejected"); + + assert_eq!(err.code(), tonic::Code::AlreadyExists); + assert_eq!(state.archive_tasks.len(), 1); + } + + #[test] + fn put_recall_task_is_idempotent_for_identical_payload() { + let mut state = MetadataState::default(); + apply_command( + &mut state, + MetadataCommand::PutRecallTask(recall_task("recall-task-1", 1)), + ) + .expect("first recall task insert"); + + apply_command( + &mut state, + MetadataCommand::PutRecallTask(recall_task("recall-task-1", 1)), + ) + .expect("second identical insert should be idempotent"); + + assert_eq!(state.recall_tasks.len(), 1); + } + + #[test] + fn put_recall_task_requires_same_payload_for_idempotent_write() { + let mut state = MetadataState::default(); + apply_command( + &mut state, + MetadataCommand::PutRecallTask(recall_task("recall-task-2", 1)), + ) + .expect("first recall task insert"); + + let mut changed = recall_task("recall-task-2", 2); + changed.checksum = "different-checksum".into(); + + let err = apply_command(&mut state, MetadataCommand::PutRecallTask(changed)) + .expect_err("different payload should be rejected"); + + assert_eq!(err.code(), tonic::Code::AlreadyExists); + assert_eq!(state.recall_tasks.len(), 1); + } + + #[test] + fn restore_waiting_for_media_can_transition_to_in_progress() { + assert!(validate_restore_transition( + Some(common::RestoreStatus::RestoreWaitingForMedia as i32), + common::RestoreStatus::RestoreInProgress as i32, + ) + .is_ok()); + } } diff --git a/crates/proto/proto/metadata.proto b/crates/proto/proto/metadata.proto index 89c0716..df42da0 100644 --- a/crates/proto/proto/metadata.proto +++ b/crates/proto/proto/metadata.proto @@ -25,6 +25,7 @@ service MetadataService { rpc ListObjects(ListObjectsRequest) returns (ListObjectsResponse); rpc UpdateStorageClass(UpdateStorageClassRequest) returns (google.protobuf.Empty); rpc UpdateArchiveLocation(UpdateArchiveLocationRequest) returns (google.protobuf.Empty); + rpc CompleteArchiveObject(CompleteArchiveObjectRequest) returns (google.protobuf.Empty); rpc UpdateRestoreStatus(UpdateRestoreStatusRequest) returns (google.protobuf.Empty); rpc ScanColdPending(ScanColdPendingRequest) returns (ScanColdPendingResponse); @@ -138,6 +139,19 @@ message UpdateArchiveLocationRequest { uint64 tape_block_offset = 6; } +message CompleteArchiveObjectRequest { + string bucket = 1; + string key = 2; + optional string version_id = 3; + optional uint64 expected_size = 4; + optional string expected_checksum = 5; + optional coldstore.common.StorageClass expected_storage_class = 6; + optional google.protobuf.Timestamp expected_updated_at = 7; + coldstore.common.ArchiveBundle bundle = 8; + uint64 tape_block_offset = 9; + coldstore.common.StorageClass storage_class = 10; +} + message UpdateRestoreStatusRequest { string bucket = 1; string key = 2; diff --git a/crates/scheduler/src/lib.rs b/crates/scheduler/src/lib.rs index 59df646..3974c2f 100644 --- a/crates/scheduler/src/lib.rs +++ b/crates/scheduler/src/lib.rs @@ -5,6 +5,9 @@ use coldstore_common::config::SchedulerConfig; use coldstore_proto::cache::cache_service_client::CacheServiceClient; use coldstore_proto::metadata::metadata_service_client::MetadataServiceClient; use coldstore_proto::tape::tape_service_client::TapeServiceClient; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use tokio::sync::{Mutex, Semaphore}; use tokio::time::{sleep, Duration}; use tonic::transport::{Channel, Server}; use tracing::{info, warn}; @@ -14,6 +17,12 @@ pub struct SchedulerState { pub cache: Option>, pub tape: Option>, pub config: SchedulerConfig, + pub active_archive_keys: Arc>>, + pub active_recall_tasks: Arc>>, + pub tape_locks: Arc>>>, + pub archive_slots: Arc, + pub recall_slots: Arc, + pub recall_task_slots: Arc, } pub async fn run(config: SchedulerConfig) -> Result<()> { @@ -42,6 +51,12 @@ pub async fn run(config: SchedulerConfig) -> Result<()> { cache: Some(cache), tape, config: config.clone(), + active_archive_keys: Arc::new(Mutex::new(HashSet::new())), + active_recall_tasks: Arc::new(Mutex::new(HashSet::new())), + tape_locks: Arc::new(Mutex::new(HashMap::new())), + archive_slots: Arc::new(Semaphore::new(config.archive.max_workers.max(1))), + recall_slots: Arc::new(Semaphore::new(config.recall.max_workers.max(1))), + recall_task_slots: Arc::new(Semaphore::new(config.recall.max_concurrent_restores.max(1))), }); service::spawn_background_loops(state.clone()); diff --git a/crates/scheduler/src/service.rs b/crates/scheduler/src/service.rs index 30b6e6d..b88724e 100644 --- a/crates/scheduler/src/service.rs +++ b/crates/scheduler/src/service.rs @@ -5,9 +5,9 @@ use coldstore_proto::cache::get_staging_response::Payload as GetStagingPayload; use coldstore_proto::cache::put_restored_request::Payload as PutRestoredPayload; use coldstore_proto::cache::put_staging_request::Payload as PutStagingPayload; use coldstore_proto::cache::{ - DeleteStagingRequest, GetRequest as CacheGetRequest, GetStagingRequest, ListStagingKeysRequest, - PutRestoredMeta, PutRestoredRequest, PutStagingMeta, PutStagingRequest, StagingKeyEntry, - StagingObjectMeta, + DeleteRequest, DeleteStagingRequest, GetRequest as CacheGetRequest, GetStagingRequest, + ListStagingKeysRequest, PutRestoredMeta, PutRestoredRequest, PutStagingMeta, PutStagingRequest, + StagingKeyEntry, StagingObjectMeta, }; use coldstore_proto::common; use coldstore_proto::scheduler::scheduler_service_server::SchedulerService; @@ -24,14 +24,16 @@ use coldstore_proto::tape::{ use prost_types::Timestamp; use sha2::{Digest, Sha256}; #[cfg(test)] -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; #[cfg(test)] use std::sync::RwLock; -use tokio::sync::mpsc; +#[cfg(test)] +use tokio::sync::Mutex; +use tokio::sync::{mpsc, OwnedSemaphorePermit, Semaphore}; use tokio::time::{interval, Duration}; use tokio_stream::wrappers::ReceiverStream; -use tonic::{Request, Response, Status, Streaming}; +use tonic::{Code, Request, Response, Status, Streaming}; use tracing::{debug, error, info, warn}; #[tonic::async_trait] @@ -70,8 +72,17 @@ pub trait Phase1SchedulerBackend: Send + Sync + 'static { bucket: &str, prefix: Option<&str>, marker: Option<&str>, + delimiter: Option<&str>, max_keys: u32, - ) -> std::result::Result, Status>; + ) -> std::result::Result; +} + +#[derive(Debug, Clone, Default)] +pub struct ListObjectsPage { + objects: Vec, + common_prefixes: Vec, + next_marker: Option, + is_truncated: bool, } #[derive(Debug, Clone)] @@ -96,6 +107,11 @@ pub struct ArchiveBatchResult { pub bundle_ids: Vec, } +type ArchiveProcessResult = std::result::Result< + (Option, Option), + (Status, Option), +>; + #[derive(Debug, Clone, Default, Eq, PartialEq)] pub struct RecallBatchResult { pub restored_objects: u32, @@ -367,6 +383,110 @@ async fn release_drive_best_effort( .await; } +#[derive(Debug)] +struct TapeLease { + _permit: OwnedSemaphorePermit, + _tape_id: String, +} + +fn phase1_archive_item_key(bucket: &str, key: &str, version_id: Option<&str>) -> String { + match version_id.filter(|version| !version.is_empty()) { + Some(version) => format!("{bucket}/{key}#{version}"), + None => format!("{bucket}/{key}"), + } +} + +fn phase1_recall_task_key(task: &common::RecallTask) -> String { + task.id.clone() +} + +const CAPACITY_REJECT_PREFIX: &str = "capacity_reject:"; + +fn capacity_reject_reason(message: &str) -> Option<&str> { + let message = message.trim(); + if !message.starts_with(CAPACITY_REJECT_PREFIX) { + return None; + } + let parts: Vec<&str> = message[CAPACITY_REJECT_PREFIX.len()..].split(':').collect(); + parts.first().cloned().filter(|reason| !reason.is_empty()) +} + +fn is_retriable_recall_processing_error(status: &Status) -> bool { + match status.code() { + Code::Unavailable => true, + Code::ResourceExhausted => { + matches!( + capacity_reject_reason(status.message()), + Some("staging_budget_exceeded") + | Some("restored_budget_exceeded") + | Some("global_capacity_exceeded") + | Some("low_watermark_exceeded") + | Some("no_eviction_candidate") + ) + } + _ => false, + } +} + +async fn acquire_archive_item(state: &SchedulerState, key: &str) -> bool { + let mut lock = state.active_archive_keys.lock().await; + if lock.contains(key) { + return false; + } + lock.insert(key.to_string()); + true +} + +async fn release_archive_item(state: &SchedulerState, key: &str) { + let mut lock = state.active_archive_keys.lock().await; + lock.remove(key); +} + +async fn acquire_recall_task(state: &SchedulerState, key: &str) -> bool { + let mut lock = state.active_recall_tasks.lock().await; + if lock.contains(key) { + return false; + } + lock.insert(key.to_string()); + true +} + +async fn release_recall_task(state: &SchedulerState, key: &str) { + let mut lock = state.active_recall_tasks.lock().await; + lock.remove(key); +} + +async fn acquire_tape_lease( + state: &SchedulerState, + tape_id: &str, +) -> std::result::Result { + let lock = state.tape_locks.clone(); + let tape_lock = { + let mut map = lock.lock().await; + map.entry(tape_id.to_string()) + .or_insert_with(|| Arc::new(Semaphore::new(1))) + .clone() + }; + + let permit = tape_lock + .acquire_owned() + .await + .map_err(|_| Status::internal("tape lease manager closed"))?; + Ok(TapeLease { + _permit: permit, + _tape_id: tape_id.to_string(), + }) +} + +fn restore_tier_rank(tier: i32) -> u8 { + match common::RestoreTier::try_from(tier) { + Ok(common::RestoreTier::Expedited) => 0, + Ok(common::RestoreTier::Standard) => 1, + Ok(common::RestoreTier::Bulk) => 2, + _ => 3, + } +} + #[tonic::async_trait] impl TapeArchiveWriter for TapeArchiveClient { async fn write_bundle( @@ -649,6 +769,7 @@ impl MetadataBackedSchedulerBackend { pub async fn archive_staging_batch( &self, + state: &SchedulerState, cache: &C, tape: &T, limit: u32, @@ -661,100 +782,197 @@ impl MetadataBackedSchedulerBackend { let mut result = ArchiveBatchResult::default(); for entry in entries { - let object = self.head_object(&entry.bucket, &entry.key).await?; - if object.storage_class != common::StorageClass::ColdPending as i32 { + let item_key = + phase1_archive_item_key(&entry.bucket, &entry.key, entry.version_id.as_deref()); + if !acquire_archive_item(state, &item_key).await { continue; } - let staged = cache - .get_staging(&entry.bucket, &entry.key, entry.version_id.as_deref()) - .await?; - if staged.meta.size != staged.data.len() as u64 { - return Err(Status::invalid_argument(format!( - "staging size mismatch for {}/{}: meta={}, bytes={}", - entry.bucket, - entry.key, - staged.meta.size, - staged.data.len() - ))); - } + let process_result: ArchiveProcessResult = async { + let object = self + .head_object(&entry.bucket, &entry.key) + .await + .map_err(|status| (status, None))?; + let bundle_id = + phase1_bundle_id(&entry.bucket, &entry.key, entry.version_id.as_deref()); + let mut task = self.phase1_archive_task( + &entry.bucket, + &entry.key, + entry.version_id.as_deref(), + &bundle_id, + &state.config.archive.tape_id, + object.size, + ); - let bundle_id = - phase1_bundle_id(&entry.bucket, &entry.key, entry.version_id.as_deref()); - let checksum = staged - .meta - .checksum - .clone() - .unwrap_or_else(|| sha256_hex(&staged.data)); - let write = tape - .write_bundle(&bundle_id, 1, staged.data.clone()) - .await?; - if write.bytes_written != staged.data.len() as u64 { - return Err(Status::internal(format!( - "tape writer reported {} bytes for {bundle_id}, expected {}", - write.bytes_written, - staged.data.len() - ))); - } + if object.storage_class != common::StorageClass::ColdPending as i32 { + self.delete_staging_best_effort( + &entry.bucket, + &entry.key, + entry.version_id.as_deref(), + ) + .await; + return Ok((None, None)); + } - let now = now_timestamp(); - let bundle = common::ArchiveBundle { - id: bundle_id.clone(), - tape_id: write.tape_id.clone(), - tape_set: write.tape_set.clone(), - entries: vec![common::BundleEntry { - bucket: entry.bucket.clone(), - key: entry.key.clone(), - version_id: entry.version_id.clone(), - size: staged.data.len() as u64, - offset_in_bundle: 0, - tape_block_offset: write.filemark_start as u64, - checksum: checksum.clone(), - }], - total_size: write.bytes_written, - filemark_start: write.filemark_start, - filemark_end: write.filemark_end, - checksum: Some(checksum), - status: common::ArchiveBundleStatus::BundleCompleted as i32, - created_at: Some(now), - completed_at: Some(now), - }; + if !self + .claim_archive_task(&mut task) + .await + .map_err(|status| (status, Some(task.clone())))? + { + return Ok((None, None)); + } - let mut client = self.metadata.clone(); - client.put_archive_bundle(Request::new(bundle)).await?; - client - .update_archive_location(Request::new( - coldstore_proto::metadata::UpdateArchiveLocationRequest { + let staged = cache + .get_staging(&entry.bucket, &entry.key, entry.version_id.as_deref()) + .await + .map_err(|status| (status, Some(task.clone())))?; + validate_staged_object_matches_metadata(&object, &staged) + .map_err(|status| (*status, Some(task.clone())))?; + + let checksum = sha256_hex(&staged.data); + let write = tape + .write_bundle(&bundle_id, 1, staged.data.clone()) + .await + .map_err(|status| (status, Some(task.clone())))?; + if write.bytes_written != staged.data.len() as u64 { + return Err(( + Status::internal(format!( + "tape writer reported {} bytes for {bundle_id}, expected {}", + write.bytes_written, + staged.data.len() + )), + Some(task), + )); + } + + let current_object = self + .head_object(&entry.bucket, &entry.key) + .await + .map_err(|status| (status, Some(task.clone())))?; + if current_object.storage_class != common::StorageClass::ColdPending as i32 { + return Err(( + Status::failed_precondition(format!( + "object {}/{} changed storage class during archive", + entry.bucket, entry.key + )), + Some(task), + )); + } + validate_staged_object_matches_metadata(¤t_object, &staged) + .map_err(|status| (*status, Some(task.clone())))?; + + let now = now_timestamp(); + let bundle = common::ArchiveBundle { + id: bundle_id.clone(), + tape_id: write.tape_id.clone(), + tape_set: write.tape_set.clone(), + entries: vec![common::BundleEntry { bucket: entry.bucket.clone(), key: entry.key.clone(), - archive_id: bundle_id.clone(), - tape_id: write.tape_id.clone(), - tape_set: write.tape_set.clone(), + version_id: entry.version_id.clone(), + size: staged.data.len() as u64, + offset_in_bundle: 0, tape_block_offset: write.filemark_start as u64, - }, - )) - .await?; - client - .update_storage_class(Request::new( - coldstore_proto::metadata::UpdateStorageClassRequest { - bucket: entry.bucket.clone(), - key: entry.key.clone(), - storage_class: common::StorageClass::Cold as i32, - }, - )) - .await?; - cache - .delete_staging(&entry.bucket, &entry.key, entry.version_id.as_deref()) - .await?; + checksum: checksum.clone(), + }], + total_size: write.bytes_written, + filemark_start: write.filemark_start, + filemark_end: write.filemark_end, + checksum: Some(checksum), + status: common::ArchiveBundleStatus::BundleCompleted as i32, + created_at: Some(now), + completed_at: Some(now), + }; - result.archived_objects += 1; - result.bytes_written += write.bytes_written; - result.bundle_ids.push(bundle_id); + let mut client = self.metadata.clone(); + client + .complete_archive_object(Request::new( + coldstore_proto::metadata::CompleteArchiveObjectRequest { + bucket: entry.bucket.clone(), + key: entry.key.clone(), + version_id: current_object.version_id.clone(), + expected_size: Some(current_object.size), + expected_checksum: Some(current_object.checksum.clone()), + expected_storage_class: Some(current_object.storage_class), + expected_updated_at: current_object.updated_at, + bundle: Some(bundle), + tape_block_offset: write.filemark_start as u64, + storage_class: common::StorageClass::Cold as i32, + }, + )) + .await + .map_err(|status| (status, Some(task.clone())))?; + cache + .delete_staging(&entry.bucket, &entry.key, entry.version_id.as_deref()) + .await + .map_err(|status| (status, Some(task.clone())))?; + + self.finalize_archive_task_success(&mut task, write.bytes_written) + .await + .map_err(|status| (status, Some(task.clone())))?; + + result.archived_objects += 1; + result.bytes_written += write.bytes_written; + result.bundle_ids.push(bundle_id); + + Ok((Some(write.bytes_written), Some(task))) + } + .await; + + match process_result { + Ok((_bytes_written, _task)) => { + release_archive_item(state, &item_key).await; + } + Err((status, maybe_task)) => { + if let Some(task) = maybe_task { + let _ = self + .mark_archive_task_failed(task, status.message().to_string()) + .await; + } + release_archive_item(state, &item_key).await; + return Err(status); + } + } } Ok(result) } + async fn delete_cached_object_best_effort( + &self, + bucket: &str, + key: &str, + version_id: Option<&str>, + ) { + if let Some(cache) = &self.cache { + let mut client = cache.clone(); + let version_id = version_id.map(str::to_string); + let _ = client + .delete(Request::new(DeleteRequest { + bucket: bucket.into(), + key: key.into(), + version_id: version_id.clone(), + })) + .await; + let _ = client + .delete_staging(Request::new(DeleteStagingRequest { + bucket: bucket.into(), + key: key.into(), + version_id, + })) + .await; + } + } + + async fn update_archive_task( + &self, + task: common::ArchiveTask, + ) -> std::result::Result<(), Status> { + let mut client = self.metadata.clone(); + client.update_archive_task(Request::new(task)).await?; + Ok(()) + } + async fn update_recall_task( &self, task: common::RecallTask, @@ -764,6 +982,217 @@ impl MetadataBackedSchedulerBackend { Ok(()) } + async fn get_archive_task( + &self, + task_id: &str, + ) -> std::result::Result { + let mut client = self.metadata.clone(); + let task = client + .get_archive_task(Request::new( + coldstore_proto::metadata::GetArchiveTaskRequest { id: task_id.into() }, + )) + .await? + .into_inner(); + Ok(task) + } + + async fn get_recall_task( + &self, + task_id: &str, + ) -> std::result::Result { + let mut client = self.metadata.clone(); + let task = client + .get_recall_task(Request::new( + coldstore_proto::metadata::GetRecallTaskRequest { id: task_id.into() }, + )) + .await? + .into_inner(); + Ok(task) + } + + fn phase1_archive_task( + &self, + bucket: &str, + key: &str, + version_id: Option<&str>, + bundle_id: &str, + tape_id: &str, + object_size: u64, + ) -> common::ArchiveTask { + let _ = (bucket, key, version_id); + common::ArchiveTask { + id: bundle_id.to_string(), + bundle_id: bundle_id.to_string(), + tape_id: tape_id.to_string(), + drive_id: None, + object_count: 1, + total_size: object_size, + bytes_written: 0, + status: common::ArchiveTaskStatus::ArchiveTaskPending as i32, + retry_count: 0, + created_at: Some(now_timestamp()), + started_at: None, + completed_at: None, + error: None, + } + } + + async fn claim_archive_task( + &self, + task: &mut common::ArchiveTask, + ) -> std::result::Result { + let mut client = self.metadata.clone(); + if let Err(status) = client.put_archive_task(Request::new(task.clone())).await { + if status.code() != tonic::Code::AlreadyExists { + return Err(status); + } + } + + let mut current = self.get_archive_task(&task.id).await?; + let current_status = common::ArchiveTaskStatus::try_from(current.status) + .map_err(|_| Status::internal("invalid archive task status"))?; + + match current_status { + common::ArchiveTaskStatus::ArchiveTaskCompleted => { + *task = current; + Ok(false) + } + common::ArchiveTaskStatus::ArchiveTaskFailed => Ok(false), + common::ArchiveTaskStatus::ArchiveTaskInProgress => { + *task = current; + Ok(false) + } + common::ArchiveTaskStatus::ArchiveTaskPending => { + current.status = common::ArchiveTaskStatus::ArchiveTaskInProgress as i32; + current.error = None; + current.started_at = Some(now_timestamp()); + if let Err(status) = client + .update_archive_task(Request::new(current.clone())) + .await + { + if status.code() == tonic::Code::FailedPrecondition { + return Ok(false); + } + return Err(status); + } + *task = current; + Ok(true) + } + _ => Err(Status::failed_precondition( + "invalid archive task state for claim", + )), + } + } + + async fn finalize_archive_task_success( + &self, + task: &mut common::ArchiveTask, + bytes_written: u64, + ) -> std::result::Result<(), Status> { + let mut current = self.get_archive_task(&task.id).await?; + let current_status = common::ArchiveTaskStatus::try_from(current.status) + .map_err(|_| Status::internal("invalid archive task status"))?; + + if current_status == common::ArchiveTaskStatus::ArchiveTaskCompleted { + return Ok(()); + } + + current.status = common::ArchiveTaskStatus::ArchiveTaskCompleted as i32; + current.bytes_written = bytes_written; + current.error = None; + current.completed_at = Some(now_timestamp()); + + if let Err(status) = self.update_archive_task(current.clone()).await { + if status.code() == tonic::Code::FailedPrecondition { + return Ok(()); + } + return Err(status); + } + *task = current; + Ok(()) + } + + async fn mark_archive_task_failed( + &self, + task: common::ArchiveTask, + error: String, + ) -> std::result::Result<(), Status> { + let mut current = self.get_archive_task(&task.id).await?; + let current_status = common::ArchiveTaskStatus::try_from(current.status) + .map_err(|_| Status::internal("invalid archive task status"))?; + + if matches!( + current_status, + common::ArchiveTaskStatus::ArchiveTaskCompleted + | common::ArchiveTaskStatus::ArchiveTaskFailed + ) { + return Ok(()); + } + + current.status = common::ArchiveTaskStatus::ArchiveTaskFailed as i32; + current.error = Some(error); + current.completed_at = Some(now_timestamp()); + current.retry_count = current.retry_count.saturating_add(1); + + self.update_archive_task(current).await + } + + async fn claim_recall_task( + &self, + task: &mut common::RecallTask, + ) -> std::result::Result { + let mut client = self.metadata.clone(); + if let Err(status) = client.put_recall_task(Request::new(task.clone())).await { + if status.code() != tonic::Code::AlreadyExists { + return Err(status); + } + } + + let mut current = self.get_recall_task(&task.id).await?; + let current_status = common::RestoreStatus::try_from(current.status) + .map_err(|_| Status::internal("invalid restore task status"))?; + + match current_status { + common::RestoreStatus::RestoreCompleted | common::RestoreStatus::RestoreExpired => { + *task = current; + Ok(false) + } + common::RestoreStatus::RestoreInProgress => { + *task = current; + Ok(false) + } + common::RestoreStatus::RestorePending + | common::RestoreStatus::RestoreWaitingForMedia => { + current.status = common::RestoreStatus::RestoreInProgress as i32; + current.error = None; + current.started_at = Some(now_timestamp()); + if let Err(status) = client + .update_recall_task(Request::new(current.clone())) + .await + { + if status.code() == tonic::Code::FailedPrecondition { + return Ok(false); + } + return Err(status); + } + self.update_object_restore_status( + ¤t.bucket, + ¤t.key, + common::RestoreStatus::RestoreInProgress, + current.expire_at, + ) + .await?; + *task = current; + Ok(true) + } + common::RestoreStatus::RestoreFailed => { + *task = current; + Ok(false) + } + common::RestoreStatus::Unspecified => Ok(false), + } + } + async fn update_object_restore_status( &self, bucket: &str, @@ -787,17 +1216,33 @@ impl MetadataBackedSchedulerBackend { async fn mark_recall_failed( &self, - mut task: common::RecallTask, + task: common::RecallTask, error: String, ) -> std::result::Result<(), Status> { - task.status = common::RestoreStatus::RestoreFailed as i32; - task.retry_count = task.retry_count.saturating_add(1); - task.completed_at = Some(now_timestamp()); - task.error = Some(error); - self.update_recall_task(task.clone()).await?; + let mut current = self.get_recall_task(&task.id).await?; + let status = common::RestoreStatus::try_from(current.status) + .map_err(|_| Status::internal("invalid restore task status"))?; + + if matches!( + status, + common::RestoreStatus::RestoreCompleted | common::RestoreStatus::RestoreFailed + ) { + return Ok(()); + } + + current.status = common::RestoreStatus::RestoreFailed as i32; + current.retry_count = current.retry_count.saturating_add(1); + current.completed_at = Some(now_timestamp()); + current.error = Some(error); + if let Err(err) = self.update_recall_task(current.clone()).await { + if err.code() != tonic::Code::FailedPrecondition { + return Err(err); + } + } + self.update_object_restore_status( - &task.bucket, - &task.key, + ¤t.bucket, + ¤t.key, common::RestoreStatus::RestoreFailed, None, ) @@ -805,8 +1250,39 @@ impl MetadataBackedSchedulerBackend { Ok(()) } + async fn retry_recall_task_later( + &self, + task: &mut common::RecallTask, + error: String, + ) -> std::result::Result<(), Status> { + let mut current = self.get_recall_task(&task.id).await?; + let current_status = common::RestoreStatus::try_from(current.status) + .map_err(|_| Status::internal("invalid recall task status"))?; + + if current_status != common::RestoreStatus::RestoreInProgress { + return Ok(()); + } + + current.status = common::RestoreStatus::RestoreWaitingForMedia as i32; + current.error = Some(error); + current.started_at = None; + current.retry_count = current.retry_count.saturating_add(1); + self.update_recall_task(current.clone()).await?; + self.update_object_restore_status( + ¤t.bucket, + ¤t.key, + common::RestoreStatus::RestoreWaitingForMedia, + current.expire_at, + ) + .await?; + task.status = current.status; + task.started_at = None; + Ok(()) + } + pub async fn recall_pending_batch( &self, + state: &SchedulerState, cache: &C, tape: &T, limit: usize, @@ -816,26 +1292,89 @@ impl MetadataBackedSchedulerBackend { T: TapeRecallReader + ?Sized, { let mut client = self.metadata.clone(); - let tasks = client + let mut tasks = client .list_pending_recall_tasks(Request::new(())) .await? .into_inner() - .tasks; + .tasks + .into_iter() + .filter(is_pending_recall_task) + .collect::>(); + tasks.sort_by(|a, b| { + let tier = restore_tier_rank(a.tier).cmp(&restore_tier_rank(b.tier)); + tier.then_with(|| a.tape_id.cmp(&b.tape_id)) + .then_with(|| a.tape_block_offset.cmp(&b.tape_block_offset)) + .then_with(|| { + a.created_at + .as_ref() + .map(|v| v.seconds) + .cmp(&b.created_at.as_ref().map(|v| v.seconds)) + }) + }); let mut result = RecallBatchResult::default(); - for task in tasks.into_iter().filter(is_pending_recall_task).take(limit) { - let restore_result = self.process_recall_task(cache, tape, task.clone()).await; + for task in tasks.into_iter().take(limit) { + let _task_slot = match state.recall_task_slots.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + release_recall_task(state, &phase1_recall_task_key(&task)).await; + return Err(Status::internal("recall task concurrency slots closed")); + } + }; + let task_key = phase1_recall_task_key(&task); + if !acquire_recall_task(state, &task_key).await { + continue; + } + + let _tape_lease = match acquire_tape_lease(state, &task.tape_id).await { + Ok(lease) => lease, + Err(status) => { + release_recall_task(state, &task_key).await; + return Err(status); + } + }; + + let mut claimed_task = task; + let claim_result = match self.claim_recall_task(&mut claimed_task).await { + Ok(claimed) => claimed, + Err(status) => { + release_recall_task(state, &task_key).await; + return Err(status); + } + }; + + if !claim_result { + release_recall_task(state, &task_key).await; + continue; + } + + let restore_result = self + .process_recall_task(cache, tape, claimed_task.clone()) + .await; match restore_result { Ok(bytes_read) => { result.restored_objects += 1; result.bytes_read += bytes_read; - result.task_ids.push(task.id); + result.task_ids.push(claimed_task.id); } Err(status) => { - self.mark_recall_failed(task, status.message().to_string()) - .await?; + let finalize_result = if is_retriable_recall_processing_error(&status) { + self.retry_recall_task_later( + &mut claimed_task, + status.message().to_string(), + ) + .await + } else { + self.mark_recall_failed(claimed_task.clone(), status.message().to_string()) + .await + }; + if let Err(mark_status) = finalize_result { + release_recall_task(state, &task_key).await; + return Err(mark_status); + } } } + release_recall_task(state, &task_key).await; } Ok(result) @@ -865,17 +1404,6 @@ impl MetadataBackedSchedulerBackend { ))); } - task.status = common::RestoreStatus::RestoreInProgress as i32; - task.started_at = Some(now_timestamp()); - self.update_recall_task(task.clone()).await?; - self.update_object_restore_status( - &task.bucket, - &task.key, - common::RestoreStatus::RestoreInProgress, - Some(task.expire_at.unwrap_or_else(|| days_from_now(1))), - ) - .await?; - let filemark = u32::try_from(task.tape_block_offset) .map_err(|_| Status::invalid_argument("recall tape offset does not fit filemark"))?; let data = tape @@ -1051,6 +1579,8 @@ impl Phase1SchedulerBackend for MetadataBackedSchedulerBackend { }, )) .await?; + self.delete_cached_object_best_effort(bucket, key, None) + .await; Ok(()) } @@ -1076,6 +1606,12 @@ impl Phase1SchedulerBackend for MetadataBackedSchedulerBackend { )); } + if tier == common::RestoreTier::Expedited { + return Err(Status::unavailable( + "glacier expedited retrieval is not available in this environment", + )); + } + let restore_status = object .restore_status .and_then(|status| common::RestoreStatus::try_from(status).ok()); @@ -1088,7 +1624,7 @@ impl Phase1SchedulerBackend for MetadataBackedSchedulerBackend { common::RestoreStatus::RestorePending | common::RestoreStatus::RestoreWaitingForMedia | common::RestoreStatus::RestoreInProgress, - ) => Ok(RestoreObjectResponse { status_code: 202 }), + ) => Ok(RestoreObjectResponse { status_code: 409 }), Some(common::RestoreStatus::RestoreExpired | common::RestoreStatus::RestoreFailed) => { Err(Status::failed_precondition( "restore_object cannot reopen expired or failed restores in phase-1 metadata-backed mode", @@ -1158,10 +1694,11 @@ impl Phase1SchedulerBackend for MetadataBackedSchedulerBackend { bucket: &str, prefix: Option<&str>, marker: Option<&str>, + delimiter: Option<&str>, max_keys: u32, - ) -> std::result::Result, Status> { + ) -> std::result::Result { let mut client = self.metadata.clone(); - Ok(client + let objects = client .list_objects(Request::new( coldstore_proto::metadata::ListObjectsRequest { bucket: bucket.into(), @@ -1172,7 +1709,10 @@ impl Phase1SchedulerBackend for MetadataBackedSchedulerBackend { )) .await? .into_inner() - .objects) + .objects; + Ok(build_list_objects_page( + bucket, objects, prefix, marker, delimiter, max_keys, + )) } } @@ -1235,6 +1775,15 @@ fn days_from_now(days: u32) -> Timestamp { } } +fn restore_expiry_http_date(expire_at: &Timestamp) -> String { + use chrono::TimeZone; + chrono::Utc + .timestamp_opt(expire_at.seconds, expire_at.nanos as u32) + .single() + .map(|value| value.format("%a, %d %b %Y %H:%M:%S GMT").to_string()) + .unwrap_or_else(|| expire_at.seconds.to_string()) +} + fn build_restore_info( restore_status: Option, restore_expire_at: Option<&Timestamp>, @@ -1248,8 +1797,8 @@ fn build_restore_info( Some(common::RestoreStatus::RestoreCompleted) => { if let Some(expire_at) = restore_expire_at { Some(format!( - "ongoing-request=\"false\", expiry-ts=\"{}\"", - expire_at.seconds + "ongoing-request=\"false\", expiry-date=\"{}\"", + restore_expiry_http_date(expire_at) )) } else { Some("ongoing-request=\"false\"".into()) @@ -1291,6 +1840,83 @@ fn build_object_entry(object: &common::ObjectMetadata) -> ObjectEntry { } } +fn build_list_objects_page( + bucket: &str, + objects: Vec, + prefix: Option<&str>, + marker: Option<&str>, + delimiter: Option<&str>, + max_keys: u32, +) -> ListObjectsPage { + let prefix = prefix.unwrap_or_default(); + let marker = marker.unwrap_or_default(); + let delimiter = delimiter.unwrap_or_default(); + let limit = if max_keys == 0 { + usize::MAX + } else { + max_keys as usize + }; + let enforce_limit = limit != usize::MAX; + + let mut filtered: Vec<_> = objects + .into_iter() + .filter(|object| object.bucket == bucket) + .filter(|object| object.key.starts_with(prefix) && object.key.as_str() > marker) + .collect(); + filtered.sort_by(|a, b| { + a.key + .cmp(&b.key) + .then_with(|| a.version_id.cmp(&b.version_id)) + }); + + let mut emitted = 0usize; + let mut next_marker = None; + let mut is_truncated = false; + let mut last_emitted_key = None; + + let mut contents = Vec::new(); + let mut common_prefixes = Vec::new(); + let mut prefix_set = std::collections::HashSet::new(); + for object in filtered { + if emitted >= limit { + is_truncated = true; + break; + } + + if !delimiter.is_empty() { + if let Some(suffix) = object.key.strip_prefix(prefix) { + if let Some(pos) = suffix.find(delimiter) { + let candidate_prefix = format!("{prefix}{}", &suffix[..pos + delimiter.len()]); + if prefix_set.insert(candidate_prefix.clone()) { + common_prefixes.push(candidate_prefix.clone()); + emitted += 1; + last_emitted_key = Some(candidate_prefix); + } + continue; + } + } + } + + contents.push(object); + emitted += 1; + if let Some(entry) = contents.last() { + last_emitted_key = Some(entry.key.clone()); + } + } + + if is_truncated && enforce_limit { + next_marker = last_emitted_key; + is_truncated = true; + } + + ListObjectsPage { + objects: contents, + common_prefixes, + next_marker, + is_truncated, + } +} + fn build_bucket_entry(bucket: &common::BucketInfo) -> BucketEntry { BucketEntry { name: bucket.name.clone(), @@ -1307,7 +1933,11 @@ fn storage_class_label(storage_class: i32) -> &'static str { } fn is_pending_recall_task(task: &common::RecallTask) -> bool { - common::RestoreStatus::try_from(task.status) == Ok(common::RestoreStatus::RestorePending) + matches!( + common::RestoreStatus::try_from(task.status), + Ok(common::RestoreStatus::RestorePending) + | Ok(common::RestoreStatus::RestoreWaitingForMedia) + ) } fn phase1_bundle_id(bucket: &str, key: &str, version_id: Option<&str>) -> String { @@ -1319,39 +1949,123 @@ fn phase1_bundle_id(bucket: &str, key: &str, version_id: Option<&str>) -> String fn phase1_recall_task_id(bucket: &str, key: &str, version_id: Option<&str>) -> String { match version_id.filter(|version| !version.is_empty()) { - Some(version) => format!( - "phase1-recall:{bucket}/{key}#{version}:{}", - uuid::Uuid::new_v4() - ), - None => format!("phase1-recall:{bucket}/{key}:{}", uuid::Uuid::new_v4()), + Some(version) => format!("phase1-recall:{bucket}/{key}#{version}"), + None => format!("phase1-recall:{bucket}/{key}"), + } +} + +fn validate_staged_object_matches_metadata( + object: &common::ObjectMetadata, + staged: &Phase1StagedObject, +) -> std::result::Result<(), Box> { + if staged.meta.bucket != object.bucket + || staged.meta.key != object.key + || staged.meta.version_id != object.version_id + { + return Err(Box::new(Status::data_loss(format!( + "staging identity mismatch: cache returned {}/{}#{:?} for metadata object {}/{}#{:?}", + staged.meta.bucket, + staged.meta.key, + staged.meta.version_id, + object.bucket, + object.key, + object.version_id + )))); + } + + if staged.meta.size != staged.data.len() as u64 { + return Err(Box::new(Status::data_loss(format!( + "staging size mismatch for {}/{}: meta={}, bytes={}", + object.bucket, + object.key, + staged.meta.size, + staged.data.len() + )))); + } + + if object.size != staged.data.len() as u64 { + return Err(Box::new(Status::data_loss(format!( + "metadata size {} for {}/{} does not match staging bytes {}", + object.size, + object.bucket, + object.key, + staged.data.len() + )))); + } + + let computed_checksum = sha256_hex(&staged.data); + if let Some(staging_checksum) = staged + .meta + .checksum + .as_deref() + .filter(|checksum| !checksum.trim().is_empty()) + { + if !checksum_matches(staging_checksum, &computed_checksum) { + return Err(Box::new(Status::data_loss(format!( + "staging checksum mismatch for {}/{}", + object.bucket, object.key + )))); + } } + + let metadata_checksum = object.checksum.trim(); + if !metadata_checksum.is_empty() && !checksum_matches(metadata_checksum, &computed_checksum) { + return Err(Box::new(Status::data_loss(format!( + "metadata checksum mismatch for {}/{}", + object.bucket, object.key + )))); + } + + Ok(()) +} + +fn checksum_matches(expected: &str, computed_hex: &str) -> bool { + let expected = expected.trim(); + let expected = expected + .strip_prefix("sha256:") + .or_else(|| expected.strip_prefix("SHA256:")) + .unwrap_or(expected); + expected.eq_ignore_ascii_case(computed_hex) } pub fn spawn_background_loops(state: Arc) { if state.config.archive.enabled { - tokio::spawn(run_archive_loop(state.clone())); + let workers = state.config.archive.max_workers.max(1); + for worker in 0..workers { + tokio::spawn(run_archive_agent(state.clone(), worker as u64)); + } } else { info!("scheduler archive background loop disabled"); } if state.config.recall.enabled { - tokio::spawn(run_recall_loop(state)); + let workers = state.config.recall.max_workers.max(1); + for worker in 0..workers { + tokio::spawn(run_recall_agent(state.clone(), worker as u64)); + } } else { info!("scheduler recall background loop disabled"); } } -async fn run_archive_loop(state: Arc) { +async fn run_archive_agent(state: Arc, agent_id: u64) { let every = Duration::from_secs(state.config.archive.scan_interval_secs.max(1)); let mut ticker = interval(every); info!( - "scheduler archive background loop enabled: interval={}s batch_size={}", + agent_id, + "scheduler archive agent enabled: interval={}s batch_size={}", every.as_secs(), state.config.archive.batch_size ); loop { ticker.tick().await; + let _permit = state + .archive_slots + .clone() + .acquire_owned() + .await + .expect("archive concurrency slot should be available"); match archive_staging_once(state.clone()).await { Ok(result) if result.archived_objects > 0 => { info!( @@ -1370,10 +2084,12 @@ async fn run_archive_loop(state: Arc) { } } -async fn run_recall_loop(state: Arc) { +async fn run_recall_agent(state: Arc, agent_id: u64) { let every = Duration::from_secs(state.config.recall.scan_interval_secs.max(1)); + let max_batch = state.config.recall.max_concurrent_restores.max(1); let mut ticker = interval(every); info!( + agent_id, "scheduler recall background loop enabled: interval={}s max_concurrent_restores={}", every.as_secs(), state.config.recall.max_concurrent_restores @@ -1381,7 +2097,13 @@ async fn run_recall_loop(state: Arc) { loop { ticker.tick().await; - match recall_pending_once(state.clone()).await { + let _permit = state + .recall_slots + .clone() + .acquire_owned() + .await + .expect("recall concurrency slot should be available"); + match recall_pending_once(state.clone(), max_batch).await { Ok(result) if result.restored_objects > 0 => { info!( restored_objects = result.restored_objects, @@ -1418,6 +2140,13 @@ pub async fn archive_staging_once( } else { state.config.archive.tape_set.clone() }; + + let tape_id = tape_set + .first() + .cloned() + .unwrap_or_else(|| state.config.archive.tape_id.clone()); + let _tape_lease = acquire_tape_lease(&state, &tape_id).await?; + let tape = TapeArchiveClient::new( tape, state.config.archive.drive_id.clone(), @@ -1427,6 +2156,7 @@ pub async fn archive_staging_once( ); backend .archive_staging_batch( + &state, &cache, &tape, state.config.archive.batch_size.min(u32::MAX as usize) as u32, @@ -1436,6 +2166,7 @@ pub async fn archive_staging_once( pub async fn recall_pending_once( state: Arc, + max_tasks: usize, ) -> std::result::Result { let cache = state .cache @@ -1450,7 +2181,12 @@ pub async fn recall_pending_once( let cache = CacheRestoreClient::new(cache); let tape = TapeRecallClient::new(tape, state.config.recall.drive_id.clone()); backend - .recall_pending_batch(&cache, &tape, state.config.recall.max_concurrent_restores) + .recall_pending_batch( + &state, + &cache, + &tape, + max_tasks.min(state.config.recall.max_concurrent_restores), + ) .await } @@ -1554,40 +2290,33 @@ impl SchedulerService for SchedulerServiceImpl { request: Request, ) -> std::result::Result, Status> { let request = request.into_inner(); - let objects = self + let page = self .backend .list_objects( &request.bucket, request.prefix.as_deref(), request.marker.as_deref(), + request.delimiter.as_deref(), request.max_keys, ) .await?; - let is_truncated = request.max_keys > 0 && objects.len() > request.max_keys as usize; - let next_marker = if is_truncated { - objects - .get(request.max_keys as usize - 1) - .map(|object| object.key.clone()) - } else { - None - }; Ok(Response::new(ListObjectsResponse { bucket: request.bucket, prefix: request.prefix, marker: request.marker, - next_marker, + next_marker: page.next_marker, max_keys: request.max_keys, - is_truncated, - contents: objects + is_truncated: page.is_truncated, + contents: page + .objects .into_iter() - .take(if request.max_keys == 0 { - usize::MAX - } else { - request.max_keys as usize - }) .map(|object| build_object_entry(&object)) .collect(), - common_prefixes: vec![], + common_prefixes: page + .common_prefixes + .into_iter() + .map(|prefix| CommonPrefix { prefix }) + .collect(), })) } @@ -1652,6 +2381,63 @@ mod tests { use coldstore_proto::tape::read_bundle_request::Location as TapeReadLocation; use coldstore_proto::tape::read_bundle_response::Payload as TapeReadPayload; use coldstore_proto::tape::tape_service_client::TapeServiceClient; + + #[test] + fn staged_object_validation_rejects_metadata_checksum_mismatch() { + let data = b"staged bytes".to_vec(); + let staged_checksum = sha256_hex(&data); + let object = common::ObjectMetadata { + bucket: "docs".into(), + key: "a.bin".into(), + version_id: None, + size: data.len() as u64, + checksum: sha256_hex(b"different bytes"), + ..Default::default() + }; + let staged = Phase1StagedObject { + meta: StagingObjectMeta { + bucket: "docs".into(), + key: "a.bin".into(), + version_id: None, + size: data.len() as u64, + checksum: Some(staged_checksum), + ..Default::default() + }, + data, + }; + + let status = validate_staged_object_matches_metadata(&object, &staged) + .expect_err("checksum mismatch must fail"); + assert_eq!(status.code(), Code::DataLoss); + } + + #[test] + fn staged_object_validation_accepts_prefixed_sha256_checksum() { + let data = b"staged bytes".to_vec(); + let checksum = sha256_hex(&data); + let object = common::ObjectMetadata { + bucket: "docs".into(), + key: "a.bin".into(), + version_id: Some("v1".into()), + size: data.len() as u64, + checksum: format!("sha256:{checksum}"), + ..Default::default() + }; + let staged = Phase1StagedObject { + meta: StagingObjectMeta { + bucket: "docs".into(), + key: "a.bin".into(), + version_id: Some("v1".into()), + size: data.len() as u64, + checksum: Some(checksum), + ..Default::default() + }, + data, + }; + + validate_staged_object_matches_metadata(&object, &staged) + .expect("matching checksum should pass"); + } use coldstore_proto::tape::tape_service_server::{ TapeService as TapeWorkerService, TapeServiceServer, }; @@ -1871,38 +2657,66 @@ mod tests { bucket: &str, prefix: Option<&str>, marker: Option<&str>, - _max_keys: u32, - ) -> std::result::Result, Status> { - let prefix = prefix.unwrap_or_default(); - let marker = marker.unwrap_or_default(); - let mut objects: Vec<_> = self + delimiter: Option<&str>, + max_keys: u32, + ) -> std::result::Result { + let objects: Vec<_> = self .objects .read() .unwrap() .values() .map(|(o, _)| o.clone()) - .filter(|o| o.bucket == bucket) - .filter(|o| o.key.starts_with(prefix)) - .filter(|o| o.key.as_str() > marker) .collect(); - objects.sort_by(|a, b| a.key.cmp(&b.key)); - Ok(objects) + Ok(build_list_objects_page( + bucket, objects, prefix, marker, delimiter, max_keys, + )) } } fn service() -> SchedulerServiceImpl { - let state = Arc::new(SchedulerState { - metadata: - coldstore_proto::metadata::metadata_service_client::MetadataServiceClient::new( - tonic::transport::Channel::from_static("http://127.0.0.1:1").connect_lazy(), - ), - cache: None, - tape: None, - config: SchedulerConfig::default(), - }); + let state = test_scheduler_state( + coldstore_proto::metadata::metadata_service_client::MetadataServiceClient::new( + tonic::transport::Channel::from_static("http://127.0.0.1:1").connect_lazy(), + ), + None, + None, + SchedulerConfig::default(), + ); SchedulerServiceImpl::new_with_backend(state, Arc::new(InMemoryBackend::with_fixture())) } + fn test_scheduler_state( + metadata: coldstore_proto::metadata::metadata_service_client::MetadataServiceClient< + tonic::transport::Channel, + >, + cache: Option< + coldstore_proto::cache::cache_service_client::CacheServiceClient< + tonic::transport::Channel, + >, + >, + tape: Option< + coldstore_proto::tape::tape_service_client::TapeServiceClient< + tonic::transport::Channel, + >, + >, + config: SchedulerConfig, + ) -> Arc { + Arc::new(SchedulerState { + metadata, + cache, + tape, + config: config.clone(), + active_archive_keys: Arc::new(Mutex::new(HashSet::new())), + active_recall_tasks: Arc::new(Mutex::new(HashSet::new())), + tape_locks: Arc::new(Mutex::new(HashMap::new())), + archive_slots: Arc::new(Semaphore::new(config.archive.max_workers.max(1))), + recall_slots: Arc::new(Semaphore::new(config.recall.max_workers.max(1))), + recall_task_slots: Arc::new(Semaphore::new( + config.recall.max_concurrent_restores.max(1), + )), + }) + } + #[test] fn helper_head_object_response_contains_restore_info() { let (object, _) = InMemoryBackend::with_fixture() @@ -1917,7 +2731,7 @@ mod tests { assert_eq!(response.etag, "etag-1"); assert_eq!( response.restore_info.as_deref(), - Some("ongoing-request=\"false\", expiry-ts=\"123\"") + Some("ongoing-request=\"false\", expiry-date=\"Thu, 01 Jan 1970 00:02:03 GMT\"") ); } @@ -2066,6 +2880,140 @@ mod tests { assert_eq!(response.contents[0].storage_class, "COLD"); } + #[test] + fn build_list_objects_page_respects_max_keys_and_delimiter() { + let objects = vec![ + common::ObjectMetadata { + bucket: "docs".into(), + key: "apple.txt".into(), + version_id: None, + size: 1, + checksum: "".into(), + content_type: None, + etag: None, + storage_class: common::StorageClass::Cold as i32, + archive_id: None, + tape_id: None, + tape_set: vec![], + tape_block_offset: None, + restore_status: None, + restore_expire_at: None, + created_at: None, + updated_at: None, + }, + common::ObjectMetadata { + bucket: "docs".into(), + key: "docs/file1".into(), + version_id: None, + size: 1, + checksum: "".into(), + content_type: None, + etag: None, + storage_class: common::StorageClass::Cold as i32, + archive_id: None, + tape_id: None, + tape_set: vec![], + tape_block_offset: None, + restore_status: None, + restore_expire_at: None, + created_at: None, + updated_at: None, + }, + common::ObjectMetadata { + bucket: "docs".into(), + key: "docs/file2".into(), + version_id: None, + size: 1, + checksum: "".into(), + content_type: None, + etag: None, + storage_class: common::StorageClass::Cold as i32, + archive_id: None, + tape_id: None, + tape_set: vec![], + tape_block_offset: None, + restore_status: None, + restore_expire_at: None, + created_at: None, + updated_at: None, + }, + common::ObjectMetadata { + bucket: "docs".into(), + key: "video.txt".into(), + version_id: None, + size: 1, + checksum: "".into(), + content_type: None, + etag: None, + storage_class: common::StorageClass::Cold as i32, + archive_id: None, + tape_id: None, + tape_set: vec![], + tape_block_offset: None, + restore_status: None, + restore_expire_at: None, + created_at: None, + updated_at: None, + }, + common::ObjectMetadata { + bucket: "docs".into(), + key: "zebra.txt".into(), + version_id: None, + size: 1, + checksum: "".into(), + content_type: None, + etag: None, + storage_class: common::StorageClass::Cold as i32, + archive_id: None, + tape_id: None, + tape_set: vec![], + tape_block_offset: None, + restore_status: None, + restore_expire_at: None, + created_at: None, + updated_at: None, + }, + ]; + + let page = build_list_objects_page("docs", objects, None, None, Some("/"), 3); + + assert_eq!(page.objects.len(), 2); + assert_eq!(page.objects[0].key, "apple.txt"); + assert_eq!(page.objects[1].key, "video.txt"); + assert_eq!(page.common_prefixes, vec!["docs/"]); + assert!(page.is_truncated); + assert_eq!(page.next_marker.as_deref(), Some("video.txt")); + } + + #[test] + fn build_list_objects_page_with_max_keys_zero_is_unbounded() { + let objects = vec![common::ObjectMetadata { + bucket: "docs".into(), + key: "alpha.txt".into(), + version_id: None, + size: 1, + checksum: "".into(), + content_type: None, + etag: None, + storage_class: common::StorageClass::Cold as i32, + archive_id: None, + tape_id: None, + tape_set: vec![], + tape_block_offset: None, + restore_status: None, + restore_expire_at: None, + created_at: None, + updated_at: None, + }]; + + let page = build_list_objects_page("docs", objects, None, None, Some("/"), 0); + + assert!(!page.is_truncated); + assert!(page.next_marker.is_none()); + assert_eq!(page.objects.len(), 1); + assert_eq!(page.objects[0].key, "alpha.txt"); + } + async fn metadata_backed_service() -> ( SchedulerServiceImpl, Arc, @@ -2104,12 +3052,7 @@ mod tests { } } let metadata = metadata_client.expect("connect metadata client"); - let state = Arc::new(SchedulerState { - metadata, - cache: None, - tape: None, - config: SchedulerConfig::default(), - }); + let state = test_scheduler_state(metadata, None, None, SchedulerConfig::default()); (SchedulerServiceImpl::new(state.clone()), state, shutdown_tx) } @@ -2441,6 +3384,35 @@ mod tests { .into_inner(); assert_eq!(restore.status_code, 202); + let restore_conflict = svc + .restore_object(Request::new(RestoreObjectRequest { + bucket: "docs".into(), + key: "guide.txt".into(), + version_id: None, + days: 3, + tier: common::RestoreTier::Standard as i32, + })) + .await + .expect("restore object again") + .into_inner(); + assert_eq!(restore_conflict.status_code, 409); + + let expedited_err = svc + .restore_object(Request::new(RestoreObjectRequest { + bucket: "docs".into(), + key: "guide.txt".into(), + version_id: None, + days: 3, + tier: common::RestoreTier::Expedited as i32, + })) + .await + .expect_err("expedited restore should be unavailable"); + assert_eq!(expedited_err.code(), tonic::Code::Unavailable); + assert!(expedited_err + .message() + .to_lowercase() + .contains("expedited retrieval")); + let restored = svc .head_object(Request::new(HeadObjectRequest { bucket: "docs".into(), @@ -2526,13 +3498,13 @@ mod tests { assert!(!put.etag.is_empty()); let listed = backend - .list_objects("docs", Some("gui"), None, 10) + .list_objects("docs", Some("gui"), None, None, 10) .await .expect("list objects through metadata backend"); - assert_eq!(listed.len(), 1); - assert_eq!(listed[0].etag.as_deref(), Some(put.etag.as_str())); + assert_eq!(listed.objects.len(), 1); + assert_eq!(listed.objects[0].etag.as_deref(), Some(put.etag.as_str())); assert_eq!( - listed[0].storage_class, + listed.objects[0].storage_class, common::StorageClass::ColdPending as i32 ); @@ -2657,6 +3629,250 @@ mod tests { metadata_shutdown_tx.send(()).ok(); } + #[tokio::test] + async fn claim_recall_task_transitions_waiting_for_media_to_in_progress() { + let (_svc, state, metadata_shutdown_tx) = metadata_backed_service().await; + let mut metadata = state.metadata.clone(); + let backend = MetadataBackedSchedulerBackend::new(state.metadata.clone()); + + backend + .create_bucket("docs") + .await + .expect("create bucket through metadata backend"); + let body = b"restore-media".to_vec(); + seed_object_metadata( + &mut metadata, + "docs", + "media.txt", + &body, + common::StorageClass::Cold, + None, + ) + .await; + metadata + .update_archive_location(Request::new( + coldstore_proto::metadata::UpdateArchiveLocationRequest { + bucket: "docs".into(), + key: "media.txt".into(), + archive_id: "archive-waiting".into(), + tape_id: "TAPE-WAIT".into(), + tape_set: vec!["TAPE-WAIT".into()], + tape_block_offset: 11, + }, + )) + .await + .expect("set archive location"); + + let task = common::RecallTask { + id: "media-recall-task".into(), + bucket: "docs".into(), + key: "media.txt".into(), + version_id: None, + archive_id: "archive-waiting".into(), + tape_id: "TAPE-WAIT".into(), + tape_set: vec!["TAPE-WAIT".into()], + tape_block_offset: 11, + object_size: body.len() as u64, + checksum: sha256_hex(&body), + tier: common::RestoreTier::Standard as i32, + days: 1, + expire_at: Some(days_from_now(1)), + status: common::RestoreStatus::RestoreWaitingForMedia as i32, + drive_id: None, + retry_count: 0, + created_at: Some(Timestamp { + seconds: 10, + nanos: 0, + }), + started_at: None, + completed_at: None, + error: None, + }; + metadata + .put_recall_task(Request::new(task.clone())) + .await + .expect("seed waiting recall task"); + + let mut claimed_task = task; + assert!(backend + .claim_recall_task(&mut claimed_task) + .await + .expect("claim recall task")); + assert_eq!( + claimed_task.status, + common::RestoreStatus::RestoreInProgress as i32 + ); + assert!(claimed_task.started_at.is_some()); + + let current_task = metadata + .get_recall_task(Request::new( + coldstore_proto::metadata::GetRecallTaskRequest { + id: "media-recall-task".into(), + }, + )) + .await + .expect("lookup updated recall task") + .into_inner(); + assert_eq!( + current_task.status, + common::RestoreStatus::RestoreInProgress as i32 + ); + + let object = backend + .head_object("docs", "media.txt") + .await + .expect("head restored object"); + assert_eq!( + object.restore_status, + Some(common::RestoreStatus::RestoreInProgress as i32) + ); + + let pending = metadata + .list_pending_recall_tasks(Request::new(())) + .await + .expect("list pending recall tasks") + .into_inner() + .tasks; + assert!(pending.is_empty()); + + let mut second_claim = claimed_task; + assert!(!backend + .claim_recall_task(&mut second_claim) + .await + .expect("re-claim should be rejected")); + + metadata_shutdown_tx.send(()).ok(); + } + + #[tokio::test] + async fn concurrent_claim_recall_task_has_single_winner() { + let (_svc, state, metadata_shutdown_tx) = metadata_backed_service().await; + let mut metadata = state.metadata.clone(); + + let backend1 = MetadataBackedSchedulerBackend::new(state.metadata.clone()); + let backend2 = MetadataBackedSchedulerBackend::new(state.metadata.clone()); + + backend1 + .create_bucket("docs") + .await + .expect("create bucket through metadata backend"); + + let body = b"parallel-restore".to_vec(); + seed_object_metadata( + &mut metadata, + "docs", + "parallel.txt", + &body, + common::StorageClass::Cold, + None, + ) + .await; + metadata + .update_archive_location(Request::new( + coldstore_proto::metadata::UpdateArchiveLocationRequest { + bucket: "docs".into(), + key: "parallel.txt".into(), + archive_id: "archive-parallel".into(), + tape_id: "TAPE-PARALLEL".into(), + tape_set: vec!["TAPE-PARALLEL".into()], + tape_block_offset: 9, + }, + )) + .await + .expect("set archive location"); + + let task = common::RecallTask { + id: "parallel-recall-task".into(), + bucket: "docs".into(), + key: "parallel.txt".into(), + version_id: None, + archive_id: "archive-parallel".into(), + tape_id: "TAPE-PARALLEL".into(), + tape_set: vec!["TAPE-PARALLEL".into()], + tape_block_offset: 9, + object_size: body.len() as u64, + checksum: sha256_hex(&body), + tier: common::RestoreTier::Standard as i32, + days: 1, + expire_at: Some(days_from_now(1)), + status: common::RestoreStatus::RestorePending as i32, + drive_id: None, + retry_count: 0, + created_at: Some(Timestamp { + seconds: 10, + nanos: 0, + }), + started_at: None, + completed_at: None, + error: None, + }; + metadata + .put_recall_task(Request::new(task.clone())) + .await + .expect("seed pending recall task"); + + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(2)); + let mut first = task.clone(); + let mut second = task.clone(); + + let barrier_a = barrier.clone(); + let barrier_b = barrier.clone(); + + let winner_a = tokio::spawn(async move { + barrier_a.wait().await; + let started_at = { + let success = backend1.claim_recall_task(&mut first).await?; + (success, first.started_at) + }; + Ok::<(bool, Option), tonic::Status>(started_at) + }); + + let winner_b = tokio::spawn(async move { + barrier_b.wait().await; + let started_at = { + let success = backend2.claim_recall_task(&mut second).await?; + (success, second.started_at) + }; + Ok::<(bool, Option), tonic::Status>(started_at) + }); + + let (result_a, result_b) = tokio::join!(winner_a, winner_b); + let result_a = result_a + .expect("agent a spawn joined") + .expect("agent a claim"); + let result_b = result_b + .expect("agent b spawn joined") + .expect("agent b claim"); + + let winner_count = (result_a.0 as u8) + (result_b.0 as u8); + assert_eq!( + winner_count, 1, + "only one agent can claim the same recall task" + ); + + let started_count = [result_a.1.is_some(), result_b.1.is_some()] + .iter() + .filter(|started| **started) + .count(); + assert_eq!(started_count, 1, "only one claim should set started_at"); + + let current_task = metadata + .get_recall_task(Request::new( + coldstore_proto::metadata::GetRecallTaskRequest { + id: "parallel-recall-task".into(), + }, + )) + .await + .expect("lookup current recall task") + .into_inner(); + assert_eq!( + current_task.status, + common::RestoreStatus::RestoreInProgress as i32 + ); + + metadata_shutdown_tx.send(()).ok(); + } + struct TestArchiveCache { staged: RwLock>, deleted: RwLock>, @@ -2846,7 +4062,7 @@ mod tests { let tape = DirectTapeWriter::loaded().await; let archived = backend - .archive_staging_batch(&cache, &tape, 10) + .archive_staging_batch(&state, &cache, &tape, 10) .await .expect("archive staging batch"); assert_eq!(archived.archived_objects, 1); @@ -2891,6 +4107,26 @@ mod tests { shutdown_tx.send(()).ok(); } + #[tokio::test] + async fn archive_staging_batch_releases_lock_on_head_object_failure() { + let (_svc, state, shutdown_tx) = metadata_backed_service().await; + let backend = MetadataBackedSchedulerBackend::new(state.metadata.clone()); + + let cache = TestArchiveCache::with_object("docs", "orphan.txt", b"orphan".to_vec()); + let tape = DirectTapeWriter::loaded().await; + let item_key = phase1_archive_item_key("docs", "orphan.txt", None); + + let result = backend + .archive_staging_batch(&state, &cache, &tape, 10) + .await; + assert!(result.is_err()); + + let active = state.active_archive_keys.lock().await; + assert!(!active.contains(&item_key)); + + shutdown_tx.send(()).ok(); + } + #[tokio::test] async fn archive_staging_batch_consumes_real_cache_service_staging() { let (_svc, state, metadata_shutdown_tx) = metadata_backed_service().await; @@ -2937,7 +4173,7 @@ mod tests { let tape = DirectTapeWriter::loaded().await; let archived = backend - .archive_staging_batch(&cache, &tape, 10) + .archive_staging_batch(&state, &cache, &tape, 10) .await .expect("archive real cache staging batch"); assert_eq!(archived.archived_objects, 1); @@ -3035,7 +4271,7 @@ mod tests { ); let archived = backend - .archive_staging_batch(&cache, &tape, 10) + .archive_staging_batch(&state, &cache, &tape, 10) .await .expect("archive through cache and tape grpc clients"); assert_eq!(archived.archived_objects, 1); @@ -3100,12 +4336,12 @@ mod tests { config.archive.tape_set = vec!["TAPE-GRPC".into()]; config.recall.max_concurrent_restores = 2; config.recall.drive_id = "drive-0".into(); - let loop_state = Arc::new(SchedulerState { - metadata: state.metadata.clone(), - cache: Some(raw_cache_client.clone()), - tape: Some(raw_tape_client.clone()), + let loop_state = test_scheduler_state( + state.metadata.clone(), + Some(raw_cache_client.clone()), + Some(raw_tape_client.clone()), config, - }); + ); let archived = archive_staging_once(loop_state.clone()) .await @@ -3129,7 +4365,12 @@ mod tests { .expect("queue restore"); assert_eq!(restore.status_code, 202); - let recalled = recall_pending_once(loop_state).await.expect("recall once"); + let recalled = recall_pending_once( + loop_state.clone(), + loop_state.config.recall.max_concurrent_restores, + ) + .await + .expect("recall once"); assert_eq!(recalled.restored_objects, 1); assert_eq!(recalled.bytes_read, body.len() as u64); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..114d350 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,278 @@ +# ColdStore 当前架构与组件职责 + +本文档描述当前代码基线，而不是完整目标态。目标态设计仍保留在 `docs/modules/` 与 `docs/DESIGN.md` 中。 + +## 1. 系统定位 + +ColdStore 是一个 S3 兼容的磁带冷归档系统。当前实现重点是把以下闭环做成可验证路径： + +- S3 HTTP 接入 +- Scheduler 业务编排 +- Metadata 状态机与持久化 +- Cache staging/restored 数据面 +- Tape 抽象读写接口 +- 归档完成的原子 metadata 提交 +- Restore 后从缓存读取 + +当前不是完整多节点分布式系统。Metadata 多节点 `persistent_raft` 当前会 fail-fast，避免多个本地状态机伪装成安全 Raft 集群。 + +## 2. 组件总览 + +```text +S3 Client + | + | HTTP/S3 + v +Gateway + | + | gRPC SchedulerService + v +Scheduler + | | | + | MetadataService | CacheService | TapeService + v v v +Metadata Cache Tape +``` + +## 3. 组件职责 + +| 组件 | crate | 职责 | 不负责 | +|------|-------|------|--------| +| Gateway | `crates/gateway` | S3 HTTP 路由、Range/条件请求解析、Restore XML 解析、S3 错误码映射、转发 Scheduler | 不直接访问 Metadata/Cache/Tape，不做归档决策 | +| Scheduler | `crates/scheduler` | 业务中枢；对象 PUT/GET/HEAD/DELETE/List；Restore 请求；归档扫描；召回调度；编排 Metadata/Cache/Tape | 不持久化对象数据，不直接管理磁盘文件 | +| Metadata | `crates/metadata` | bucket/object/task/tape/worker 元数据状态机；归档完成 CAS；单节点持久化路径；worker 注册与心跳 | 当前不提供真实多节点 Raft 复制 | +| Cache | `crates/cache` | staging 写入暂存；restored 取回缓存；容量预算；restored 淘汰；staging 背压 | 不保存权威元数据，不淘汰 staging | +| Tape | `crates/tape` | TapeService 抽象；bundle 写入/读取；驱动加载/释放接口 | 当前真实硬件接入不是完整生产态 | +| Common | `crates/common` | 配置结构、默认值、共享类型 | 不包含业务状态机 | +| Proto | `crates/proto` | gRPC/protobuf 契约 | 不包含运行时逻辑 | + +## 4. 默认拓扑 + +当前默认配置是安全单节点 metadata 拓扑： + +```text +Metadata: 127.0.0.1:21001 +Scheduler: 127.0.0.1:22001 +Cache: 127.0.0.1:23001 +Tape: 127.0.0.1:24001 +Gateway: 0.0.0.0:9000 +``` + +默认 `MetadataConfig`： + +- `consensus_mode = persistent_raft` +- `cluster = 1:127.0.0.1:21001` +- `raft_state_path = data_path/node-{node_id}/raft`，除非显式配置 + +多节点 metadata cluster 当前不是默认路径。配置多节点 `persistent_raft` 会拒绝启动，因为真实 distributed Raft runtime 尚未完成。 + +## 5. 核心数据流 + +### 5.1 PutObject 写入 + +```text +Client PUT + -> Gateway + -> Scheduler.PutObject + -> Cache.PutStaging + -> Metadata.PutObject(ColdPending) + -> Response(etag/version_id) +``` + +关键语义： + +- 写入后对象是 `ColdPending`。 +- 数据进入 Cache 的 staging 区。 +- staging 属于写入路径 durable buffer，不能被 eviction 自动淘汰。 +- staging 超预算时返回容量背压，上游应重试或等待归档 drain。 + +### 5.2 Archive 归档 + +```text +Scheduler archive loop + -> Cache.ListStagingKeys + -> Metadata.HeadObject + -> Cache.GetStaging + -> 校验 staging identity/size/checksum 与 metadata 一致 + -> Tape.WriteBundle + -> Metadata.CompleteArchiveObject(expected_*) + -> Cache.DeleteStaging +``` + +`CompleteArchiveObject` 是归档完成的关键一致性边界。它在 Metadata 状态机单次 apply 内完成： + +- 检查 object version +- 检查 expected size +- 检查 expected checksum +- 检查 expected storage class +- 检查 expected updated_at，作为 generation token +- 写入 archive bundle +- 更新 object archive location +- 将 object storage class 切换为 `Cold` + +如果 tape 写成功但 CAS 失败，metadata 不会错误提交归档结果。物理 tape 上可能留下未引用 bundle，这是后续 audit/GC 需要处理的残余问题。 + +### 5.3 RestoreObject 取回 + +```text +Client RestoreObject + -> Gateway + -> Scheduler.RestoreObject + -> Metadata 更新 restore 状态 / 创建 RecallTask + -> Response 202/200/409 + +Scheduler recall loop + -> Metadata.ListPendingRecallTasks + -> Tape.ReadBundle + -> 校验 checksum/size + -> Cache.PutRestored + -> Metadata 更新 task/object restore 状态 +``` + +关键语义： + +- Expedited 不可用时映射为 S3 `GlacierExpeditedRetrievalNotAvailable`。 +- Cache restored 超预算属于可重试背压。 +- Restored 数据有 TTL，到期后可被回收。 + +### 5.4 GetObject 读取 + +```text +Client GET + -> Gateway + -> Scheduler.GetObject + -> Metadata.Head/GetObject + -> 若对象未 Restore completed: InvalidObjectState + -> Cache.Get + -> Gateway HTTP response +``` + +当前限制： + +- Gateway 和 Scheduler 的流式接口仍存在部分全量 `Vec` 聚合路径，大对象内存放大仍需后续重构。 +- Range 读取当前不是完整下推到后端的生产态实现。 + +### 5.5 ListObjects + +```text +Client ListObjects + -> Gateway parse prefix/marker/delimiter/max-keys + -> Scheduler.ListObjects + -> Metadata.ListObjects + -> Gateway XML response +``` + +当前实现支持基础 V1 风格 list、delimiter/common prefixes 和最大 `max-keys` 边界。完整 ListObjectsV2 仍是后续任务。 + +## 6. 一致性边界 + +| 场景 | 当前策略 | +------|----------| +| Metadata 多节点 | 未完成真实分布式 Raft，多节点 fail-fast | +| Archive 提交 | 使用 `CompleteArchiveObject(expected_*)` 单状态机命令 | +| Staging 容量压力 | 不淘汰 staging，返回 `ResourceExhausted` | +| Restored 容量压力 | 可淘汰 restored；无 victim 时返回背压 | +| Cache 覆盖写 | 先写新 storage，成功后原子切索引，再 best-effort 删除旧 storage | +| Cache 重建重复 key | 选择较新 entry，duplicate loser best-effort 清理并 warning | +| Gateway 节流 | S3 数据面返回 `503 SlowDown + Retry-After` | + +## 7. 组件间接口 + +### Gateway -> Scheduler + +接口：`SchedulerService` + +用途： + +- PUT/GET/HEAD/DELETE +- RestoreObject +- ListObjects/ListBuckets +- Bucket CRUD + +Gateway 只做协议适配，不绕过 Scheduler。 + +### Scheduler -> Metadata + +接口：`MetadataService` + +用途： + +- bucket/object 元数据读写 +- archive bundle/task/recall task 管理 +- `CompleteArchiveObject` 原子归档提交 +- worker/tape 状态读写 + +Scheduler 是业务路径中的唯一 metadata 写入编排者。 + +### Scheduler -> Cache + +接口：`CacheService` + +用途： + +- `PutStaging`：写入待归档数据 +- `GetStaging`：归档时读取 staging +- `DeleteStaging`：归档提交成功后清理 staging +- `PutRestored`：召回后写入 restored +- `Get`：GET 已解冻对象 +- `Stats`：缓存统计 + +### Scheduler -> Tape + +接口：`TapeService` + +用途： + +- Acquire/Release drive +- Load/Unload tape +- WriteBundle +- ReadBundle + +Tape 不直接写 Metadata。写 tape 成功后由 Scheduler 调用 Metadata 的原子提交接口。 + +## 8. 状态模型 + +### Object storage class + +```text +ColdPending -> Cold +``` + +- `ColdPending`：对象已写入 staging，等待归档。 +- `Cold`：对象已归档，metadata 中有 archive location。 + +### Restore status + +```text +Pending -> WaitingForMedia -> InProgress -> Completed + | + v + Failed +Completed -> Expired +``` + +当前实现已有基础状态流转，但 task 与 object restore 状态仍需要进一步原子化/reconciler 补强。 + +### Cache category + +```text +Staging: write path buffer, no eviction +Restored: read cache, evictable +``` + +## 9. 当前仍需补强的事项 + +| 优先级 | 事项 | +--------|------| +| P0 | 真正 distributed Raft runtime、leader redirect、quorum commit、log replay | +| P0 | Restore task 与 object restore 状态原子化，避免 pending-without-task | +| P1 | Gateway/Scheduler/Cache 真流式大对象读写，降低内存放大 | +| P1 | Tape orphan bundle audit/GC，处理 tape 写成功但 metadata CAS 失败的残留 | +| P1 | ListObjectsV2、DeleteObject 幂等、更多 S3 兼容细节 | +| P2 | Admin Console、完整 OpenTelemetry、真实 SPDK/真实磁带库生产验证 | + +## 10. 文档使用说明 + +- 判断当前实现时，以本文档和代码为准。 +- `docs/modules/*` 保留更完整目标态设计，可能包含当前未实现能力。 +- 如果文档与代码冲突，应更新本文档或在模块文档中标记“目标态”。 diff --git a/docs/DESIGN.md b/docs/DESIGN.md index e8fae75..8a3d1d3 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -3,6 +3,8 @@ > 版本：1.1 > 更新日期：2025-02-27 +> 当前实现说明：本文档包含历史设计和目标态内容，部分章节已经超前于当前代码。当前组件职责、联动关系和一致性边界以 [ARCHITECTURE.md](./ARCHITECTURE.md) 为准。 + ## 1. 文档概述 本文档描述 ColdStore 冷存储系统的详细技术设计方案，重点涵盖： diff --git a/docs/README.md b/docs/README.md index b8ca9fc..a74c89a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,28 +1,40 @@ -# ColdStore 设计文档 +# ColdStore 文档总览 -## 文档索引 +本目录文档按“当前实现优先、目标设计分层保留”的原则重新整理。 -| 文档 | 说明 | +## 当前权威文档 + +| 文档 | 定位 | |------|------| -| [DESIGN.md](./DESIGN.md) | **总架构设计** - 协议、元数据集群、缓存层、磁带层完整设计 | -| [modules/](./modules/) | **模块设计** - 按架构分层拆分的独立设计文档 | +| [ARCHITECTURE.md](./ARCHITECTURE.md) | 当前代码基线的组件、职责、联动关系、一致性边界 | +| [DESIGN.md](./DESIGN.md) | 较完整的历史设计文档，包含目标态说明，部分内容可能超前于当前实现 | +| [plans/](./plans/) | 阶段性实施计划与演进记录 | ## 模块设计文档 -| 模块 | 文档 | -|------|------| -| 接入层 | [01-access-layer.md](./modules/01-access-layer.md) | -| 协议适配层 | [02-protocol-adapter.md](./modules/02-protocol-adapter.md) | -| 元数据集群 | [03-metadata-cluster.md](./modules/03-metadata-cluster.md) | -| 数据缓存层 | [04-cache-layer.md](./modules/04-cache-layer.md) | -| 归档取回调度层 | [05-scheduler-layer.md](./modules/05-scheduler-layer.md) | -| 磁带管理层 | [06-tape-layer.md](./modules/06-tape-layer.md) | +[modules/](./modules/) 下的文档保留为分层设计参考。它们包含部分目标态内容，例如真实多节点 Raft、完整 SPDK、真实磁带库、Admin Console 和 OpenTelemetry 全量接入。 + +阅读优先级： + +1. 先读 [ARCHITECTURE.md](./ARCHITECTURE.md)，了解当前实现。 +2. 再读 [modules/README.md](./modules/README.md)，定位某个模块的目标设计。 +3. 涉及实现判断时，以 `crates/*/src` 和 `crates/proto/proto` 为准。 + +## 当前实现状态摘要 -## 设计要点摘要 +| 组件 | 当前状态 | +|------|----------| +| Gateway | Axum S3 HTTP 网关，全部业务请求转发 Scheduler | +| Scheduler | 业务编排中心，负责 PUT/GET/Restore/List、归档扫描、取回调度 | +| Metadata | 单节点持久化状态机路径；多节点 `persistent_raft` 当前 fail-fast，避免伪集群 | +| Cache | staging/restored 双区缓存，支持容量预算、restored 淘汰、staging 背压 | +| Tape | TapeService 抽象与模拟/基础接口，真实设备集成仍是目标态 | +| Proto | gRPC 契约已包含 `CompleteArchiveObject` 原子归档提交接口 | -- **协议**：兼容 S3 Glacier 冷归档协议（RestoreObject、x-amz-restore、取回层级） -- **元数据**：[OpenRaft](https://github.com/databendlabs/openraft) + [openraft-rocksstore](https://crates.io/crates/openraft-rocksstore)，强一致性集群 -- **缓存**：[async-spdk](https://github.com/madsys-dev/async-spdk) 用户态 NVMe 缓存，原生 async/await -- **磁带**：自研 SDK 抽象层，前期对接 Linux SCSI（st 驱动 + MTIO） +## 重要一致性约束 -详见 [DESIGN.md](./DESIGN.md) 与 [modules/README.md](./modules/README.md)。 +- `Staging` 是写入路径数据，不允许被容量淘汰；超预算时必须背压。 +- `Restored` 是可重建缓存，可以按 LRU/LFU/TTL/容量回收。 +- Scheduler 归档完成必须通过 Metadata 的 `CompleteArchiveObject` 单命令提交，不能拆成多个无条件 RPC。 +- Metadata 当前默认单节点，默认 worker metadata 地址也为单节点。 +- 多节点 metadata 复制尚未完成，不能将 `persistent_raft` 多节点配置视为生产可用。 diff --git a/docs/modules/README.md b/docs/modules/README.md index dc7f828..721a07a 100644 --- a/docs/modules/README.md +++ b/docs/modules/README.md @@ -1,6 +1,8 @@ # ColdStore 模块设计文档 -按架构分层拆分的独立模块设计文档，与 [总架构设计](../DESIGN.md) 配套使用。 +本目录保留按架构分层拆分的模块设计文档。注意：这些文档同时包含当前实现和目标态设计，部分内容已经超前于当前代码。 + +当前实现的权威说明见 [../ARCHITECTURE.md](../ARCHITECTURE.md)。涉及实现判断时，优先以该文档和 `crates/*/src` 为准。 ## 文档索引 @@ -8,27 +10,30 @@ |------|------|------|------| | 01 | 接入层 | [01-access-layer.md](./01-access-layer.md) | S3 HTTP 服务、路由、Axum | | 02 | 协议适配层 | [02-protocol-adapter.md](./02-protocol-adapter.md) | StorageClass 映射、RestoreRequest、x-amz-restore、错误码 | -| 03 | 元数据集群 | [03-metadata-cluster.md](./03-metadata-cluster.md) | OpenRaft + RocksDB、强一致性 | -| 04 | 数据缓存层 | [04-cache-layer.md](./04-cache-layer.md) | async-spdk、解冻数据缓存 | +| 03 | 元数据集群 | [03-metadata-cluster.md](./03-metadata-cluster.md) | 当前单节点持久化路径；OpenRaft 多节点是目标态 | +| 04 | 数据缓存层 | [04-cache-layer.md](./04-cache-layer.md) | 当前 staging/restored 缓存；完整 async-spdk 是目标态 | | 05 | 归档取回调度层 | [05-scheduler-layer.md](./05-scheduler-layer.md) | Archive Scheduler、Recall Scheduler | | 06 | 磁带管理层 | [06-tape-layer.md](./06-tape-layer.md) | 自研 SDK、Linux SCSI | | 07 | 跨层一致性与性能 | [07-consistency-performance.md](./07-consistency-performance.md) | Saga 模式、并发控制、故障矩阵、性能优化 | | 08 | 可观测性与链路追踪 | [08-observability.md](./08-observability.md) | OpenTelemetry、Traces、Metrics、Logs、告警 | | 09 | 管控面 (Admin Console) | [09-admin-console.md](./09-admin-console.md) | Web UI、Admin API、集群/磁带/任务管理 | -## 架构层次关系 +## 当前实现联动关系 ``` -接入层 (01) - │ - ▼ -协议适配层 (02) - │ - ├──────────────────┬──────────────────┬──────────────────┐ - ▼ ▼ ▼ ▼ -元数据集群 (03) 数据缓存层 (04) 归档取回调度层 (05) - │ │ - │ ▼ - │ 磁带管理层 (06) - └──────────────────┘ +Gateway(01/02) + | + v +Scheduler(05) + | | | + v v v +Metadata(03) Cache(04) Tape(06) ``` + +Gateway 不直接连接 Metadata/Cache/Tape。Scheduler 是业务中枢，负责把协议请求编排成 metadata/cache/tape 操作。 + +## 已知过时点 + +- 真实多节点 Metadata Raft 仍是目标态；当前多节点 `persistent_raft` 会 fail-fast。 +- 完整 SPDK、真实磁带库、Admin Console、OpenTelemetry 全量链路仍是目标态。 +- Multipart Upload、ListObjectsV2、完整 S3 兼容仍需继续实现。 From dd7eac158b12b239f87b877dc24c1d519c7d77ff Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Sun, 5 Jul 2026 20:42:26 +0800 Subject: [PATCH 4/4] Install libclang for CI builds --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0ddfd8e..e2d6829 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -16,8 +16,8 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install protoc - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler clang libclang-dev llvm-dev - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable