Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README_RUST.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,18 @@ ColdStore 是**纯冷归档系统**(类似 AWS Glacier Deep Archive):
- 不提供在线热存储层,不做生命周期管理
- 外部热存储系统在需要冷归档时调用 ColdStore 的 PutObject API

## 当前实现状态

当前代码基线已经进入 Phase 1 本地闭环,并开始 Phase 2A Metadata 单节点持久化:

| 分期 | 状态 | 已落地能力 |
|------|------|------------|
| Phase 1 | 已落地 / 可单测 | Gateway/Scheduler/Metadata/Cache 本地闭环;bucket/object CRUD;Put/Head/Get/Delete/Restore/List;HDD Cache staging/restored;Phase-1 archive 标记 Cold 并清理 staging |
| Phase 2A | 已启动 / 可单测 | Metadata opt-in 二进制 snapshot:`MetadataServiceImpl::new_with_snapshot(config, path)` 支持写入后保存、重启后恢复 bucket/object/task/worker/tape 等状态 |
| Phase 2B | 下一步 | 将 Metadata 状态机接入 OpenRaft + RocksDB/openraft-rocksstore,补齐安全的 Raft 状态机单测和多节点一致性测试 |

当前仍不会访问真实磁带设备,也不默认执行集成测试。

## Workspace 结构

| Crate | 类型 | 说明 |
Expand Down
2 changes: 1 addition & 1 deletion crates/cache/src/service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -458,7 +458,7 @@ impl CacheService for CacheServiceImpl {
.filter(|(key, _)| key.as_cursor() > after)
.map(|(key, entry)| (key.clone(), entry.clone()))
.collect();
entries.sort_by(|(a, _), (b, _)| a.as_cursor().cmp(&b.as_cursor()));
entries.sort_by_key(|(key, _)| key.as_cursor());

let has_more = entries.len() > limit;
let response_entries = entries
Expand Down
13 changes: 10 additions & 3 deletions crates/metadata/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ license.workspace = true
name = "coldstore-metadata"
path = "src/main.rs"

[features]
default = []
metadata-raft = ["dep:openraft"]
metadata-raft-rocksdb = ["metadata-raft", "dep:rocksdb"]

[dependencies]
coldstore-proto = { workspace = true }
coldstore-common = { workspace = true }
Expand All @@ -25,6 +30,8 @@ chrono = { workspace = true }
uuid = { workspace = true }
config = { workspace = true }

# TODO: Phase 2 - Raft + RocksDB
# openraft = "0.10"
# rocksdb = "0.22"
# Phase 2A starts with opt-in local binary snapshot persistence in service.rs.
# Phase 2B keeps OpenRaft + RocksDB behind the `metadata-raft` feature until the
# command/state-machine boundary is stable enough to become the default backend.
openraft = { version = "0.10.0-alpha.18", optional = true }
rocksdb = { version = "0.24", default-features = false, optional = true }
29 changes: 29 additions & 0 deletions crates/metadata/src/command.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
use coldstore_proto::common;
use coldstore_proto::metadata::*;

#[derive(Debug, Clone)]
pub enum MetadataCommand {
PutObject(common::ObjectMetadata),
DeleteObject(DeleteObjectRequest),
UpdateStorageClass(UpdateStorageClassRequest),
UpdateArchiveLocation(UpdateArchiveLocationRequest),
UpdateRestoreStatus(UpdateRestoreStatusRequest),
CreateBucket(common::BucketInfo),
DeleteBucket(DeleteBucketRequest),
PutArchiveBundle(common::ArchiveBundle),
UpdateArchiveBundleStatus(UpdateArchiveBundleStatusRequest),
PutArchiveTask(common::ArchiveTask),
UpdateArchiveTask(common::ArchiveTask),
PutRecallTask(common::RecallTask),
UpdateRecallTask(common::RecallTask),
PutTape(common::TapeInfo),
UpdateTape(common::TapeInfo),
RegisterSchedulerWorker(common::SchedulerWorkerInfo),
DeregisterSchedulerWorker(DeregisterWorkerRequest),
RegisterCacheWorker(common::CacheWorkerInfo),
DeregisterCacheWorker(DeregisterWorkerRequest),
RegisterTapeWorker(common::TapeWorkerInfo),
DeregisterTapeWorker(DeregisterWorkerRequest),
UpdateWorkerStatus(UpdateWorkerStatusRequest),
Heartbeat(HeartbeatRequest),
}
7 changes: 7 additions & 0 deletions crates/metadata/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
pub mod command;
pub mod service;
pub mod state_machine;

#[cfg(feature = "metadata-raft")]
pub mod raft;
#[cfg(feature = "metadata-raft-rocksdb")]
pub mod raft_storage;

use anyhow::Result;
use coldstore_common::config::MetadataConfig;
Expand Down
72 changes: 72 additions & 0 deletions crates/metadata/src/raft.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
//! Feature-gated local propose backend for metadata Raft integration.
//!
//! This is not a complete OpenRaft runtime. It provides an explicit, opt-in
//! proposal boundary so `MetadataServiceImpl` can route writes through the same
//! command/state-machine path that a real Raft state machine will use later.

use std::sync::Arc;
use tokio::sync::RwLock;
use tonic::Status;

use crate::command::MetadataCommand;
use crate::state_machine::{apply_command, MetadataState};

pub type ColdStoreNodeId = u64;
pub type ColdStoreNode = openraft::BasicNode;

#[derive(Debug, Default)]
pub struct RaftMetadataBackend {
proposed_commands: RwLock<u64>,
}

impl RaftMetadataBackend {
pub fn new() -> Self {
Self::default()
}

pub async fn proposed_commands(&self) -> u64 {
*self.proposed_commands.read().await
}

pub async fn propose_local_apply(
&self,
state: &Arc<RwLock<MetadataState>>,
command: MetadataCommand,
) -> std::result::Result<(), Status> {
let mut guard = state.write().await;
apply_command(&mut guard, command)?;
*self.proposed_commands.write().await += 1;
Ok(())
}
}

#[cfg(test)]
mod tests {
use super::*;
use coldstore_proto::common;

fn test_bucket(name: &str) -> common::BucketInfo {
common::BucketInfo {
name: name.into(),
created_at: None,
owner: Some("tester".into()),
versioning_enabled: false,
object_count: 0,
total_size: 0,
}
}

#[tokio::test]
async fn raft_backend_proposes_commands_through_state_machine_apply_path() {
let state = Arc::new(RwLock::new(MetadataState::default()));
let backend = RaftMetadataBackend::new();

backend
.propose_local_apply(&state, MetadataCommand::CreateBucket(test_bucket("docs")))
.await
.expect("command should apply");

assert_eq!(backend.proposed_commands().await, 1);
assert!(state.read().await.buckets.contains_key("docs"));
}
}
Loading
Loading