diff --git a/02_docs/architecture/23-sentinel_game_features.md b/02_docs/architecture/23-sentinel_game_features.md new file mode 100644 index 000000000..7a15b1690 --- /dev/null +++ b/02_docs/architecture/23-sentinel_game_features.md @@ -0,0 +1,143 @@ +# SentinelOS Game & Simulation Micro-Features + +This file catalogs micro-features for the game/simulation layer that sits between Physics and AI in the roadmap. These features form building blocks for the Security Simulation Engine, Multi-Agent Cyber World, Cyber Range, and Training systems. + +## Simulation primitives (GM-001..GM-015) +- GM-001: World tick manager +- GM-002: Simulation clock +- GM-003: Pause simulation +- GM-004: Replay speed control +- GM-005: Time rewind marker +- GM-006: Frame snapshot +- GM-007: Checkpoint save +- GM-008: Scenario reset +- GM-009: Random seed control +- GM-010: Deterministic mode +- GM-011: Fast forward +- GM-012: Event inject +- GM-013: Simulation branch +- GM-014: State rollback +- GM-015: Experiment replay + +## Agent features (AG-001..AG-015) +- AG-001: Attacker agent +- AG-002: Defender agent +- AG-003: Observer agent +- AG-004: Scout behavior +- AG-005: Exploit attempt +- AG-006: Defense action +- AG-007: Retreat action +- AG-008: Risk memory +- AG-009: Goal selection +- AG-010: Strategy update +- AG-011: Path planning +- AG-012: Cooperation score +- AG-013: Competition score +- AG-014: Trust value +- AG-015: Learning state + +## Environment templates (ENV-001..ENV-015) +- ENV-001: Virtual host +- ENV-002: Fake network +- ENV-003: Honey service +- ENV-004: Dummy user +- ENV-005: Credential cache +- ENV-006: Firewall node +- ENV-007: Server template +- ENV-008: Client template +- ENV-009: Database node +- ENV-010: IoT node +- ENV-011: PLC node +- ENV-012: Hospital model +- ENV-013: Cloud node +- ENV-014: Edge node +- ENV-015: SOC station + +## Strategy primitives (STR-001..STR-015) +- STR-001: Attack objective +- STR-002: Defense objective +- STR-003: Mission score +- STR-004: Victory condition +- STR-005: Loss condition +- STR-006: Risk reward +- STR-007: Budget limit +- STR-008: Resource cap +- STR-009: Adaptive strategy +- STR-010: Node priority +- STR-011: Escalation level +- STR-012: Threat tier +- STR-013: Mission timer +- STR-014: Action utility +- STR-015: Reward update + +## Training primitives (TRN-001..TRN-015) +- TRN-001: Blue team mode +- TRN-002: Red team mode +- TRN-003: Purple team mode +- TRN-004: Incident injection +- TRN-005: Malware simulation +- TRN-006: Phishing scenario +- TRN-007: Lateral movement test +- TRN-008: Ransomware drill +- TRN-009: Recovery drill +- TRN-010: SOC evaluation +- TRN-011: Response timer +- TRN-012: Detection score +- TRN-013: False positive score +- TRN-014: Replay grading +- TRN-015: Team leaderboard + +## Economy primitives (ECO-001..ECO-015) +- ECO-001: Defense budget +- ECO-002: Attack budget +- ECO-003: Node cost +- ECO-004: Resource auction +- ECO-005: Trust credits +- ECO-006: Security tokens +- ECO-007: Honeypot value +- ECO-008: AI cost +- ECO-009: Logging cost +- ECO-010: Isolation cost +- ECO-011: Recovery reward +- ECO-012: Risk tax +- ECO-013: Penalty score +- ECO-014: Investment weight +- ECO-015: Utility matrix + +## World evolution (WRD-001..WRD-015) +- WRD-001: Threat weather +- WRD-002: Attack season +- WRD-003: Node mutation +- WRD-004: Malware evolution +- WRD-005: Defense adaptation +- WRD-006: Entropy drift +- WRD-007: Trust decay +- WRD-008: Population growth +- WRD-009: Cluster instability +- WRD-010: Node aging +- WRD-011: Recovery growth +- WRD-012: Failure propagation +- WRD-013: Pressure field +- WRD-014: Chaos event +- WRD-015: Equilibrium detection + +## Visualization (VIS-001..VIS-015) +- VIS-001: Attack map +- VIS-002: Heat zones +- VIS-003: Node animation +- VIS-004: Timeline player +- VIS-005: Replay camera +- VIS-006: Risk overlay +- VIS-007: Trust graph +- VIS-008: Threat wave +- VIS-009: Evidence explorer +- VIS-010: Cluster map +- VIS-011: Agent tracker +- VIS-012: Mission view +- VIS-013: Physics display +- VIS-014: Entropy field +- VIS-015: Strategy board + +## Integration notes +- These primitives are intentionally small and composable. Each micro-feature should map to an issue and a test harness. Combined, they form larger systems like the Security Simulation Engine, Cyber Range, and Training Platform. +- Suggested immediate work: implement deterministic replay, world clock, checkpointing, and a minimal attacker/defender agent for early validation. diff --git a/02_docs/architecture/24-cloud-distributed_features.md b/02_docs/architecture/24-cloud-distributed_features.md new file mode 100644 index 000000000..e3c503224 --- /dev/null +++ b/02_docs/architecture/24-cloud-distributed_features.md @@ -0,0 +1,126 @@ +# AI / Cloud / Distributed Micro-Features Catalog + +This document catalogs AI/ML, Cyber, Cloud, and Distributed micro-features mapped to the extended roadmap. These features support building the Cloud Distributed OS and Sentinel Cloud capabilities. + +## AI / ML features (AI-ML-001..AI-ML-025) +- AI-ML-001: Anomaly detector +- AI-ML-002: Risk classifier +- AI-ML-003: Threat labeler +- AI-ML-004: Log embedding +- AI-ML-005: Graph embedding +- AI-ML-006: Sequence predictor +- AI-ML-007: Time series model +- AI-ML-008: False positive scorer +- AI-ML-009: Confidence estimator +- AI-ML-010: Policy recommender +- AI-ML-011: Node ranking +- AI-ML-012: Attack prediction +- AI-ML-013: Model registry +- AI-ML-014: Model version tracker +- AI-ML-015: Model rollback +- AI-ML-016: Online learning +- AI-ML-017: Federated update +- AI-ML-018: Drift detector +- AI-ML-019: Feature store +- AI-ML-020: Training queue +- AI-ML-021: Inference cache +- AI-ML-022: Prompt memory +- AI-ML-023: RAG index +- AI-ML-024: Replay explanation +- AI-ML-025: SOC copilot + +## Cybersecurity features (CYB-001..CYB-025) +- CYB-001: Process trust score +- CYB-002: Binary reputation +- CYB-003: Command anomaly +- CYB-004: Privilege jump detector +- CYB-005: Credential tracker +- CYB-006: Token misuse detector +- CYB-007: Persistence scanner +- CYB-008: Registry watcher +- CYB-009: Kernel event monitor +- CYB-010: File entropy monitor +- CYB-011: Ransom burst detector +- CYB-012: Lateral movement graph +- CYB-013: Beacon detector +- CYB-014: C2 suspicion score +- CYB-015: DNS anomaly +- CYB-016: Threat timeline +- CYB-017: MITRE mapper +- CYB-018: Evidence signer +- CYB-019: Incident builder +- CYB-020: Attack replay +- CYB-021: Sandbox launcher +- CYB-022: Honey token +- CYB-023: Decoy node +- CYB-024: Auto isolate +- CYB-025: Recovery planner + +## Cloud features (CLD-001..CLD-025) +- CLD-001: VM launcher +- CLD-002: Container launcher +- CLD-003: Image registry +- CLD-004: Snapshot manager +- CLD-005: Volume attach +- CLD-006: Volume replicate +- CLD-007: Cloud scheduler +- CLD-008: Workload placement +- CLD-009: Policy push +- CLD-010: Global config +- CLD-011: Secrets manager +- CLD-012: Identity provider +- CLD-013: API gateway +- CLD-014: Load balancer +- CLD-015: Service discovery +- CLD-016: Autoscaling +- CLD-017: Cold migration +- CLD-018: Hot migration +- CLD-019: Cloud replay +- CLD-020: Telemetry aggregation +- CLD-021: Cluster dashboard +- CLD-022: Remote evidence +- CLD-023: Cloud SOC +- CLD-024: Distributed storage +- CLD-025: Cloud shell + +## Distributed system features (DST-001..DST-025) +- DST-001: Node heartbeat +- DST-002: Leader election +- DST-003: Consensus vote +- DST-004: Node trust +- DST-005: Node join +- DST-006: Node leave +- DST-007: Replica tracker +- DST-008: Event replication +- DST-009: Distributed replay +- DST-010: Global process ID +- DST-011: Cross host lineage +- DST-012: Remote execution +- DST-013: Task migration +- DST-014: Failure detector +- DST-015: Clock sync +- DST-016: State sync +- DST-017: Shard manager +- DST-018: Partition detector +- DST-019: Recovery node +- DST-020: Distributed cache +- DST-021: Distributed graph +- DST-022: Evidence federation +- DST-023: Cluster entropy +- DST-024: Topology mapper +- DST-025: Global namespace + +## Mapping to phases & priorities +- Phase 5 (Distributed Telemetry Layer): DST-008, DST-009, DST-011, DST-021, DST-022, DST-023 +- Phase 6 (Cloud Control Plane): CLD-007, CLD-008, CLD-009, CLD-020, CLD-022, CLD-024 +- Phase 11 (AI Runtime): AI-ML-001..AI-ML-025 (start with anomaly, drift, model registry) +- Cross-cutting Cyber basics (P0): CYB-001..CYB-010 + +## Implementation guidance +- Start with minimal, well-tested primitives: `DST-001` (heartbeat), `DST-015` (clock sync), `DST-016` (state sync), `CLD-020` (telemetry aggregation), `AI-ML-001` (anomaly detector), and `CYB-010` (file entropy monitor). +- Gate distributed/Cloud rollout behind replay determinism and evidence federation tests. +- Treat AI models as replaceable components with a model registry and versioning; enable model rollback and federated update for safety. + +## Notes +- Each micro-feature should be tracked as an individual issue with an owner, tests, and benchmark targets. +- These lists are intentionally exhaustive; prioritize and stage implementation using the recommended P0–P3 buckets in the roadmap. diff --git a/02_docs/architecture/25-cyber_micro_features.md b/02_docs/architecture/25-cyber_micro_features.md new file mode 100644 index 000000000..6b769bbdc --- /dev/null +++ b/02_docs/architecture/25-cyber_micro_features.md @@ -0,0 +1,157 @@ +# Cybersecurity Micro-Features Catalog + +This document catalogs passive defense, active defense, attack-analysis, attack-simulation, support/resilience, deception, and strategic micro-features for SentinelOS. These map to the Cyber Operations layer and the roadmap's phased implementation. + +## Passive defense features (PDEF-001..PDEF-020) +- PDEF-001: Process watcher +- PDEF-002: File watcher +- PDEF-003: Registry monitor +- PDEF-004: DNS observer +- PDEF-005: TLS metadata capture +- PDEF-006: Network flow collector +- PDEF-007: Kernel event tap +- PDEF-008: Memory pressure tracker +- PDEF-009: Credential usage tracker +- PDEF-010: Session observer +- PDEF-011: API invocation logger +- PDEF-012: Privilege escalation watcher +- PDEF-013: Entropy scanner +- PDEF-014: Persistence monitor +- PDEF-015: USB observer +- PDEF-016: Cloud audit collector +- PDEF-017: Container monitor +- PDEF-018: Namespace observer +- PDEF-019: Process lineage builder +- PDEF-020: Trust scorer + +Combines into: Passive Security Runtime + +## Active defense features (ADEF-001..ADEF-020) +- ADEF-001: Process throttle +- ADEF-002: Priority reduction +- ADEF-003: Network isolate +- ADEF-004: Remote freeze +- ADEF-005: Node quarantine +- ADEF-006: Auto snapshot +- ADEF-007: Policy injection +- ADEF-008: Dynamic firewall +- ADEF-009: Credential revoke +- ADEF-010: Session kill +- ADEF-011: Container pause +- ADEF-012: Decoy deploy +- ADEF-013: Honey credential +- ADEF-014: Traffic redirect +- ADEF-015: Replay trigger +- ADEF-016: Recovery switch +- ADEF-017: Safe mode entry +- ADEF-018: Cluster isolation +- ADEF-019: Rollback action +- ADEF-020: Defense escalation + +Combines into: Adaptive Defense Engine + +## Passive attack-analysis features (PATT-001..PATT-020) +- PATT-001: Recon pattern detector +- PATT-002: Port scan tracker +- PATT-003: DNS anomaly detector +- PATT-004: Beacon interval detector +- PATT-005: Lateral path graph +- PATT-006: Credential abuse score +- PATT-007: Privilege chain tracker +- PATT-008: File mutation graph +- PATT-009: Command rarity +- PATT-010: Payload classifier +- PATT-011: Timeline generator +- PATT-012: MITRE mapper +- PATT-013: IOC matcher +- PATT-014: Campaign correlator +- PATT-015: Risk propagation +- PATT-016: Exfiltration estimator +- PATT-017: Evidence builder +- PATT-018: Threat confidence +- PATT-019: Host impact score +- PATT-020: Replay reconstruction + +Combines into: Threat Intelligence Runtime + +## Active attack simulation features (ASIM-001..ASIM-020) +- ASIM-001: Recon simulator +- ASIM-002: Phishing scenario +- ASIM-003: Credential attack model +- ASIM-004: Privilege escalation simulation +- ASIM-005: Lateral movement simulation +- ASIM-006: Ransom scenario +- ASIM-007: Persistence simulation +- ASIM-008: Beacon emulation +- ASIM-009: Exfiltration scenario +- ASIM-010: Insider threat model +- ASIM-011: APT workflow simulation +- ASIM-012: SOC stress mode +- ASIM-013: Recovery drill +- ASIM-014: Incident replay +- ASIM-015: Blue team challenge +- ASIM-016: Red team arena +- ASIM-017: Purple team mode +- ASIM-018: Digital twin attack +- ASIM-019: Cloud breach scenario +- ASIM-020: Distributed incident replay + +Combines into: Cyber Range System + +## Support / resilience features (SUP-001..SUP-020) +- SUP-001: Evidence archive +- SUP-002: Replay database +- SUP-003: Threat memory +- SUP-004: Knowledge graph +- SUP-005: Case similarity search +- SUP-006: Recovery planner +- SUP-007: Incident notebook +- SUP-008: SOC dashboard +- SUP-009: Risk history +- SUP-010: Policy library +- SUP-011: Rule versioning +- SUP-012: Model registry +- SUP-013: Experiment storage +- SUP-014: Forensic export +- SUP-015: Trust ledger +- SUP-016: Audit chain +- SUP-017: Cluster history +- SUP-018: Validation store +- SUP-019: Decision log +- SUP-020: Playbook engine + +Combines into: Security Support Layer + +## Deception features (DEC-001..DEC-015) +- DEC-001: Honey files +- DEC-002: Honey users +- DEC-003: Honey credentials +- DEC-004: Honey shares +- DEC-005: Honey process +- DEC-006: Fake services +- DEC-007: Virtual host +- DEC-008: Decoy database +- DEC-009: Shadow node +- DEC-010: Mirror network +- DEC-011: Synthetic traffic +- DEC-012: Fake telemetry +- DEC-013: False asset map +- DEC-014: Trap session +- DEC-015: Adaptive deception + +Combines into: Deception Runtime + +## Strategic layer +Attack analysis + Passive defense + Active defense + Simulation + Support + Deception = Cyber Operations Layer + +## Implementation order (recommended) +- P0: Passive defense, Support, Evidence +- P1: Threat analysis, Replay, Deception +- P2: Active defense +- P3: Simulation range +- P4: Autonomous cyber operations + +## Notes +- Track each micro-feature as an independent issue with owner, tests, and benchmarks. +- Prioritize P0 features to establish secure telemetry and evidence first. +- Avoid shipping in-kernel enforcement until replay determinism and evidence verification are in place. diff --git a/02_docs/architecture/26-brain_architecture.md b/02_docs/architecture/26-brain_architecture.md new file mode 100644 index 000000000..2ed617abd --- /dev/null +++ b/02_docs/architecture/26-brain_architecture.md @@ -0,0 +1,56 @@ +# SentinelOS LLM Brain — Architecture Summary + +This document describes the LLM Brain concept, responsibilities, modules, and integration points with SentinelOS. It is a concise reference for architects and implementers. + +Purpose +- Provide a cognitive coordination layer that understands telemetry, stitches evidence, reasons over attack graphs, plans defenses, and executes (safely) through the control runtime. + +Stack placement +Telemetry → Evidence → Graph → Math → Physics → Game → Cyber → Cloud/Distributed → **LLM Brain** → Control → Kernel/Hybrid + +Brain capabilities +- Observation: normalize and enrich telemetry into structured world-state snapshots. +- Memory: maintain session and long-term stores (vector DB, graph DB, replay DB, evidence DB). +- Reasoning: LLM + symbolic rules + graph traversal for explainable outputs. +- Planning: produce stepwise response plans and risk-adjusted actions. +- Execution: safe enactment APIs with human-in-the-loop controls and rollback. +- Learning: closed-loop improvement from replay and SOC feedback. + +Multi-agent design +- Chief Agent coordinates policies and approvals. +- Domain agents (SOC, DFIR, Cloud, Game, Physics, Graph, Recovery, Research) implement specialised reasoning and actions. + +Model tiers +- Fast: low-latency classification and routing +- Medium: plan and analysis +- Deep: research and simulation + +Safety & governance +- Policy DB enforces allowed actions and risk budgets. +- All actions have `shadow`, `canary`, and `enforce` modes. +- Evidence-first: all decisions are recorded, signed, and reproducible via replay. + +Implementation milestones +1. Observation adapters + mini LLM classification prototype (shadow-only). +2. RAG + graph connector to support enriched incident analysis. +3. SOC Agent + human approval UI. Integrate with `agents/surface/orchestrator` API. +4. DFIR Agent: replay export, forensics, and evidence stitching. +5. Execution wiring to `07_security/control` with canary rollout. +6. Federated learning & model governance. + +Repository suggestion +- `20_brain/` (see README for layout) +- Keep documentation under `02_docs/architecture/26-brain_architecture.md` for cross-references. + +Security notes +- LLM outputs must be treated as recommendations; do not allow blind automatic enforcement in Phase 0–3. +- Model explainability and decision logs are required for audit and compliance. + +References +- See `02_docs/architecture/13-roadmap.md` for phase alignment and gating rules. +- See `02_docs/architecture/23-sentinel_game_features.md` and `02_docs/architecture/24-cloud-distributed_features.md` for micro-feature catalogs. + +Questions for the team +- Which model runtimes are acceptable for the fast/medium/deep tiers (local vs cloud)? +- Preferred vector DB and graph DB stacks for low-latency RAG? +- Human approval workflows and SOC operator UX requirements. diff --git a/02_docs/validation/architecture_health.md b/02_docs/validation/architecture_health.md new file mode 100644 index 000000000..ca55b094c --- /dev/null +++ b/02_docs/validation/architecture_health.md @@ -0,0 +1,14 @@ +# Architecture Health + +Summary +- Layering: Mostly consistent with telemetry → normalization → graph → evidence → control → physics → game → AI → swarm. +- Observations: Some documents place control and game ordering ambiguously; kernel work is scheduled before full replay/evidence validation in places — suggest reordering to always validate in userspace first. + +Checks +- Circular dependencies: None obvious in docs, but code coupling between `phoenix_os/warden` and kernel hooks should be audited. +- Module isolation: Good separation between telemetry, trace, and evidence; control currently mixes strategy and actuation in some modules. + +Recommendations +- Enforce clear contracts: `telemetry schema` (RFC-001), `trace ingestion` (RFC-006), `warden API` (RFC-005). +- Add interface mocks for kernel behavior to allow offline testing. +- Gate kernel patches behind evidence replay and determinism tests. diff --git a/02_docs/validation/dependency_graph.md b/02_docs/validation/dependency_graph.md new file mode 100644 index 000000000..9cc152ce4 --- /dev/null +++ b/02_docs/validation/dependency_graph.md @@ -0,0 +1,21 @@ +# Dependency Graph (high-level) + +Nodes (modules) +- `phoenix_os/monitor` — telemetry ingestion +- `phoenix_os/bus` — event bus +- `phoenix_os/trace` — lineage DAG & storage +- `phoenix_os/ledger` — evidence chain +- `07_security/control` — warden/actuation +- `phoenix_os/warden` — prototype controller +- `10_kernel` — eBPF hooks and kernel scheduler patches +- `07_security/nexus` — swarm +- `agents/surface/orchestrator` — control plane API + +Edges (direction) +- monitor → bus → trace → ledger → control → kernel +- control → agents/surface/orchestrator (cloud control) +- nexus ↔ agents/surface/orchestrator (gossip/control plane) + +Notes +- Keep dependencies acyclic where possible; prefer pub/sub and RPC contracts between layers. +- Add an explicit schema-and-contract artifact: `02_docs/specifications/telemetry_agent_spec.md` is authoritative. diff --git a/02_docs/validation/doc_change_log.md b/02_docs/validation/doc_change_log.md new file mode 100644 index 000000000..4a97479ca --- /dev/null +++ b/02_docs/validation/doc_change_log.md @@ -0,0 +1,5 @@ +# Documentation Change Log + +- 2026-05-21: Created `02_docs/validation/` with validation artifacts by AI assistant. +- 2026-05-21: Noted that `22-mathematical-physical-game-architecture.md` was updated to include distributed/cloud phases. +- Keep this log for future validations and updates to architecture or RFCs. diff --git a/02_docs/validation/feasibility_report.md b/02_docs/validation/feasibility_report.md new file mode 100644 index 000000000..3c6ccca55 --- /dev/null +++ b/02_docs/validation/feasibility_report.md @@ -0,0 +1,21 @@ +# Feasibility Report + +Overall Flag: PHASED + +Technical feasibility +- Telemetry, entropy/Kalman math, and DAG lineage: READY — implemented in userspace and tested. +- Strategic game and control loops: EXPERIMENTAL — prototype code exists but needs hardened performance testing. +- Kernel scheduler modifications and custom kernel: HIGH_RISK — requires deep kernel expertise and incremental verification. + +Team & maintenance +- Maintenance cost: High for kernel-level changes; moderate for userspace modules. +- Need for specialists: kernel engineers, control theorists, ML engineers, distributed systems. + +Timeline & costs (rough) +- MVP (userspace + eBPF hooks): 3–6 months (small team). +- Kernel-weighted scheduler: 6–18 months with safeguards and formal verification. +- Cluster/swarm: additive after stable telemetry & control (12–24 months). + +MVP viability +- Yes: userspace-only MVP with eBPF probes and centralized cloud control plane simulation. +- Validate via simulation and replay before kernel push. diff --git a/02_docs/validation/idea_validation.md b/02_docs/validation/idea_validation.md new file mode 100644 index 000000000..714048373 --- /dev/null +++ b/02_docs/validation/idea_validation.md @@ -0,0 +1,17 @@ +# Idea Validation + +Verdict: PARTIAL — Novel and defensible, but scope, kernel work, and operational costs require staged validation. + +Summary +- Problem: Real-time autonomous containment of fast-moving host threats (ransomware, supply-chain attacks). Problem exists; target users include enterprise SOCs, cloud providers, and high-assurance environments. +- Frequency & Impact: High-impact but moderate frequency; cascade failures are rare but costly. +- Need for custom OS / kernel: Partial — many goals can be achieved in userspace + eBPF; full kernel replacement is high-risk and should be postponed until userspace models and replay/forensics mature. +- Need for AI: Useful for correlation and policy recommendation; not required for core telemetry, SDI, or control loops. + +Alternatives +- Existing IDS/EDR + SIEM stacks (Velociraptor, OSQuery, Zeek, sysdig) for telemetry and detection. +- eBPF-based in-kernel filtering + userspace graph/evidence chain is a pragmatic compromise. + +Recommendation +- Continue: telemetry, trace, evidence ledger, warden controls in userspace with eBPF hooks. +- Defer: invasive kernel scheduler changes until replay and end-to-end validation exist (Phase E scheduled but gated). diff --git a/02_docs/validation/implementation_order.md b/02_docs/validation/implementation_order.md new file mode 100644 index 000000000..3f5af91c2 --- /dev/null +++ b/02_docs/validation/implementation_order.md @@ -0,0 +1,15 @@ +# Implementation Order (recommended) + +1. Core telemetry ingestion & normalization (RFC-001) — userspace, eBPF probes for collection. +2. Trace & evidence ledger (RFC-006, ledger) — deterministic ingestion and replay tests. +3. Signal processing & SDI math (Kalman, entropy, RFC-001C) — validate on replay data. +4. Warden control loops & FSM (RFC-005) — userspace actuators tied to cgroups/eBPF. +5. Strategic game (Arbiter) — policy manager, economic allocation. +6. Performance benchmarking & CI integration — synthetic and real-world traces. +7. Kernel scheduler telemetry + safe, feature-flagged in-kernel helpers — gated by replay validation. +8. Distributed node runtime & cloud control plane — Nexus and API orchestration. +9. Packaging and bootable hybrid image (Phase F+) — after stability and security audits. + +Notes +- Each step must include tests, benchmarks, and a rollback plan. +- Gate kernel patches behind deterministic replay and canary rollout. diff --git a/02_docs/validation/project_status.md b/02_docs/validation/project_status.md new file mode 100644 index 000000000..495c79f1b --- /dev/null +++ b/02_docs/validation/project_status.md @@ -0,0 +1,20 @@ +# Project Status — PhoenixOS + +Short summary +- Vision: Autonomous, thermodynamics-inspired OS security stack using telemetry → graph → physics → control → game → cloud. +- Current state: Mixed — telemetry, trace, basic SDI math reported "STABLE" in README; control/warden and kernel patches are partial or experimental. +- Branch activity: active PRs (e.g., Kalman drift PR on branch `feature/issue-706-kalman-drift-detection`). +- Tests: Unit tests exist for FSM/warden area; test coverage across repo is partial. + +Key modules +- Telemetry (L3): `phoenix_os/monitor`, `02_docs/specifications/telemetry_agent_spec.md` +- Trace/Graph (L4): `phoenix_os/trace` +- Evidence Ledger: `phoenix_os/ledger` +- Warden/Control (L5): `07_security/control`, `phoenix_os/warden` +- Kernel/Evangelism (L1/L2): `10_kernel/`, eBPF fast paths +- Nexus/Swarm (L7): `07_security/nexus` (gossip, MARL) — in progress + +Next immediate actions +- Create a PR grouping documentation validation artifacts and the small FSM threshold refactor (if not already included). +- Run repository-wide CI and unit tests; collect coverage. +- Triage and prioritize P0/P1 risks: kernel patch safety, evidence integrity, and CI stabilization. diff --git a/02_docs/validation/risk_register.md b/02_docs/validation/risk_register.md new file mode 100644 index 000000000..b841c3d0f --- /dev/null +++ b/02_docs/validation/risk_register.md @@ -0,0 +1,23 @@ +# Risk Register (summary) + +P0 Critical +- Evidence chain integrity compromise (P0): If ledger can be spoofed, all actions are unverifiable. + - Mitigation: Audit cryptographic signing; require remote attestation; add evidence replay tests. + +- Kernel Fast-Path errors causing service disruption (P0): In-kernel blocking may accidentally block benign workloads. + - Mitigation: Feature-flag kernel changes, staged rollout, canary nodes, kill-switch and whitelist. + +P1 Major +- Scheduler weight misconfiguration causing DoS (P1): Threat-weighted scheduling could starve critical processes. + - Mitigation: Safe defaults, max-throttle limits, circuit-breaker, observability. + +- Distributed control inconsistency (P1): Divergent controller state across nodes may cause instability. + - Mitigation: Consensus protocols, leader leases, versioned policies. + +P2 Moderate +- Performance regressions from additional telemetry (P2). + - Mitigation: Sampling, adaptive telemetry rates, overhead budgets. + +P3 Future +- Regulatory and privacy compliance for telemetry (P3). + - Mitigation: Data minimization, on-device aggregation, PI redaction. diff --git a/02_docs/validation/validation_summary.md b/02_docs/validation/validation_summary.md new file mode 100644 index 000000000..875354225 --- /dev/null +++ b/02_docs/validation/validation_summary.md @@ -0,0 +1,16 @@ +# Validation Summary + +Coverage snapshot +- Unit tests: Present for FSM/warden components. Run results: 4 tests passed (local run context). +- Integration tests: Limited; no full end-to-end CI verification of kernel patches or swarm coordination. +- Benchmarks: Throughput claim (10M+ events/sec) in README — need traceable benchmark artifacts. + +Missing validation +- Replay determinism tests for evidence ledger and trace ingestion. +- Performance microbenchmarks for eBPF filters under realistic load. +- Fault-injection tests for controller stability under adversarial load. + +Recommendations +- Add test harness for replay/forensic validation. +- Add benchmark artifacts to `02_docs/benchmarks/` and CI job to run them. +- Expand unit tests to cover telemetry normalization and Kalman drift detectors. diff --git a/06_ai/importance_scoring/scorer.go b/06_ai/importance_scoring/scorer.go new file mode 100644 index 000000000..4451818a4 --- /dev/null +++ b/06_ai/importance_scoring/scorer.go @@ -0,0 +1,18 @@ +package importance_scoring + +// Scorer calculates the Importance Score based on weighted metrics. +type Scorer struct { + Weights [3]float64 +} + +// NewScorer creates a new scorer with default weights. +func NewScorer() *Scorer { + return &Scorer{ + Weights: [3]float64{0.5, 0.3, 0.2}, // Entropy, Network, Resource + } +} + +// Calculate returns a normalized score [0, 1]. +func (s *Scorer) Calculate(entropy, network, resource float64) float64 { + return (entropy * s.Weights[0]) + (network * s.Weights[1]) + (resource * s.Weights[2]) +} diff --git a/06_ai/importance_scoring/scorer_test.go b/06_ai/importance_scoring/scorer_test.go new file mode 100644 index 000000000..f90934d9e --- /dev/null +++ b/06_ai/importance_scoring/scorer_test.go @@ -0,0 +1,12 @@ +package importance_scoring + +import "testing" + +func TestScorer(t *testing.T) { + s := NewScorer() + score := s.Calculate(1.0, 0.5, 0.0) + expected := 0.5*1.0 + 0.3*0.5 + 0.2*0.0 + if score != expected { + t.Errorf("Expected %f, got %f", expected, score) + } +} diff --git a/20_brain/README.md b/20_brain/README.md new file mode 100644 index 000000000..2fedd0849 --- /dev/null +++ b/20_brain/README.md @@ -0,0 +1,90 @@ +# SentinelOS Brain — Design and Repository Layout + +Purpose +- Define the LLM "Brain" that coordinates SentinelOS: observation, memory, reasoning, planning, execution, and learning. +- Provide a practical repo layout for implementation and incremental prototyping. + +Overview +The Brain sits above the telemetry, evidence, graph, math, physics, game, cyber, cloud and distribution layers and acts as a multi-agent coordination and decision-making layer. It is intentionally gated: decisions that affect kernel or production policy require policy constraints, simulation-first checks and human approval flows. + +High-level flow +Users / Nodes → Telemetry → Evidence → Graph → Math → Physics → Game → Cyber → Cloud/Distributed → Brain → Control → Execution (Kernel/Hybrid) + +Brain lifecycle +- Observe → Understand → Reason → Predict → Plan → Execute → Learn + +Core modules (20_brain/) +- `observation/` — adapters, ingest transforms, schema normalization +- `memory/` — session cache, replay store, vector store connectors, graph DB connectors +- `rag/` — retrieval-augmented components, indexing, chunking +- `reasoning/` — LLM wrappers, symbolic rules, graph reasoning APIs +- `planning/` — plan synthesizers, step generators, policy composer +- `execution/` — safe action runner, approvals, shadow execution, rollbacks +- `learning/` — offline training, federated update orchestrator, drift detectors +- `agents/` — SOC, DFIR, Cloud, Graph, Physics, Game, Recovery, Research agents +- `models/` — curated model artifacts (small/medium/large) and deploy configs +- `knowledge/` — policies, replay, evidence, research artifacts + +Memory architecture +- Realtime cache: low-latency in-memory store for current session state +- Session memory: short-lived context for ongoing incidents +- Replay memory: deterministic replay artifacts and checkpoints +- Evidence memory: signed, immutable evidence records +- Graph memory: graph DB snapshot and indexes +- Long-term knowledge: vector DB + policy DB + research archive + +Model tiers and roles +- Fast (1B–8B): routing, classification, short reasoning, SOC realtime +- Medium (8B–30B): incident analysis, plan generation, graph reasoning +- Deep (30B+): research, simulation, global policy synthesis + +Agents and responsibilities +- Chief Agent: policy, risk budget, approvals +- SOC Agent: alert triage, summarization, MITRE mapping +- DFIR Agent: evidence stitching, timeline, export +- Cloud Agent: scheduler advice, placement, migration plans +- Game Agent: defense allocation, honeypot placement +- Physics Agent: entropy/propagation analysis +- Graph Agent: attack chain reasoning and query execution +- Recovery Agent: rollback planning and choreography +- Research Agent: experiment orchestration, model training + +Safety and gating +- All effectful actions must check: policy constraints, safety levels, simulation pass, and optionally require human approval. +- Modes: `shadow` (dry-run), `canary` (limited small group), `enforce` (full action) +- Rollback: all actions create a reversible checkpoint and decision log. + +Integration points +- Telemetry adapters: `phoenix_os/monitor` → `20_brain/observation` connectors +- Evidence linkage: `phoenix_os/ledger` → `20_brain/memory` evidence ingestion +- Graph queries: `phoenix_os/trace` → `20_brain/reasoning` traversal API +- Execution: `20_brain/execution` → `07_security/control` & `phoenix_os/warden` → kernel helpers + +Initial implementation plan (phased) +1. Prototype observation adapters, realtime cache, and a tiny LLM wrapper (fast model) for classification and alert summarization. +2. Add RAG index + graph connector for incident context enrichment. +3. Implement SOC Agent with shadow-mode actions (no enforcement) and a human approval flow. +4. Implement DFIR Agent with replay-execution and forensic export. +5. Hardening: policy DB, model registry, drift detectors, federated update. +6. Gate and scale with canary and feature flags before enabling active enforcement. + +Repository layout (suggested) +- `20_brain/` + - `observation/` + - `memory/` + - `rag/` + - `reasoning/` + - `planning/` + - `execution/` + - `learning/` + - `agents/` + - `models/` + - `knowledge/` + +Next steps +- Create a minimal prototype: ingest a telemetry sample, enrich with RAG, generate an incident summary, and propose a plan in `shadow` mode. +- Optionally, create a small CI job to run the prototype using a tiny open LLM (or local LLM runtime) for reproducible testing. + +Notes +- Keep the brain modular and replaceable: LLM weights, vector DB, and policy engine must be pluggable. +- Document every decision in `20_brain/DECISION_LOG.md` and keep an audit trail for changes. diff --git a/agents/surface/orchestrator/README.brain.md b/agents/surface/orchestrator/README.brain.md new file mode 100644 index 000000000..af4480b93 --- /dev/null +++ b/agents/surface/orchestrator/README.brain.md @@ -0,0 +1,26 @@ +Minimal Brain Prototype +====================== + +This tiny prototype exposes the deterministic `MinimalBrain` for quick experimentation. + +CLI +--- +Run locally: + +```bash +PYTHONPATH=$(pwd) python agents/surface/orchestrator/brain_cli.py --signal telemetry-spike --score 0.92 --evidence 3 +``` + +API +--- +Start the FastAPI app and POST to `/brain/evaluate` (the service is small and safe): + +```bash +PYTHONPATH=$(pwd) uvicorn agents.surface.orchestrator.api:app --reload --port 8001 +curl -s -X POST "http://localhost:8001/brain/evaluate" -H "Content-Type: application/json" -d '{"signal":"telemetry-spike","score":0.92,"evidence_count":3}' | jq +``` + +Notes +----- +- The brain is intentionally deterministic and shadow-mode by default. +- Use this as a harness to prototype RAG, memory, and policy gates incrementally. diff --git a/agents/surface/orchestrator/__pycache__/api.cpython-313.pyc b/agents/surface/orchestrator/__pycache__/api.cpython-313.pyc new file mode 100644 index 000000000..dbf8c910a Binary files /dev/null and b/agents/surface/orchestrator/__pycache__/api.cpython-313.pyc differ diff --git a/agents/surface/orchestrator/__pycache__/github_cli.cpython-313.pyc b/agents/surface/orchestrator/__pycache__/github_cli.cpython-313.pyc new file mode 100644 index 000000000..a2d05c9c1 Binary files /dev/null and b/agents/surface/orchestrator/__pycache__/github_cli.cpython-313.pyc differ diff --git a/agents/surface/orchestrator/__pycache__/minimal_brain.cpython-313.pyc b/agents/surface/orchestrator/__pycache__/minimal_brain.cpython-313.pyc new file mode 100644 index 000000000..5959cb5a7 Binary files /dev/null and b/agents/surface/orchestrator/__pycache__/minimal_brain.cpython-313.pyc differ diff --git a/agents/surface/orchestrator/__pycache__/minimal_brain_test.cpython-313-pytest-9.0.3.pyc b/agents/surface/orchestrator/__pycache__/minimal_brain_test.cpython-313-pytest-9.0.3.pyc new file mode 100644 index 000000000..9cb16dc15 Binary files /dev/null and b/agents/surface/orchestrator/__pycache__/minimal_brain_test.cpython-313-pytest-9.0.3.pyc differ diff --git a/agents/surface/orchestrator/__pycache__/service.cpython-313.pyc b/agents/surface/orchestrator/__pycache__/service.cpython-313.pyc new file mode 100644 index 000000000..c30506a81 Binary files /dev/null and b/agents/surface/orchestrator/__pycache__/service.cpython-313.pyc differ diff --git a/agents/surface/orchestrator/brain_cli.py b/agents/surface/orchestrator/brain_cli.py new file mode 100644 index 000000000..cd17b84a0 --- /dev/null +++ b/agents/surface/orchestrator/brain_cli.py @@ -0,0 +1,35 @@ +"""CLI runner for the MinimalBrain prototype. + +Usage: + python agents/surface/orchestrator/brain_cli.py --signal telemetry-spike --score 0.92 --evidence 3 +""" +from __future__ import annotations + +import argparse +import json + +from agents.surface.orchestrator.minimal_brain import BrainSample, build_minimal_brain + + +def main(): + p = argparse.ArgumentParser(description="Minimal Brain CLI") + p.add_argument("--signal", required=True) + p.add_argument("--score", type=float, required=True) + p.add_argument("--evidence", type=int, default=0) + p.add_argument("--mode", default="shadow") + args = p.parse_args() + + brain = build_minimal_brain() + sample = BrainSample(signal=args.signal, score=args.score, evidence_count=args.evidence, mode=args.mode) + dec = brain.evaluate(sample) + out = { + "action": dec.action, + "risk": dec.risk, + "explanation": dec.explanation, + "trace": list(dec.trace), + } + print(json.dumps(out, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/agents/surface/orchestrator/minimal_brain.py b/agents/surface/orchestrator/minimal_brain.py new file mode 100644 index 000000000..3fa24f879 --- /dev/null +++ b/agents/surface/orchestrator/minimal_brain.py @@ -0,0 +1,70 @@ +"""Minimal brain bootstrap for SentinelOS prototyping. + +This keeps the first runnable slice intentionally small: score a telemetry +sample, choose a coarse action, and emit a trace that can be debugged. +""" +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class BrainSample: + """Input sample for the minimal brain setup.""" + + signal: str + score: float + evidence_count: int = 0 + mode: str = "shadow" + + +@dataclass(frozen=True) +class BrainDecision: + """Decision produced by the minimal brain setup.""" + + action: str + risk: str + explanation: str + trace: tuple[str, ...] + + +class MinimalBrain: + """Tiny deterministic brain used for the first runnable prototype.""" + + def __init__(self, investigate_threshold: float = 0.4, escalate_threshold: float = 0.8): + self.investigate_threshold = investigate_threshold + self.escalate_threshold = escalate_threshold + + def evaluate(self, sample: BrainSample) -> BrainDecision: + if sample.score >= self.escalate_threshold: + action = "escalate" + risk = "high" + elif sample.score >= self.investigate_threshold: + action = "investigate" + risk = "medium" + else: + action = "observe" + risk = "low" + + explanation = ( + f"signal={sample.signal} score={sample.score:.2f} " + f"evidence={sample.evidence_count} mode={sample.mode}" + ) + trace = ( + f"thresholds investigate={self.investigate_threshold:.2f} escalate={self.escalate_threshold:.2f}", + f"sample signal={sample.signal} score={sample.score:.2f} evidence={sample.evidence_count} mode={sample.mode}", + f"decision action={action} risk={risk}", + ) + return BrainDecision(action=action, risk=risk, explanation=explanation, trace=trace) + + def debug_trace(self, sample: BrainSample) -> str: + """Render a compact debug trace for manual inspection.""" + + decision = self.evaluate(sample) + return "\n".join(decision.trace) + + +def build_minimal_brain() -> MinimalBrain: + """Factory for the default minimal brain configuration.""" + + return MinimalBrain() \ No newline at end of file diff --git a/agents/surface/orchestrator/minimal_brain_test.py b/agents/surface/orchestrator/minimal_brain_test.py new file mode 100644 index 000000000..08cf87286 --- /dev/null +++ b/agents/surface/orchestrator/minimal_brain_test.py @@ -0,0 +1,19 @@ +from agents.surface.orchestrator.minimal_brain import BrainSample, build_minimal_brain + + +def test_minimal_brain_escalates_high_risk_signal(): + brain = build_minimal_brain() + decision = brain.evaluate(BrainSample(signal="telemetry-spike", score=0.92, evidence_count=3)) + + assert decision.action == "escalate" + assert decision.risk == "high" + assert "score=0.92" in decision.explanation + assert any("decision action=escalate" in line for line in decision.trace) + + +def test_minimal_brain_debug_trace_is_readable(): + brain = build_minimal_brain() + trace = brain.debug_trace(BrainSample(signal="steady-state", score=0.12, evidence_count=1)) + + assert "thresholds investigate=0.40 escalate=0.80" in trace + assert "decision action=observe risk=low" in trace \ No newline at end of file diff --git a/go.work b/go.work index b9e69fbdd..cffaf109a 100644 --- a/go.work +++ b/go.work @@ -1,34 +1,35 @@ go 1.26 use ( - ./06_ai - ./07_security - ./09_telemetry - ./10_kernel - ./14_experiments/sentinel_mvp - ./14_experiments/telemetry_replay - ./agents - ./custom/control_plane - ./phoenix_os/agents/external/benchmark - ./phoenix_os/agents/external/build - ./phoenix_os/agents/external/common - ./phoenix_os/agents/external/intel - ./phoenix_os/agents/external/research - ./phoenix_os/agents/internal/common - ./phoenix_os/agents/internal/game/marl - ./phoenix_os/agents/internal/healer - ./phoenix_os/agents/internal/optimizer - ./phoenix_os/agents/internal/security - ./phoenix_os/agents/internal/swarm - ./phoenix_os/agents/internal/swarm/governance - ./phoenix_os/arbiter - ./phoenix_os/bus - ./phoenix_os/common - ./phoenix_os/guard - ./phoenix_os/ledger - ./phoenix_os/monitor - ./phoenix_os/nexus - ./phoenix_os/sentinel - ./phoenix_os/trace - ./phoenix_os/warden + ./06_ai + ./07_security + ./09_telemetry + ./10_kernel + ./14_experiments/sentinel_mvp + ./14_experiments/telemetry_replay + ./agents + ./custom/control_plane + ./phoenix_os/agents/external/benchmark + ./phoenix_os/agents/external/build + ./phoenix_os/agents/external/common + ./phoenix_os/agents/external/intel + ./phoenix_os/agents/external/research + ./phoenix_os/agents/internal/common + ./phoenix_os/agents/internal/game/marl + ./phoenix_os/agents/internal/healer + ./phoenix_os/agents/internal/optimizer + ./phoenix_os/agents/internal/security + ./phoenix_os/agents/internal/swarm + ./phoenix_os/agents/internal/swarm/governance + ./phoenix_os/agents/internal/telemetry/math + ./phoenix_os/arbiter + ./phoenix_os/bus + ./phoenix_os/common + ./phoenix_os/guard + ./phoenix_os/ledger + ./phoenix_os/monitor + ./phoenix_os/nexus + ./phoenix_os/sentinel + ./phoenix_os/trace + ./phoenix_os/warden ) diff --git a/phoenix_os/agents/internal/swarm/swarm_agent.go b/phoenix_os/agents/internal/swarm/agent/swarm_agent.go similarity index 96% rename from phoenix_os/agents/internal/swarm/swarm_agent.go rename to phoenix_os/agents/internal/swarm/agent/swarm_agent.go index 7b3945816..ecd622e9b 100644 --- a/phoenix_os/agents/internal/swarm/swarm_agent.go +++ b/phoenix_os/agents/internal/swarm/agent/swarm_agent.go @@ -1,4 +1,4 @@ -package main +package agent import ( "fmt" diff --git a/phoenix_os/agents/internal/swarm/consensus.go b/phoenix_os/agents/internal/swarm/consensus.go new file mode 100644 index 000000000..bc9def579 --- /dev/null +++ b/phoenix_os/agents/internal/swarm/consensus.go @@ -0,0 +1,40 @@ +package swarm + +import ( + "sync" +) + +// ConsensusModule handles PoA and Weighted Quorum for node agreement. +type ConsensusModule struct { + mu sync.RWMutex + NodeReputation map[string]float64 + QuorumWeight float64 +} + +// NewConsensusModule initializes consensus with default weight. +func NewConsensusModule(quorumWeight float64) *ConsensusModule { + return &ConsensusModule{ + NodeReputation: make(map[string]float64), + QuorumWeight: quorumWeight, + } +} + +// Propose evaluates a proposal based on weighted reputation quorum. +func (cm *ConsensusModule) Propose(nodeID string, reputation float64) bool { + cm.mu.Lock() + defer cm.mu.Unlock() + + cm.NodeReputation[nodeID] = reputation + + // Calculate total weight + var totalWeight float64 + for _, r := range cm.NodeReputation { + totalWeight += r + } + + // Agreement reached if proposer has sufficient reputation and total quorum is met + if reputation > 0.5 && totalWeight >= cm.QuorumWeight { + return true + } + return false +} diff --git a/phoenix_os/agents/internal/swarm/consensus_test.go b/phoenix_os/agents/internal/swarm/consensus_test.go new file mode 100644 index 000000000..0331538d9 --- /dev/null +++ b/phoenix_os/agents/internal/swarm/consensus_test.go @@ -0,0 +1,19 @@ +package swarm + +import ( + "testing" +) + +func TestConsensus(t *testing.T) { + cm := NewConsensusModule(1.5) // Need 1.5 total reputation + + // Node 1 proposes + if cm.Propose("node-1", 0.6) { + t.Error("Proposal should fail due to insufficient quorum") + } + + // Node 2 joins + if !cm.Propose("node-2", 1.0) { + t.Error("Proposal should pass with sufficient quorum") + } +} diff --git a/phoenix_os/common/scoring.go b/phoenix_os/common/scoring.go new file mode 100644 index 000000000..ed8fe026e --- /dev/null +++ b/phoenix_os/common/scoring.go @@ -0,0 +1,8 @@ +package common + +// CalculateImportanceScore calculates the Importance Score based on weighted metrics. +// Experts recommend weighting criticality and entropy contribution highest. +// Criticality (0.6) is the primary driver to prevent mimicry. +func CalculateImportanceScore(rank, criticality, entropy, spread, depth float64) float64 { + return 0.1*rank + 0.6*criticality + 0.1*entropy + 0.1*spread + 0.1*depth +} diff --git a/swarm b/swarm new file mode 100755 index 000000000..5dcfb234b Binary files /dev/null and b/swarm differ