diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8da601d..c5bd929 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,9 @@ jobs: if: ${{ !cancelled() }} run: cargo clippy --workspace --all-targets -- -D warnings + - name: Test compatibility harness + run: cargo test -p splice-compat + - name: Test if: ${{ !cancelled() }} run: cargo test --workspace diff --git a/Cargo.lock b/Cargo.lock index f3f9d59..f20d479 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3364,6 +3364,14 @@ dependencies = [ "windows", ] +[[package]] +name = "splice-compat" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "splice-core" version = "0.1.0" @@ -3378,7 +3386,7 @@ dependencies = [ [[package]] name = "splice-shell-desktop" -version = "0.1.0" +version = "0.2.0" dependencies = [ "log", "serde", diff --git a/Cargo.toml b/Cargo.toml index fbe2655..cc64442 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ "apps/desktop/src-tauri", "crates/splice-core", + "crates/splice-compat", "crates/splice-pty", "crates/splice-clipboard" ] diff --git a/crates/splice-compat/Cargo.toml b/crates/splice-compat/Cargo.toml new file mode 100644 index 0000000..6189cd5 --- /dev/null +++ b/crates/splice-compat/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "splice-compat" +version = "0.1.0" +description = "Evidence-based AI CLI compatibility matrix" +license.workspace = true +repository.workspace = true +edition.workspace = true + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/crates/splice-compat/src/lib.rs b/crates/splice-compat/src/lib.rs new file mode 100644 index 0000000..3482036 --- /dev/null +++ b/crates/splice-compat/src/lib.rs @@ -0,0 +1,280 @@ +use serde::Serialize; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::{collections::BTreeMap, ffi::OsStr}; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Tier { + Verified, + Qualified, + Fallback, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EvidenceConfidence { + Unevidenced, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct CliDescriptor { + pub id: &'static str, + pub command: Option<&'static str>, + pub tier: Tier, +} +const DESCRIPTORS: [CliDescriptor; 7] = [ + descriptor("codex", Some("codex")), + descriptor("claude", Some("claude")), + descriptor("opencode", Some("opencode")), + descriptor("gemini", Some("gemini")), + descriptor("aider", Some("aider")), + descriptor("agy", Some("agy")), + descriptor("generic-tui", None), +]; +const fn descriptor(id: &'static str, command: Option<&'static str>) -> CliDescriptor { + CliDescriptor { + id, + command, + tier: Tier::Fallback, + } +} +pub fn descriptors() -> &'static [CliDescriptor] { + &DESCRIPTORS +} +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Behavior { + Startup, + Input, + Interrupt, + Liveness, + ImagePaste, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case", tag = "status")] +pub enum Probe { + Passed, + Skipped { reason: String }, + Unsupported { reason: String }, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Observation { + available: bool, + version: Option, + prerequisites: Vec, + skip: Option, +} +impl Observation { + pub fn available(version: impl Into, prerequisites: I) -> Self + where + I: IntoIterator, + S: Into, + { + Self { + available: true, + version: Some(version.into()), + prerequisites: prerequisites.into_iter().map(Into::into).collect(), + skip: None, + } + } + pub fn unavailable(reason: impl Into) -> Self { + Self { + available: false, + version: None, + prerequisites: Vec::new(), + skip: Some(reason.into()), + } + } +} +pub fn discover( + command: &str, + path: &OsStr, + version: Option, + prerequisites: Vec, +) -> Observation { + let found = std::env::split_paths(path).any(|directory| command_exists(&directory, command)); + if found { + Observation { + available: true, + version, + prerequisites, + skip: None, + } + } else { + Observation::unavailable(format!("{command} not found on PATH")) + } +} +fn command_exists(directory: &std::path::Path, command: &str) -> bool { + #[cfg(windows)] + { + if std::path::Path::new(command).extension().is_some() { + return directory.join(command).is_file(); + } + let extensions = + std::env::var("PATHEXT").unwrap_or_else(|_| ".COM;.EXE;.BAT;.CMD".to_owned()); + extensions + .split(';') + .any(|extension| directory.join(format!("{command}{extension}")).is_file()) + } + #[cfg(unix)] + { + directory + .join(command) + .metadata() + .is_ok_and(|metadata| metadata.is_file() && metadata.permissions().mode() & 0o111 != 0) + } +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct CapabilityReport { + pub descriptor: CliDescriptor, + pub tier: Tier, + pub os: &'static str, + pub shell: &'static str, + pub version: Option, + pub prerequisites: Vec, + pub constraints: Vec, + pub evidence_confidence: EvidenceConfidence, + pub skip: Option, + pub probes: BTreeMap, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct MatrixReport { + pub rows: Vec, +} +impl MatrixReport { + pub fn json(&self) -> Result { + serde_json::to_string_pretty(self) + } + pub fn human(&self) -> String { + let tier = |tier| match tier { + Tier::Verified => "verified", + Tier::Qualified => "qualified", + Tier::Fallback => "fallback", + }; + self.rows + .iter() + .map(|row| { + format!( + "{} tier={} os={} shell={} version={} prerequisites={:?} constraints={:?} confidence={:?} skip={} probes={:?}", + row.descriptor.id, + tier(row.tier), + row.os, + row.shell, + row.version.as_deref().unwrap_or("unknown"), + row.prerequisites, + row.constraints, + row.evidence_confidence, + row.skip.as_deref().unwrap_or("none"), + row.probes + ) + }) + .collect::>() + .join("\n") + } +} +pub fn matrix(observations: &BTreeMap) -> MatrixReport { + MatrixReport { + rows: descriptors() + .iter() + .cloned() + .map(|descriptor| { + let observation = observations + .get(descriptor.id) + .cloned() + .unwrap_or_else(|| Observation::unavailable("no runtime evidence collected")); + CapabilityReport { + tier: Tier::Fallback, + descriptor, + os: "unknown", + shell: "unknown", + version: observation.version, + prerequisites: observation.prerequisites, + constraints: vec!["runtime behavior is not evidenced".to_owned()], + evidence_confidence: EvidenceConfidence::Unevidenced, + skip: observation + .skip + .or_else(|| (!observation.available).then(|| "not available".to_owned())), + probes: unevidenced_probes(), + } + }) + .collect(), + } +} +pub fn harness( + path: &OsStr, + version: Option, + prerequisites: Vec, + tui: FakeTui, +) -> MatrixReport { + let observations = descriptors() + .iter() + .filter_map(|descriptor| { + descriptor.command.map(|command| { + ( + descriptor.id.to_owned(), + discover(command, path, version.clone(), prerequisites.clone()), + ) + }) + }) + .collect(); + let probes = tui.probe(); + let mut report = matrix(&observations); + for row in &mut report.rows { + if observations + .get(row.descriptor.id) + .is_some_and(|observation| observation.available) + { + row.probes.extend(probes.clone()); + } + } + report +} +fn unevidenced_probes() -> BTreeMap { + let skipped = || Probe::Skipped { + reason: "no runtime evidence".to_owned(), + }; + [ + (Behavior::Startup, skipped()), + (Behavior::Input, skipped()), + (Behavior::Interrupt, skipped()), + (Behavior::Liveness, skipped()), + ( + Behavior::ImagePaste, + Probe::Unsupported { + reason: "no evidence".to_owned(), + }, + ), + ] + .into_iter() + .collect() +} +pub struct FakeTui([bool; 4]); +impl FakeTui { + pub fn new(startup: bool, input: bool, interrupt: bool, liveness: bool) -> Self { + Self([startup, input, interrupt, liveness]) + } + pub fn probe(self) -> BTreeMap { + [ + (Behavior::Startup, "fake TUI did not start"), + (Behavior::Input, "fake TUI did not accept input"), + ( + Behavior::Interrupt, + "fake TUI did not acknowledge interrupt", + ), + (Behavior::Liveness, "fake TUI was not live"), + ] + .into_iter() + .zip(self.0) + .map(|((behavior, reason), passed)| { + ( + behavior, + if passed { + Probe::Passed + } else { + Probe::Skipped { + reason: reason.to_owned(), + } + }, + ) + }) + .collect() + } +} diff --git a/crates/splice-compat/tests/matrix.rs b/crates/splice-compat/tests/matrix.rs new file mode 100644 index 0000000..feebddb --- /dev/null +++ b/crates/splice-compat/tests/matrix.rs @@ -0,0 +1,94 @@ +use splice_compat::{ + descriptors, discover, harness, matrix, Behavior, FakeTui, Observation, Probe, Tier, +}; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::{collections::BTreeMap, fs}; +#[test] +fn fixed_descriptors_default_to_fallback_without_runtime_evidence() { + assert_eq!( + descriptors() + .iter() + .map(|descriptor| descriptor.id) + .collect::>(), + [ + "codex", + "claude", + "opencode", + "gemini", + "aider", + "agy", + "generic-tui" + ] + ); + let report = matrix(&BTreeMap::new()); + let row = &report.rows[0]; + assert_eq!((row.os, row.shell), ("unknown", "unknown")); + assert_eq!(row.constraints, ["runtime behavior is not evidenced"]); +} +#[test] +fn unrecognized_observations_keep_generic_fallback_and_image_paste_unsupported() { + let report = matrix(&BTreeMap::from([( + "unrecognized-cli".to_owned(), + Observation::available("9.9.9", ["not a supported descriptor"]), + )])); + let generic = report.rows.last().unwrap(); + assert_eq!( + (report.rows.len(), generic.tier, generic.skip.as_deref()), + (7, Tier::Fallback, Some("no runtime evidence collected")) + ); + assert!(report.rows.iter().all(|row| row.tier == Tier::Fallback)); + assert!(matches!( + generic.probes.get(&Behavior::ImagePaste), + Some(Probe::Unsupported { .. }) + )); +} +#[test] +fn deterministic_harness_discovers_fixture_and_renders_semantic_evidence() { + let fixture = std::env::temp_dir().join(format!("splice-compat-{}", std::process::id())); + let _ = fs::remove_dir_all(&fixture); + fs::create_dir(&fixture).unwrap(); + fs::write(fixture.join("codex"), "fixture").unwrap(); + let path = std::env::join_paths([&fixture]).unwrap(); + #[cfg(unix)] + { + let missing = discover("codex", path.as_os_str(), None, vec![]); + assert_eq!(missing, Observation::unavailable("codex not found on PATH")); + fs::set_permissions(fixture.join("codex"), fs::Permissions::from_mode(0o700)).unwrap(); + } + #[cfg(windows)] + { + fs::rename(fixture.join("codex"), fixture.join("codex.CMD")).unwrap(); + std::env::set_var("PATHEXT", ".EXE;.CMD"); + let direct = discover("codex.CMD", path.as_os_str(), Some("direct".into()), vec![]); + assert_eq!(direct, Observation::available("direct", [] as [&str; 0])); + } + let found = discover( + "codex", + path.as_os_str(), + Some("0.42.0".into()), + vec!["node >= 20".into()], + ); + assert_eq!(found, Observation::available("0.42.0", ["node >= 20"])); + assert_eq!( + discover("claude", path.as_os_str(), None, vec![]), + Observation::unavailable("claude not found on PATH") + ); + let report = harness( + path.as_os_str(), + Some("0.42.0".into()), + vec!["node >= 20".into()], + FakeTui::new(true, true, false, true), + ); + let human = report.human(); + assert!( + human.contains("prerequisites=[\"node >= 20\"]") + && human.contains("Startup: Passed") + && human.contains("Input: Passed") + && human.contains("Interrupt: Skipped") + && human.contains("Liveness: Passed") + ); + let json = report.json().unwrap(); + assert!(json.contains("\"evidence_confidence\": \"unevidenced\"")); + fs::remove_dir_all(fixture).unwrap(); +}