use std::collections::BTreeMap;
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::configfile::{resolve, ResolveOptions, Resolved};
use crate::presets;
use crate::runtime::{RuntimeBackend, RuntimeCapabilities};
use crate::tools::ToolRegistry;
pub const LEDGER_JSON: &str = include_str!("parity/ledger.json");
pub const ORCHESTRATION_JSON: &str = include_str!("parity/orchestration.json");
pub const PARITY_PRESETS: &[(&str, &str)] = &[("cc", "cc-parity"), ("cx", "cx-parity")];
pub const ORCHESTRATION_PRESETS: &[&str] = &["hermes", "openclaw"];
pub const ORCHESTRATOR_PRESET: &str = "orchestrator";
const HERMES_HELP_FIXTURE: &str = include_str!("parity/fixtures/hermes-help.txt");
const OPENCLAW_HELP_FIXTURE: &str = include_str!("parity/fixtures/openclaw-help.txt");
const STORE_SEARCH_ROOTS: &[&str] = &["crates/interchange/src", "crates/harness/src"];
const STORE_OPEN_CALLS: &[&str] = &["SELECT", "open(", "open_with_flags(", ".join("];
pub fn preset_names() -> Vec<&'static str> {
PARITY_PRESETS
.iter()
.map(|(_, preset)| *preset)
.chain(ORCHESTRATION_PRESETS.iter().copied())
.chain(std::iter::once(ORCHESTRATOR_PRESET))
.collect()
}
pub fn help_fixture(harness: &str) -> Option<&'static str> {
match harness {
"hermes" => Some(HERMES_HELP_FIXTURE),
"openclaw" => Some(OPENCLAW_HELP_FIXTURE),
_ => None,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Has {
Yes,
Variant,
Extension,
No,
}
impl Has {
pub fn present(self) -> bool {
!matches!(self, Has::No)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Status {
Implemented,
Partial,
Absent,
Irreducible,
NotApplicable,
Unaudited,
}
impl Status {
pub fn is_gap(self) -> bool {
matches!(
self,
Status::Partial | Status::Absent | Status::Irreducible | Status::Unaudited
)
}
pub fn requires_evidence(self) -> bool {
matches!(self, Status::Implemented | Status::Partial)
}
fn label(self) -> &'static str {
match self {
Status::Implemented => "implemented",
Status::Partial => "partial",
Status::Absent => "absent",
Status::Irreducible => "irreducible",
Status::NotApplicable => "not_applicable",
Status::Unaudited => "unaudited",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Cost {
Trivial,
Architectural,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum Evidence {
Tool {
name: String,
},
Module {
name: String,
},
Config {
key: String,
},
Runtime {
capability: String,
},
Code {
path: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
symbol: Option<String>,
},
CliVerb {
harness: String,
verb: String,
},
Store {
harness: String,
path: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Row {
pub id: String,
pub domain: u8,
pub domain_name: String,
pub capability: String,
pub semantics: String,
pub cc: Has,
pub cx: Has,
pub cc_detail: String,
pub cx_detail: String,
pub catalog_supercode_today: String,
pub provenance: String,
pub status: Status,
#[serde(default)]
pub evidence: Vec<Evidence>,
#[serde(default)]
pub note: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cost: Option<Cost>,
}
impl Row {
pub fn has(&self, column: &str) -> bool {
match column {
"cc" => self.cc.present(),
"cx" => self.cx.present(),
_ => false,
}
}
pub fn applicable_presets(&self) -> Vec<&'static str> {
PARITY_PRESETS
.iter()
.filter(|(column, _)| self.has(column))
.map(|(_, preset)| *preset)
.collect()
}
}
pub fn ledger() -> Vec<Row> {
serde_json::from_str(LEDGER_JSON).expect("embedded parity ledger is valid JSON")
}
pub const ORCHESTRATION_DOMAIN: u8 = 11;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct OrchestrationRow {
pub id: String,
pub concept: String,
pub verbs: String,
pub capability: String,
pub semantics: String,
pub hermes: Has,
pub openclaw: Has,
pub orchestrator: Has,
pub hermes_detail: String,
pub openclaw_detail: String,
pub orchestrator_detail: String,
#[serde(default)]
pub hermes_evidence: Vec<Evidence>,
#[serde(default)]
pub openclaw_evidence: Vec<Evidence>,
pub orchestrator_status: Status,
#[serde(default)]
pub orchestrator_evidence: Vec<Evidence>,
pub orchestrator_note: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub orchestrator_cost: Option<Cost>,
pub provenance: String,
pub status: Status,
#[serde(default)]
pub evidence: Vec<Evidence>,
#[serde(default)]
pub note: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cost: Option<Cost>,
}
impl OrchestrationRow {
pub fn has(&self, harness: &str) -> bool {
self.column(harness)
.is_some_and(|(has, _, _)| has.present())
}
pub fn column(&self, harness: &str) -> Option<(Has, &str, &[Evidence])> {
match harness {
"hermes" => Some((
self.hermes,
self.hermes_detail.as_str(),
self.hermes_evidence.as_slice(),
)),
"openclaw" => Some((
self.openclaw,
self.openclaw_detail.as_str(),
self.openclaw_evidence.as_slice(),
)),
_ => None,
}
}
}
pub fn orchestration_ledger() -> Vec<OrchestrationRow> {
serde_json::from_str(ORCHESTRATION_JSON).expect("embedded orchestration ledger is valid JSON")
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PresetSummary {
pub preset: String,
pub harness_column: String,
pub rows: usize,
pub counts: BTreeMap<String, usize>,
pub gaps: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct GapRow {
pub id: String,
pub domain: u8,
pub capability: String,
pub status: Status,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cost: Option<Cost>,
pub note: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PresetReport {
pub summary: PresetSummary,
pub gaps: Vec<GapRow>,
}
pub fn report(preset: &str) -> Option<PresetReport> {
if let Some((column, _)) = PARITY_PRESETS.iter().find(|(_, p)| *p == preset) {
return Some(catalog_report(preset, column));
}
if preset == ORCHESTRATOR_PRESET {
return Some(orchestrator_report());
}
if ORCHESTRATION_PRESETS.contains(&preset) {
return Some(orchestration_report(preset));
}
None
}
fn catalog_report(preset: &str, column: &str) -> PresetReport {
let rows: Vec<Row> = ledger().into_iter().filter(|r| r.has(column)).collect();
let mut counts: BTreeMap<String, usize> = BTreeMap::new();
let mut gaps = Vec::new();
for row in &rows {
*counts.entry(row.status.label().to_string()).or_default() += 1;
if row.status.is_gap() {
gaps.push(GapRow {
id: row.id.clone(),
domain: row.domain,
capability: row.capability.clone(),
status: row.status,
cost: row.cost,
note: row.note.clone(),
});
}
}
PresetReport {
summary: PresetSummary {
preset: preset.to_string(),
harness_column: column.to_string(),
rows: rows.len(),
counts,
gaps: gaps.len(),
},
gaps,
}
}
fn orchestration_report(harness: &str) -> PresetReport {
let rows: Vec<OrchestrationRow> = orchestration_ledger()
.into_iter()
.filter(|r| r.has(harness))
.collect();
let mut counts: BTreeMap<String, usize> = BTreeMap::new();
let mut gaps = Vec::new();
for row in &rows {
*counts.entry(row.status.label().to_string()).or_default() += 1;
if row.status.is_gap() {
gaps.push(GapRow {
id: row.id.clone(),
domain: ORCHESTRATION_DOMAIN,
capability: row.capability.clone(),
status: row.status,
cost: row.cost,
note: row.note.clone(),
});
}
}
PresetReport {
summary: PresetSummary {
preset: harness.to_string(),
harness_column: harness.to_string(),
rows: rows.len(),
counts,
gaps: gaps.len(),
},
gaps,
}
}
fn orchestrator_report() -> PresetReport {
let rows = orchestration_ledger();
let mut counts: BTreeMap<String, usize> = BTreeMap::new();
let mut gaps = Vec::new();
for row in &rows {
*counts
.entry(row.orchestrator_status.label().to_string())
.or_default() += 1;
if row.orchestrator_status.is_gap() {
gaps.push(GapRow {
id: row.id.clone(),
domain: ORCHESTRATION_DOMAIN,
capability: row.capability.clone(),
status: row.orchestrator_status,
cost: row.orchestrator_cost,
note: row.orchestrator_note.clone(),
});
}
}
PresetReport {
summary: PresetSummary {
preset: ORCHESTRATOR_PRESET.to_string(),
harness_column: ORCHESTRATOR_PRESET.to_string(),
rows: rows.len(),
counts,
gaps: gaps.len(),
},
gaps,
}
}
pub fn render(report: &PresetReport) -> String {
let s = &report.summary;
let mut out = format!("{}: {} rows · {} gaps", s.preset, s.rows, s.gaps);
for status in [
Status::Implemented,
Status::Partial,
Status::Absent,
Status::Irreducible,
Status::NotApplicable,
Status::Unaudited,
] {
if let Some(n) = s.counts.get(status.label()) {
out.push_str(&format!(" · {n} {}", status.label()));
}
}
out.push('\n');
let mut domain = 0u8;
for gap in &report.gaps {
if gap.domain != domain {
domain = gap.domain;
out.push_str(&format!("\nDomain {domain}\n"));
}
let cost = match gap.cost {
Some(Cost::Trivial) => " [trivial]",
Some(Cost::Architectural) => " [architectural]",
None => "",
};
out.push_str(&format!(
" {:<12}{cost} {} ({})",
gap.status.label(),
gap.capability,
gap.id
));
if !gap.note.is_empty() {
out.push_str(&format!(" — {}", gap.note));
}
out.push('\n');
}
out
}
pub fn resolve_preset(preset: &str) -> Result<Resolved, String> {
let toml = presets::lookup(preset).ok_or_else(|| format!("unknown preset `{preset}`"))?;
resolve(toml, None, &ResolveOptions { strict: true }).map_err(|e| e.to_string())
}
fn runtime_flag(capabilities: &RuntimeCapabilities, flag: &str) -> Option<bool> {
Some(match flag {
"start_session" => capabilities.start_session,
"resume_session" => capabilities.resume_session,
"attach_existing_process" => capabilities.attach_existing_process,
"send_input" => capabilities.send_input,
"stream_events" => capabilities.stream_events,
"interrupt" => capabilities.interrupt,
"steer" => capabilities.steer,
"respond_to_requests" => capabilities.respond_to_requests,
_ => return None,
})
}
fn backend_capabilities(column: &str) -> RuntimeCapabilities {
match column {
"cc" => crate::runtime::ClaudeCodeRuntimeBackend::default().capabilities(),
"cx" => crate::runtime::CodexRuntimeBackend::default().capabilities(),
other => panic!("no runtime backend for column `{other}`"),
}
}
fn check_cli_verb(harness: &str, verb: &str) -> Result<(), String> {
let fixture = help_fixture(harness)
.ok_or_else(|| format!("no committed help fixture for harness `{harness}`"))?;
let mut parts: Vec<&str> = verb.split_whitespace().collect();
let leaf = parts.pop().ok_or_else(|| "empty cli verb".to_string())?;
let header = if parts.is_empty() {
format!("$ {harness} --help")
} else {
format!("$ {harness} {} --help", parts.join(" "))
};
let mut in_section = false;
let mut saw_section = false;
for line in fixture.lines() {
if line.starts_with("$ ") {
in_section = line.trim() == header;
saw_section |= in_section;
continue;
}
if !in_section || line.starts_with('#') {
continue;
}
let indent = line.len() - line.trim_start().len();
if !(2..=6).contains(&indent) {
continue;
}
let Some(token) = line.split_whitespace().next() else {
continue;
};
if token.starts_with('-') {
continue;
}
if token.split('|').any(|alias| alias == leaf) {
return Ok(());
}
}
if !saw_section {
return Err(format!(
"`{header}` is not a section of the {harness} help fixture"
));
}
Err(format!(
"`{harness} {verb}` is not advertised under `{header}`"
))
}
fn push_rust_sources(dir: &Path, out: &mut Vec<std::path::PathBuf>) {
let Ok(entries) = std::fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
push_rust_sources(&path, out);
} else if path.extension().is_some_and(|ext| ext == "rs") {
out.push(path);
}
}
}
fn store_segment_is_opened(segment: &str, workspace_root: &Path) -> bool {
let mut files = Vec::new();
for root in STORE_SEARCH_ROOTS {
push_rust_sources(&workspace_root.join(root), &mut files);
}
for file in files {
let Ok(text) = std::fs::read_to_string(&file) else {
continue;
};
let lines: Vec<&str> = text.lines().collect();
for (index, line) in lines.iter().enumerate() {
if !line.contains(segment) || line.trim_start().starts_with("//") {
continue;
}
let start = index.saturating_sub(3);
let context = lines[start..=index].join("\n");
if STORE_OPEN_CALLS.iter().any(|call| context.contains(call)) {
return true;
}
}
}
false
}
fn check_store(harness: &str, path: &str, workspace_root: &Path) -> Result<(), String> {
if !ORCHESTRATION_PRESETS.contains(&harness) && harness != ORCHESTRATOR_PRESET {
return Err(format!(
"`{harness}` is not an orchestration harness or the orchestrator"
));
}
let mut checked = 0usize;
for segment in path.split('/') {
if segment.is_empty() || (segment.starts_with('<') && segment.ends_with('>')) {
continue;
}
if segment.len() < 3 {
return Err(format!(
"store segment `{segment}` is too short to identify a store"
));
}
if !store_segment_is_opened(segment, workspace_root) {
return Err(format!(
"no loader under {} opens `{segment}` (from {harness} store `{path}`)",
STORE_SEARCH_ROOTS.join(", ")
));
}
checked += 1;
}
if checked == 0 {
return Err(format!("store path `{path}` names no concrete segment"));
}
Ok(())
}
fn toml_has_key(doc: &toml::Value, key: &str) -> bool {
let mut cur = doc;
for part in key.split('.') {
match cur.get(part) {
Some(next) => cur = next,
None => return false,
}
}
true
}
pub fn check_evidence(
row: &Row,
evidence: &Evidence,
workspace_root: &std::path::Path,
) -> Result<(), String> {
let applicable: Vec<(&str, &str)> = PARITY_PRESETS
.iter()
.filter(|(column, _)| row.has(column))
.map(|(c, p)| (*c, *p))
.collect();
if applicable.is_empty() {
return Err("row has no applicable preset (neither cc nor cx has it)".into());
}
match evidence {
Evidence::Tool { name } => {
for (_, preset) in &applicable {
let resolved = resolve_preset(preset)?;
let registry = ToolRegistry::from_config(&resolved.config);
if registry.get(name).is_none() {
return Err(format!("tool `{name}` is not registered under `{preset}`"));
}
}
Ok(())
}
Evidence::Module { name } => {
for (_, preset) in &applicable {
let resolved = resolve_preset(preset)?;
match resolved.modules.get(name) {
Some(true) => {}
Some(false) => {
return Err(format!("module `{name}` is disabled under `{preset}`"))
}
None => return Err(format!("module `{name}` is not a known module")),
}
}
Ok(())
}
Evidence::Config { key } => {
for (_, preset) in &applicable {
let text =
presets::lookup(preset).ok_or_else(|| format!("unknown preset `{preset}`"))?;
let doc: toml::Value = toml::from_str(text).map_err(|e| e.to_string())?;
if !toml_has_key(&doc, key) {
return Err(format!("`{preset}` does not set `{key}`"));
}
}
Ok(())
}
Evidence::Runtime { capability } => {
for (column, _) in &applicable {
let caps = backend_capabilities(column);
match runtime_flag(&caps, capability) {
Some(true) => {}
Some(false) => {
return Err(format!(
"runtime capability `{capability}` is false for `{column}`"
))
}
None => return Err(format!("`{capability}` is not a runtime capability flag")),
}
}
Ok(())
}
Evidence::Code { .. } | Evidence::CliVerb { .. } | Evidence::Store { .. } => {
check_source_evidence(evidence, workspace_root)
}
}
}
pub fn check_source_evidence(
evidence: &Evidence,
workspace_root: &std::path::Path,
) -> Result<(), String> {
match evidence {
Evidence::Code { path, symbol } => {
let full = workspace_root.join(path);
let text =
std::fs::read_to_string(&full).map_err(|e| format!("cannot read `{path}`: {e}"))?;
if let Some(symbol) = symbol {
if !text.contains(symbol.as_str()) {
return Err(format!("`{path}` does not contain `{symbol}`"));
}
}
Ok(())
}
Evidence::CliVerb { harness, verb } => check_cli_verb(harness, verb),
Evidence::Store { harness, path } => check_store(harness, path, workspace_root),
other => Err(format!(
"{other:?} needs a preset context; use `check_evidence`"
)),
}
}
pub fn check_orchestration_evidence(
lane: OrchestrationLane<'_>,
evidence: &Evidence,
workspace_root: &std::path::Path,
) -> Result<(), String> {
match (lane, evidence) {
(OrchestrationLane::Column(harness), Evidence::CliVerb { harness: cited, .. }) => {
if cited != harness {
return Err(format!(
"the {harness} column cites a `{cited}` verb ({evidence:?})"
));
}
check_source_evidence(evidence, workspace_root)
}
(OrchestrationLane::Column(harness), other) => Err(format!(
"the {harness} column may only cite `cli_verb`, not {other:?}"
)),
(OrchestrationLane::Supercode, Evidence::Store { .. } | Evidence::Code { .. }) => {
check_source_evidence(evidence, workspace_root)
}
(OrchestrationLane::Supercode, other) => Err(format!(
"a supercode status may only cite `store` or `code`, not {other:?}"
)),
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OrchestrationLane<'a> {
Column(&'a str),
Supercode,
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashSet;
use std::path::PathBuf;
fn workspace_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../..")
.canonicalize()
.unwrap()
}
#[test]
fn ledger_parses_with_unique_ids_and_full_catalog() {
let rows = ledger();
assert_eq!(rows.len(), 263, "one row per catalog capability");
let ids: HashSet<&str> = rows.iter().map(|r| r.id.as_str()).collect();
assert_eq!(ids.len(), rows.len(), "row ids must be unique");
for row in &rows {
assert!(
(1..=11).contains(&row.domain),
"{}: domain out of range",
row.id
);
}
}
#[test]
fn not_applicable_rows_are_exactly_those_neither_harness_has() {
for row in ledger() {
let neither = !row.cc.present() && !row.cx.present();
assert_eq!(
row.status == Status::NotApplicable,
neither,
"{}: not_applicable must mean neither cc nor cx has it",
row.id
);
}
}
#[test]
fn no_row_remains_unaudited() {
let mut stale: Vec<String> = ledger()
.into_iter()
.filter(|r| r.status == Status::Unaudited)
.map(|r| r.id)
.collect();
stale.extend(
orchestration_ledger()
.into_iter()
.filter(|r| r.status == Status::Unaudited)
.map(|r| r.id),
);
assert!(stale.is_empty(), "unaudited rows: {stale:?}");
}
#[test]
fn both_parity_presets_resolve_strictly() {
for (_, preset) in PARITY_PRESETS {
resolve_preset(preset).unwrap_or_else(|e| panic!("{preset}: {e}"));
}
}
#[test]
fn every_audited_claim_is_backed_by_resolvable_evidence() {
let root = workspace_root();
let mut failures = Vec::new();
for row in ledger() {
if row.status.requires_evidence() && row.evidence.is_empty() {
failures.push(format!(
"{}: `{}` cites no evidence",
row.id,
row.status.label()
));
}
if row.status == Status::Irreducible && row.note.is_empty() {
failures.push(format!("{}: irreducible without a note", row.id));
}
if row.status == Status::Absent && row.cost.is_none() {
failures.push(format!("{}: absent without a cost class", row.id));
}
for ev in &row.evidence {
if let Err(reason) = check_evidence(&row, ev, &root) {
failures.push(format!("{}: {reason}", row.id));
}
}
}
assert!(
failures.is_empty(),
"ledger evidence failures:\n{}",
failures.join("\n")
);
}
#[test]
fn evidence_gate_rejects_unresolvable_citations() {
let root = workspace_root();
let mut row = ledger().into_iter().find(|r| r.cc.present()).unwrap();
row.cx = Has::No;
let bad = [
Evidence::Tool {
name: "no_such_tool".into(),
},
Evidence::Module {
name: "no_such_module".into(),
},
Evidence::Module {
name: "model_oauth".into(),
}, Evidence::Config {
key: "capabilities.no_such.key".into(),
},
Evidence::Runtime {
capability: "attach_existing_process".into(),
}, Evidence::Runtime {
capability: "not_a_flag".into(),
},
Evidence::Code {
path: "crates/harness/src/no_such_file.rs".into(),
symbol: None,
},
Evidence::Code {
path: "crates/harness/src/parity.rs".into(),
symbol: Some(["ZZZ_NOT", "_PRESENT_ZZZ"].concat()),
},
];
for ev in bad {
assert!(
check_evidence(&row, &ev, &root).is_err(),
"{ev:?} must be rejected"
);
}
let good = [
Evidence::Tool {
name: "read_file".into(),
},
Evidence::Module {
name: "subagents".into(),
},
Evidence::Config {
key: "capabilities.subagents".into(),
},
Evidence::Runtime {
capability: "steer".into(),
},
Evidence::Code {
path: "crates/harness/src/parity.rs".into(),
symbol: Some("pub fn check_evidence".into()),
},
];
for ev in good {
check_evidence(&row, &ev, &root).unwrap_or_else(|e| panic!("{ev:?}: {e}"));
}
row.cc = Has::No;
assert!(check_evidence(
&row,
&Evidence::Tool {
name: "read_file".into()
},
&root
)
.is_err());
}
#[test]
fn report_counts_add_up() {
for preset in preset_names() {
let r = report(preset).unwrap();
let total: usize = r.summary.counts.values().sum();
assert_eq!(total, r.summary.rows);
assert_eq!(r.gaps.len(), r.summary.gaps);
assert!(!render(&r).is_empty());
}
assert!(report("pi-core").is_none());
}
#[test]
fn preset_names_cover_both_ledgers() {
assert_eq!(
preset_names(),
vec![
"cc-parity",
"cx-parity",
"hermes",
"openclaw",
"orchestrator"
]
);
for harness in ORCHESTRATION_PRESETS {
let r = report(harness).unwrap();
assert_eq!(&r.summary.preset, harness);
assert_eq!(&r.summary.harness_column, harness);
assert!(r.summary.rows > 0, "{harness}: empty denominator");
let rendered = render(&r);
assert!(
rendered.starts_with(&format!("{harness}: {} rows · ", r.summary.rows)),
"{harness}: unexpected headline: {rendered}"
);
assert!(rendered.contains("\nDomain 11\n"), "{harness}: {rendered}");
}
}
#[test]
fn the_orchestrator_column_is_graded_on_every_row_with_resolvable_evidence() {
let root = workspace_root();
let rows = orchestration_ledger();
let report = report(ORCHESTRATOR_PRESET).unwrap();
assert_eq!(
report.summary.rows,
rows.len(),
"the orchestrator is graded on every row, never a filtered subset"
);
let mut failures = Vec::new();
for row in &rows {
if row.orchestrator.present() && row.orchestrator_detail.is_empty() {
failures.push(format!("{}: orchestrator column has no detail", row.id));
}
if !row.orchestrator.present() && row.orchestrator_status != Status::NotApplicable {
failures.push(format!(
"{}: the orchestrator lacks this row but is graded `{}`",
row.id,
row.orchestrator_status.label()
));
}
if !row.orchestrator.present() && !row.orchestrator_evidence.is_empty() {
failures.push(format!("{}: a `no` column cites evidence", row.id));
}
if row.orchestrator_status.requires_evidence() && row.orchestrator_evidence.is_empty() {
failures.push(format!(
"{}: orchestrator `{}` cites no evidence",
row.id,
row.orchestrator_status.label()
));
}
if row.orchestrator_status == Status::Absent && row.orchestrator_cost.is_none() {
failures.push(format!(
"{}: orchestrator absent without a cost class",
row.id
));
}
if row.orchestrator_note.is_empty() {
failures.push(format!("{}: no orchestrator note", row.id));
}
for ev in &row.orchestrator_evidence {
if let Err(reason) =
check_orchestration_evidence(OrchestrationLane::Supercode, ev, &root)
{
failures.push(format!("{}: {reason}", row.id));
}
}
}
assert!(
failures.is_empty(),
"orchestrator column failures:\n{}",
failures.join("\n")
);
let rendered = render(&report);
assert!(
rendered.starts_with(&format!(
"orchestrator: {} rows · {} gaps",
report.summary.rows, report.summary.gaps
)),
"{rendered}"
);
}
#[test]
fn orchestrator_store_citations_resolve_like_every_other_supercode_status() {
let root = workspace_root();
check_orchestration_evidence(
OrchestrationLane::Supercode,
&Evidence::Store {
harness: ORCHESTRATOR_PRESET.into(),
path: "cron/jobs.json".into(),
},
&root,
)
.unwrap();
assert!(check_orchestration_evidence(
OrchestrationLane::Supercode,
&Evidence::Store {
harness: ORCHESTRATOR_PRESET.into(),
path: "cron/no_such_store.json".into(),
},
&root,
)
.is_err());
}
#[test]
fn orchestration_ledger_parses_with_unique_ids_and_every_concept() {
let rows = orchestration_ledger();
let ids: HashSet<&str> = rows.iter().map(|r| r.id.as_str()).collect();
assert_eq!(ids.len(), rows.len(), "row ids must be unique");
let concepts: HashSet<&str> = rows.iter().map(|r| r.concept.as_str()).collect();
let expected: HashSet<&str> = crate::support::ORCHESTRATION_CONCEPTS
.iter()
.copied()
.collect();
assert_eq!(
concepts, expected,
"every ORCHESTRATION_CONCEPTS entry needs at least one row, and no others"
);
for row in &rows {
assert!(!row.verbs.is_empty(), "{}: no verb group", row.id);
assert!(!row.semantics.is_empty(), "{}: no semantics", row.id);
assert!(!row.provenance.is_empty(), "{}: no provenance", row.id);
}
}
#[test]
fn orchestration_not_applicable_rows_are_exactly_those_neither_harness_has() {
for row in orchestration_ledger() {
let neither = !row.hermes.present() && !row.openclaw.present();
assert_eq!(
row.status == Status::NotApplicable,
neither,
"{}: not_applicable must mean neither hermes nor openclaw has it",
row.id
);
}
}
#[test]
fn every_orchestration_claim_is_backed_by_resolvable_evidence() {
let root = workspace_root();
let mut failures = Vec::new();
for row in orchestration_ledger() {
for harness in ORCHESTRATION_PRESETS {
let (has, detail, evidence) = row.column(harness).unwrap();
if has.present() {
if detail.is_empty() {
failures.push(format!("{}: {harness} column has no detail", row.id));
}
if evidence.is_empty() {
failures.push(format!("{}: {harness} column cites no verb", row.id));
}
} else if !evidence.is_empty() {
failures.push(format!(
"{}: {harness} lacks the row but cites {evidence:?}",
row.id
));
}
for ev in evidence {
if let Err(reason) =
check_orchestration_evidence(OrchestrationLane::Column(harness), ev, &root)
{
failures.push(format!("{}: {reason}", row.id));
}
}
}
if row.status.requires_evidence() && row.evidence.is_empty() {
failures.push(format!(
"{}: `{}` cites no evidence",
row.id,
row.status.label()
));
}
if row.status == Status::Absent && row.cost.is_none() {
failures.push(format!("{}: absent without a cost class", row.id));
}
if row.note.is_empty() {
failures.push(format!("{}: no note", row.id));
}
for ev in &row.evidence {
if let Err(reason) =
check_orchestration_evidence(OrchestrationLane::Supercode, ev, &root)
{
failures.push(format!("{}: {reason}", row.id));
}
}
}
assert!(
failures.is_empty(),
"orchestration ledger evidence failures:\n{}",
failures.join("\n")
);
}
#[test]
fn orchestration_evidence_gate_rejects_unresolvable_citations() {
let root = workspace_root();
let bad = [
Evidence::CliVerb {
harness: "hermes".into(),
verb: "cron teleport".into(),
},
Evidence::CliVerb {
harness: "hermes".into(),
verb: "approvals list".into(),
},
Evidence::CliVerb {
harness: "openclaw".into(),
verb: "sessions archive".into(),
},
Evidence::CliVerb {
harness: "openclaw".into(),
verb: "approvals resolve".into(),
},
Evidence::CliVerb {
harness: "hermes".into(),
verb: "kanban list".into(),
},
Evidence::CliVerb {
harness: "claude-code".into(),
verb: "cron list".into(),
},
Evidence::Store {
harness: "openclaw".into(),
path: "state/openclaw.sqlite/delivery_queue_entries".into(),
},
Evidence::Store {
harness: "openclaw".into(),
path: "cron_run_receipts".into(),
},
Evidence::Store {
harness: "grok".into(),
path: "state.db".into(),
},
Evidence::Store {
harness: "hermes".into(),
path: "<agentId>".into(),
},
];
for ev in &bad {
let lane = match ev {
Evidence::CliVerb { harness, .. } => OrchestrationLane::Column(harness),
_ => OrchestrationLane::Supercode,
};
assert!(
check_orchestration_evidence(lane, ev, &root).is_err(),
"{ev:?} must be rejected"
);
}
let good = [
Evidence::CliVerb {
harness: "hermes".into(),
verb: "cron list".into(),
},
Evidence::CliVerb {
harness: "openclaw".into(),
verb: "agents bindings".into(),
},
Evidence::Store {
harness: "hermes".into(),
path: "state.db/sessions/session_key".into(),
},
Evidence::Store {
harness: "hermes".into(),
path: "profiles/<name>/cron/jobs.json".into(),
},
Evidence::Store {
harness: "openclaw".into(),
path: "cron/jobs.json".into(),
},
Evidence::Store {
harness: "hermes".into(),
path: "cron/executions.db".into(),
},
Evidence::Store {
harness: "openclaw".into(),
path: "state/openclaw.sqlite/cron_run_logs".into(),
},
Evidence::Store {
harness: "hermes".into(),
path: "state.db/delivery_obligations".into(),
},
];
for ev in &good {
let lane = match ev {
Evidence::CliVerb { harness, .. } => OrchestrationLane::Column(harness),
_ => OrchestrationLane::Supercode,
};
check_orchestration_evidence(lane, ev, &root).unwrap_or_else(|e| panic!("{ev:?}: {e}"));
}
assert!(
check_orchestration_evidence(OrchestrationLane::Supercode, &good[0], &root).is_err()
);
assert!(
check_orchestration_evidence(OrchestrationLane::Column("hermes"), &good[2], &root)
.is_err()
);
assert!(check_orchestration_evidence(
OrchestrationLane::Column("openclaw"),
&good[0],
&root
)
.is_err());
}
#[test]
fn help_fixtures_record_the_pin_and_how_to_recapture() {
for harness in ORCHESTRATION_PRESETS {
let fixture = help_fixture(harness).expect("committed fixture");
let mut lines = fixture.lines();
let first = lines.next().unwrap_or_default();
assert!(
first.starts_with(&format!("# fixture: {harness} CLI help @ ")),
"{harness}: first line must name the harness and the pinned version: {first}"
);
assert!(
first.trim_end().len() > format!("# fixture: {harness} CLI help @ ").len(),
"{harness}: no pinned version in `{first}`"
);
let recapture = fixture
.lines()
.find(|line| line.starts_with("# recapture:"))
.unwrap_or_else(|| panic!("{harness}: no `# recapture:` header line"));
assert!(
recapture.contains("--help"),
"{harness}: recapture line names no command: {recapture}"
);
assert!(
fixture
.lines()
.any(|line| line.starts_with("# provenance:")),
"{harness}: no `# provenance:` header line"
);
assert!(
fixture
.lines()
.any(|line| line.starts_with(&format!("$ {harness} "))),
"{harness}: fixture captures no `$ {harness} ... --help` section"
);
}
assert!(help_fixture("claude-code").is_none());
}
}