use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::PathBuf;
use std::process::Command;
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use supercode_harness::{
core_messages, harness_support_registry, measure_fidelity, messages_equal_multimodal,
replay_eligible, ChatMessage, DiscoveryQuery, HarnessCatalog, HarnessHomes, HarnessId,
ImplementationKind, Role, Session, SessionFormat,
};
const SCHEMA: &str = "supercode.real-corpus-fidelity-matrix.v2";
const REQUIRED_COVERAGE: [&str; 7] = [
"compaction_or_rollback",
"errors",
"native_metadata",
"subagents_or_lineage",
"text",
"thinking",
"tools",
];
fn argument<T: std::str::FromStr>(name: &str, default: Option<T>) -> T {
let mut arguments = std::env::args();
while let Some(argument) = arguments.next() {
if argument == name {
return arguments
.next()
.unwrap_or_else(|| panic!("{name} requires a value"))
.parse()
.unwrap_or_else(|_| panic!("invalid value for {name}"));
}
}
default.unwrap_or_else(|| panic!("required argument {name} is missing"))
}
fn optional_argument(name: &str) -> Option<String> {
let mut arguments = std::env::args();
while let Some(argument) = arguments.next() {
if argument == name {
return Some(
arguments
.next()
.unwrap_or_else(|| panic!("{name} requires a value")),
);
}
}
None
}
fn format_for(harness: &str) -> SessionFormat {
match harness {
HarnessId::CLAUDE_CODE => SessionFormat::ClaudeCode,
HarnessId::CODEX => SessionFormat::Codex,
HarnessId::OPENCODE => SessionFormat::OpenCode,
HarnessId::PI => SessionFormat::Pi,
HarnessId::GROK => SessionFormat::Grok,
HarnessId::GEMINI => SessionFormat::Gemini,
HarnessId::GOOSE => SessionFormat::Goose,
other => panic!("unsupported built-in harness {other}"),
}
}
fn format_name(format: SessionFormat) -> &'static str {
match format {
SessionFormat::ClaudeCode => HarnessId::CLAUDE_CODE,
SessionFormat::Codex => HarnessId::CODEX,
SessionFormat::OpenCode => HarnessId::OPENCODE,
SessionFormat::Pi => HarnessId::PI,
SessionFormat::Grok => HarnessId::GROK,
SessionFormat::Gemini => HarnessId::GEMINI,
SessionFormat::Goose => HarnessId::GOOSE,
}
}
#[derive(Clone)]
struct Sample {
session: Session,
report: SampleReport,
features: BTreeSet<String>,
}
#[derive(Clone, Serialize)]
struct SampleReport {
fingerprint: String,
native_bytes: usize,
messages: usize,
raw_records: usize,
features: Vec<String>,
}
#[derive(Default, Serialize)]
struct CorpusReport {
status: String,
provenance: &'static str,
observed: usize,
eligible: usize,
scanned: usize,
loaded: usize,
load_failures: usize,
selected: usize,
selected_messages: usize,
selected_native_bytes: usize,
selected_coverage: Vec<String>,
supplemental_coverage: Vec<String>,
coverage: Vec<String>,
samples: Vec<SampleReport>,
supplemental_samples: Vec<SampleReport>,
}
#[derive(Default, Serialize)]
struct CellReport {
source: String,
target: String,
sessions: usize,
export_failures: usize,
source_messages: usize,
matched_messages: usize,
message_fidelity_percent: f64,
replay_messages: usize,
replay_matched_messages: usize,
replay_fidelity_percent: f64,
intentional_replay_exclusions: usize,
intentional_field_exclusions: BTreeMap<String, usize>,
hard_loss_messages: usize,
hard_loss_samples: Vec<String>,
hard_loss_shapes: BTreeMap<String, usize>,
semantic_fields_total: usize,
semantic_fields_retained: usize,
semantic_field_fidelity_percent: f64,
metadata_values_total: usize,
metadata_values_retained: usize,
metadata_fidelity_percent: f64,
field_residue: BTreeMap<String, usize>,
opaque_residue_keys: Vec<String>,
}
#[derive(Default, Serialize)]
struct RoundTripReport {
source: String,
intermediate: String,
sessions: usize,
semantic_exact: usize,
metadata_exact: usize,
raw_byte_exact: usize,
raw_value_exact: usize,
hard_loss_messages: usize,
field_residue: BTreeMap<String, usize>,
discrepancies: BTreeMap<String, usize>,
discrepancy_samples: BTreeMap<String, Vec<String>>,
}
#[derive(Serialize)]
struct VersionReport {
program: String,
available: bool,
version: Option<String>,
}
#[derive(Serialize)]
struct Report {
schema: &'static str,
scope: &'static str,
capture_date: String,
capture_unix_ms: u128,
content_emitted: bool,
sample_size: usize,
scan_limit: usize,
max_sample_bytes: usize,
versions: BTreeMap<String, VersionReport>,
corpora: BTreeMap<String, CorpusReport>,
cells: Vec<CellReport>,
round_trips: Vec<RoundTripReport>,
support_claim_gate: SupportClaimGate,
}
#[derive(Serialize)]
struct SupportClaimGate {
corpus_complete: bool,
matrix_complete: bool,
round_trips_complete: bool,
translation_semantic_lossless: bool,
translation_metadata_exact: bool,
round_trips_semantic_exact: bool,
round_trips_metadata_exact: bool,
round_trips_metadata_residue_named: bool,
native_diagonal_raw_exact: bool,
cross_format_raw_residue_named: bool,
}
#[derive(Deserialize)]
struct BaselineReport {
corpora: BTreeMap<String, BaselineCorpus>,
cells: Vec<BaselineCell>,
support_claim_gate: BaselineGate,
}
#[derive(Deserialize)]
struct BaselineCorpus {
samples: Vec<BaselineSample>,
}
#[derive(Deserialize)]
struct BaselineSample {
fingerprint: String,
native_bytes: usize,
}
#[derive(Deserialize)]
struct BaselineCell {
source: String,
target: String,
semantic_fields_total: usize,
semantic_fields_retained: usize,
hard_loss_messages: usize,
}
#[derive(Deserialize)]
struct BaselineGate {
translation_semantic_lossless: bool,
round_trips_semantic_exact: bool,
}
fn main() {
let sample_size = argument("--sample-size", Some(5_usize));
let max_sample_bytes = argument("--max-bytes", Some(4_194_304_usize));
let scan_limit = argument(
"--scan-limit",
Some(sample_size.saturating_mul(4).max(sample_size)),
);
let output = optional_argument("--output").map(PathBuf::from);
let baseline_path = optional_argument("--baseline").map(PathBuf::from);
let baseline = baseline_path.as_ref().map(|path| load_baseline(path));
assert!(sample_size > 0, "--sample-size must be positive");
assert!(
scan_limit >= sample_size,
"--scan-limit must cover --sample-size"
);
let capture_unix_ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis();
let registry = harness_support_registry();
let native_harnesses = registry
.harnesses
.iter()
.filter(|harness| {
harness.id.as_str() != HarnessId::SUPERCODE
&& harness.native.load != ImplementationKind::Absent
})
.collect::<Vec<_>>();
let versions = registry
.harnesses
.iter()
.map(|harness| {
let program = harness
.runtime
.default_launch
.as_ref()
.map(|launch| launch.program.clone())
.unwrap_or_else(|| harness.id.0.clone());
(harness.id.0.clone(), executable_version(program))
})
.collect::<BTreeMap<_, _>>();
let catalog = HarnessCatalog::new();
let homes = HarnessHomes::default();
let mut corpora = BTreeMap::new();
let mut samples = BTreeMap::<String, Vec<Sample>>::new();
for harness in &native_harnesses {
let (selected, corpus) = select_samples(
&catalog,
&homes,
&harness.id,
sample_size,
scan_limit,
max_sample_bytes,
baseline
.as_ref()
.and_then(|report| report.corpora.get(&harness.id.0))
.map(|corpus| corpus.samples.as_slice()),
);
samples.insert(harness.id.0.clone(), selected);
corpora.insert(harness.id.0.clone(), corpus);
}
let mut cells = Vec::new();
let mut round_trips = Vec::new();
for source in &native_harnesses {
let source_format = format_for(&source.id.0);
let source_samples = samples
.get(&source.id.0)
.expect("sample map follows registry");
for target in &native_harnesses {
let target_format = format_for(&target.id.0);
cells.push(measure_cell_corpus(
source_format,
target_format,
source_samples,
));
round_trips.push(measure_round_trip_corpus(
source_format,
target_format,
source_samples,
));
}
}
let corpus_complete = corpora.values().all(|corpus| {
corpus.selected == sample_size
&& corpus.load_failures == 0
&& REQUIRED_COVERAGE
.iter()
.all(|feature| corpus.coverage.iter().any(|present| present == feature))
});
let expected_cells = native_harnesses.len() * native_harnesses.len();
let matrix_complete = cells.len() == expected_cells
&& cells
.iter()
.all(|cell| cell.sessions == sample_size && cell.export_failures == 0);
let round_trips_complete = round_trips.len() == expected_cells
&& round_trips.iter().all(|cell| cell.sessions == sample_size);
let translation_semantic_lossless = cells.iter().all(|cell| {
cell.export_failures == 0
&& cell.hard_loss_messages == 0
&& cell.semantic_fields_retained == cell.semantic_fields_total
&& cell.field_residue.is_empty()
});
let translation_metadata_exact = cells
.iter()
.all(|cell| cell.metadata_values_retained == cell.metadata_values_total);
let round_trips_semantic_exact = round_trips
.iter()
.all(|cell| cell.semantic_exact == cell.sessions);
let round_trips_metadata_exact = round_trips
.iter()
.all(|cell| cell.metadata_exact == cell.sessions);
let round_trips_metadata_residue_named = round_trips.iter().all(|cell| {
cell.metadata_exact == cell.sessions || cell.discrepancies.contains_key("metadata_values")
});
let native_diagonal_raw_exact = round_trips.iter().all(|cell| {
cell.source != cell.intermediate
|| cell.raw_byte_exact == cell.sessions
|| cell.raw_value_exact == cell.sessions
});
let cross_format_raw_residue_named = round_trips.iter().all(|cell| {
cell.source == cell.intermediate
|| cell.raw_byte_exact == cell.sessions
|| cell.raw_value_exact == cell.sessions
|| cell.discrepancies.contains_key("raw_bytes_only")
|| cell.discrepancies.contains_key("raw_or_residue_values")
});
let report = Report {
schema: SCHEMA,
scope: "local-read-only-bounded-content-free",
capture_date: capture_date(),
capture_unix_ms,
content_emitted: false,
sample_size,
scan_limit,
max_sample_bytes,
versions,
corpora,
cells,
round_trips,
support_claim_gate: SupportClaimGate {
corpus_complete,
matrix_complete,
round_trips_complete,
translation_semantic_lossless,
translation_metadata_exact,
round_trips_semantic_exact,
round_trips_metadata_exact,
round_trips_metadata_residue_named,
native_diagonal_raw_exact,
cross_format_raw_residue_named,
},
};
if let (Some(path), Some(baseline)) = (baseline_path.as_ref(), baseline.as_ref()) {
verify_baseline(&report, baseline, path);
}
let encoded = serde_json::to_string_pretty(&report).unwrap() + "\n";
if let Some(path) = output {
fs::write(&path, encoded).unwrap_or_else(|error| {
panic!("cannot write fidelity report {}: {error}", path.display())
});
eprintln!("wrote content-free fidelity report to {}", path.display());
} else {
print!("{encoded}");
}
assert!(corpus_complete, "real-corpus sample is incomplete");
assert!(
matrix_complete,
"one or more directed matrix cells did not execute"
);
assert!(
round_trips_complete,
"one or more round-trip cells did not execute"
);
assert!(
translation_semantic_lossless,
"one or more translation cells lost a measured semantic field"
);
assert!(
round_trips_semantic_exact,
"one or more round-trip cells changed canonical semantics"
);
assert!(
native_diagonal_raw_exact,
"one or more native diagonal round trips changed raw values"
);
assert!(
cross_format_raw_residue_named,
"one or more cross-format raw mismatches lacks named residue"
);
}
fn load_baseline(path: &std::path::Path) -> BaselineReport {
serde_json::from_str(
&fs::read_to_string(path)
.unwrap_or_else(|error| panic!("cannot read baseline {}: {error}", path.display())),
)
.unwrap_or_else(|error| panic!("invalid baseline {}: {error}", path.display()))
}
fn verify_baseline(report: &Report, baseline: &BaselineReport, path: &std::path::Path) {
for (format, current) in &report.corpora {
let previous = baseline
.corpora
.get(format)
.unwrap_or_else(|| panic!("baseline has no {format} corpus"));
let current_fingerprints = current
.samples
.iter()
.map(|sample| sample.fingerprint.as_str())
.collect::<Vec<_>>();
let previous_fingerprints = previous
.samples
.iter()
.map(|sample| sample.fingerprint.as_str())
.collect::<Vec<_>>();
assert_eq!(
current_fingerprints, previous_fingerprints,
"{format} corpus changed; capture a new versioned baseline instead of comparing different sessions"
);
}
for current in &report.cells {
let previous = baseline
.cells
.iter()
.find(|cell| cell.source == current.source && cell.target == current.target)
.unwrap_or_else(|| {
panic!(
"baseline has no {}->{} fidelity cell",
current.source, current.target
)
});
assert_eq!(
current.semantic_fields_total, previous.semantic_fields_total,
"{}->{} semantic field population changed",
current.source, current.target
);
assert!(
current.semantic_fields_retained >= previous.semantic_fields_retained,
"{}->{} silently dropped a previously retained semantic field",
current.source,
current.target
);
assert!(
current.hard_loss_messages <= previous.hard_loss_messages,
"{}->{} introduced additional hard message loss",
current.source,
current.target
);
}
if baseline.support_claim_gate.translation_semantic_lossless {
assert!(
report.support_claim_gate.translation_semantic_lossless,
"translation semantic-lossless gate regressed from the baseline"
);
}
if baseline.support_claim_gate.round_trips_semantic_exact {
assert!(
report.support_claim_gate.round_trips_semantic_exact,
"round-trip semantic-exact gate regressed from the baseline"
);
}
eprintln!("baseline fidelity retained: {}", path.display());
}
fn select_samples(
catalog: &HarnessCatalog,
homes: &HarnessHomes,
harness: &HarnessId,
sample_size: usize,
scan_limit: usize,
max_sample_bytes: usize,
baseline_samples: Option<&[BaselineSample]>,
) -> (Vec<Sample>, CorpusReport) {
let discovered = catalog
.discover(&DiscoveryQuery {
harnesses: vec![harness.clone()],
homes: homes.clone(),
limit: Some(scan_limit),
..DiscoveryQuery::default()
})
.unwrap_or_else(|error| panic!("{} discovery failed: {error}", harness.0));
let observed = discovered.len();
let mut candidates = Vec::new();
let mut failures = 0;
for descriptor in discovered {
match catalog.load(&descriptor.locator) {
Ok(discovered_session) => {
let discovered_native = discovered_session.to_native_jsonl();
let snapshot = match baseline_samples {
Some(expected) => captured_baseline_snapshot(
&discovered_native,
expected,
format_for(harness.as_str()),
),
None => Some((discovered_session, discovered_native)),
};
if let Some((session, native)) = snapshot {
if native.len() > max_sample_bytes || session.messages.is_empty() {
continue;
}
let features = session_features(&session);
let fingerprint = blake3::hash(native.as_bytes()).to_hex().to_string();
if std::env::var_os("SUPERCODE_FIDELITY_DEBUG_LOCATORS").is_some() {
eprintln!(
"fidelity-debug {} {} {}",
harness.0,
fingerprint,
descriptor.locator.storage.path().display()
);
}
if candidates
.iter()
.any(|sample: &Sample| sample.report.fingerprint == fingerprint)
{
continue;
}
candidates.push(Sample {
report: SampleReport {
fingerprint,
native_bytes: native.len(),
messages: session.messages.len(),
raw_records: session.raw.len(),
features: features.iter().cloned().collect(),
},
session,
features,
});
}
}
Err(_) => failures += 1,
}
}
let eligible = candidates.len();
let selected = match baseline_samples {
Some(expected) => {
candidates.sort_by_key(|sample| {
expected
.iter()
.position(|item| item.fingerprint == sample.report.fingerprint)
.unwrap_or(usize::MAX)
});
candidates.into_iter().take(sample_size).collect()
}
None => coverage_seeking_sample(candidates, sample_size),
};
let selected_coverage = selected
.iter()
.flat_map(|sample| sample.features.iter().cloned())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
let supplemental = supplemental_samples(harness);
let supplemental_coverage = supplemental
.iter()
.flat_map(|sample| sample.features.iter().cloned())
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
let coverage = selected_coverage
.iter()
.chain(&supplemental_coverage)
.cloned()
.collect::<BTreeSet<_>>()
.into_iter()
.collect();
let report = CorpusReport {
status: if selected.len() == sample_size && failures == 0 {
"verified".into()
} else {
"insufficient".into()
},
provenance: "unspecified-by-probe",
observed,
eligible,
scanned: observed,
loaded: eligible,
load_failures: failures,
selected: selected.len(),
selected_messages: selected.iter().map(|sample| sample.report.messages).sum(),
selected_native_bytes: selected
.iter()
.map(|sample| sample.report.native_bytes)
.sum(),
selected_coverage,
supplemental_coverage,
coverage,
samples: selected
.iter()
.map(|sample| sample.report.clone())
.collect(),
supplemental_samples: supplemental
.iter()
.map(|sample| sample.report.clone())
.collect(),
};
(selected, report)
}
fn captured_baseline_snapshot(
native: &str,
expected: &[BaselineSample],
format: SessionFormat,
) -> Option<(Session, String)> {
expected.iter().find_map(|sample| {
if sample.native_bytes > native.len() || !native.is_char_boundary(sample.native_bytes) {
return None;
}
let prefix = &native[..sample.native_bytes];
let fingerprint = blake3::hash(prefix.as_bytes()).to_hex().to_string();
if fingerprint != sample.fingerprint {
return None;
}
let session = Session::load_str(prefix, format).ok()?;
Some((session, prefix.to_string()))
})
}
fn supplemental_samples(harness: &HarnessId) -> Vec<Sample> {
let fixtures = match harness.as_str() {
HarnessId::CLAUDE_CODE => vec![
"claude_code_session.jsonl",
"claude_code_session_compacted_lineage.jsonl",
],
HarnessId::CODEX => vec![
"codex_real_rollout_tools.jsonl",
"codex_real_rollout_compacted.jsonl",
"codex_real_rollout_patch.jsonl",
],
HarnessId::OPENCODE => vec!["opencode_session.jsonl"],
HarnessId::PI => vec![
"pi_session.jsonl",
"pi_real_corpus_plain_text.jsonl",
"pi_real_corpus_thinking.jsonl",
"pi_real_corpus_tool_call.jsonl",
"pi_real_corpus_error_retry.jsonl",
"pi_real_corpus_aborted.jsonl",
],
HarnessId::GROK => vec!["grok_session/chat_history.jsonl"],
HarnessId::GEMINI => vec!["gemini_session.jsonl"],
HarnessId::GOOSE => vec!["goose_session.json"],
_ => Vec::new(),
};
fixtures
.into_iter()
.map(|relative| {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures")
.join(relative);
let format = format_for(harness.as_str());
let session = match format {
SessionFormat::ClaudeCode => Session::from_claude_code(path),
SessionFormat::Codex => Session::from_codex(path),
SessionFormat::OpenCode => Session::from_opencode(path),
SessionFormat::Pi => Session::from_pi(path),
SessionFormat::Grok => Session::from_grok(path),
SessionFormat::Gemini => Session::from_gemini(path),
SessionFormat::Goose => Session::from_goose(path),
}
.unwrap_or_else(|error| panic!("supplemental fixture {relative} failed: {error}"));
let native = session.to_native_jsonl();
let features = session_features(&session);
Sample {
report: SampleReport {
fingerprint: blake3::hash(native.as_bytes()).to_hex().to_string(),
native_bytes: native.len(),
messages: session.messages.len(),
raw_records: session.raw.len(),
features: features.iter().cloned().collect(),
},
session,
features,
}
})
.collect()
}
fn coverage_seeking_sample(mut candidates: Vec<Sample>, count: usize) -> Vec<Sample> {
let mut selected = Vec::new();
let mut covered = BTreeSet::new();
while selected.len() < count && !candidates.is_empty() {
let best = candidates
.iter()
.enumerate()
.max_by_key(|(index, sample)| {
let gain = sample.features.difference(&covered).count();
(
gain,
std::cmp::Reverse(sample.report.native_bytes),
std::cmp::Reverse(*index),
)
})
.map(|(index, _)| index)
.unwrap();
let sample = candidates.remove(best);
covered.extend(sample.features.iter().cloned());
selected.push(sample);
}
selected
}
fn session_features(session: &Session) -> BTreeSet<String> {
let mut features = BTreeSet::new();
let discriminants = raw_discriminants(session);
if session.messages.iter().any(|message| {
message
.content
.as_deref()
.is_some_and(|text| !text.is_empty())
|| message.content_parts.as_ref().is_some_and(|parts| {
parts
.iter()
.any(|part| part.get("type").and_then(Value::as_str) == Some("text"))
})
}) {
features.insert("text".into());
}
if discriminants.iter().any(|value| {
value.contains("thinking") || value.contains("reasoning") || value.contains("analysis")
}) || session.messages.iter().any(|message| {
message.metadata.keys().any(|key| {
key.contains("thinking") || key.contains("reasoning") || key.contains("analysis")
})
}) {
features.insert("thinking".into());
}
if discriminants.iter().any(|value| {
value.contains("tool")
|| value.contains("function")
|| value.contains("command")
|| value.contains("bash")
}) || session.messages.iter().any(|message| {
message.role == Role::Tool
|| !message.tool_calls().is_empty()
|| message.tool_call_id.is_some()
}) {
features.insert("tools".into());
}
if discriminants
.iter()
.any(|value| value.contains("error") || value.contains("fail") || value.contains("abort"))
|| session.messages.iter().any(|message| {
message.metadata.iter().any(|(key, value)| {
key.contains("error")
|| key.contains("failed")
|| value.eq_ignore_ascii_case("error")
})
})
{
features.insert("errors".into());
}
if discriminants.iter().any(|value| {
value.contains("compact")
|| value.contains("summary")
|| value.contains("rollback")
|| value.contains("revert")
}) || session.messages.iter().any(|message| {
message.metadata.keys().any(|key| {
key.contains("compact") || key.contains("rollback") || key.contains("revert")
})
}) {
features.insert("compaction_or_rollback".into());
}
if !session.subagents.is_empty()
|| session.meta.agent_id.is_some()
|| session.meta.parent_tool_use_id.is_some()
|| !session.meta.lineage.is_empty()
|| discriminants
.iter()
.any(|value| matches!(value.as_str(), "agent" | "task" | "subtask"))
{
features.insert("subagents_or_lineage".into());
}
if session.meta.model.is_some()
|| session.meta.system_prompt.is_some()
|| !session.meta.codex_headers.is_empty()
|| !session.meta.opencode_headers.is_empty()
|| session
.messages
.iter()
.any(|message| !message.metadata.is_empty())
{
features.insert("native_metadata".into());
}
features
}
fn raw_discriminants(session: &Session) -> BTreeSet<String> {
fn visit(value: &Value, output: &mut BTreeSet<String>) {
match value {
Value::Object(object) => {
for (key, value) in object {
if matches!(
key.as_str(),
"type"
| "subtype"
| "kind"
| "role"
| "status"
| "finish"
| "finish_reason"
| "name"
) {
if let Some(value) = value.as_str() {
output.insert(value.to_ascii_lowercase());
}
}
visit(value, output);
}
}
Value::Array(values) => {
for value in values {
visit(value, output);
}
}
_ => {}
}
}
let mut output = BTreeSet::new();
for raw in &session.raw {
if let Ok(value) = serde_json::from_str::<Value>(raw) {
visit(&value, &mut output);
}
}
output
}
fn measure_cell_corpus(
source: SessionFormat,
target: SessionFormat,
samples: &[Sample],
) -> CellReport {
let mut report = CellReport {
source: format_name(source).into(),
target: format_name(target).into(),
..CellReport::default()
};
let mut residue = BTreeSet::new();
for sample in samples {
let source_messages = core_messages(&sample.session.messages);
let exported = match sample.session.to_jsonl(target) {
Ok(exported) => exported,
Err(_) => {
report.export_failures += 1;
continue;
}
};
let reloaded = match Session::load_str(&exported, target) {
Ok(reloaded) => reloaded,
Err(_) => {
report.export_failures += 1;
continue;
}
};
report.sessions += 1;
let target_messages = core_messages(&reloaded.messages);
let metric = measure_fidelity(&source_messages, &target_messages);
report.source_messages += metric.total;
report.matched_messages += metric.matched;
report.intentional_replay_exclusions += metric.residue.compacted_out_excluded;
let replay = replay_eligible(&source_messages);
let replay_metric = measure_fidelity(&replay, &target_messages);
report.replay_messages += replay_metric.total;
report.replay_matched_messages += replay_metric.matched;
let fields = field_fidelity(&source_messages, &target_messages);
if std::env::var_os("SUPERCODE_FIDELITY_DEBUG_LOCATORS").is_some()
&& (fields.hard_loss_messages > 0 || !fields.residue.is_empty())
{
debug_semantic_differences(
&sample.report.fingerprint,
source,
target,
&source_messages,
&target_messages,
);
}
report.hard_loss_messages += fields.hard_loss_messages;
if fields.hard_loss_messages > 0 {
report
.hard_loss_samples
.push(sample.report.fingerprint.clone());
}
report.semantic_fields_total += fields.semantic_total;
report.semantic_fields_retained += fields.semantic_retained;
report.metadata_values_total += fields.metadata_total;
report.metadata_values_retained += fields.metadata_retained;
for (field, count) in fields.residue {
*report.field_residue.entry(field).or_default() += count;
}
for (field, count) in fields.intentional_exclusions {
*report
.intentional_field_exclusions
.entry(field)
.or_default() += count;
}
for (shape, count) in fields.hard_loss_shapes {
*report.hard_loss_shapes.entry(shape).or_default() += count;
}
residue.extend(fields.metadata_residue);
}
report.message_fidelity_percent = percent(report.matched_messages, report.source_messages);
report.replay_fidelity_percent =
percent(report.replay_matched_messages, report.replay_messages);
report.semantic_field_fidelity_percent = percent(
report.semantic_fields_retained,
report.semantic_fields_total,
);
report.metadata_fidelity_percent = percent(
report.metadata_values_retained,
report.metadata_values_total,
);
report.opaque_residue_keys = residue.into_iter().collect();
report
}
#[derive(Default)]
struct FieldCounts {
semantic_total: usize,
semantic_retained: usize,
metadata_total: usize,
metadata_retained: usize,
hard_loss_messages: usize,
residue: BTreeMap<String, usize>,
intentional_exclusions: BTreeMap<String, usize>,
metadata_residue: BTreeSet<String>,
hard_loss_shapes: BTreeMap<String, usize>,
}
fn debug_semantic_differences(
fingerprint: &str,
source_format: SessionFormat,
target_format: SessionFormat,
source: &[ChatMessage],
target: &[ChatMessage],
) {
let mut used_targets = vec![false; target.len()];
let mut last_target_index: Option<usize> = None;
for (source_index, source_message) in source.iter().enumerate() {
if replay_excluded(source_message) {
continue;
}
let found = find_unused_match(source_message, target, &used_targets);
let Some(index) = found else {
let window = target
.iter()
.enumerate()
.skip(last_target_index.unwrap_or_default().saturating_sub(2))
.take(6)
.map(|(index, message)| {
format!(
"{index}:{}:{}:{}",
message_shape(message),
optional_text_hash(message.content.as_deref()),
message_identity_hash(message),
)
})
.collect::<Vec<_>>()
.join("|");
eprintln!(
"fidelity-diff {fingerprint} {}->{} source[{source_index}] unaligned {} content_hash={} identity_hash={} target_window={window}",
format_name(source_format),
format_name(target_format),
message_shape(source_message),
optional_text_hash(source_message.content.as_deref()),
message_identity_hash(source_message),
);
continue;
};
used_targets[index] = true;
let target_message = &target[index];
if last_target_index.is_some_and(|previous| index < previous) {
eprintln!(
"fidelity-diff {fingerprint} {}->{} source[{source_index}]->target[{index}] reordered shape={} content_hash={} identity_hash={}",
format_name(source_format),
format_name(target_format),
message_shape(source_message),
optional_text_hash(source_message.content.as_deref()),
message_identity_hash(source_message),
);
}
if source_message.content != target_message.content
|| source_message.content_parts != target_message.content_parts
|| source_message.name != target_message.name
{
eprintln!(
"fidelity-diff {fingerprint} {}->{} source[{source_index}]->target[{index}] shape={}->{} content={}->{} parts={}->{} name={}->{}",
format_name(source_format),
format_name(target_format),
message_shape(source_message),
message_shape(target_message),
optional_text_hash(source_message.content.as_deref()),
optional_text_hash(target_message.content.as_deref()),
source_message.content_parts.as_ref().map_or(0, Vec::len),
target_message.content_parts.as_ref().map_or(0, Vec::len),
source_message.name.is_some(),
target_message.name.is_some(),
);
}
last_target_index = Some(index);
}
}
fn optional_text_hash(value: Option<&str>) -> String {
value
.map(|value| blake3::hash(value.as_bytes()).to_hex()[..12].to_string())
.unwrap_or_else(|| "none".to_string())
}
fn message_identity_hash(message: &ChatMessage) -> String {
let value = message
.tool_call_id
.as_deref()
.map(str::to_string)
.or_else(|| {
(!message.tool_calls().is_empty()).then(|| {
message
.tool_calls()
.iter()
.map(|call| call.id.as_str())
.collect::<Vec<_>>()
.join("|")
})
});
optional_text_hash(value.as_deref())
}
fn field_fidelity(source: &[ChatMessage], target: &[ChatMessage]) -> FieldCounts {
let mut counts = FieldCounts::default();
let mut used_targets = vec![false; target.len()];
let mut last_target_index = None;
let source_tool_names = source
.iter()
.flat_map(|message| message.tool_calls())
.map(|call| (call.id.as_str(), call.function.name.as_str()))
.collect::<BTreeMap<_, _>>();
for source_message in source {
if replay_excluded(source_message) {
continue;
}
let tool_result_name_is_redundant = source_message.role == Role::Tool
&& source_message
.tool_call_id
.as_deref()
.zip(source_message.name.as_deref())
.is_some_and(|(id, name)| source_tool_names.get(id).copied() == Some(name));
counts.semantic_total +=
semantic_field_count(source_message, tool_result_name_is_redundant);
counts.metadata_total += source_message.metadata.len();
let found = find_unused_match(source_message, target, &used_targets);
if let Some(index) = found {
used_targets[index] = true;
if last_target_index.is_some_and(|previous| index < previous) {
*counts
.residue
.entry("message_reordered".into())
.or_default() += 1;
}
compare_message_fields(
source_message,
&target[index],
tool_result_name_is_redundant,
&mut counts,
);
last_target_index = Some(index);
} else {
counts.hard_loss_messages += 1;
*counts
.hard_loss_shapes
.entry(message_shape(source_message))
.or_default() += 1;
*counts
.residue
.entry("message_unaligned".into())
.or_default() += 1;
for key in source_message.metadata.keys() {
counts.metadata_residue.insert(key.clone());
}
}
}
counts
}
fn find_unused_match(
source: &ChatMessage,
target: &[ChatMessage],
used_targets: &[bool],
) -> Option<usize> {
target
.iter()
.enumerate()
.find(|(index, message)| !used_targets[*index] && semantic_messages_equal(source, message))
.map(|(index, _)| index)
.or_else(|| {
target
.iter()
.enumerate()
.find(|(index, message)| !used_targets[*index] && messages_align(source, message))
.map(|(index, _)| index)
})
}
fn message_shape(message: &ChatMessage) -> String {
format!(
"role={:?};content={};parts={};calls={};tool_call_id={};name={}",
message.role,
message.content.is_some(),
message.content_parts.as_ref().map_or(0, Vec::len),
message.tool_calls().len(),
message.tool_call_id.is_some(),
message.name.is_some(),
)
.to_ascii_lowercase()
}
fn messages_align(source: &ChatMessage, target: &ChatMessage) -> bool {
if semantic_messages_equal(source, target) {
return true;
}
if source.role != target.role {
return false;
}
if source.content == target.content && source.content.is_some() {
return true;
}
if source.tool_call_id.is_some() && source.tool_call_id == target.tool_call_id {
return true;
}
let source_calls = source.tool_calls();
let target_calls = target.tool_calls();
if source.content.is_none()
&& target.content.is_none()
&& source_calls.is_empty()
&& target_calls.is_empty()
{
return true;
}
!source_calls.is_empty()
&& source_calls.iter().any(|source_call| {
target_calls
.iter()
.any(|target_call| source_call.id == target_call.id)
})
}
fn semantic_messages_equal(source: &ChatMessage, target: &ChatMessage) -> bool {
if source.role != Role::Tool {
return messages_equal_multimodal(source, target);
}
let mut target_without_redundant_name = target.clone();
target_without_redundant_name.name = source.name.clone();
messages_equal_multimodal(source, &target_without_redundant_name)
}
fn semantic_sequences_equal(source: &[ChatMessage], target: &[ChatMessage]) -> bool {
if source.len() != target.len() {
return false;
}
let source_tool_names = source
.iter()
.flat_map(|message| message.tool_calls())
.map(|call| (call.id.as_str(), call.function.name.as_str()))
.collect::<BTreeMap<_, _>>();
source.iter().zip(target).all(|(left, right)| {
let redundant_name = left.role == Role::Tool
&& left.tool_call_id.as_deref().is_some_and(|id| {
source_tool_names.get(id).is_some_and(|call_name| {
left.name.as_deref().is_none_or(|name| name == *call_name)
&& right.name.as_deref().is_none_or(|name| name == *call_name)
})
});
if !redundant_name {
return messages_equal_multimodal(left, right);
}
let mut right_without_redundant_name = right.clone();
right_without_redundant_name.name = left.name.clone();
messages_equal_multimodal(left, &right_without_redundant_name)
})
}
fn compare_message_fields(
source: &ChatMessage,
target: &ChatMessage,
tool_result_name_is_redundant: bool,
counts: &mut FieldCounts,
) {
compare_field(counts, "role", source.role == target.role);
if source.content.is_some() {
compare_field(counts, "content", source.content == target.content);
}
if source.name.is_some() && tool_result_name_is_redundant {
*counts
.intentional_exclusions
.entry("tool_result.name_redundant_with_call".into())
.or_default() += 1;
} else if source.name.is_some() {
compare_field(counts, "name", source.name == target.name);
}
if source.tool_call_id.is_some() {
compare_field(
counts,
"tool_call_id",
source.tool_call_id == target.tool_call_id,
);
}
if let Some(parts) = &source.content_parts {
for (index, part) in parts.iter().enumerate() {
compare_field(
counts,
"content_part",
target
.content_parts
.as_ref()
.and_then(|target_parts| target_parts.get(index))
== Some(part),
);
}
}
let target_calls = target.tool_calls();
for source_call in source.tool_calls() {
let target_call = target_calls
.iter()
.find(|target_call| target_call.id == source_call.id);
compare_field(counts, "tool_call.id", target_call.is_some());
compare_field(
counts,
"tool_call.name",
target_call
.is_some_and(|target_call| target_call.function.name == source_call.function.name),
);
let source_arguments = source_call.function.parsed_arguments().ok();
let target_arguments =
target_call.and_then(|target_call| target_call.function.parsed_arguments().ok());
if source_arguments == target_arguments {
counts.semantic_retained += 1;
} else {
let field = format!(
"tool_call.arguments:{}->{}",
json_kind(source_arguments.as_ref()),
json_kind(target_arguments.as_ref())
);
*counts.residue.entry(field).or_default() += 1;
}
}
for (key, value) in &source.metadata {
if target.metadata.get(key) == Some(value) {
counts.metadata_retained += 1;
} else {
counts.metadata_residue.insert(key.clone());
}
}
}
fn json_kind(value: Option<&Value>) -> &'static str {
match value {
None => "invalid_or_absent",
Some(Value::Null) => "null",
Some(Value::Bool(_)) => "boolean",
Some(Value::Number(_)) => "number",
Some(Value::String(_)) => "string",
Some(Value::Array(_)) => "array",
Some(Value::Object(_)) => "object",
}
}
fn compare_field(counts: &mut FieldCounts, field: &str, retained: bool) {
if retained {
counts.semantic_retained += 1;
} else {
*counts.residue.entry(field.into()).or_default() += 1;
}
}
fn replay_excluded(message: &ChatMessage) -> bool {
message.metadata.get("compacted_out").map(String::as_str) == Some("true")
|| message
.metadata
.get("pi_exclude_from_context")
.map(String::as_str)
== Some("true")
}
fn semantic_field_count(message: &ChatMessage, tool_result_name_is_redundant: bool) -> usize {
1 + usize::from(message.content.is_some())
+ usize::from(message.name.is_some() && !tool_result_name_is_redundant)
+ usize::from(message.tool_call_id.is_some())
+ message.content_parts.as_ref().map_or(0, Vec::len)
+ message.tool_calls().len().saturating_mul(3)
}
fn measure_round_trip_corpus(
source: SessionFormat,
intermediate: SessionFormat,
samples: &[Sample],
) -> RoundTripReport {
let mut report = RoundTripReport {
source: format_name(source).into(),
intermediate: format_name(intermediate).into(),
..RoundTripReport::default()
};
for sample in samples {
report.sessions += 1;
let restored = match restore_round_trip(&sample.session, source, intermediate) {
Ok(restored) => restored,
Err(kind) => {
record_round_trip_discrepancy(&mut report, kind, sample);
continue;
}
};
let source_core = core_messages(&sample.session.messages);
let expected = if source == intermediate {
source_core
} else {
replay_eligible(&source_core)
};
let got = core_messages(&restored.messages);
let semantic_exact = semantic_sequences_equal(&expected, &got);
if semantic_exact {
report.semantic_exact += 1;
} else {
record_round_trip_discrepancy(&mut report, "semantic_messages", sample);
}
let fields = field_fidelity(&expected, &got);
report.hard_loss_messages += fields.hard_loss_messages;
for (field, count) in fields.residue {
*report.field_residue.entry(field).or_default() += count;
}
let metadata_exact = semantic_exact
&& expected
.iter()
.zip(&got)
.all(|(left, right)| left.metadata == right.metadata);
if metadata_exact {
report.metadata_exact += 1;
} else {
record_round_trip_discrepancy(&mut report, "metadata_values", sample);
}
let original_native = native_body(&sample.session.to_native_jsonl());
let restored_native = native_body(&restored.to_native_jsonl());
if original_native.as_bytes() == restored_native.as_bytes() {
report.raw_byte_exact += 1;
report.raw_value_exact += 1;
} else if native_values_equal(&original_native, &restored_native) {
report.raw_value_exact += 1;
record_round_trip_discrepancy(&mut report, "raw_bytes_only", sample);
} else {
record_round_trip_discrepancy(&mut report, "raw_or_residue_values", sample);
}
}
report
}
fn restore_round_trip(
session: &Session,
source: SessionFormat,
intermediate: SessionFormat,
) -> Result<Session, &'static str> {
if source == intermediate {
return Session::from_native_str(&session.to_native_jsonl())
.map_err(|_| "native_reload_error");
}
let first = session
.to_jsonl(intermediate)
.map_err(|_| "first_export_error")?;
let mid = Session::load_str(&first, intermediate).map_err(|_| "intermediate_load_error")?;
let back = mid.to_jsonl(source).map_err(|_| "return_export_error")?;
Session::load_str(&back, source).map_err(|_| "return_load_error")
}
fn record_round_trip_discrepancy(report: &mut RoundTripReport, kind: &str, sample: &Sample) {
*report.discrepancies.entry(kind.into()).or_default() += 1;
report
.discrepancy_samples
.entry(kind.into())
.or_default()
.push(sample.report.fingerprint.clone());
}
fn native_body(native: &str) -> String {
native
.split_once('\n')
.map(|(_, body)| body)
.unwrap_or_default()
.to_string()
}
fn native_values_equal(left: &str, right: &str) -> bool {
let parse = |input: &str| {
input
.lines()
.filter(|line| !line.trim().is_empty())
.map(serde_json::from_str::<Value>)
.collect::<Result<Vec<_>, _>>()
};
matches!((parse(left), parse(right)), (Ok(left), Ok(right)) if left == right)
}
fn percent(retained: usize, total: usize) -> f64 {
if total == 0 {
100.0
} else {
retained as f64 / total as f64 * 100.0
}
}
fn executable_version(program: String) -> VersionReport {
let output = Command::new(&program).arg("--version").output();
match output {
Ok(output) if output.status.success() => {
let version = String::from_utf8_lossy(&output.stdout).trim().to_string();
VersionReport {
program,
available: true,
version: Some(if version.is_empty() {
String::from_utf8_lossy(&output.stderr).trim().to_string()
} else {
version
}),
}
}
_ => VersionReport {
program,
available: false,
version: None,
},
}
}
fn capture_date() -> String {
Command::new("date")
.args(["-u", "+%Y-%m-%dT%H:%M:%SZ"])
.output()
.ok()
.filter(|output| output.status.success())
.map(|output| String::from_utf8_lossy(&output.stdout).trim().to_string())
.filter(|value| !value.is_empty())
.unwrap_or_else(|| "unavailable".into())
}
#[cfg(test)]
mod tests {
use super::*;
use supercode_harness::{FunctionCall, ToolCall};
fn message(role: Role, name: Option<&str>, tool_call_id: Option<&str>) -> ChatMessage {
ChatMessage {
role,
content: Some("payload".into()),
content_parts: None,
tool_calls: None,
tool_call_id: tool_call_id.map(str::to_string),
name: name.map(str::to_string),
metadata: BTreeMap::new(),
}
}
fn call(name: &str) -> ChatMessage {
ChatMessage {
role: Role::Assistant,
content: None,
content_parts: None,
tool_calls: Some(vec![ToolCall {
id: "call-1".into(),
kind: "function".into(),
function: FunctionCall {
name: name.into(),
arguments: "{}".into(),
},
}]),
tool_call_id: None,
name: None,
metadata: BTreeMap::new(),
}
}
#[test]
fn baseline_snapshot_reconstructs_an_append_only_session_prefix() {
let captured = concat!(
"{\"type\":\"user\",\"message\":{\"role\":\"user\",\"content\":\"before\"},",
"\"uuid\":\"u1\",\"sessionId\":\"s1\"}\n"
);
let current = format!(
"{captured}{}",
"{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"content\":\"after\"},\"uuid\":\"a1\",\"parentUuid\":\"u1\",\"sessionId\":\"s1\"}\n"
);
let baseline = BaselineSample {
fingerprint: blake3::hash(captured.as_bytes()).to_hex().to_string(),
native_bytes: captured.len(),
};
let (session, native) =
captured_baseline_snapshot(¤t, &[baseline], SessionFormat::ClaudeCode)
.expect("append-only current log must reproduce the captured prefix");
assert_eq!(native, captured);
assert_eq!(session.messages.len(), 1);
assert_eq!(session.messages[0].content.as_deref(), Some("before"));
}
#[test]
fn round_trip_equivalence_allows_only_names_redundant_with_a_tool_call() {
let source = vec![call("read_file"), message(Role::Tool, None, Some("call-1"))];
let enriched = vec![
call("read_file"),
message(Role::Tool, Some("read_file"), Some("call-1")),
];
let changed = vec![
call("read_file"),
message(Role::Tool, Some("write_file"), Some("call-1")),
];
assert!(semantic_sequences_equal(&source, &enriched));
assert!(!semantic_sequences_equal(&source, &changed));
}
#[test]
fn round_trip_equivalence_preserves_orphan_tool_result_names() {
let source = vec![message(Role::Tool, Some("read_file"), Some("orphan"))];
let missing_name = vec![message(Role::Tool, None, Some("orphan"))];
assert!(!semantic_sequences_equal(&source, &missing_name));
let fields = field_fidelity(&source, &missing_name);
assert_eq!(fields.semantic_total, 4);
assert_eq!(fields.semantic_retained, 3);
assert_eq!(fields.residue.get("name"), Some(&1));
}
}