use anyhow::Result;
use super::types::{CodingBenchFailureReason, CodingMemoryAttribution};
use super::*;
const TEST_RUNS_PER_CONDITION: usize = MIN_RUNS_PER_CONDITION;
const TEST_FINAL_HEAD_SHA: &str = "0123456789abcdef0123456789abcdef01234567";
fn score_evidence() -> CodingBenchRunScoreEvidence {
CodingBenchRunScoreEvidence {
commands: vec![CodingBenchScoreCommandEvidence {
command: vec![
"cargo".to_string(),
"test".to_string(),
"--test".to_string(),
"fixture".to_string(),
],
exit_code: 0,
stdout: Some("test result: ok".to_string()),
stderr: None,
output_artifact_path: None,
}],
}
}
fn metrics() -> CodingBenchRunMetrics {
CodingBenchRunMetrics {
tokens_input: Some(100),
tokens_output: Some(25),
tokens_total: Some(125),
token_accounting_unsupported_reason: None,
turns: Some(3),
wall_time_ms: Some(4_000),
}
}
fn memory_contract() -> CodingMemoryAttribution {
CodingMemoryAttribution {
injected_memory_ids: vec![1],
used_memory_ids: vec![1],
citation_precision: 1.0,
citation_recall: 1.0,
stale_used_count: 0,
irrelevant_injection_count: 0,
missing_relevant_memory_count: 0,
memory_helped: true,
memory_hurt: false,
}
}
fn context_audit_snapshot() -> Result<RememContextAuditSnapshot> {
let entry = crate::context_bundle::AuditEntry {
stable_key: "memory:1".to_string(),
channel: crate::context_bundle::ChannelKind::Core,
source_kind: crate::context_bundle::SourceKind::Canonical,
validity: crate::context_bundle::ItemValidity::Current,
selected: true,
reason: "selected_channel".to_string(),
relevance_score: Some(0.75),
token_estimate: 10,
};
let audit = crate::context_bundle::ContextAudit {
schema_version: crate::context_bundle::CONTEXT_BUNDLE_SCHEMA_VERSION,
policy_version: "retrieval_router_v2".to_string(),
relevance_policy_version: "sessionstart_significant_token_v1".to_string(),
plan_hash: "a".repeat(64),
degraded_mode: crate::context_bundle::DegradedMode::Full,
candidates_considered: 1,
selected_count: 1,
dropped_count: 0,
token_estimate: 10,
token_budget: 100,
truncation_reason: None,
entries: vec![entry],
};
let (canonical_audit_json, audit_hash) =
crate::context_bundle::persistence::canonical_context_audit(
&audit,
crate::retrieval_router::RETRIEVAL_PLAN_SCHEMA_VERSION,
)?;
let injection_run_id = "coding-bench-run-1".to_string();
let injection_binding_hash =
super::audit_contract::context_audit_binding_hash(&injection_run_id, &audit_hash);
Ok(RememContextAuditSnapshot {
injection_run_id,
bundle_schema_version: audit.schema_version,
plan_schema_version: crate::retrieval_router::RETRIEVAL_PLAN_SCHEMA_VERSION,
policy_version: audit.policy_version,
relevance_policy_version: audit.relevance_policy_version,
plan_hash: audit.plan_hash,
audit_hash,
injection_binding_hash,
degraded_mode: audit.degraded_mode,
candidates_considered: audit.candidates_considered,
selected_count: audit.selected_count,
dropped_count: audit.dropped_count,
token_budget: audit.token_budget,
token_estimate: audit.token_estimate,
truncation_reason: audit.truncation_reason,
canonical_audit_json,
})
}
fn valid_remem_contract_snapshot(
contract_report: crate::eval::current_memory_contracts::CurrentMemoryContractEvalReport,
captured_at_epoch: i64,
) -> Result<RememContractSnapshot> {
Ok(build_remem_contract_snapshot(
contract_report,
captured_at_epoch,
Some(context_audit_snapshot()?),
))
}
fn remem_run(snapshot: RememContractSnapshot) -> CodingBenchRunReport {
remem_run_for_task(snapshot, "fixture-task", 0)
}
fn remem_run_for_task(
snapshot: RememContractSnapshot,
task_id: &str,
run_index: usize,
) -> CodingBenchRunReport {
CodingBenchRunReport {
condition: CodingBenchCondition::RememSeededSessionStart,
task_id: task_id.to_string(),
run_index,
task_success: true,
task_failure_reason: None,
memory_contract_status: if snapshot.contract_health.all_checks_passed {
CodingBenchMemoryContractStatus::Passed
} else {
CodingBenchMemoryContractStatus::Failed
},
runtime_contract_failure: !snapshot.contract_health.all_checks_passed,
runtime_contract_failure_reason: (!snapshot.contract_health.all_checks_passed)
.then(|| snapshot.contract_health.failing_examples.join("; ")),
context_audit_status: RememContextAuditStatus::Verified,
context_audit_failure_reason: None,
score: score_evidence(),
metrics: metrics(),
final_head_sha: Some(TEST_FINAL_HEAD_SHA.to_string()),
patch_artifact_path: None,
unauthorized_path_changes: Vec::new(),
memory_contract: Some(memory_contract()),
remem_contract_snapshot: Some(snapshot),
}
}
fn control_run(
condition: CodingBenchCondition,
task_id: &str,
run_index: usize,
) -> CodingBenchRunReport {
CodingBenchRunReport {
condition,
task_id: task_id.to_string(),
run_index,
task_success: true,
task_failure_reason: None,
memory_contract_status: CodingBenchMemoryContractStatus::NotApplicable,
runtime_contract_failure: false,
runtime_contract_failure_reason: None,
context_audit_status: RememContextAuditStatus::NotApplicable,
context_audit_failure_reason: None,
score: score_evidence(),
metrics: metrics(),
final_head_sha: Some(TEST_FINAL_HEAD_SHA.to_string()),
patch_artifact_path: None,
unauthorized_path_changes: Vec::new(),
memory_contract: None,
remem_contract_snapshot: None,
}
}
fn condition_report(
name: CodingBenchCondition,
task_id: &str,
) -> Result<CodingBenchConditionReport> {
let runs = match name {
CodingBenchCondition::RememSeededSessionStart => {
let snapshot = valid_remem_contract_snapshot(
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?,
1_800_000_000,
)?;
(0..TEST_RUNS_PER_CONDITION)
.map(|run_index| remem_run_for_task(snapshot.clone(), task_id, run_index))
.collect()
}
CodingBenchCondition::NoMemory | CodingBenchCondition::CuratedFileExpert => (0
..TEST_RUNS_PER_CONDITION)
.map(|run_index| control_run(name, task_id, run_index))
.collect(),
};
Ok(CodingBenchConditionReport { name, runs })
}
fn report_with_run(run: CodingBenchRunReport) -> Result<CodingBenchReport> {
let task_id = run.task_id.clone();
let mut report = CodingBenchReport {
schema_version: 1,
benchmark_spec_path: CODING_AGENT_AB_SPEC_PATH,
current_memory_contract_spec_path: CURRENT_MEMORY_CONTRACT_SPEC_PATH,
runs_per_condition: TEST_RUNS_PER_CONDITION,
conditions: vec![
condition_report(CodingBenchCondition::RememSeededSessionStart, &task_id)?,
condition_report(CodingBenchCondition::NoMemory, &task_id)?,
condition_report(CodingBenchCondition::CuratedFileExpert, &task_id)?,
],
};
let target_condition = report
.conditions
.iter_mut()
.find(|condition| condition.name == run.condition);
if let Some(condition) = target_condition {
let run_index = run.run_index;
if run_index < condition.runs.len() {
condition.runs[run_index] = run;
} else {
condition.runs.push(run);
}
}
Ok(report)
}
#[test]
fn remem_run_artifact_includes_current_memory_contract_snapshot() -> Result<()> {
let contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
let snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
let report = report_with_run(remem_run(snapshot))?;
validate_contract_snapshots(&report)?;
let json = serde_json::to_value(&report)?;
let run = &json["conditions"][0]["runs"][0];
assert_eq!(json["conditions"][0]["name"], "remem_seeded_sessionstart");
assert_eq!(run["condition"], "remem_seeded_sessionstart");
let run_object = run.as_object().expect("run serializes as an object");
assert_eq!(run["resolved"], true);
assert!(!run_object.contains_key("task_success"));
assert_eq!(run["metrics"]["tokens_total"], 125);
assert_eq!(run["metrics"]["turns"], 3);
assert_eq!(run["metrics"]["wall_time_ms"], 4_000);
assert_eq!(run["score"]["commands"][0]["command"][0], "cargo");
assert_eq!(run["score"]["commands"][0]["stdout"], "test result: ok");
assert_eq!(run["final_head_sha"], TEST_FINAL_HEAD_SHA);
assert_eq!(run["patch_artifact_path"], serde_json::Value::Null);
assert!(run["unauthorized_path_changes"]
.as_array()
.is_some_and(|changes| changes.is_empty()));
assert_eq!(run["memory_contract_status"], "passed");
assert_eq!(run["context_audit_status"], "verified");
assert_eq!(
run["remem_contract_snapshot"]["context_audit"]["plan_hash"],
"a".repeat(64)
);
assert_eq!(
run["remem_contract_snapshot"]["context_audit"]["selected_count"],
1
);
assert_eq!(
run["remem_contract_snapshot"]["contract_health"]["all_checks_passed"],
true
);
assert_eq!(
run["remem_contract_snapshot"]["citation_precision"]["rate"],
1.0
);
assert_eq!(
run["remem_contract_snapshot"]["staleness_handling"]["verify_before_trust"]["rate"],
1.0
);
assert_eq!(
run["remem_contract_snapshot"]["staleness_handling"]["history_tracked"]["rate"],
1.0
);
assert_eq!(
run["remem_contract_snapshot"]["temporal_fact_eligibility"]["invalidated_fact_exclusion"]
["rate"],
1.0
);
assert_eq!(
run["remem_contract_snapshot"]["injected_memory_audit"]["injected"]["rate"],
1.0
);
assert_eq!(
run["remem_contract_snapshot"]["usage_feedback_coverage"]
["usage_event_linked_to_injection_item"]["rate"],
1.0
);
assert_eq!(
run["remem_contract_snapshot"]["current_memory_contracts"]["metrics"]["all_checks_passed"],
true
);
Ok(())
}
#[test]
fn missing_context_audit_is_a_remem_contract_failure() -> Result<()> {
let contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
let mut snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
snapshot.context_audit = None;
let mut run = remem_run(snapshot);
run.context_audit_status = RememContextAuditStatus::ContractFailure;
run.context_audit_failure_reason = Some("missing persisted ContextAudit".to_string());
run.memory_contract_status = CodingBenchMemoryContractStatus::Failed;
run.runtime_contract_failure = true;
run.runtime_contract_failure_reason = Some("missing persisted ContextAudit".to_string());
let report = report_with_run(run)?;
validate_contract_snapshots(&report)?;
Ok(())
}
#[test]
fn verifier_rejects_tampered_context_audit_hash() -> Result<()> {
let contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
let mut snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
snapshot
.context_audit
.as_mut()
.expect("fixture audit")
.audit_hash = "b".repeat(64);
let report = report_with_run(remem_run(snapshot))?;
assert!(validate_contract_snapshots(&report)
.unwrap_err()
.to_string()
.contains("hash mismatch"));
Ok(())
}
#[test]
fn runtime_contract_failure_is_distinct_from_agent_task_failure() -> Result<()> {
let mut contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
contract_report.metrics.all_checks_passed = false;
contract_report
.failing_examples
.push("usage.citation_event_matched expected 1.0 but got 0.0".to_string());
let snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
let report = report_with_run(remem_run(snapshot))?;
validate_contract_snapshots(&report)?;
let json = serde_json::to_value(&report)?;
let run = &json["conditions"][0]["runs"][0];
assert_eq!(run["resolved"], true);
assert_eq!(run["failure_reason"], serde_json::Value::Null);
assert!(run
.as_object()
.is_some_and(|object| !object.contains_key("task_failure_reason")));
assert_eq!(run["memory_contract_status"], "failed");
assert_eq!(run["runtime_contract_failure"], true);
assert!(run["runtime_contract_failure_reason"]
.as_str()
.is_some_and(|reason| reason.contains("usage.citation_event_matched")));
Ok(())
}
#[test]
fn validator_rejects_stale_runtime_contract_failure_reason() -> Result<()> {
let contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
let snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
let mut run = remem_run(snapshot);
run.runtime_contract_failure = false;
run.runtime_contract_failure_reason = Some("stale contract failure".to_string());
let report = report_with_run(run)?;
assert!(validate_contract_snapshots(&report)
.unwrap_err()
.to_string()
.contains("stale runtime_contract_failure_reason"));
Ok(())
}
#[test]
fn validator_rejects_blank_runtime_contract_failure_reason() -> Result<()> {
let mut contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
contract_report.metrics.all_checks_passed = false;
contract_report
.failing_examples
.push("usage.citation_event_matched expected 1.0 but got 0.0".to_string());
let snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
let mut run = remem_run(snapshot);
run.runtime_contract_failure_reason = Some(" \n\t ".to_string());
let report = report_with_run(run)?;
assert!(validate_contract_snapshots(&report)
.unwrap_err()
.to_string()
.contains("runtime contract failure without reason"));
Ok(())
}
#[test]
fn validator_rejects_stale_task_failure_reason_on_resolved_run() -> Result<()> {
let contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
let snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
let mut run = remem_run(snapshot);
run.task_success = true;
run.task_failure_reason = Some(CodingBenchFailureReason::TestFailure);
let report = report_with_run(run)?;
assert!(validate_contract_snapshots(&report)
.unwrap_err()
.to_string()
.contains("stale task_failure_reason"));
Ok(())
}
#[test]
fn validator_requires_score_and_patch_evidence() -> Result<()> {
let contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
let snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
let mut missing_score = remem_run(snapshot.clone());
missing_score.score.commands.clear();
assert!(
validate_contract_snapshots(&report_with_run(missing_score)?)
.unwrap_err()
.to_string()
.contains("missing score command evidence")
);
let mut missing_score_output = remem_run(snapshot.clone());
missing_score_output.score.commands[0].stdout = Some(" ".to_string());
missing_score_output.score.commands[0].stderr = None;
missing_score_output.score.commands[0].output_artifact_path = None;
assert!(
validate_contract_snapshots(&report_with_run(missing_score_output)?)
.unwrap_err()
.to_string()
.contains("score command 0 has no output evidence")
);
let mut patch_only = remem_run(snapshot.clone());
patch_only.final_head_sha = None;
patch_only.patch_artifact_path = Some("artifacts/fixture-task-0.patch".to_string());
validate_contract_snapshots(&report_with_run(patch_only)?)?;
let mut missing_patch_evidence = remem_run(snapshot.clone());
missing_patch_evidence.final_head_sha = None;
missing_patch_evidence.patch_artifact_path = None;
assert!(
validate_contract_snapshots(&report_with_run(missing_patch_evidence)?)
.unwrap_err()
.to_string()
.contains("missing final_head_sha or patch_artifact_path")
);
let mut blank_unauthorized_path = remem_run(snapshot);
blank_unauthorized_path
.unauthorized_path_changes
.push(" ".to_string());
assert!(
validate_contract_snapshots(&report_with_run(blank_unauthorized_path)?)
.unwrap_err()
.to_string()
.contains("blank unauthorized path change")
);
Ok(())
}
#[test]
fn validator_requires_token_accounting_or_unsupported_provider_note() -> Result<()> {
let contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
let snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
let mut missing_tokens = remem_run(snapshot.clone());
missing_tokens.metrics.tokens_input = None;
missing_tokens.metrics.tokens_output = None;
missing_tokens.metrics.tokens_total = None;
assert!(
validate_contract_snapshots(&report_with_run(missing_tokens.clone())?)
.unwrap_err()
.to_string()
.contains("missing token accounting without token_accounting_unsupported_reason")
);
let mut unsupported_provider = missing_tokens;
unsupported_provider
.metrics
.token_accounting_unsupported_reason =
Some("provider does not expose token usage for coding-bench runs".to_string());
validate_contract_snapshots(&report_with_run(unsupported_provider)?)?;
let mut partial_tokens = remem_run(snapshot.clone());
partial_tokens.metrics.tokens_output = None;
assert!(
validate_contract_snapshots(&report_with_run(partial_tokens)?)
.unwrap_err()
.to_string()
.contains("complete token accounting")
);
let mut mismatched_total = remem_run(snapshot);
mismatched_total.metrics.tokens_total = Some(126);
assert!(
validate_contract_snapshots(&report_with_run(mismatched_total)?)
.unwrap_err()
.to_string()
.contains("tokens_total=126 does not equal tokens_input + tokens_output")
);
Ok(())
}
#[test]
fn validator_requires_turns_and_wall_time_metrics() -> Result<()> {
let contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
let snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
let mut missing_turns = remem_run(snapshot.clone());
missing_turns.metrics.turns = None;
assert!(
validate_contract_snapshots(&report_with_run(missing_turns)?)
.unwrap_err()
.to_string()
.contains("missing turns")
);
let mut missing_wall_time = remem_run(snapshot);
missing_wall_time.metrics.wall_time_ms = None;
assert!(
validate_contract_snapshots(&report_with_run(missing_wall_time)?)
.unwrap_err()
.to_string()
.contains("missing wall_time_ms")
);
Ok(())
}
#[test]
fn validator_requires_full_condition_matrix() -> Result<()> {
let contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
let snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
let report = report_with_run(remem_run(snapshot))?;
let mut missing_condition = report.clone();
missing_condition
.conditions
.retain(|condition| condition.name != CodingBenchCondition::NoMemory);
assert!(validate_contract_snapshots(&missing_condition)
.unwrap_err()
.to_string()
.contains("missing required NoMemory condition"));
let mut too_few_repeats = report.clone();
for condition in &mut too_few_repeats.conditions {
if condition.name == CodingBenchCondition::CuratedFileExpert {
condition.runs.pop();
}
}
assert!(validate_contract_snapshots(&too_few_repeats)
.unwrap_err()
.to_string()
.contains("CuratedFileExpert condition missing run fixture-task#2"));
let mut invalid_repeat_contract = report;
invalid_repeat_contract.runs_per_condition = MIN_RUNS_PER_CONDITION - 1;
assert!(validate_contract_snapshots(&invalid_repeat_contract)
.unwrap_err()
.to_string()
.contains("below required minimum"));
Ok(())
}
#[test]
fn validator_derives_contract_health_from_embedded_report() -> Result<()> {
let contract_report =
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?;
let mut snapshot = valid_remem_contract_snapshot(contract_report, 1_800_000_000)?;
snapshot.current_memory_contracts.metrics.all_checks_passed = false;
let report = report_with_run(remem_run(snapshot))?;
assert!(validate_contract_snapshots(&report)
.unwrap_err()
.to_string()
.contains("contract_health does not match embedded current_memory_contracts"));
Ok(())
}
#[test]
fn validator_requires_snapshots_only_for_remem_runs() -> Result<()> {
let missing_snapshot = report_with_run(CodingBenchRunReport {
condition: CodingBenchCondition::RememSeededSessionStart,
task_id: "missing-snapshot".to_string(),
run_index: 0,
task_success: true,
task_failure_reason: None,
memory_contract_status: CodingBenchMemoryContractStatus::Passed,
runtime_contract_failure: false,
runtime_contract_failure_reason: None,
context_audit_status: RememContextAuditStatus::Verified,
context_audit_failure_reason: None,
score: score_evidence(),
metrics: metrics(),
final_head_sha: Some(TEST_FINAL_HEAD_SHA.to_string()),
patch_artifact_path: None,
unauthorized_path_changes: Vec::new(),
memory_contract: Some(memory_contract()),
remem_contract_snapshot: None,
})?;
assert!(validate_contract_snapshots(&missing_snapshot)
.unwrap_err()
.to_string()
.contains("missing current memory contract snapshot"));
let snapshot = valid_remem_contract_snapshot(
crate::eval::current_memory_contracts::run_current_memory_contracts_eval()?,
1_800_000_000,
)?;
let non_remem_snapshot = report_with_run(CodingBenchRunReport {
condition: CodingBenchCondition::NoMemory,
task_id: "bad-no-memory".to_string(),
run_index: 0,
task_success: true,
task_failure_reason: None,
memory_contract_status: CodingBenchMemoryContractStatus::NotApplicable,
runtime_contract_failure: false,
runtime_contract_failure_reason: None,
context_audit_status: RememContextAuditStatus::NotApplicable,
context_audit_failure_reason: None,
score: score_evidence(),
metrics: metrics(),
final_head_sha: Some(TEST_FINAL_HEAD_SHA.to_string()),
patch_artifact_path: None,
unauthorized_path_changes: Vec::new(),
memory_contract: None,
remem_contract_snapshot: Some(snapshot),
})?;
assert!(validate_contract_snapshots(&non_remem_snapshot)
.unwrap_err()
.to_string()
.contains("must not carry a remem contract snapshot"));
let non_remem_contract_failure = report_with_run(CodingBenchRunReport {
condition: CodingBenchCondition::CuratedFileExpert,
task_id: "bad-curated-file".to_string(),
run_index: 0,
task_success: true,
task_failure_reason: None,
memory_contract_status: CodingBenchMemoryContractStatus::NotApplicable,
runtime_contract_failure: true,
runtime_contract_failure_reason: Some("current-memory contract failed".to_string()),
context_audit_status: RememContextAuditStatus::NotApplicable,
context_audit_failure_reason: None,
score: score_evidence(),
metrics: metrics(),
final_head_sha: Some(TEST_FINAL_HEAD_SHA.to_string()),
patch_artifact_path: None,
unauthorized_path_changes: Vec::new(),
memory_contract: None,
remem_contract_snapshot: None,
})?;
assert!(validate_contract_snapshots(&non_remem_contract_failure)
.unwrap_err()
.to_string()
.contains("must not report remem runtime contract failure"));
Ok(())
}