1use anyhow::{bail, Result};
2use serde::Serialize;
3use std::collections::{BTreeMap, BTreeSet};
4
5use crate::eval::current_memory_contracts::{
6 CurrentMemoryContractEvalReport, CurrentMemoryContractRateMetric,
7};
8
9use super::types::{CodingBenchFailureReason, CodingMemoryAttribution};
10
11pub const CODING_AGENT_AB_SPEC_PATH: &str = "docs/specs/issue385-coding-agent-ab/TECH.md";
12pub const CURRENT_MEMORY_CONTRACT_SPEC_PATH: &str = "docs/specs/current-memory-contracts/TECH.md";
13pub const MIN_RUNS_PER_CONDITION: usize = 3;
14const REQUIRED_CONDITIONS: [CodingBenchCondition; 3] = [
15 CodingBenchCondition::Remem,
16 CodingBenchCondition::NoMemory,
17 CodingBenchCondition::CuratedFile,
18];
19
20#[derive(Debug, Clone, Serialize, PartialEq)]
21pub struct CodingBenchReport {
22 pub schema_version: u32,
23 pub benchmark_spec_path: &'static str,
24 pub current_memory_contract_spec_path: &'static str,
25 pub runs_per_condition: usize,
26 pub conditions: Vec<CodingBenchConditionReport>,
27}
28
29#[derive(Debug, Clone, Serialize, PartialEq)]
30pub struct CodingBenchConditionReport {
31 pub name: CodingBenchCondition,
32 pub runs: Vec<CodingBenchRunReport>,
33}
34
35#[derive(Debug, Clone, Serialize, PartialEq)]
36pub struct CodingBenchRunReport {
37 pub condition: CodingBenchCondition,
38 pub task_id: String,
39 pub run_index: usize,
40 #[serde(rename = "resolved")]
41 pub task_success: bool,
42 #[serde(rename = "failure_reason")]
43 pub task_failure_reason: Option<CodingBenchFailureReason>,
44 pub memory_contract_status: CodingBenchMemoryContractStatus,
45 pub runtime_contract_failure: bool,
46 pub runtime_contract_failure_reason: Option<String>,
47 pub score: CodingBenchRunScoreEvidence,
48 pub metrics: CodingBenchRunMetrics,
49 pub final_head_sha: Option<String>,
50 pub patch_artifact_path: Option<String>,
51 pub unauthorized_path_changes: Vec<String>,
52 #[serde(skip_serializing_if = "Option::is_none")]
53 pub memory_contract: Option<CodingMemoryAttribution>,
54 pub remem_contract_snapshot: Option<RememContractSnapshot>,
55}
56
57#[derive(Debug, Clone, Serialize, PartialEq)]
58pub struct CodingBenchRunScoreEvidence {
59 pub commands: Vec<CodingBenchScoreCommandEvidence>,
60}
61
62#[derive(Debug, Clone, Serialize, PartialEq)]
63pub struct CodingBenchScoreCommandEvidence {
64 pub command: Vec<String>,
65 pub exit_code: i32,
66 pub stdout: Option<String>,
67 pub stderr: Option<String>,
68 pub output_artifact_path: Option<String>,
69}
70
71#[derive(Debug, Clone, Serialize, PartialEq)]
72pub struct CodingBenchRunMetrics {
73 pub tokens_input: Option<u64>,
74 pub tokens_output: Option<u64>,
75 pub tokens_total: Option<u64>,
76 pub token_accounting_unsupported_reason: Option<String>,
77 pub turns: Option<u64>,
78 pub wall_time_ms: Option<u64>,
79}
80
81#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq, PartialOrd, Ord)]
82#[serde(rename_all = "snake_case")]
83pub enum CodingBenchCondition {
84 NoMemory,
85 Remem,
86 CuratedFile,
87}
88
89#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
90#[serde(rename_all = "snake_case")]
91pub enum CodingBenchMemoryContractStatus {
92 Passed,
93 Failed,
94 NotApplicable,
95}
96
97#[derive(Debug, Clone, Serialize, PartialEq)]
98pub struct RememContractSnapshot {
99 pub schema_version: u32,
100 pub source: &'static str,
101 pub spec_path: &'static str,
102 pub captured_at_epoch: i64,
103 pub contract_health: RememContractHealth,
104 pub citation_precision: CurrentMemoryContractRateMetric,
105 pub staleness_handling: RememStalenessHandlingSnapshot,
106 pub temporal_fact_eligibility: RememTemporalFactEligibilitySnapshot,
107 pub injected_memory_audit: RememInjectedMemoryAuditSnapshot,
108 pub usage_feedback_coverage: RememUsageFeedbackCoverageSnapshot,
109 pub current_memory_contracts: CurrentMemoryContractEvalReport,
110}
111
112#[derive(Debug, Clone, Serialize, PartialEq)]
113pub struct RememContractHealth {
114 pub all_checks_passed: bool,
115 pub failing_examples: Vec<String>,
116 pub warning_count: usize,
117 pub warnings: Vec<RememContractWarning>,
118}
119
120#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
121pub struct RememContractWarning {
122 pub code: &'static str,
123 pub message: String,
124}
125
126#[derive(Debug, Clone, Serialize, PartialEq)]
127pub struct RememStalenessHandlingSnapshot {
128 pub tracked: CurrentMemoryContractRateMetric,
129 pub untracked: CurrentMemoryContractRateMetric,
130 pub history_tracked: CurrentMemoryContractRateMetric,
131 pub verify_before_trust: CurrentMemoryContractRateMetric,
132 pub error: CurrentMemoryContractRateMetric,
133}
134
135#[derive(Debug, Clone, Serialize, PartialEq)]
136pub struct RememTemporalFactEligibilitySnapshot {
137 pub invalidated_fact_exclusion: CurrentMemoryContractRateMetric,
138 pub expired_fact_exclusion: CurrentMemoryContractRateMetric,
139 pub as_of_fact_retrieval: CurrentMemoryContractRateMetric,
140}
141
142#[derive(Debug, Clone, Serialize, PartialEq)]
143pub struct RememInjectedMemoryAuditSnapshot {
144 pub injected: CurrentMemoryContractRateMetric,
145 pub dropped: CurrentMemoryContractRateMetric,
146 pub abstained: CurrentMemoryContractRateMetric,
147}
148
149#[derive(Debug, Clone, Serialize, PartialEq)]
150pub struct RememUsageFeedbackCoverageSnapshot {
151 pub citation_event_matched: CurrentMemoryContractRateMetric,
152 pub usage_event_linked_to_injection_item: CurrentMemoryContractRateMetric,
153}
154
155pub fn build_remem_contract_snapshot(
156 contract_report: CurrentMemoryContractEvalReport,
157 captured_at_epoch: i64,
158) -> RememContractSnapshot {
159 let mut warnings = Vec::new();
160 let mut failing_examples = contract_report.failing_examples.clone();
161 if contract_report.metadata.real_db_touched {
162 warnings.push(RememContractWarning {
163 code: "current_memory_contract_real_db_touched",
164 message: "current-memory-contract eval touched the real runtime database".to_string(),
165 });
166 failing_examples.push("current-memory-contract eval touched real runtime database".into());
167 }
168 let all_checks_passed =
169 contract_report.metrics.all_checks_passed && failing_examples.is_empty();
170
171 RememContractSnapshot {
172 schema_version: 1,
173 source: "current_memory_contracts",
174 spec_path: CURRENT_MEMORY_CONTRACT_SPEC_PATH,
175 captured_at_epoch,
176 contract_health: RememContractHealth {
177 all_checks_passed,
178 failing_examples,
179 warning_count: warnings.len(),
180 warnings,
181 },
182 citation_precision: contract_report.metrics.usage.citation_event_matched.clone(),
183 staleness_handling: RememStalenessHandlingSnapshot {
184 tracked: contract_report.metrics.staleness.tracked.clone(),
185 untracked: contract_report.metrics.staleness.untracked.clone(),
186 history_tracked: contract_report.metrics.staleness.history_tracked.clone(),
187 verify_before_trust: contract_report
188 .metrics
189 .staleness
190 .verify_before_trust
191 .clone(),
192 error: contract_report.metrics.staleness.error.clone(),
193 },
194 temporal_fact_eligibility: RememTemporalFactEligibilitySnapshot {
195 invalidated_fact_exclusion: contract_report
196 .metrics
197 .temporal
198 .invalidated_fact_exclusion
199 .clone(),
200 expired_fact_exclusion: contract_report
201 .metrics
202 .temporal
203 .expired_fact_exclusion
204 .clone(),
205 as_of_fact_retrieval: contract_report
206 .metrics
207 .temporal
208 .as_of_fact_retrieval
209 .clone(),
210 },
211 injected_memory_audit: RememInjectedMemoryAuditSnapshot {
212 injected: contract_report.metrics.injection.audit_injected.clone(),
213 dropped: contract_report.metrics.injection.audit_dropped.clone(),
214 abstained: contract_report.metrics.injection.audit_abstained.clone(),
215 },
216 usage_feedback_coverage: RememUsageFeedbackCoverageSnapshot {
217 citation_event_matched: contract_report.metrics.usage.citation_event_matched.clone(),
218 usage_event_linked_to_injection_item: contract_report
219 .metrics
220 .usage
221 .usage_event_linked_to_injection_item
222 .clone(),
223 },
224 current_memory_contracts: contract_report,
225 }
226}
227
228pub fn validate_contract_snapshots(report: &CodingBenchReport) -> Result<()> {
229 validate_condition_matrix(report)?;
230
231 for condition in &report.conditions {
232 for run in &condition.runs {
233 if run.condition != condition.name {
234 bail!(
235 "coding bench run {}#{} condition {:?} is nested under {:?}",
236 run.task_id,
237 run.run_index,
238 run.condition,
239 condition.name
240 );
241 }
242 if run.task_success && run.task_failure_reason.is_some() {
243 bail!(
244 "coding bench run {}#{} has task_success=true with stale task_failure_reason",
245 run.task_id,
246 run.run_index
247 );
248 }
249 if !run.task_success && run.task_failure_reason.is_none() {
250 bail!(
251 "coding bench run {}#{} failed task without task_failure_reason",
252 run.task_id,
253 run.run_index
254 );
255 }
256 if run.runtime_contract_failure
257 && run
258 .runtime_contract_failure_reason
259 .as_deref()
260 .is_none_or(|reason| reason.trim().is_empty())
261 {
262 bail!(
263 "coding bench run {}#{} has runtime contract failure without reason",
264 run.task_id,
265 run.run_index
266 );
267 }
268 if !run.runtime_contract_failure && run.runtime_contract_failure_reason.is_some() {
269 bail!(
270 "coding bench run {}#{} has runtime_contract_failure=false with stale runtime_contract_failure_reason",
271 run.task_id,
272 run.run_index
273 );
274 }
275 validate_score_and_patch_evidence(run)?;
276 validate_token_accounting(run)?;
277 validate_required_run_metrics(run)?;
278
279 match run.condition {
280 CodingBenchCondition::Remem => validate_remem_run_contract(run)?,
281 CodingBenchCondition::NoMemory | CodingBenchCondition::CuratedFile => {
282 if run.memory_contract_status != CodingBenchMemoryContractStatus::NotApplicable
283 {
284 bail!(
285 "{:?} run {}#{} must mark memory_contract_status as not_applicable",
286 run.condition,
287 run.task_id,
288 run.run_index
289 );
290 }
291 if run.remem_contract_snapshot.is_some() {
292 bail!(
293 "{:?} run {}#{} must not carry a remem contract snapshot",
294 run.condition,
295 run.task_id,
296 run.run_index
297 );
298 }
299 if run.runtime_contract_failure || run.runtime_contract_failure_reason.is_some()
300 {
301 bail!(
302 "{:?} run {}#{} must not report remem runtime contract failure",
303 run.condition,
304 run.task_id,
305 run.run_index
306 );
307 }
308 if run.memory_contract.is_some() {
309 bail!(
310 "{:?} run {}#{} must not carry memory_contract attribution",
311 run.condition,
312 run.task_id,
313 run.run_index
314 );
315 }
316 }
317 }
318 }
319 }
320 Ok(())
321}
322
323fn validate_condition_matrix(report: &CodingBenchReport) -> Result<()> {
324 if report.runs_per_condition < MIN_RUNS_PER_CONDITION {
325 bail!(
326 "coding bench report runs_per_condition={} is below required minimum {MIN_RUNS_PER_CONDITION}",
327 report.runs_per_condition
328 );
329 }
330
331 let mut reports_by_condition = BTreeMap::new();
332 for condition in &report.conditions {
333 if reports_by_condition
334 .insert(condition.name, condition)
335 .is_some()
336 {
337 bail!(
338 "coding bench report contains duplicate {:?} condition",
339 condition.name
340 );
341 }
342 }
343 for required in REQUIRED_CONDITIONS {
344 if !reports_by_condition.contains_key(&required) {
345 bail!(
346 "coding bench report missing required {:?} condition",
347 required
348 );
349 }
350 }
351 if reports_by_condition.len() != REQUIRED_CONDITIONS.len() {
352 bail!(
353 "coding bench report condition count={} does not match required condition count={}",
354 reports_by_condition.len(),
355 REQUIRED_CONDITIONS.len()
356 );
357 }
358
359 let mut task_ids = BTreeSet::new();
360 for condition in reports_by_condition.values() {
361 for run in &condition.runs {
362 task_ids.insert(run.task_id.clone());
363 }
364 }
365 if task_ids.is_empty() {
366 bail!("coding bench report has no task runs");
367 }
368
369 for required in REQUIRED_CONDITIONS {
370 let Some(condition) = reports_by_condition.get(&required) else {
371 bail!(
372 "coding bench report missing required {:?} condition",
373 required
374 );
375 };
376 let mut seen = BTreeSet::new();
377 for run in &condition.runs {
378 if run.run_index >= report.runs_per_condition {
379 bail!(
380 "{:?} run {}#{} is outside runs_per_condition={}",
381 required,
382 run.task_id,
383 run.run_index,
384 report.runs_per_condition
385 );
386 }
387 if !seen.insert((run.task_id.clone(), run.run_index)) {
388 bail!(
389 "{:?} condition repeats run {}#{}",
390 required,
391 run.task_id,
392 run.run_index
393 );
394 }
395 }
396 for task_id in &task_ids {
397 for run_index in 0..report.runs_per_condition {
398 if !seen.contains(&(task_id.clone(), run_index)) {
399 bail!(
400 "{:?} condition missing run {}#{}",
401 required,
402 task_id,
403 run_index
404 );
405 }
406 }
407 }
408 }
409
410 Ok(())
411}
412
413fn validate_score_and_patch_evidence(run: &CodingBenchRunReport) -> Result<()> {
414 if run.score.commands.is_empty() {
415 bail!(
416 "coding bench run {}#{} is missing score command evidence",
417 run.task_id,
418 run.run_index
419 );
420 }
421 for (command_index, command) in run.score.commands.iter().enumerate() {
422 if command.command.is_empty() || command.command.iter().any(|part| part.trim().is_empty()) {
423 bail!(
424 "coding bench run {}#{} score command {command_index} has blank command argv",
425 run.task_id,
426 run.run_index
427 );
428 }
429 let has_inline_output = command
430 .stdout
431 .as_deref()
432 .is_some_and(|stdout| !stdout.trim().is_empty())
433 || command
434 .stderr
435 .as_deref()
436 .is_some_and(|stderr| !stderr.trim().is_empty());
437 let has_output_artifact = command
438 .output_artifact_path
439 .as_deref()
440 .is_some_and(|path| !path.trim().is_empty());
441 if !has_inline_output && !has_output_artifact {
442 bail!(
443 "coding bench run {}#{} score command {command_index} has no output evidence",
444 run.task_id,
445 run.run_index
446 );
447 }
448 }
449
450 let has_final_head_sha = run
451 .final_head_sha
452 .as_deref()
453 .is_some_and(|sha| !sha.trim().is_empty());
454 let has_patch_artifact = run
455 .patch_artifact_path
456 .as_deref()
457 .is_some_and(|path| !path.trim().is_empty());
458 if !has_final_head_sha && !has_patch_artifact {
459 bail!(
460 "coding bench run {}#{} is missing final_head_sha or patch_artifact_path",
461 run.task_id,
462 run.run_index
463 );
464 }
465 if let Some(sha) = run.final_head_sha.as_deref() {
466 let sha = sha.trim();
467 if sha.len() != 40 || !sha.chars().all(|ch| ch.is_ascii_hexdigit()) {
468 bail!(
469 "coding bench run {}#{} final_head_sha is not a full git SHA",
470 run.task_id,
471 run.run_index
472 );
473 }
474 }
475 if run
476 .unauthorized_path_changes
477 .iter()
478 .any(|path| path.trim().is_empty())
479 {
480 bail!(
481 "coding bench run {}#{} has blank unauthorized path change",
482 run.task_id,
483 run.run_index
484 );
485 }
486
487 Ok(())
488}
489
490fn validate_token_accounting(run: &CodingBenchRunReport) -> Result<()> {
491 let token_fields = [
492 run.metrics.tokens_input,
493 run.metrics.tokens_output,
494 run.metrics.tokens_total,
495 ];
496 let token_fields_present = token_fields.iter().filter(|value| value.is_some()).count();
497 let unsupported_reason = run
498 .metrics
499 .token_accounting_unsupported_reason
500 .as_deref()
501 .map(str::trim);
502 let has_unsupported_reason = unsupported_reason.is_some_and(|reason| !reason.is_empty());
503
504 match (token_fields_present, has_unsupported_reason) {
505 (3, false) => {
506 let input = run.metrics.tokens_input.unwrap_or_default();
507 let output = run.metrics.tokens_output.unwrap_or_default();
508 let total = run.metrics.tokens_total.unwrap_or_default();
509 if input.saturating_add(output) != total {
510 bail!(
511 "coding bench run {}#{} tokens_total={} does not equal tokens_input + tokens_output ({input} + {output})",
512 run.task_id,
513 run.run_index,
514 total
515 );
516 }
517 Ok(())
518 }
519 (3, true) => bail!(
520 "coding bench run {}#{} has token metrics and token_accounting_unsupported_reason",
521 run.task_id,
522 run.run_index
523 ),
524 (0, true) => Ok(()),
525 (0, false) => bail!(
526 "coding bench run {}#{} is missing token accounting without token_accounting_unsupported_reason",
527 run.task_id,
528 run.run_index
529 ),
530 (_, _) => bail!(
531 "coding bench run {}#{} must record complete token accounting or token_accounting_unsupported_reason",
532 run.task_id,
533 run.run_index
534 ),
535 }
536}
537
538fn validate_required_run_metrics(run: &CodingBenchRunReport) -> Result<()> {
539 if run.metrics.turns.is_none() {
540 bail!(
541 "coding bench run {}#{} is missing turns",
542 run.task_id,
543 run.run_index
544 );
545 }
546 if run.metrics.wall_time_ms.is_none() {
547 bail!(
548 "coding bench run {}#{} is missing wall_time_ms",
549 run.task_id,
550 run.run_index
551 );
552 }
553 Ok(())
554}
555
556fn validate_remem_run_contract(run: &CodingBenchRunReport) -> Result<()> {
557 validate_memory_attribution(run)?;
558 let snapshot = run.remem_contract_snapshot.as_ref().ok_or_else(|| {
559 anyhow::anyhow!(
560 "remem run {}#{} is missing current memory contract snapshot",
561 run.task_id,
562 run.run_index
563 )
564 })?;
565 let embedded_contract_passed = snapshot.current_memory_contracts.metrics.all_checks_passed
566 && snapshot
567 .current_memory_contracts
568 .failing_examples
569 .is_empty()
570 && !snapshot.current_memory_contracts.metadata.real_db_touched;
571 if snapshot.contract_health.all_checks_passed != embedded_contract_passed {
572 bail!(
573 "remem run {}#{} contract_health does not match embedded current_memory_contracts",
574 run.task_id,
575 run.run_index
576 );
577 }
578 let contract_failed = !embedded_contract_passed;
579 let expected_status = if contract_failed {
580 CodingBenchMemoryContractStatus::Failed
581 } else {
582 CodingBenchMemoryContractStatus::Passed
583 };
584 if run.memory_contract_status != expected_status {
585 bail!(
586 "remem run {}#{} memory_contract_status={:?} does not match contract health={}",
587 run.task_id,
588 run.run_index,
589 run.memory_contract_status,
590 snapshot.contract_health.all_checks_passed
591 );
592 }
593 if contract_failed != run.runtime_contract_failure {
594 bail!(
595 "remem run {}#{} runtime_contract_failure={} does not match contract health={}",
596 run.task_id,
597 run.run_index,
598 run.runtime_contract_failure,
599 snapshot.contract_health.all_checks_passed
600 );
601 }
602 Ok(())
603}
604
605fn validate_memory_attribution(run: &CodingBenchRunReport) -> Result<()> {
606 let attribution = run.memory_contract.as_ref().ok_or_else(|| {
607 anyhow::anyhow!(
608 "remem run {}#{} is missing memory_contract attribution",
609 run.task_id,
610 run.run_index
611 )
612 })?;
613 validate_rate(attribution.citation_precision, "citation_precision", run)?;
614 validate_rate(attribution.citation_recall, "citation_recall", run)?;
615 ensure_unique_positive_ids(&attribution.injected_memory_ids, "injected_memory_ids", run)?;
616 ensure_unique_positive_ids(&attribution.used_memory_ids, "used_memory_ids", run)?;
617 if attribution.memory_helped && attribution.memory_hurt {
618 bail!(
619 "remem run {}#{} memory_contract cannot mark both memory_helped and memory_hurt",
620 run.task_id,
621 run.run_index
622 );
623 }
624 if run
625 .task_failure_reason
626 .is_some_and(CodingBenchFailureReason::is_memory_specific)
627 && !attribution.memory_hurt
628 {
629 bail!(
630 "remem run {}#{} has memory-specific failure without memory_hurt=true",
631 run.task_id,
632 run.run_index
633 );
634 }
635 Ok(())
636}
637
638fn validate_rate(value: f64, field: &str, run: &CodingBenchRunReport) -> Result<()> {
639 if !(0.0..=1.0).contains(&value) || !value.is_finite() {
640 bail!(
641 "remem run {}#{} memory_contract {field} must be a finite rate between 0 and 1",
642 run.task_id,
643 run.run_index
644 );
645 }
646 Ok(())
647}
648
649fn ensure_unique_positive_ids(ids: &[i64], field: &str, run: &CodingBenchRunReport) -> Result<()> {
650 let mut seen = BTreeSet::new();
651 for id in ids {
652 if *id <= 0 {
653 bail!(
654 "remem run {}#{} memory_contract {field} contains non-positive id",
655 run.task_id,
656 run.run_index
657 );
658 }
659 if !seen.insert(*id) {
660 bail!(
661 "remem run {}#{} memory_contract {field} contains duplicate id",
662 run.task_id,
663 run.run_index
664 );
665 }
666 }
667 Ok(())
668}