1use std::collections::{BTreeMap, BTreeSet};
2use std::fs;
3use std::path::{Path, PathBuf};
4
5use anyhow::{Context, Result};
6use serde::Serialize;
7use serde_json::Value;
8
9use super::types::{
10 BenchVerifyOptions, BenchVerifyReport, BenchmarkLayer, PublicBenchmarkReport, RunEnvironment,
11 VerifiedArtifact, VerifiedBenchmarkArtifacts,
12};
13
14pub(super) mod matrix;
15mod statistics;
16
17use statistics::coding_variance;
18pub(in crate::eval::bench_artifact) use statistics::{
19 coding_paired_statistics, coding_report_structurally_complete,
20};
21pub use statistics::{CodingConditionVariance, CodingPairedStatistic};
22
23#[derive(Debug, Clone)]
24pub struct BenchReportOptions {
25 pub root: PathBuf,
26 pub claim_registry_path: PathBuf,
27 pub json_out: PathBuf,
28 pub markdown_out: PathBuf,
29}
30
31#[derive(Debug, Clone, Serialize)]
32pub struct PublicBaselineReport {
33 pub schema_version: u32,
34 pub report_id: String,
35 pub report_kind: String,
36 pub root: String,
37 pub created_at_epoch: i64,
38 pub claim_level: String,
39 #[serde(serialize_with = "super::types::serialize_persisted_bench_verify_report")]
40 pub artifact_verifier: BenchVerifyReport,
41 pub summary: BaselineSummary,
42 pub reports: Vec<BaselineReportEntry>,
43 pub memory_task_outcomes: Vec<MemoryTaskOutcome>,
44 pub coding_task_outcomes: Vec<CodingTaskOutcome>,
45 pub coding_condition_variance: Vec<CodingConditionVariance>,
46 pub coding_paired_statistics: Vec<CodingPairedStatistic>,
47 pub failure_decomposition: FailureDecomposition,
48 pub reproducibility: ReproducibilitySummary,
49 pub claim_gate: ClaimGateSummary,
50 pub reproduction_commands: Vec<String>,
51}
52
53#[derive(Debug, Clone, Serialize)]
54pub struct BaselineSummary {
55 pub memory_system: BaselineLayerSummary,
56 pub coding_agent: BaselineLayerSummary,
57 pub manifest_count: usize,
58 pub report_count: usize,
59 pub run_artifact_count: usize,
60}
61
62#[derive(Debug, Clone, Serialize)]
63pub struct BaselineLayerSummary {
64 pub status: String,
65 pub report_count: usize,
66 pub run_artifact_count: usize,
67 pub benchmark_ids: Vec<String>,
68 pub conditions: Vec<String>,
69 pub claim_levels: Vec<String>,
70 pub notes: Vec<String>,
71}
72
73#[derive(Debug, Clone, Serialize)]
74pub struct BaselineReportEntry {
75 pub path: String,
76 pub benchmark_id: String,
77 pub benchmark_version: String,
78 pub layer: BenchmarkLayer,
79 pub conditions: Vec<String>,
80 pub run_artifact_count: usize,
81 pub claim_level: String,
82 pub aggregate_metrics: Value,
83}
84
85#[derive(Debug, Clone, Serialize)]
86pub struct MemoryTaskOutcome {
87 pub report_path: String,
88 pub suite: String,
89 pub condition: String,
90 pub task_id: String,
91 pub run_index: u32,
92 pub answer_score: Option<f64>,
93 pub support_coverage: Option<f64>,
94 pub citation_recall: Option<f64>,
95 pub write_side_gap: bool,
96 pub retrieval_side_gap: bool,
97 pub reader_gap: bool,
98 pub policy_abstention: bool,
99}
100
101#[derive(Debug, Clone, Serialize)]
102pub struct CodingTaskOutcome {
103 pub report_path: String,
104 pub benchmark_id: String,
105 pub benchmark_version: String,
106 pub run_phase: String,
107 pub matrix_namespace: String,
108 pub condition: String,
109 pub task_id: String,
110 pub run_index: u32,
111 pub attempt_id: Option<String>,
112 pub target_started: Option<bool>,
113 pub resolved: bool,
114 pub failure_reason: Option<String>,
115 pub tokens_total: Option<u64>,
116 pub turns: Option<u64>,
117 pub wall_time_ms: Option<u64>,
118 pub memory_helped: Option<bool>,
119 pub memory_hurt: Option<bool>,
120}
121
122#[derive(Debug, Clone, Default, Serialize)]
123pub struct FailureDecomposition {
124 pub coding_failure_counts: BTreeMap<String, usize>,
125 pub coding_memory_failure_counts: BTreeMap<String, usize>,
126 pub memory_gap_counts: BTreeMap<String, usize>,
127}
128
129#[derive(Debug, Clone, Serialize)]
130pub struct ReproducibilitySummary {
131 pub remem_commits: Vec<String>,
132 pub fixture_revisions: Vec<String>,
133 pub docker_image_digests: Vec<String>,
134 pub repo_base_commits: Vec<String>,
135 pub prompt_hashes: Vec<String>,
136 pub models: Vec<String>,
137}
138
139#[derive(Debug, Clone, Serialize)]
140pub struct ClaimGateSummary {
141 pub artifact_verifier_passed: bool,
142 pub coding_claim_level: String,
143 pub coding_outcome_stop_loss_status: String,
144 pub public_sota_status: String,
145 pub notes: Vec<String>,
146}
147
148#[derive(Default)]
149struct BuildState {
150 manifest_count: usize,
151 report_paths: BTreeSet<PathBuf>,
152 reports: Vec<BaselineReportEntry>,
153 memory_outcomes: Vec<MemoryTaskOutcome>,
154 coding_outcomes: Vec<CodingTaskOutcome>,
155 memory_benchmarks: BTreeSet<String>,
156 coding_benchmarks: BTreeSet<String>,
157 memory_conditions: BTreeSet<String>,
158 coding_conditions: BTreeSet<String>,
159 memory_claim_levels: BTreeSet<String>,
160 coding_claim_levels: BTreeSet<String>,
161 failure_decomposition: FailureDecomposition,
162 remem_commits: BTreeSet<String>,
163 fixture_revisions: BTreeSet<String>,
164 docker_image_digests: BTreeSet<String>,
165 repo_base_commits: BTreeSet<String>,
166 prompt_hashes: BTreeSet<String>,
167 models: BTreeSet<String>,
168 max_created_at_epoch: i64,
169}
170
171pub fn write_public_baseline_report(options: BenchReportOptions) -> Result<PublicBaselineReport> {
172 let report = generate_public_baseline_report(&options.root, &options.claim_registry_path)?;
173 write_text_file(&options.json_out, &serde_json::to_string_pretty(&report)?)?;
174 write_text_file(
175 &options.markdown_out,
176 &render_public_baseline_markdown(&report),
177 )?;
178 Ok(report)
179}
180
181pub fn generate_public_baseline_report(
182 root: &Path,
183 claim_registry_path: &Path,
184) -> Result<PublicBaselineReport> {
185 let artifact_verifier = super::verify::verify_benchmark_artifacts(BenchVerifyOptions::new(
186 root.to_path_buf(),
187 claim_registry_path.to_path_buf(),
188 ))?;
189 generate_public_baseline_report_from_verified(root, artifact_verifier)
190}
191
192pub(super) fn generate_public_baseline_report_from_verified(
193 root: &Path,
194 artifact_verifier: BenchVerifyReport,
195) -> Result<PublicBaselineReport> {
196 let verified = &artifact_verifier.verified_artifacts;
197
198 let mut state = BuildState {
199 max_created_at_epoch: 0,
200 ..BuildState::default()
201 };
202
203 state.manifest_count = verified.manifests.len();
204 state.max_created_at_epoch = verified
205 .manifests
206 .iter()
207 .map(|manifest| manifest.value.created_at_epoch)
208 .max()
209 .unwrap_or_default();
210 for report in &verified.reports {
211 let logical_path = PathBuf::from(&report.path);
212 if state.report_paths.insert(logical_path) {
213 load_verified_report(report, verified, &mut state)?;
214 }
215 }
216
217 let coding_condition_variance = coding_variance(&state.coding_outcomes);
218 let coding_paired_statistics =
219 coding_paired_statistics(&state.coding_outcomes, artifact_verifier.passed);
220 let claim_gate = claim_gate(&artifact_verifier, &state);
221 let memory_summary = layer_summary(
222 "directional_memory_system_evidence",
223 &state.memory_benchmarks,
224 &state.memory_conditions,
225 &state.memory_claim_levels,
226 state.memory_outcomes.len(),
227 &[
228 "Memory-system capability results are separate from coding-agent outcomes.".to_string(),
229 "Committed memory suites are directional until public claim gates pass.".to_string(),
230 ],
231 );
232 let coding_summary = layer_summary(
233 "smoke_coding_outcome_evidence",
234 &state.coding_benchmarks,
235 &state.coding_conditions,
236 &state.coding_claim_levels,
237 state.coding_outcomes.len(),
238 &[
239 "Current committed coding artifacts are smoke-only.".to_string(),
240 "The #931 claim gate requires no_memory, remem_e2e, and curated_file_budgeted across the registered 16-task fixture with exactly run indices 0, 1, and 2 per task and condition.".to_string(),
241 ],
242 );
243
244 Ok(PublicBaselineReport {
245 schema_version: 1,
246 report_id: "public-baseline-directional-v1".to_string(),
247 report_kind: "baseline_directional_public_benchmark".to_string(),
248 root: root.to_string_lossy().to_string(),
249 created_at_epoch: state.max_created_at_epoch,
250 claim_level: "directional_only_no_public_claim".to_string(),
251 summary: BaselineSummary {
252 memory_system: memory_summary,
253 coding_agent: coding_summary,
254 manifest_count: state.manifest_count,
255 report_count: state.reports.len(),
256 run_artifact_count: state.memory_outcomes.len() + state.coding_outcomes.len(),
257 },
258 reports: state.reports,
259 memory_task_outcomes: state.memory_outcomes,
260 coding_task_outcomes: state.coding_outcomes,
261 coding_condition_variance,
262 coding_paired_statistics,
263 failure_decomposition: state.failure_decomposition,
264 reproducibility: ReproducibilitySummary {
265 remem_commits: sorted_vec(state.remem_commits),
266 fixture_revisions: sorted_vec(state.fixture_revisions),
267 docker_image_digests: sorted_vec(state.docker_image_digests),
268 repo_base_commits: sorted_vec(state.repo_base_commits),
269 prompt_hashes: sorted_vec(state.prompt_hashes),
270 models: sorted_vec(state.models),
271 },
272 claim_gate,
273 reproduction_commands: reproduction_commands(),
274 artifact_verifier,
275 })
276}
277
278pub fn render_public_baseline_markdown(report: &PublicBaselineReport) -> String {
279 let mut out = String::new();
280 out.push_str("# remem Public Baseline Directional Report\n\n");
281 out.push_str("Claim level: `");
282 out.push_str(&report.claim_level);
283 out.push_str("`.\n\n");
284 out.push_str("This report separates memory-system capability evidence from coding-agent outcome evidence. It is directional only and does not support SOTA, broad superiority, or coding-task superiority claims.\n\n");
285
286 out.push_str("## Artifact Verification\n\n");
287 out.push_str(&format!(
288 "- Passed: `{}`\n- Manifests checked: `{}`\n- Reports checked: `{}`\n- Run artifacts checked: `{}`\n- Artifact files checked: `{}`\n\n",
289 report.artifact_verifier.passed,
290 report.artifact_verifier.manifests_checked,
291 report.artifact_verifier.reports_checked,
292 report.artifact_verifier.run_artifacts_checked,
293 report.artifact_verifier.artifact_files_checked
294 ));
295
296 out.push_str("## Memory-System Capability\n\n");
297 out.push_str("| Report | Runs | Claim level | Answer score | Support coverage | Citation recall | Non-retention leak rate |\n");
298 out.push_str("|---|---:|---|---:|---:|---:|---:|\n");
299 for entry in report
300 .reports
301 .iter()
302 .filter(|entry| entry.layer == BenchmarkLayer::MemorySystemCapability)
303 {
304 out.push_str(&format!(
305 "| `{}` | {} | `{}` | {} | {} | {} | {} |\n",
306 escape_md(&entry.benchmark_id),
307 entry.run_artifact_count,
308 escape_md(&entry.claim_level),
309 fmt_metric(metric_path(
310 &entry.aggregate_metrics,
311 &["overall", "answer_score"]
312 )),
313 fmt_metric(metric_path(
314 &entry.aggregate_metrics,
315 &["overall", "support_coverage"]
316 )),
317 fmt_metric(metric_path(
318 &entry.aggregate_metrics,
319 &["overall", "citation_recall"]
320 )),
321 fmt_metric(metric_path(
322 &entry.aggregate_metrics,
323 &["policy", "non_retention_leak_rate"]
324 ))
325 ));
326 }
327 out.push('\n');
328
329 out.push_str("## Coding-Agent Outcome\n\n");
330 out.push_str("| Condition | Runs | Resolved rate | Token mean | Token variance | Wall-time mean ms | Variance status |\n");
331 out.push_str("|---|---:|---:|---:|---:|---:|---|\n");
332 for variance in &report.coding_condition_variance {
333 out.push_str(&format!(
334 "| `{}` | {} | {:.3} | {} | {} | {} | `{}` |\n",
335 escape_md(&variance.condition),
336 variance.runs,
337 variance.resolved_rate,
338 fmt_metric(variance.tokens_total_mean),
339 fmt_metric(variance.tokens_total_sample_variance),
340 fmt_metric(variance.wall_time_ms_mean),
341 escape_md(&variance.variance_status)
342 ));
343 }
344 out.push('\n');
345
346 out.push_str("## Paired Coding Statistics\n\n");
347 out.push_str("| Comparison | Report | Status | Treatment rate | Control rate | Effect pp | 95% CI pp | Method | Reason |\n");
348 out.push_str("|---|---|---|---:|---:|---:|---:|---|---|\n");
349 for stat in &report.coding_paired_statistics {
350 out.push_str(&format!(
351 "| `{}` | {} | `{}` | {} | {} | {} | {} | `{}` | {} |\n",
352 escape_md(&stat.comparison_id),
353 stat.report_path
354 .as_deref()
355 .map(|path| format!("`{}`", escape_md(path)))
356 .unwrap_or_else(|| "`n/a`".to_string()),
357 escape_md(&stat.status),
358 fmt_metric(stat.treatment_resolved_rate),
359 fmt_metric(stat.control_resolved_rate),
360 fmt_metric(stat.effect_pp),
361 fmt_ci(stat.ci_lower_pp, stat.ci_upper_pp),
362 escape_md(&stat.algorithm),
363 stat.insufficient_reason
364 .as_deref()
365 .map(escape_md)
366 .unwrap_or_else(|| "none".to_string())
367 ));
368 }
369 out.push('\n');
370
371 out.push_str("## Coding Task Outcomes\n\n");
372 out.push_str("| Task | Condition | Run | Resolved | Failure reason | Tokens | Wall time ms | Memory helped | Memory hurt |\n");
373 out.push_str("|---|---|---:|---|---|---:|---:|---|---|\n");
374 for run in &report.coding_task_outcomes {
375 out.push_str(&format!(
376 "| `{}` | `{}` | {} | `{}` | {} | {} | {} | {} | {} |\n",
377 escape_md(&run.task_id),
378 escape_md(&run.condition),
379 run.run_index,
380 run.resolved,
381 run.failure_reason
382 .as_deref()
383 .map(|value| format!("`{}`", escape_md(value)))
384 .unwrap_or_else(|| "`none`".to_string()),
385 fmt_u64(run.tokens_total),
386 fmt_u64(run.wall_time_ms),
387 fmt_bool(run.memory_helped),
388 fmt_bool(run.memory_hurt)
389 ));
390 }
391 out.push('\n');
392
393 out.push_str("## Failure Decomposition\n\n");
394 out.push_str("Coding failure counts:\n\n");
395 append_count_map(
396 &mut out,
397 &report.failure_decomposition.coding_failure_counts,
398 );
399 out.push_str("\nCoding memory-specific failure counts:\n\n");
400 append_count_map(
401 &mut out,
402 &report.failure_decomposition.coding_memory_failure_counts,
403 );
404 out.push_str("\nMemory gap counts:\n\n");
405 append_count_map(&mut out, &report.failure_decomposition.memory_gap_counts);
406 out.push('\n');
407
408 out.push_str("## Reproducibility\n\n");
409 out.push_str("Run these commands from a clean checkout:\n\n");
410 out.push_str("```bash\n");
411 for command in &report.reproduction_commands {
412 out.push_str(command);
413 out.push('\n');
414 }
415 out.push_str("```\n\n");
416 out.push_str("Locks and evidence are recorded in the JSON report under `reproducibility`, including remem commits, fixture revisions, Docker image digests, prompt hashes, model labels, and repo base commits when present.\n\n");
417
418 out.push_str("## Claim Gate\n\n");
419 out.push_str(&format!(
420 "- Artifact verifier passed: `{}`\n- Coding outcome stop-loss status: `{}`\n- Public SOTA status: `{}`\n",
421 report.claim_gate.artifact_verifier_passed,
422 report.claim_gate.coding_outcome_stop_loss_status,
423 report.claim_gate.public_sota_status
424 ));
425 for note in &report.claim_gate.notes {
426 out.push_str("- ");
427 out.push_str(note);
428 out.push('\n');
429 }
430
431 out
432}
433
434fn load_verified_report(
435 artifact: &VerifiedArtifact<PublicBenchmarkReport>,
436 verified: &VerifiedBenchmarkArtifacts,
437 state: &mut BuildState,
438) -> Result<()> {
439 let report = &artifact.value;
440 let report_path = artifact.path.clone();
441 match report.layer {
442 BenchmarkLayer::MemorySystemCapability => {
443 state.memory_benchmarks.insert(report.benchmark_id.clone());
444 state.memory_claim_levels.insert(report.claim_level.clone());
445 for condition in &report.conditions {
446 state.memory_conditions.insert(condition.clone());
447 }
448 load_memory_runs(&report_path, report, verified, state)?;
449 }
450 BenchmarkLayer::CodingAgentOutcome => {
451 state.coding_benchmarks.insert(report.benchmark_id.clone());
452 state.coding_claim_levels.insert(report.claim_level.clone());
453 for condition in &report.conditions {
454 state.coding_conditions.insert(condition.clone());
455 }
456 load_coding_runs(&report_path, report, verified, state)?;
457 }
458 }
459 state.reports.push(BaselineReportEntry {
460 path: report_path,
461 benchmark_id: report.benchmark_id.clone(),
462 benchmark_version: report.benchmark_version.clone(),
463 layer: report.layer,
464 conditions: report.conditions.clone(),
465 run_artifact_count: report.run_artifacts.len(),
466 claim_level: report.claim_level.clone(),
467 aggregate_metrics: report.aggregate_metrics.clone(),
468 });
469 Ok(())
470}
471
472fn load_memory_runs(
473 report_path: &str,
474 report: &PublicBenchmarkReport,
475 verified: &VerifiedBenchmarkArtifacts,
476 state: &mut BuildState,
477) -> Result<()> {
478 for run_path in &report.run_artifacts {
479 let run = verified
480 .memory_runs
481 .iter()
482 .find(|artifact| artifact.path == *run_path)
483 .with_context(|| format!("verified memory run is missing: {run_path}"))?
484 .value
485 .clone();
486 observe_environment(&run.environment, state);
487 observe_model(&run.reader_model, state);
488 observe_prompt_hash(&run.reader_model, state);
489 if run.diagnosis.write_side_gap {
490 increment(
491 &mut state.failure_decomposition.memory_gap_counts,
492 "write_side_gap",
493 );
494 }
495 if run.diagnosis.retrieval_side_gap {
496 increment(
497 &mut state.failure_decomposition.memory_gap_counts,
498 "retrieval_side_gap",
499 );
500 }
501 if run.diagnosis.reader_gap {
502 increment(
503 &mut state.failure_decomposition.memory_gap_counts,
504 "reader_gap",
505 );
506 }
507 if run.diagnosis.policy_abstention {
508 increment(
509 &mut state.failure_decomposition.memory_gap_counts,
510 "policy_abstention",
511 );
512 }
513 state.memory_outcomes.push(MemoryTaskOutcome {
514 report_path: report_path.to_string(),
515 suite: run.suite,
516 condition: run.condition,
517 task_id: run.task_id,
518 run_index: run.run_index,
519 answer_score: metric_path(&run.metrics, &["answer_score"]),
520 support_coverage: metric_path(&run.metrics, &["support_coverage"]),
521 citation_recall: metric_path(&run.metrics, &["citation_recall"]),
522 write_side_gap: run.diagnosis.write_side_gap,
523 retrieval_side_gap: run.diagnosis.retrieval_side_gap,
524 reader_gap: run.diagnosis.reader_gap,
525 policy_abstention: run.diagnosis.policy_abstention,
526 });
527 }
528 Ok(())
529}
530
531fn load_coding_runs(
532 report_path: &str,
533 report: &PublicBenchmarkReport,
534 verified: &VerifiedBenchmarkArtifacts,
535 state: &mut BuildState,
536) -> Result<()> {
537 for run_path in &report.run_artifacts {
538 let run = verified
539 .coding_runs
540 .iter()
541 .find(|artifact| artifact.path == *run_path)
542 .with_context(|| format!("verified coding run is missing: {run_path}"))?
543 .value
544 .clone();
545 observe_environment(&run.environment, state);
546 observe_model(&run.model, state);
547 observe_prompt_hash(&run.model, state);
548 if let Some(reason) = &run.failure_reason {
549 increment(
550 &mut state.failure_decomposition.coding_failure_counts,
551 reason,
552 );
553 if is_memory_specific_failure(reason) {
554 increment(
555 &mut state.failure_decomposition.coding_memory_failure_counts,
556 reason,
557 );
558 }
559 }
560 state.coding_outcomes.push(CodingTaskOutcome {
561 report_path: report_path.to_string(),
562 benchmark_id: run.benchmark_id,
563 benchmark_version: run.benchmark_version,
564 run_phase: run.run_phase,
565 matrix_namespace: run.matrix_namespace,
566 condition: run.condition,
567 task_id: run.task_id,
568 run_index: run.run_index,
569 attempt_id: run.attempt_id,
570 target_started: run.target_started,
571 resolved: run.resolved,
572 failure_reason: run.failure_reason,
573 tokens_total: run.metrics.tokens_total,
574 turns: run.metrics.turns,
575 wall_time_ms: run.metrics.wall_time_ms,
576 memory_helped: run
577 .memory_contract
578 .as_ref()
579 .map(|contract| contract.memory_helped),
580 memory_hurt: run
581 .memory_contract
582 .as_ref()
583 .map(|contract| contract.memory_hurt),
584 });
585 }
586 Ok(())
587}
588
589fn claim_gate(artifact_verifier: &BenchVerifyReport, state: &BuildState) -> ClaimGateSummary {
590 let matrix = matrix::coding_matrix_readiness(&state.coding_outcomes);
591 let coding_outcome_stop_loss_status = if matrix::has_claim_ready_coding_matrix(
592 artifact_verifier.passed,
593 &state.coding_outcomes,
594 ) {
595 "ready_for_stop_loss_evaluation"
596 } else {
597 "not_evaluated_insufficient_coding_matrix"
598 };
599 let mut notes = vec![
600 "This baseline is directional only and must not be used for coding-task superiority claims.".to_string(),
601 "README and release wording must not claim SOTA or coding outcome improvement from this report.".to_string(),
602 ];
603 if !artifact_verifier.passed {
604 notes.push("Coding artifacts must pass the public artifact verifier.".to_string());
605 }
606 if !matrix.has_registered_identity {
607 notes.push(
608 "Coding artifacts are not the registered issue385-v1/official-v1 official matrix."
609 .to_string(),
610 );
611 } else if !matrix.has_required_conditions {
612 notes.push(
613 "Coding artifacts do not yet include no_memory, remem_e2e, and curated_file_budgeted conditions."
614 .to_string(),
615 );
616 } else if !matrix.has_identical_task_sets {
617 notes.push(
618 "Claim-bearing coding conditions must use the exact registered 16-task set in one report."
619 .to_string(),
620 );
621 } else if !matrix.has_three_runs_per_task {
622 notes.push(
623 "Coding artifacts do not yet have exactly the registered run indices 0, 1, and 2 per task and condition."
624 .to_string(),
625 );
626 } else if !matrix.has_aggregate_ready_attempts {
627 notes.push(
628 "Every official tuple must carry a unique verified attempt_id and target_started=true; pre-target failures make the matrix insufficient."
629 .to_string(),
630 );
631 }
632 ClaimGateSummary {
633 artifact_verifier_passed: artifact_verifier.passed,
634 coding_claim_level: "directional_only_no_public_claim".to_string(),
635 coding_outcome_stop_loss_status: coding_outcome_stop_loss_status.to_string(),
636 public_sota_status: "not_evaluated_no_public_sota_claim".to_string(),
637 notes,
638 }
639}
640
641fn layer_summary(
642 status: &str,
643 benchmark_ids: &BTreeSet<String>,
644 conditions: &BTreeSet<String>,
645 claim_levels: &BTreeSet<String>,
646 run_artifact_count: usize,
647 notes: &[String],
648) -> BaselineLayerSummary {
649 BaselineLayerSummary {
650 status: status.to_string(),
651 report_count: claim_levels.len().max(benchmark_ids.len()),
652 run_artifact_count,
653 benchmark_ids: sorted_vec(benchmark_ids.clone()),
654 conditions: sorted_vec(conditions.clone()),
655 claim_levels: sorted_vec(claim_levels.clone()),
656 notes: notes.to_vec(),
657 }
658}
659
660fn write_text_file(path: &Path, content: &str) -> Result<()> {
661 if let Some(parent) = path.parent() {
662 if !parent.as_os_str().is_empty() {
663 fs::create_dir_all(parent)
664 .with_context(|| format!("create report directory {}", parent.display()))?;
665 }
666 }
667 fs::write(path, content).with_context(|| format!("write {}", path.display()))
668}
669
670fn observe_environment(environment: &RunEnvironment, state: &mut BuildState) {
671 insert_non_empty(&mut state.remem_commits, &environment.remem_commit);
672 if let Some(value) = &environment.fixture_revision {
673 insert_non_empty(&mut state.fixture_revisions, value);
674 }
675 if let Some(value) = &environment.docker_image_digest {
676 insert_non_empty(&mut state.docker_image_digests, value);
677 }
678 if let Some(value) = &environment.repo_base_commit {
679 insert_non_empty(&mut state.repo_base_commits, value);
680 }
681}
682
683fn observe_model(model: &Value, state: &mut BuildState) {
684 let provider = value_string(model, "provider")
685 .or_else(|| value_string(model, "agent"))
686 .unwrap_or_else(|| "unknown".to_string());
687 let name = value_string(model, "model").unwrap_or_else(|| "unknown".to_string());
688 insert_non_empty(&mut state.models, &format!("{provider}/{name}"));
689}
690
691fn observe_prompt_hash(model: &Value, state: &mut BuildState) {
692 if let Some(value) = value_string(model, "prompt_hash") {
693 insert_non_empty(&mut state.prompt_hashes, &value);
694 }
695}
696
697fn value_string(value: &Value, key: &str) -> Option<String> {
698 value
699 .get(key)
700 .and_then(Value::as_str)
701 .map(ToString::to_string)
702}
703
704fn insert_non_empty(set: &mut BTreeSet<String>, value: &str) {
705 if !value.trim().is_empty() {
706 set.insert(value.to_string());
707 }
708}
709
710fn metric_path(value: &Value, path: &[&str]) -> Option<f64> {
711 let mut cursor = value;
712 for segment in path {
713 cursor = cursor.get(*segment)?;
714 }
715 cursor.as_f64()
716}
717
718fn increment(map: &mut BTreeMap<String, usize>, key: &str) {
719 *map.entry(key.to_string()).or_default() += 1;
720}
721
722fn is_memory_specific_failure(reason: &str) -> bool {
723 matches!(
724 reason,
725 "ignored_memory"
726 | "missing_memory"
727 | "stale_memory_followed"
728 | "irrelevant_memory_distracted"
729 | "agent_hallucinated_memory"
730 )
731}
732
733fn sorted_vec(set: BTreeSet<String>) -> Vec<String> {
734 set.into_iter().collect()
735}
736
737fn reproduction_commands() -> Vec<String> {
738 vec![
739 "cargo run -- bench verify --root eval/public --json-out /tmp/remem-public-bench-verify.json".to_string(),
740 "cargo run -- bench report --root eval/public --json-out eval/public/reports/baseline.json --markdown-out eval/public/reports/baseline.md".to_string(),
741 "cargo run -- bench coding --suite issue385-v1 --dry-run --json-out /tmp/remem-issue385-v1-dry-run.json".to_string(),
742 "cargo run -- bench memory --suite remem-code-memory --condition remem_default --root eval/public --artifact-prefix memory/artifacts/remem-code-memory-v1 --json-out eval/public/memory/reports/remem-code-memory-v1.json".to_string(),
743 "cargo run -- bench memory --suite adversarial-policy --condition remem_default --root eval/public --artifact-prefix memory/artifacts/adversarial-policy-v1 --json-out eval/public/memory/reports/adversarial-policy-v1.json".to_string(),
744 ]
745}
746
747fn escape_md(value: &str) -> String {
748 value.replace('|', "\\|")
749}
750
751fn fmt_metric(value: Option<f64>) -> String {
752 value
753 .map(|value| format!("{value:.3}"))
754 .unwrap_or_else(|| "n/a".to_string())
755}
756
757fn fmt_u64(value: Option<u64>) -> String {
758 value
759 .map(|value| value.to_string())
760 .unwrap_or_else(|| "n/a".to_string())
761}
762
763fn fmt_bool(value: Option<bool>) -> String {
764 value
765 .map(|value| format!("`{value}`"))
766 .unwrap_or_else(|| "`n/a`".to_string())
767}
768
769fn fmt_ci(lower: Option<f64>, upper: Option<f64>) -> String {
770 match (lower, upper) {
771 (Some(lower), Some(upper)) => format!("{lower:.3} to {upper:.3}"),
772 _ => "n/a".to_string(),
773 }
774}
775
776fn append_count_map(out: &mut String, map: &BTreeMap<String, usize>) {
777 if map.is_empty() {
778 out.push_str("- none\n");
779 return;
780 }
781 for (key, count) in map {
782 out.push_str(&format!("- `{}`: {}\n", escape_md(key), count));
783 }
784}