1use std::collections::{BTreeMap, BTreeSet};
2use std::fs;
3use std::path::{Path, PathBuf};
4
5use anyhow::{Context, Result};
6use serde::de::DeserializeOwned;
7use serde::Serialize;
8use serde_json::Value;
9
10use super::types::{
11 BenchVerifyOptions, BenchVerifyReport, BenchmarkLayer, CodingRunArtifact, MemoryRunArtifact,
12 PublicBenchmarkManifest, PublicBenchmarkReport, RunEnvironment,
13};
14
15#[derive(Debug, Clone)]
16pub struct BenchReportOptions {
17 pub root: PathBuf,
18 pub json_out: PathBuf,
19 pub markdown_out: PathBuf,
20}
21
22#[derive(Debug, Clone, Serialize)]
23pub struct PublicBaselineReport {
24 pub schema_version: u32,
25 pub report_id: String,
26 pub report_kind: String,
27 pub root: String,
28 pub created_at_epoch: i64,
29 pub claim_level: String,
30 pub artifact_verifier: BenchVerifyReport,
31 pub summary: BaselineSummary,
32 pub reports: Vec<BaselineReportEntry>,
33 pub memory_task_outcomes: Vec<MemoryTaskOutcome>,
34 pub coding_task_outcomes: Vec<CodingTaskOutcome>,
35 pub coding_condition_variance: Vec<CodingConditionVariance>,
36 pub failure_decomposition: FailureDecomposition,
37 pub reproducibility: ReproducibilitySummary,
38 pub claim_gate: ClaimGateSummary,
39 pub reproduction_commands: Vec<String>,
40}
41
42#[derive(Debug, Clone, Serialize)]
43pub struct BaselineSummary {
44 pub memory_system: BaselineLayerSummary,
45 pub coding_agent: BaselineLayerSummary,
46 pub manifest_count: usize,
47 pub report_count: usize,
48 pub run_artifact_count: usize,
49}
50
51#[derive(Debug, Clone, Serialize)]
52pub struct BaselineLayerSummary {
53 pub status: String,
54 pub report_count: usize,
55 pub run_artifact_count: usize,
56 pub benchmark_ids: Vec<String>,
57 pub conditions: Vec<String>,
58 pub claim_levels: Vec<String>,
59 pub notes: Vec<String>,
60}
61
62#[derive(Debug, Clone, Serialize)]
63pub struct BaselineReportEntry {
64 pub path: String,
65 pub benchmark_id: String,
66 pub benchmark_version: String,
67 pub layer: BenchmarkLayer,
68 pub conditions: Vec<String>,
69 pub run_artifact_count: usize,
70 pub claim_level: String,
71 pub aggregate_metrics: Value,
72}
73
74#[derive(Debug, Clone, Serialize)]
75pub struct MemoryTaskOutcome {
76 pub report_path: String,
77 pub suite: String,
78 pub condition: String,
79 pub task_id: String,
80 pub run_index: u32,
81 pub answer_score: Option<f64>,
82 pub support_coverage: Option<f64>,
83 pub citation_recall: Option<f64>,
84 pub write_side_gap: bool,
85 pub retrieval_side_gap: bool,
86 pub reader_gap: bool,
87 pub policy_abstention: bool,
88}
89
90#[derive(Debug, Clone, Serialize)]
91pub struct CodingTaskOutcome {
92 pub report_path: String,
93 pub condition: String,
94 pub task_id: String,
95 pub run_index: u32,
96 pub resolved: bool,
97 pub failure_reason: Option<String>,
98 pub tokens_total: Option<u64>,
99 pub turns: Option<u64>,
100 pub wall_time_ms: Option<u64>,
101 pub memory_helped: Option<bool>,
102 pub memory_hurt: Option<bool>,
103}
104
105#[derive(Debug, Clone, Serialize)]
106pub struct CodingConditionVariance {
107 pub condition: String,
108 pub runs: usize,
109 pub resolved_rate: f64,
110 pub tokens_total_mean: Option<f64>,
111 pub tokens_total_sample_variance: Option<f64>,
112 pub wall_time_ms_mean: Option<f64>,
113 pub wall_time_ms_sample_variance: Option<f64>,
114 pub variance_status: String,
115}
116
117#[derive(Debug, Clone, Default, Serialize)]
118pub struct FailureDecomposition {
119 pub coding_failure_counts: BTreeMap<String, usize>,
120 pub coding_memory_failure_counts: BTreeMap<String, usize>,
121 pub memory_gap_counts: BTreeMap<String, usize>,
122}
123
124#[derive(Debug, Clone, Serialize)]
125pub struct ReproducibilitySummary {
126 pub remem_commits: Vec<String>,
127 pub fixture_revisions: Vec<String>,
128 pub docker_image_digests: Vec<String>,
129 pub repo_base_commits: Vec<String>,
130 pub prompt_hashes: Vec<String>,
131 pub models: Vec<String>,
132}
133
134#[derive(Debug, Clone, Serialize)]
135pub struct ClaimGateSummary {
136 pub artifact_verifier_passed: bool,
137 pub coding_outcome_stop_loss_status: String,
138 pub public_sota_status: String,
139 pub notes: Vec<String>,
140}
141
142#[derive(Default)]
143struct BuildState {
144 manifest_count: usize,
145 report_paths: BTreeSet<PathBuf>,
146 reports: Vec<BaselineReportEntry>,
147 memory_outcomes: Vec<MemoryTaskOutcome>,
148 coding_outcomes: Vec<CodingTaskOutcome>,
149 memory_benchmarks: BTreeSet<String>,
150 coding_benchmarks: BTreeSet<String>,
151 memory_conditions: BTreeSet<String>,
152 coding_conditions: BTreeSet<String>,
153 memory_claim_levels: BTreeSet<String>,
154 coding_claim_levels: BTreeSet<String>,
155 failure_decomposition: FailureDecomposition,
156 remem_commits: BTreeSet<String>,
157 fixture_revisions: BTreeSet<String>,
158 docker_image_digests: BTreeSet<String>,
159 repo_base_commits: BTreeSet<String>,
160 prompt_hashes: BTreeSet<String>,
161 models: BTreeSet<String>,
162 max_created_at_epoch: i64,
163}
164
165pub fn write_public_baseline_report(options: BenchReportOptions) -> Result<PublicBaselineReport> {
166 let report = generate_public_baseline_report(&options.root)?;
167 write_text_file(&options.json_out, &serde_json::to_string_pretty(&report)?)?;
168 write_text_file(
169 &options.markdown_out,
170 &render_public_baseline_markdown(&report),
171 )?;
172 Ok(report)
173}
174
175pub fn generate_public_baseline_report(root: &Path) -> Result<PublicBaselineReport> {
176 let artifact_verifier = super::verify::verify_benchmark_artifacts(BenchVerifyOptions {
177 root: root.to_path_buf(),
178 })?;
179 let manifest_paths = super::verify::collect_manifest_paths(root)?;
180
181 let mut state = BuildState {
182 max_created_at_epoch: 0,
183 ..BuildState::default()
184 };
185
186 for manifest_path in manifest_paths {
187 let manifest: PublicBenchmarkManifest = read_json(&manifest_path)?;
188 state.manifest_count += 1;
189 state.max_created_at_epoch = state.max_created_at_epoch.max(manifest.created_at_epoch);
190 for report_path in &manifest.reports {
191 let full_report_path = root.join(report_path);
192 if state.report_paths.insert(full_report_path.clone()) {
193 load_report(root, &full_report_path, &mut state)?;
194 }
195 }
196 }
197
198 let coding_condition_variance = coding_variance(&state.coding_outcomes);
199 let claim_gate = claim_gate(&artifact_verifier, &state, &coding_condition_variance);
200 let memory_summary = layer_summary(
201 "directional_memory_system_evidence",
202 &state.memory_benchmarks,
203 &state.memory_conditions,
204 &state.memory_claim_levels,
205 state.memory_outcomes.len(),
206 &[
207 "Memory-system capability results are separate from coding-agent outcomes.".to_string(),
208 "Committed memory suites are directional until public claim gates pass.".to_string(),
209 ],
210 );
211 let coding_summary = layer_summary(
212 "smoke_coding_outcome_evidence",
213 &state.coding_benchmarks,
214 &state.coding_conditions,
215 &state.coding_claim_levels,
216 state.coding_outcomes.len(),
217 &[
218 "Current committed coding artifacts are smoke-only.".to_string(),
219 "The #385 claim gate requires no_memory, remem, and curated_file with at least three runs per condition.".to_string(),
220 ],
221 );
222
223 Ok(PublicBaselineReport {
224 schema_version: 1,
225 report_id: "public-baseline-directional-v1".to_string(),
226 report_kind: "baseline_directional_public_benchmark".to_string(),
227 root: root.to_string_lossy().to_string(),
228 created_at_epoch: state.max_created_at_epoch,
229 claim_level: "directional_only_no_public_claim".to_string(),
230 summary: BaselineSummary {
231 memory_system: memory_summary,
232 coding_agent: coding_summary,
233 manifest_count: state.manifest_count,
234 report_count: state.reports.len(),
235 run_artifact_count: state.memory_outcomes.len() + state.coding_outcomes.len(),
236 },
237 reports: state.reports,
238 memory_task_outcomes: state.memory_outcomes,
239 coding_task_outcomes: state.coding_outcomes,
240 coding_condition_variance,
241 failure_decomposition: state.failure_decomposition,
242 reproducibility: ReproducibilitySummary {
243 remem_commits: sorted_vec(state.remem_commits),
244 fixture_revisions: sorted_vec(state.fixture_revisions),
245 docker_image_digests: sorted_vec(state.docker_image_digests),
246 repo_base_commits: sorted_vec(state.repo_base_commits),
247 prompt_hashes: sorted_vec(state.prompt_hashes),
248 models: sorted_vec(state.models),
249 },
250 claim_gate,
251 reproduction_commands: reproduction_commands(),
252 artifact_verifier,
253 })
254}
255
256pub fn render_public_baseline_markdown(report: &PublicBaselineReport) -> String {
257 let mut out = String::new();
258 out.push_str("# remem Public Baseline Directional Report\n\n");
259 out.push_str("Claim level: `");
260 out.push_str(&report.claim_level);
261 out.push_str("`.\n\n");
262 out.push_str("This report separates memory-system capability evidence from coding-agent outcome evidence. It is directional only and does not support SOTA, broad superiority, or coding-task superiority claims.\n\n");
263
264 out.push_str("## Artifact Verification\n\n");
265 out.push_str(&format!(
266 "- Passed: `{}`\n- Manifests checked: `{}`\n- Reports checked: `{}`\n- Run artifacts checked: `{}`\n- Artifact files checked: `{}`\n\n",
267 report.artifact_verifier.passed,
268 report.artifact_verifier.manifests_checked,
269 report.artifact_verifier.reports_checked,
270 report.artifact_verifier.run_artifacts_checked,
271 report.artifact_verifier.artifact_files_checked
272 ));
273
274 out.push_str("## Memory-System Capability\n\n");
275 out.push_str("| Report | Runs | Claim level | Answer score | Support coverage | Citation recall | Non-retention leak rate |\n");
276 out.push_str("|---|---:|---|---:|---:|---:|---:|\n");
277 for entry in report
278 .reports
279 .iter()
280 .filter(|entry| entry.layer == BenchmarkLayer::MemorySystemCapability)
281 {
282 out.push_str(&format!(
283 "| `{}` | {} | `{}` | {} | {} | {} | {} |\n",
284 escape_md(&entry.benchmark_id),
285 entry.run_artifact_count,
286 escape_md(&entry.claim_level),
287 fmt_metric(metric_path(
288 &entry.aggregate_metrics,
289 &["overall", "answer_score"]
290 )),
291 fmt_metric(metric_path(
292 &entry.aggregate_metrics,
293 &["overall", "support_coverage"]
294 )),
295 fmt_metric(metric_path(
296 &entry.aggregate_metrics,
297 &["overall", "citation_recall"]
298 )),
299 fmt_metric(metric_path(
300 &entry.aggregate_metrics,
301 &["policy", "non_retention_leak_rate"]
302 ))
303 ));
304 }
305 out.push('\n');
306
307 out.push_str("## Coding-Agent Outcome\n\n");
308 out.push_str("| Condition | Runs | Resolved rate | Token mean | Token variance | Wall-time mean ms | Variance status |\n");
309 out.push_str("|---|---:|---:|---:|---:|---:|---|\n");
310 for variance in &report.coding_condition_variance {
311 out.push_str(&format!(
312 "| `{}` | {} | {:.3} | {} | {} | {} | `{}` |\n",
313 escape_md(&variance.condition),
314 variance.runs,
315 variance.resolved_rate,
316 fmt_metric(variance.tokens_total_mean),
317 fmt_metric(variance.tokens_total_sample_variance),
318 fmt_metric(variance.wall_time_ms_mean),
319 escape_md(&variance.variance_status)
320 ));
321 }
322 out.push('\n');
323
324 out.push_str("## Coding Task Outcomes\n\n");
325 out.push_str("| Task | Condition | Run | Resolved | Failure reason | Tokens | Wall time ms | Memory helped | Memory hurt |\n");
326 out.push_str("|---|---|---:|---|---|---:|---:|---|---|\n");
327 for run in &report.coding_task_outcomes {
328 out.push_str(&format!(
329 "| `{}` | `{}` | {} | `{}` | {} | {} | {} | {} | {} |\n",
330 escape_md(&run.task_id),
331 escape_md(&run.condition),
332 run.run_index,
333 run.resolved,
334 run.failure_reason
335 .as_deref()
336 .map(|value| format!("`{}`", escape_md(value)))
337 .unwrap_or_else(|| "`none`".to_string()),
338 fmt_u64(run.tokens_total),
339 fmt_u64(run.wall_time_ms),
340 fmt_bool(run.memory_helped),
341 fmt_bool(run.memory_hurt)
342 ));
343 }
344 out.push('\n');
345
346 out.push_str("## Failure Decomposition\n\n");
347 out.push_str("Coding failure counts:\n\n");
348 append_count_map(
349 &mut out,
350 &report.failure_decomposition.coding_failure_counts,
351 );
352 out.push_str("\nCoding memory-specific failure counts:\n\n");
353 append_count_map(
354 &mut out,
355 &report.failure_decomposition.coding_memory_failure_counts,
356 );
357 out.push_str("\nMemory gap counts:\n\n");
358 append_count_map(&mut out, &report.failure_decomposition.memory_gap_counts);
359 out.push('\n');
360
361 out.push_str("## Reproducibility\n\n");
362 out.push_str("Run these commands from a clean checkout:\n\n");
363 out.push_str("```bash\n");
364 for command in &report.reproduction_commands {
365 out.push_str(command);
366 out.push('\n');
367 }
368 out.push_str("```\n\n");
369 out.push_str("Locks and evidence are recorded in the JSON report under `reproducibility`, including remem commits, fixture revisions, Docker image digests, prompt hashes, model labels, and repo base commits when present.\n\n");
370
371 out.push_str("## Claim Gate\n\n");
372 out.push_str(&format!(
373 "- Artifact verifier passed: `{}`\n- Coding outcome stop-loss status: `{}`\n- Public SOTA status: `{}`\n",
374 report.claim_gate.artifact_verifier_passed,
375 report.claim_gate.coding_outcome_stop_loss_status,
376 report.claim_gate.public_sota_status
377 ));
378 for note in &report.claim_gate.notes {
379 out.push_str("- ");
380 out.push_str(note);
381 out.push('\n');
382 }
383
384 out
385}
386
387fn load_report(root: &Path, path: &Path, state: &mut BuildState) -> Result<()> {
388 let report: PublicBenchmarkReport = read_json(path)?;
389 let report_path = relative_path(root, path);
390 match report.layer {
391 BenchmarkLayer::MemorySystemCapability => {
392 state.memory_benchmarks.insert(report.benchmark_id.clone());
393 state.memory_claim_levels.insert(report.claim_level.clone());
394 for condition in &report.conditions {
395 state.memory_conditions.insert(condition.clone());
396 }
397 load_memory_runs(root, &report_path, &report, state)?;
398 }
399 BenchmarkLayer::CodingAgentOutcome => {
400 state.coding_benchmarks.insert(report.benchmark_id.clone());
401 state.coding_claim_levels.insert(report.claim_level.clone());
402 for condition in &report.conditions {
403 state.coding_conditions.insert(condition.clone());
404 }
405 load_coding_runs(root, &report_path, &report, state)?;
406 }
407 }
408 state.reports.push(BaselineReportEntry {
409 path: report_path,
410 benchmark_id: report.benchmark_id,
411 benchmark_version: report.benchmark_version,
412 layer: report.layer,
413 conditions: report.conditions,
414 run_artifact_count: report.run_artifacts.len(),
415 claim_level: report.claim_level,
416 aggregate_metrics: report.aggregate_metrics,
417 });
418 Ok(())
419}
420
421fn load_memory_runs(
422 root: &Path,
423 report_path: &str,
424 report: &PublicBenchmarkReport,
425 state: &mut BuildState,
426) -> Result<()> {
427 for run_path in &report.run_artifacts {
428 let run: MemoryRunArtifact = read_json(&root.join(run_path))?;
429 observe_environment(&run.environment, state);
430 observe_model(&run.reader_model, state);
431 observe_prompt_hash(&run.reader_model, state);
432 if run.diagnosis.write_side_gap {
433 increment(
434 &mut state.failure_decomposition.memory_gap_counts,
435 "write_side_gap",
436 );
437 }
438 if run.diagnosis.retrieval_side_gap {
439 increment(
440 &mut state.failure_decomposition.memory_gap_counts,
441 "retrieval_side_gap",
442 );
443 }
444 if run.diagnosis.reader_gap {
445 increment(
446 &mut state.failure_decomposition.memory_gap_counts,
447 "reader_gap",
448 );
449 }
450 if run.diagnosis.policy_abstention {
451 increment(
452 &mut state.failure_decomposition.memory_gap_counts,
453 "policy_abstention",
454 );
455 }
456 state.memory_outcomes.push(MemoryTaskOutcome {
457 report_path: report_path.to_string(),
458 suite: run.suite,
459 condition: run.condition,
460 task_id: run.task_id,
461 run_index: run.run_index,
462 answer_score: metric_path(&run.metrics, &["answer_score"]),
463 support_coverage: metric_path(&run.metrics, &["support_coverage"]),
464 citation_recall: metric_path(&run.metrics, &["citation_recall"]),
465 write_side_gap: run.diagnosis.write_side_gap,
466 retrieval_side_gap: run.diagnosis.retrieval_side_gap,
467 reader_gap: run.diagnosis.reader_gap,
468 policy_abstention: run.diagnosis.policy_abstention,
469 });
470 }
471 Ok(())
472}
473
474fn load_coding_runs(
475 root: &Path,
476 report_path: &str,
477 report: &PublicBenchmarkReport,
478 state: &mut BuildState,
479) -> Result<()> {
480 for run_path in &report.run_artifacts {
481 let run: CodingRunArtifact = read_json(&root.join(run_path))?;
482 observe_environment(&run.environment, state);
483 observe_model(&run.model, state);
484 observe_prompt_hash(&run.model, state);
485 if let Some(reason) = &run.failure_reason {
486 increment(
487 &mut state.failure_decomposition.coding_failure_counts,
488 reason,
489 );
490 if is_memory_specific_failure(reason) {
491 increment(
492 &mut state.failure_decomposition.coding_memory_failure_counts,
493 reason,
494 );
495 }
496 }
497 state.coding_outcomes.push(CodingTaskOutcome {
498 report_path: report_path.to_string(),
499 condition: run.condition,
500 task_id: run.task_id,
501 run_index: run.run_index,
502 resolved: run.resolved,
503 failure_reason: run.failure_reason,
504 tokens_total: run.metrics.tokens_total,
505 turns: run.metrics.turns,
506 wall_time_ms: run.metrics.wall_time_ms,
507 memory_helped: run
508 .memory_contract
509 .as_ref()
510 .map(|contract| contract.memory_helped),
511 memory_hurt: run
512 .memory_contract
513 .as_ref()
514 .map(|contract| contract.memory_hurt),
515 });
516 }
517 Ok(())
518}
519
520fn coding_variance(outcomes: &[CodingTaskOutcome]) -> Vec<CodingConditionVariance> {
521 let mut grouped: BTreeMap<String, Vec<&CodingTaskOutcome>> = BTreeMap::new();
522 for outcome in outcomes {
523 grouped
524 .entry(outcome.condition.clone())
525 .or_default()
526 .push(outcome);
527 }
528 grouped
529 .into_iter()
530 .map(|(condition, runs)| {
531 let resolved = runs.iter().filter(|run| run.resolved).count();
532 let tokens = runs
533 .iter()
534 .filter_map(|run| run.tokens_total.map(|value| value as f64))
535 .collect::<Vec<_>>();
536 let wall = runs
537 .iter()
538 .filter_map(|run| run.wall_time_ms.map(|value| value as f64))
539 .collect::<Vec<_>>();
540 let variance_status = if runs.len() >= 3 {
541 "satisfied"
542 } else {
543 "insufficient_runs_for_variance"
544 }
545 .to_string();
546 CodingConditionVariance {
547 condition,
548 runs: runs.len(),
549 resolved_rate: resolved as f64 / runs.len() as f64,
550 tokens_total_mean: mean(&tokens),
551 tokens_total_sample_variance: sample_variance(&tokens),
552 wall_time_ms_mean: mean(&wall),
553 wall_time_ms_sample_variance: sample_variance(&wall),
554 variance_status,
555 }
556 })
557 .collect()
558}
559
560fn claim_gate(
561 artifact_verifier: &BenchVerifyReport,
562 state: &BuildState,
563 variance: &[CodingConditionVariance],
564) -> ClaimGateSummary {
565 let required_conditions = ["no_memory", "remem", "curated_file"];
566 let has_required_conditions = required_conditions
567 .iter()
568 .all(|condition| state.coding_conditions.contains(*condition));
569 let has_three_runs = variance.iter().all(|entry| entry.runs >= 3);
570 let coding_outcome_stop_loss_status = if has_required_conditions && has_three_runs {
571 "ready_for_stop_loss_evaluation"
572 } else {
573 "not_evaluated_insufficient_coding_matrix"
574 };
575 let mut notes = vec![
576 "This baseline is directional only and must not be used for coding-task superiority claims.".to_string(),
577 "README and release wording must not claim SOTA or coding outcome improvement from this report.".to_string(),
578 ];
579 if !has_required_conditions {
580 notes.push(
581 "Coding artifacts do not yet include no_memory, remem, and curated_file conditions."
582 .to_string(),
583 );
584 }
585 if !has_three_runs {
586 notes.push(
587 "Coding artifacts do not yet have at least three runs per condition.".to_string(),
588 );
589 }
590 ClaimGateSummary {
591 artifact_verifier_passed: artifact_verifier.passed,
592 coding_outcome_stop_loss_status: coding_outcome_stop_loss_status.to_string(),
593 public_sota_status: "not_evaluated_no_public_sota_claim".to_string(),
594 notes,
595 }
596}
597
598fn layer_summary(
599 status: &str,
600 benchmark_ids: &BTreeSet<String>,
601 conditions: &BTreeSet<String>,
602 claim_levels: &BTreeSet<String>,
603 run_artifact_count: usize,
604 notes: &[String],
605) -> BaselineLayerSummary {
606 BaselineLayerSummary {
607 status: status.to_string(),
608 report_count: claim_levels.len().max(benchmark_ids.len()),
609 run_artifact_count,
610 benchmark_ids: sorted_vec(benchmark_ids.clone()),
611 conditions: sorted_vec(conditions.clone()),
612 claim_levels: sorted_vec(claim_levels.clone()),
613 notes: notes.to_vec(),
614 }
615}
616
617fn read_json<T: DeserializeOwned>(path: &Path) -> Result<T> {
618 let content = fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?;
619 serde_json::from_str(&content).with_context(|| format!("parse {}", path.display()))
620}
621
622fn write_text_file(path: &Path, content: &str) -> Result<()> {
623 if let Some(parent) = path.parent() {
624 if !parent.as_os_str().is_empty() {
625 fs::create_dir_all(parent)
626 .with_context(|| format!("create report directory {}", parent.display()))?;
627 }
628 }
629 fs::write(path, content).with_context(|| format!("write {}", path.display()))
630}
631
632fn observe_environment(environment: &RunEnvironment, state: &mut BuildState) {
633 insert_non_empty(&mut state.remem_commits, &environment.remem_commit);
634 if let Some(value) = &environment.fixture_revision {
635 insert_non_empty(&mut state.fixture_revisions, value);
636 }
637 if let Some(value) = &environment.docker_image_digest {
638 insert_non_empty(&mut state.docker_image_digests, value);
639 }
640 if let Some(value) = &environment.repo_base_commit {
641 insert_non_empty(&mut state.repo_base_commits, value);
642 }
643}
644
645fn observe_model(model: &Value, state: &mut BuildState) {
646 let provider = value_string(model, "provider")
647 .or_else(|| value_string(model, "agent"))
648 .unwrap_or_else(|| "unknown".to_string());
649 let name = value_string(model, "model").unwrap_or_else(|| "unknown".to_string());
650 insert_non_empty(&mut state.models, &format!("{provider}/{name}"));
651}
652
653fn observe_prompt_hash(model: &Value, state: &mut BuildState) {
654 if let Some(value) = value_string(model, "prompt_hash") {
655 insert_non_empty(&mut state.prompt_hashes, &value);
656 }
657}
658
659fn value_string(value: &Value, key: &str) -> Option<String> {
660 value
661 .get(key)
662 .and_then(Value::as_str)
663 .map(ToString::to_string)
664}
665
666fn insert_non_empty(set: &mut BTreeSet<String>, value: &str) {
667 if !value.trim().is_empty() {
668 set.insert(value.to_string());
669 }
670}
671
672fn metric_path(value: &Value, path: &[&str]) -> Option<f64> {
673 let mut cursor = value;
674 for segment in path {
675 cursor = cursor.get(*segment)?;
676 }
677 cursor.as_f64()
678}
679
680fn mean(values: &[f64]) -> Option<f64> {
681 (!values.is_empty()).then(|| values.iter().sum::<f64>() / values.len() as f64)
682}
683
684fn sample_variance(values: &[f64]) -> Option<f64> {
685 if values.len() < 2 {
686 return None;
687 }
688 let average = mean(values)?;
689 Some(
690 values
691 .iter()
692 .map(|value| {
693 let delta = value - average;
694 delta * delta
695 })
696 .sum::<f64>()
697 / (values.len() - 1) as f64,
698 )
699}
700
701fn increment(map: &mut BTreeMap<String, usize>, key: &str) {
702 *map.entry(key.to_string()).or_default() += 1;
703}
704
705fn is_memory_specific_failure(reason: &str) -> bool {
706 matches!(
707 reason,
708 "ignored_memory"
709 | "missing_memory"
710 | "stale_memory_followed"
711 | "irrelevant_memory_distracted"
712 | "agent_hallucinated_memory"
713 )
714}
715
716fn sorted_vec(set: BTreeSet<String>) -> Vec<String> {
717 set.into_iter().collect()
718}
719
720fn relative_path(root: &Path, path: &Path) -> String {
721 path.strip_prefix(root)
722 .unwrap_or(path)
723 .to_string_lossy()
724 .replace('\\', "/")
725}
726
727fn reproduction_commands() -> Vec<String> {
728 vec![
729 "cargo run -- bench verify --root eval/public --json-out /tmp/remem-public-bench-verify.json".to_string(),
730 "cargo run -- bench report --root eval/public --json-out eval/public/reports/baseline.json --markdown-out eval/public/reports/baseline.md".to_string(),
731 "cargo run -- bench coding --suite issue385-v1 --dry-run --json-out /tmp/remem-issue385-v1-dry-run.json".to_string(),
732 "cargo run -- bench memory --suite remem-code-memory --condition remem_default --root eval/public --artifact-prefix memory/artifacts/remem-code-memory-v1 --json-out eval/public/memory/reports/remem-code-memory-v1.json".to_string(),
733 "cargo run -- bench memory --suite adversarial-policy --condition remem_default --root eval/public --artifact-prefix memory/artifacts/adversarial-policy-v1 --json-out eval/public/memory/reports/adversarial-policy-v1.json".to_string(),
734 ]
735}
736
737fn escape_md(value: &str) -> String {
738 value.replace('|', "\\|")
739}
740
741fn fmt_metric(value: Option<f64>) -> String {
742 value
743 .map(|value| format!("{value:.3}"))
744 .unwrap_or_else(|| "n/a".to_string())
745}
746
747fn fmt_u64(value: Option<u64>) -> String {
748 value
749 .map(|value| value.to_string())
750 .unwrap_or_else(|| "n/a".to_string())
751}
752
753fn fmt_bool(value: Option<bool>) -> String {
754 value
755 .map(|value| format!("`{value}`"))
756 .unwrap_or_else(|| "`n/a`".to_string())
757}
758
759fn append_count_map(out: &mut String, map: &BTreeMap<String, usize>) {
760 if map.is_empty() {
761 out.push_str("- none\n");
762 return;
763 }
764 for (key, count) in map {
765 out.push_str(&format!("- `{}`: {}\n", escape_md(key), count));
766 }
767}