remem/eval/bench_artifact/
types.rs

1use serde::{Deserialize, Serialize};
2use serde_json::Value;
3use std::collections::BTreeMap;
4use std::path::PathBuf;
5
6#[derive(Debug, Clone)]
7pub struct BenchVerifyOptions {
8    pub root: PathBuf,
9}
10
11#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
12pub struct BenchVerifyFailure {
13    pub path: String,
14    pub message: String,
15}
16
17#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
18pub struct BenchVerifyReport {
19    pub schema_version: u32,
20    pub root: String,
21    pub passed: bool,
22    pub manifests_checked: usize,
23    pub reports_checked: usize,
24    pub run_artifacts_checked: usize,
25    pub artifact_files_checked: usize,
26    pub failures: Vec<BenchVerifyFailure>,
27}
28
29#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq)]
30#[serde(rename_all = "snake_case")]
31pub enum BenchmarkLayer {
32    MemorySystemCapability,
33    CodingAgentOutcome,
34}
35
36#[derive(Debug, Clone, Deserialize, Serialize)]
37pub struct PublicBenchmarkManifest {
38    pub schema_version: u32,
39    pub benchmark_id: String,
40    pub layer: BenchmarkLayer,
41    pub version: String,
42    pub created_at_epoch: i64,
43    pub source_policy: SourcePolicy,
44    #[serde(default)]
45    pub conditions: Vec<String>,
46    #[serde(default)]
47    pub reports: Vec<String>,
48}
49
50#[derive(Debug, Clone, Deserialize, Serialize)]
51pub struct SourcePolicy {
52    pub private_user_memory_allowed: bool,
53    pub requires_temp_remem_data_dir: bool,
54    pub external_dataset_revision: Option<String>,
55}
56
57#[derive(Debug, Clone, Deserialize, Serialize)]
58pub struct PublicBenchmarkReport {
59    pub schema_version: u32,
60    pub benchmark_id: String,
61    pub benchmark_version: String,
62    pub layer: BenchmarkLayer,
63    #[serde(default)]
64    pub conditions: Vec<String>,
65    #[serde(default)]
66    pub schema_refs: Vec<String>,
67    #[serde(default)]
68    pub run_artifacts: Vec<String>,
69    #[serde(default)]
70    pub aggregate_metrics: Value,
71    pub claim_level: String,
72    pub verifier: ReportVerifierMetadata,
73}
74
75#[derive(Debug, Clone, Deserialize, Serialize)]
76pub struct ReportVerifierMetadata {
77    pub required: bool,
78    pub schema_version: u32,
79}
80
81#[derive(Debug, Clone, Deserialize, Serialize)]
82pub struct RunEnvironment {
83    pub os: String,
84    pub arch: String,
85    pub remem_commit: String,
86    pub remem_data_dir: String,
87    #[serde(default)]
88    pub docker_image_digest: Option<String>,
89    #[serde(default)]
90    pub fixture_revision: Option<String>,
91    #[serde(default)]
92    pub repo_base_commit: Option<String>,
93}
94
95#[derive(Debug, Clone, Deserialize, Serialize)]
96pub struct MemoryRunArtifact {
97    pub schema_version: u32,
98    pub benchmark_version: String,
99    pub layer: BenchmarkLayer,
100    pub suite: String,
101    pub condition: String,
102    pub task_id: String,
103    pub run_index: u32,
104    pub reference_time_epoch: i64,
105    #[serde(default)]
106    pub reader_model: Value,
107    pub environment: RunEnvironment,
108    #[serde(default)]
109    pub answer: Value,
110    pub retrieval: MemoryRetrievalEvidence,
111    pub evidence: MemoryCitationEvidence,
112    #[serde(default)]
113    pub metrics: Value,
114    pub diagnosis: MemoryDiagnosis,
115    #[serde(default)]
116    pub artifacts: BTreeMap<String, String>,
117}
118
119#[derive(Debug, Clone, Deserialize, Serialize)]
120pub struct MemoryRetrievalEvidence {
121    #[serde(default)]
122    pub retrieved_memory_ids: Vec<i64>,
123    #[serde(default)]
124    pub retrieved_supporting_evidence_ids: Vec<String>,
125    #[serde(default)]
126    pub gold_supporting_event_ids: Vec<String>,
127    #[serde(default)]
128    pub missing_supporting_evidence_ids: Vec<String>,
129}
130
131#[derive(Debug, Clone, Deserialize, Serialize)]
132pub struct MemoryCitationEvidence {
133    #[serde(default)]
134    pub cited_memory_ids: Vec<i64>,
135    #[serde(default)]
136    pub cited_event_ids: Vec<String>,
137}
138
139#[derive(Debug, Clone, Deserialize, Serialize)]
140pub struct MemoryDiagnosis {
141    pub write_side_gap: bool,
142    pub retrieval_side_gap: bool,
143    pub reader_gap: bool,
144    pub policy_abstention: bool,
145    #[serde(default)]
146    pub notes: Vec<String>,
147}
148
149#[derive(Debug, Clone, Deserialize, Serialize)]
150pub struct CodingRunArtifact {
151    pub schema_version: u32,
152    pub benchmark_version: String,
153    pub layer: BenchmarkLayer,
154    pub condition: String,
155    pub task_id: String,
156    pub run_index: u32,
157    #[serde(default)]
158    pub model: Value,
159    pub environment: RunEnvironment,
160    pub resolved: bool,
161    pub failure_reason: Option<String>,
162    pub metrics: CodingRunMetrics,
163    #[serde(default)]
164    pub memory_contract: Option<CodingMemoryContract>,
165    #[serde(default)]
166    pub artifacts: BTreeMap<String, String>,
167}
168
169#[derive(Debug, Clone, Deserialize, Serialize)]
170pub struct CodingRunMetrics {
171    pub tokens_input: Option<u64>,
172    pub tokens_output: Option<u64>,
173    pub tokens_total: Option<u64>,
174    pub turns: Option<u64>,
175    pub wall_time_ms: Option<u64>,
176    pub tool_calls: Option<u64>,
177    pub commands_run: Option<u64>,
178}
179
180#[derive(Debug, Clone, Deserialize, Serialize)]
181pub struct CodingMemoryContract {
182    #[serde(default)]
183    pub injected_memory_ids: Vec<i64>,
184    #[serde(default)]
185    pub used_memory_ids: Vec<i64>,
186    pub citation_precision: f64,
187    pub citation_recall: f64,
188    pub stale_used_count: u64,
189    pub irrelevant_injection_count: u64,
190    pub missing_relevant_memory_count: u64,
191    pub memory_helped: bool,
192    pub memory_hurt: bool,
193}
remem/eval/bench_artifact/types.rs

remem/eval/bench_artifact/
types.rs
