1use std::collections::BTreeMap;
2
3use serde::{Deserialize, Serialize};
4
5use super::audit_contract::{RememContextAuditSnapshot, RememContextAuditStatus};
6
7#[derive(Debug, Clone)]
8pub struct CodingBenchOptions {
9 pub fixture_path: String,
10 pub runs_per_condition: usize,
11 pub json_out: String,
12 pub condition: Option<String>,
13 pub matrix: String,
14 pub task: Option<String>,
15 pub task_set: String,
16 pub keep_workdirs: bool,
17 pub dry_run: bool,
18 pub runner: String,
19 pub codex_bin: String,
20 pub model: String,
21 pub provider: Option<String>,
22 pub reasoning_effort: String,
23 pub ignore_budget: bool,
24 pub curator_root: Option<String>,
25 pub memory_config: Option<String>,
26 pub run_phase: String,
27 pub matrix_namespace: String,
28 pub verify_live_approval_only: bool,
29 pub live_approval: Option<String>,
30 pub approval_trust_root: Option<String>,
31 pub supervisor_attestation: Option<String>,
32 pub supervisor_bin: Option<String>,
33}
34
35#[derive(Debug, Clone, Deserialize)]
36pub struct CodingBenchFixture {
37 pub version: u32,
38 pub repo: FixtureRepo,
39 #[serde(default)]
40 pub curated_context: Option<String>,
41 pub tasks: Vec<CodingBenchTask>,
42}
43
44#[derive(Debug, Clone, Deserialize)]
45pub struct FixtureRepo {
46 pub kind: String,
47 pub base_commit: Option<String>,
48 pub fixture_revision: Option<String>,
49 #[serde(default)]
50 pub files: BTreeMap<String, String>,
51}
52
53#[derive(Debug, Clone, Deserialize)]
54pub struct CodingBenchTask {
55 pub id: String,
56 pub category: String,
57 #[serde(default)]
58 pub smoke: bool,
59 pub prompt: String,
60 #[serde(default = "default_timeout_ms")]
61 pub timeout_ms: u64,
62 #[serde(default)]
63 pub allowed_paths: Vec<String>,
64 #[serde(default)]
65 pub forbidden_paths: Vec<String>,
66 pub score: ScoreSpec,
67 #[serde(default)]
68 pub history_episodes: Vec<HistoryEpisode>,
69 #[serde(default)]
70 pub memories: Vec<SeedMemory>,
71 #[serde(default)]
72 pub curated_context: Option<String>,
73 #[serde(default)]
74 pub gold_memory: GoldMemory,
75}
76
77#[derive(Debug, Clone, Deserialize)]
78pub struct ScoreSpec {
79 #[serde(default)]
80 pub commands: Vec<Vec<String>>,
81 #[serde(default)]
82 pub hidden_files: BTreeMap<String, String>,
83 #[serde(default)]
84 pub required_patch_patterns: Vec<String>,
85 #[serde(default)]
86 pub forbidden_patch_patterns: Vec<String>,
87}
88
89#[derive(Debug, Clone, Deserialize)]
90pub struct HistoryEpisode {
91 pub episode_id: String,
92 pub reference_time_epoch: i64,
93 pub summary: String,
94 #[serde(default)]
95 pub expected_memory_facts: Vec<String>,
96 #[serde(default)]
97 pub memories: Vec<SeedMemory>,
98 pub raw_events: Vec<RawHistoryEvent>,
99}
100
101#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
102#[serde(deny_unknown_fields)]
103pub struct RawHistoryEvent {
104 pub event_id: String,
105 pub timestamp_epoch: i64,
106 pub role: String,
107 pub sanitized_content: String,
108 pub tool_name: Option<String>,
109 pub sanitized_tool_input: Option<String>,
110 pub sanitized_tool_output: Option<String>,
111 pub host_boundary: String,
112}
113
114#[derive(Debug, Clone, Deserialize)]
115pub struct SeedMemory {
116 pub title: String,
117 pub text: String,
118 #[serde(default)]
119 pub memory_type: Option<String>,
120 #[serde(default)]
121 pub topic_key: Option<String>,
122 #[serde(default)]
123 pub files: Vec<String>,
124}
125
126#[derive(Debug, Clone, Default, Deserialize)]
127pub struct GoldMemory {
128 #[serde(default)]
129 pub required_facts: Vec<String>,
130 #[serde(default)]
131 pub forbidden_facts: Vec<String>,
132 #[serde(default)]
133 pub supporting_event_ids: Vec<String>,
134}
135
136impl CodingBenchTask {
137 pub fn seed_memories(&self) -> Vec<&SeedMemory> {
138 self.history_episodes
139 .iter()
140 .flat_map(|episode| episode.memories.iter())
141 .chain(self.memories.iter())
142 .collect()
143 }
144}
145
146#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
147#[serde(rename_all = "snake_case")]
148pub enum BenchCondition {
149 NoMemory,
150 CuratedFileBudgeted,
151 RememE2e,
152 #[serde(rename = "remem_seeded_sessionstart")]
153 RememSeededSessionStart,
154 CuratedFileExpert,
155 OracleEvidence,
156 RememOracleRetrieval,
157 FullHistory,
158 RememNoEnrichment,
159 RememFtsOnly,
160}
161
162impl BenchCondition {
163 pub const PRIMARY: [Self; 3] = [Self::NoMemory, Self::CuratedFileBudgeted, Self::RememE2e];
164 pub const DIAGNOSTIC: [Self; 7] = [
165 Self::RememSeededSessionStart,
166 Self::CuratedFileExpert,
167 Self::OracleEvidence,
168 Self::RememOracleRetrieval,
169 Self::FullHistory,
170 Self::RememNoEnrichment,
171 Self::RememFtsOnly,
172 ];
173 pub const IMPLEMENTED: [Self; 5] = [
174 Self::NoMemory,
175 Self::CuratedFileBudgeted,
176 Self::RememE2e,
177 Self::RememSeededSessionStart,
178 Self::CuratedFileExpert,
179 ];
180 pub const ALL: [Self; 10] = [
181 Self::NoMemory,
182 Self::CuratedFileBudgeted,
183 Self::RememE2e,
184 Self::RememSeededSessionStart,
185 Self::CuratedFileExpert,
186 Self::OracleEvidence,
187 Self::RememOracleRetrieval,
188 Self::FullHistory,
189 Self::RememNoEnrichment,
190 Self::RememFtsOnly,
191 ];
192
193 pub const fn as_str(self) -> &'static str {
194 match self {
195 Self::NoMemory => "no_memory",
196 Self::CuratedFileBudgeted => "curated_file_budgeted",
197 Self::RememE2e => "remem_e2e",
198 Self::RememSeededSessionStart => "remem_seeded_sessionstart",
199 Self::CuratedFileExpert => "curated_file_expert",
200 Self::OracleEvidence => "oracle_evidence",
201 Self::RememOracleRetrieval => "remem_oracle_retrieval",
202 Self::FullHistory => "full_history",
203 Self::RememNoEnrichment => "remem_no_enrichment",
204 Self::RememFtsOnly => "remem_fts_only",
205 }
206 }
207
208 pub fn parse(value: &str) -> Option<Self> {
209 match value {
210 "no_memory" => Some(Self::NoMemory),
211 "curated_file_budgeted" => Some(Self::CuratedFileBudgeted),
212 "remem_e2e" => Some(Self::RememE2e),
213 "remem_seeded_sessionstart" => Some(Self::RememSeededSessionStart),
214 "curated_file_expert" => Some(Self::CuratedFileExpert),
215 "oracle_evidence" => Some(Self::OracleEvidence),
216 "remem_oracle_retrieval" => Some(Self::RememOracleRetrieval),
217 "full_history" => Some(Self::FullHistory),
218 "remem_no_enrichment" => Some(Self::RememNoEnrichment),
219 "remem_fts_only" => Some(Self::RememFtsOnly),
220 _ => None,
221 }
222 }
223
224 pub const fn supports_live_execution(self) -> bool {
225 matches!(
226 self,
227 Self::NoMemory
228 | Self::CuratedFileBudgeted
229 | Self::RememE2e
230 | Self::RememSeededSessionStart
231 | Self::CuratedFileExpert
232 )
233 }
234
235 pub const fn uses_remem_attribution(self) -> bool {
236 matches!(self, Self::RememE2e | Self::RememSeededSessionStart)
237 }
238}
239
240#[cfg(test)]
241mod condition_identity_tests {
242 use super::BenchCondition;
243
244 #[test]
245 fn seeded_sessionstart_has_distinct_nonlegacy_identity() -> serde_json::Result<()> {
246 assert_eq!(
247 BenchCondition::parse("remem_seeded_sessionstart"),
248 Some(BenchCondition::RememSeededSessionStart)
249 );
250 assert_eq!(
251 BenchCondition::RememSeededSessionStart.as_str(),
252 "remem_seeded_sessionstart"
253 );
254 assert_eq!(BenchCondition::parse("remem"), None);
255 assert_eq!(BenchCondition::parse("remem_preloaded"), None);
256 assert_eq!(BenchCondition::parse("curated_file"), None);
257 assert_eq!(
258 serde_json::to_value(BenchCondition::RememSeededSessionStart)?,
259 "remem_seeded_sessionstart"
260 );
261 Ok(())
262 }
263}
264
265#[derive(Debug, Clone, Serialize)]
266pub struct CodingBenchReport {
267 pub schema_version: u32,
268 pub generated_at_epoch: i64,
269 pub fixture_path: String,
270 pub fixture_sha256: String,
271 pub remem_rev: String,
272 pub source_dirty: Option<bool>,
273 pub command: Vec<String>,
274 pub artifact_policy: String,
275 pub runner: RunnerReport,
276 pub runs_per_condition: usize,
277 pub ignore_budget: bool,
278 pub conditions: Vec<ConditionReport>,
279}
280
281#[derive(Debug, Clone, Serialize)]
282pub struct RunnerReport {
283 pub provider: String,
284 pub model: String,
285 pub runner: String,
286 pub version: Option<String>,
287}
288
289#[derive(Debug, Clone, Serialize)]
290pub struct ConditionReport {
291 pub name: BenchCondition,
292 pub summary: ConditionSummary,
293 pub runs: Vec<RunReport>,
294}
295
296#[derive(Debug, Clone, Serialize, Default)]
297pub struct ConditionSummary {
298 pub resolution_rate: f64,
299 pub tokens_total_mean: f64,
300 pub tokens_total_stddev: f64,
301 pub turns_mean: Option<f64>,
302 pub wall_time_ms_mean: f64,
303 pub wall_time_ms_p95: f64,
304 pub failure_counts: BTreeMap<CodingBenchFailureReason, usize>,
305 pub memory_failure_counts: BTreeMap<CodingBenchFailureReason, usize>,
306 pub human_maintenance_minutes_per_100_sessions: Option<f64>,
307}
308
309#[derive(Debug, Clone, Serialize)]
310pub struct RunReport {
311 pub condition: BenchCondition,
312 pub task_id: String,
313 pub run_index: usize,
314 pub resolved: bool,
315 pub failure_reason: Option<CodingBenchFailureReason>,
316 pub usage: BenchTokenUsage,
317 pub turns: Option<usize>,
318 pub wall_time_ms: u128,
319 pub final_head_sha: Option<String>,
320 pub changed_paths: Vec<String>,
321 pub unauthorized_path_changes: Vec<String>,
322 pub runner_exit_code: Option<i32>,
323 pub runner_timed_out: bool,
324 pub runtime_contract_failure: bool,
325 pub runtime_contract_failure_reason: Option<String>,
326 pub context_audit_status: RememContextAuditStatus,
327 pub context_audit_failure_reason: Option<String>,
328 pub remem_context_audit: Option<RememContextAuditSnapshot>,
329 #[serde(skip_serializing_if = "Option::is_none")]
330 pub curator_log: Option<CuratorLogAttachment>,
331 #[serde(skip_serializing_if = "Option::is_none")]
332 pub e2e_pipeline: Option<super::e2e::E2ePipelineTrace>,
333 pub score_commands: Vec<CommandReport>,
334 #[serde(skip_serializing_if = "Option::is_none")]
335 pub memory_contract: Option<CodingMemoryAttribution>,
336 #[serde(skip)]
337 pub artifacts: RunArtifacts,
338 pub workdir: Option<String>,
339}
340
341#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
342#[serde(rename_all = "snake_case")]
343pub enum CodingBenchFailureReason {
344 TestFailure,
345 Timeout,
346 CompileFailure,
347 WrongFileModified,
348 IgnoredMemory,
349 MissingMemory,
350 StaleMemoryFollowed,
351 IrrelevantMemoryDistracted,
352 OverContextBudget,
353 AgentHallucinatedMemory,
354 OracleInconclusive,
355}
356
357impl CodingBenchFailureReason {
358 pub const fn is_memory_specific(self) -> bool {
359 matches!(
360 self,
361 Self::IgnoredMemory
362 | Self::MissingMemory
363 | Self::StaleMemoryFollowed
364 | Self::IrrelevantMemoryDistracted
365 | Self::AgentHallucinatedMemory
366 )
367 }
368}
369
370#[derive(Debug, Clone, Default)]
371pub struct CodingMemoryAttributionInput {
372 pub injected_memory_ids: Vec<i64>,
373 pub relevant_memory_ids: Vec<i64>,
374 pub forbidden_memory_ids: Vec<i64>,
375 pub gold_required_facts: Vec<String>,
376 pub gold_forbidden_facts: Vec<String>,
377}
378
379#[derive(Debug, Clone, Serialize, PartialEq)]
380pub struct CodingMemoryAttribution {
381 pub injected_memory_ids: Vec<i64>,
382 pub used_memory_ids: Vec<i64>,
383 pub citation_precision: f64,
384 pub citation_recall: f64,
385 pub stale_used_count: usize,
386 pub irrelevant_injection_count: usize,
387 pub missing_relevant_memory_count: usize,
388 pub memory_helped: bool,
389 pub memory_hurt: bool,
390}
391
392#[derive(Debug, Clone, Copy, Serialize, Default)]
393pub struct BenchTokenUsage {
394 pub input_tokens: u64,
395 pub output_tokens: u64,
396 pub total_tokens: u64,
397}
398
399#[derive(Debug, Clone, Serialize)]
400pub struct CommandReport {
401 pub command: Vec<String>,
402 pub exit_code: Option<i32>,
403 pub timed_out: bool,
404 #[serde(skip)]
405 pub stdout_artifact: String,
406 #[serde(skip)]
407 pub stderr_artifact: String,
408}
409
410#[derive(Debug, Clone, Serialize)]
411pub struct RunArtifacts {
412 pub runner_stdout: String,
413 pub runner_stderr: String,
414 pub final_diff: String,
415}
416
417#[derive(Debug, Clone, Serialize, PartialEq)]
418pub struct CuratorLogAttachment {
419 pub schema_version: u32,
420 pub task_id: String,
421 pub target_blind: bool,
422 pub memory_sha256: String,
423 pub curator_log_sha256: String,
424 pub final_char_count: usize,
425 pub history_session_count: usize,
426 pub maintenance_minutes: f64,
427 pub update_count: u64,
428 pub deletion_count: u64,
429 pub conflict_resolution_count: u64,
430}
431
432fn default_timeout_ms() -> u64 {
433 900_000
434}