1use std::collections::BTreeMap;
2
3use serde::{Deserialize, Serialize};
4
5#[derive(Debug, Clone)]
6pub struct CodingBenchOptions {
7 pub fixture_path: String,
8 pub runs_per_condition: usize,
9 pub json_out: String,
10 pub condition: Option<String>,
11 pub task: Option<String>,
12 pub task_set: String,
13 pub keep_workdirs: bool,
14 pub dry_run: bool,
15 pub runner: String,
16 pub codex_bin: String,
17 pub model: String,
18 pub provider: Option<String>,
19 pub reasoning_effort: String,
20 pub ignore_budget: bool,
21}
22
23#[derive(Debug, Clone, Deserialize)]
24pub struct CodingBenchFixture {
25 pub version: u32,
26 pub repo: FixtureRepo,
27 #[serde(default)]
28 pub curated_context: Option<String>,
29 pub tasks: Vec<CodingBenchTask>,
30}
31
32#[derive(Debug, Clone, Deserialize)]
33pub struct FixtureRepo {
34 pub kind: String,
35 pub base_commit: Option<String>,
36 pub fixture_revision: Option<String>,
37 #[serde(default)]
38 pub files: BTreeMap<String, String>,
39}
40
41#[derive(Debug, Clone, Deserialize)]
42pub struct CodingBenchTask {
43 pub id: String,
44 pub category: String,
45 #[serde(default)]
46 pub smoke: bool,
47 pub prompt: String,
48 #[serde(default = "default_timeout_ms")]
49 pub timeout_ms: u64,
50 #[serde(default)]
51 pub allowed_paths: Vec<String>,
52 #[serde(default)]
53 pub forbidden_paths: Vec<String>,
54 pub score: ScoreSpec,
55 #[serde(default)]
56 pub history_episodes: Vec<HistoryEpisode>,
57 #[serde(default)]
58 pub memories: Vec<SeedMemory>,
59 #[serde(default)]
60 pub curated_context: Option<String>,
61 #[serde(default)]
62 pub gold_memory: GoldMemory,
63}
64
65#[derive(Debug, Clone, Deserialize)]
66pub struct ScoreSpec {
67 #[serde(default)]
68 pub commands: Vec<Vec<String>>,
69 #[serde(default)]
70 pub hidden_files: BTreeMap<String, String>,
71 #[serde(default)]
72 pub required_patch_patterns: Vec<String>,
73 #[serde(default)]
74 pub forbidden_patch_patterns: Vec<String>,
75}
76
77#[derive(Debug, Clone, Deserialize)]
78pub struct HistoryEpisode {
79 pub episode_id: String,
80 pub reference_time_epoch: i64,
81 pub summary: String,
82 #[serde(default)]
83 pub expected_memory_facts: Vec<String>,
84 #[serde(default)]
85 pub memories: Vec<SeedMemory>,
86}
87
88#[derive(Debug, Clone, Deserialize)]
89pub struct SeedMemory {
90 pub title: String,
91 pub text: String,
92 #[serde(default)]
93 pub memory_type: Option<String>,
94 #[serde(default)]
95 pub topic_key: Option<String>,
96 #[serde(default)]
97 pub files: Vec<String>,
98}
99
100#[derive(Debug, Clone, Default, Deserialize)]
101pub struct GoldMemory {
102 #[serde(default)]
103 pub required_facts: Vec<String>,
104 #[serde(default)]
105 pub forbidden_facts: Vec<String>,
106 #[serde(default)]
107 pub supporting_event_ids: Vec<String>,
108}
109
110impl CodingBenchTask {
111 pub fn seed_memories(&self) -> Vec<&SeedMemory> {
112 self.history_episodes
113 .iter()
114 .flat_map(|episode| episode.memories.iter())
115 .chain(self.memories.iter())
116 .collect()
117 }
118}
119
120#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
121#[serde(rename_all = "snake_case")]
122pub enum BenchCondition {
123 NoMemory,
124 Remem,
125 CuratedFile,
126}
127
128impl BenchCondition {
129 pub const ALL: [Self; 3] = [Self::NoMemory, Self::Remem, Self::CuratedFile];
130
131 pub const fn as_str(self) -> &'static str {
132 match self {
133 Self::NoMemory => "no_memory",
134 Self::Remem => "remem",
135 Self::CuratedFile => "curated_file",
136 }
137 }
138
139 pub fn parse(value: &str) -> Option<Self> {
140 match value {
141 "no_memory" => Some(Self::NoMemory),
142 "remem" => Some(Self::Remem),
143 "curated_file" => Some(Self::CuratedFile),
144 _ => None,
145 }
146 }
147}
148
149#[derive(Debug, Clone, Serialize)]
150pub struct CodingBenchReport {
151 pub schema_version: u32,
152 pub generated_at_epoch: i64,
153 pub fixture_path: String,
154 pub fixture_sha256: String,
155 pub remem_rev: String,
156 pub source_dirty: Option<bool>,
157 pub command: Vec<String>,
158 pub artifact_policy: String,
159 pub runner: RunnerReport,
160 pub runs_per_condition: usize,
161 pub ignore_budget: bool,
162 pub conditions: Vec<ConditionReport>,
163}
164
165#[derive(Debug, Clone, Serialize)]
166pub struct RunnerReport {
167 pub provider: String,
168 pub model: String,
169 pub runner: String,
170 pub version: Option<String>,
171}
172
173#[derive(Debug, Clone, Serialize)]
174pub struct ConditionReport {
175 pub name: BenchCondition,
176 pub summary: ConditionSummary,
177 pub runs: Vec<RunReport>,
178}
179
180#[derive(Debug, Clone, Serialize, Default)]
181pub struct ConditionSummary {
182 pub resolution_rate: f64,
183 pub tokens_total_mean: f64,
184 pub tokens_total_stddev: f64,
185 pub turns_mean: Option<f64>,
186 pub wall_time_ms_mean: f64,
187 pub wall_time_ms_p95: f64,
188 pub failure_counts: BTreeMap<CodingBenchFailureReason, usize>,
189 pub memory_failure_counts: BTreeMap<CodingBenchFailureReason, usize>,
190}
191
192#[derive(Debug, Clone, Serialize)]
193pub struct RunReport {
194 pub condition: BenchCondition,
195 pub task_id: String,
196 pub run_index: usize,
197 pub resolved: bool,
198 pub failure_reason: Option<CodingBenchFailureReason>,
199 pub usage: BenchTokenUsage,
200 pub turns: Option<usize>,
201 pub wall_time_ms: u128,
202 pub final_head_sha: Option<String>,
203 pub changed_paths: Vec<String>,
204 pub unauthorized_path_changes: Vec<String>,
205 pub runner_exit_code: Option<i32>,
206 pub runner_timed_out: bool,
207 pub score_commands: Vec<CommandReport>,
208 #[serde(skip_serializing_if = "Option::is_none")]
209 pub memory_contract: Option<CodingMemoryAttribution>,
210 #[serde(skip)]
211 pub artifacts: RunArtifacts,
212 pub workdir: Option<String>,
213}
214
215#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
216#[serde(rename_all = "snake_case")]
217pub enum CodingBenchFailureReason {
218 TestFailure,
219 Timeout,
220 CompileFailure,
221 WrongFileModified,
222 IgnoredMemory,
223 MissingMemory,
224 StaleMemoryFollowed,
225 IrrelevantMemoryDistracted,
226 OverContextBudget,
227 AgentHallucinatedMemory,
228 OracleInconclusive,
229}
230
231impl CodingBenchFailureReason {
232 pub const fn is_memory_specific(self) -> bool {
233 matches!(
234 self,
235 Self::IgnoredMemory
236 | Self::MissingMemory
237 | Self::StaleMemoryFollowed
238 | Self::IrrelevantMemoryDistracted
239 | Self::AgentHallucinatedMemory
240 )
241 }
242}
243
244#[derive(Debug, Clone, Default)]
245pub struct CodingMemoryAttributionInput {
246 pub injected_memory_ids: Vec<i64>,
247 pub relevant_memory_ids: Vec<i64>,
248 pub forbidden_memory_ids: Vec<i64>,
249 pub gold_required_facts: Vec<String>,
250 pub gold_forbidden_facts: Vec<String>,
251}
252
253#[derive(Debug, Clone, Serialize, PartialEq)]
254pub struct CodingMemoryAttribution {
255 pub injected_memory_ids: Vec<i64>,
256 pub used_memory_ids: Vec<i64>,
257 pub citation_precision: f64,
258 pub citation_recall: f64,
259 pub stale_used_count: usize,
260 pub irrelevant_injection_count: usize,
261 pub missing_relevant_memory_count: usize,
262 pub memory_helped: bool,
263 pub memory_hurt: bool,
264}
265
266#[derive(Debug, Clone, Copy, Serialize, Default)]
267pub struct BenchTokenUsage {
268 pub input_tokens: u64,
269 pub output_tokens: u64,
270 pub total_tokens: u64,
271}
272
273#[derive(Debug, Clone, Serialize)]
274pub struct CommandReport {
275 pub command: Vec<String>,
276 pub exit_code: Option<i32>,
277 pub timed_out: bool,
278 #[serde(skip)]
279 pub stdout_artifact: String,
280 #[serde(skip)]
281 pub stderr_artifact: String,
282}
283
284#[derive(Debug, Clone, Serialize)]
285pub struct RunArtifacts {
286 pub runner_stdout: String,
287 pub runner_stderr: String,
288 pub final_diff: String,
289}
290
291fn default_timeout_ms() -> u64 {
292 900_000
293}