1use std::collections::{BTreeMap, BTreeSet};
2
3use serde::Serialize;
4
5use super::{matrix, CodingTaskOutcome};
6
7const CODING_PAIRED_BOOTSTRAP_REPLICATES: usize = 10_000;
8const CODING_PAIRED_BOOTSTRAP_SEED: u64 = 931;
9const CODING_PAIRED_BOOTSTRAP_CI_LEVEL: f64 = 0.95;
10const CODING_PAIRED_BOOTSTRAP_METHOD: &str = "task-cluster paired bootstrap";
11const CODING_PAIRED_BOOTSTRAP_ALGORITHM: &str = "task_cluster_paired_bootstrap_v1";
12const CODING_PAIRED_BOOTSTRAP_PERCENTILE_RULE: &str =
13 "sorted floor(alpha/2 * n), ceil((1 - alpha/2) * n) - 1";
14
15const CODING_PAIRED_COMPARISONS: [(&str, &str, &str); 2] = [
16 ("remem-e2e-vs-no-memory-v1", "remem_e2e", "no_memory"),
17 (
18 "remem-e2e-vs-curated-file-budgeted-v1",
19 "remem_e2e",
20 "curated_file_budgeted",
21 ),
22];
23
24#[derive(Debug, Clone, Serialize)]
25pub struct CodingConditionVariance {
26 pub condition: String,
27 pub runs: usize,
28 pub resolved_rate: f64,
29 pub tokens_total_mean: Option<f64>,
30 pub tokens_total_sample_variance: Option<f64>,
31 pub wall_time_ms_mean: Option<f64>,
32 pub wall_time_ms_sample_variance: Option<f64>,
33 pub variance_status: String,
34}
35
36#[derive(Debug, Clone, Serialize)]
37pub struct CodingPairedStatistic {
38 pub comparison_id: String,
39 pub treatment: String,
40 pub control: String,
41 pub metric: String,
42 pub report_path: Option<String>,
43 pub status: String,
44 pub insufficient_reason: Option<String>,
45 pub tasks: usize,
46 pub runs_per_task: usize,
47 pub treatment_resolved_rate: Option<f64>,
48 pub control_resolved_rate: Option<f64>,
49 pub effect_pp: Option<f64>,
50 pub ci_level: f64,
51 pub ci_lower_pp: Option<f64>,
52 pub ci_upper_pp: Option<f64>,
53 pub bootstrap_replicates: usize,
54 pub bootstrap_seed: u64,
55 pub statistical_unit: String,
56 pub method: String,
57 pub algorithm: String,
58 pub percentile_rule: String,
59}
60
61pub(super) fn coding_variance(outcomes: &[CodingTaskOutcome]) -> Vec<CodingConditionVariance> {
62 let mut grouped: BTreeMap<String, Vec<&CodingTaskOutcome>> = BTreeMap::new();
63 for outcome in outcomes {
64 grouped
65 .entry(outcome.condition.clone())
66 .or_default()
67 .push(outcome);
68 }
69 grouped
70 .into_iter()
71 .map(|(condition, runs)| {
72 let resolved = runs.iter().filter(|run| run.resolved).count();
73 let tokens = runs
74 .iter()
75 .filter_map(|run| run.tokens_total.map(|value| value as f64))
76 .collect::<Vec<_>>();
77 let wall = runs
78 .iter()
79 .filter_map(|run| run.wall_time_ms.map(|value| value as f64))
80 .collect::<Vec<_>>();
81 let variance_status = if runs.len() >= 3 {
82 "satisfied"
83 } else {
84 "insufficient_runs_for_variance"
85 }
86 .to_string();
87 CodingConditionVariance {
88 condition,
89 runs: runs.len(),
90 resolved_rate: resolved as f64 / runs.len() as f64,
91 tokens_total_mean: mean(&tokens),
92 tokens_total_sample_variance: sample_variance(&tokens),
93 wall_time_ms_mean: mean(&wall),
94 wall_time_ms_sample_variance: sample_variance(&wall),
95 variance_status,
96 }
97 })
98 .collect()
99}
100
101pub(in crate::eval::bench_artifact) fn coding_paired_statistics(
102 outcomes: &[CodingTaskOutcome],
103 artifact_verifier_passed: bool,
104) -> Vec<CodingPairedStatistic> {
105 let mut by_report: BTreeMap<&str, Vec<&CodingTaskOutcome>> = BTreeMap::new();
106 for outcome in outcomes {
107 by_report
108 .entry(&outcome.report_path)
109 .or_default()
110 .push(outcome);
111 }
112
113 let complete_reports = by_report
114 .into_iter()
115 .filter(|(_, report_outcomes)| report_outcomes_structurally_complete(report_outcomes))
116 .collect::<Vec<_>>();
117
118 if complete_reports.is_empty() {
119 return CODING_PAIRED_COMPARISONS
120 .into_iter()
121 .map(|(comparison_id, treatment, control)| {
122 insufficient_coding_paired_statistic(
123 comparison_id,
124 treatment,
125 control,
126 None,
127 0,
128 0,
129 "requires one verified issue385-v1/official-v1 report containing no_memory, remem_e2e, and curated_file_budgeted for all 16 registered tasks with run indices 0, 1, and 2",
130 )
131 })
132 .collect();
133 }
134
135 let mut statistics = Vec::new();
136 for (report_path, report_outcomes) in complete_reports {
137 for (comparison_id, treatment, control) in CODING_PAIRED_COMPARISONS {
138 if !artifact_verifier_passed {
139 statistics.push(insufficient_coding_paired_statistic(
140 comparison_id,
141 treatment,
142 control,
143 Some(report_path),
144 matrix::CLAIM_BEARING_TASK_IDS.len(),
145 matrix::REGISTERED_RUN_INDICES.len(),
146 "the benchmark artifact verifier did not pass; integrity-invalid tuples cannot be aggregated",
147 ));
148 } else if !matrix::report_attempts_ready_for_aggregation(&report_outcomes) {
149 statistics.push(insufficient_coding_paired_statistic(
150 comparison_id,
151 treatment,
152 control,
153 Some(report_path),
154 matrix::CLAIM_BEARING_TASK_IDS.len(),
155 matrix::REGISTERED_RUN_INDICES.len(),
156 "one or more tuples lack a unique verified attempt_id or target_started=true; pre-target failures cannot be scored as zero",
157 ));
158 } else {
159 statistics.push(compute_coding_paired_statistic(
160 report_path,
161 &report_outcomes,
162 comparison_id,
163 treatment,
164 control,
165 ));
166 }
167 }
168 }
169 statistics
170}
171
172pub(in crate::eval::bench_artifact) fn coding_report_structurally_complete(
173 outcomes: &[CodingTaskOutcome],
174) -> bool {
175 report_outcomes_structurally_complete(&outcomes.iter().collect::<Vec<_>>())
176}
177
178fn insufficient_coding_paired_statistic(
179 comparison_id: &str,
180 treatment: &str,
181 control: &str,
182 report_path: Option<&str>,
183 tasks: usize,
184 runs_per_task: usize,
185 reason: &str,
186) -> CodingPairedStatistic {
187 CodingPairedStatistic {
188 comparison_id: comparison_id.to_string(),
189 treatment: treatment.to_string(),
190 control: control.to_string(),
191 metric: "resolved_rate".to_string(),
192 report_path: report_path.map(ToString::to_string),
193 status: "insufficient".to_string(),
194 insufficient_reason: Some(reason.to_string()),
195 tasks,
196 runs_per_task,
197 treatment_resolved_rate: None,
198 control_resolved_rate: None,
199 effect_pp: None,
200 ci_level: CODING_PAIRED_BOOTSTRAP_CI_LEVEL,
201 ci_lower_pp: None,
202 ci_upper_pp: None,
203 bootstrap_replicates: CODING_PAIRED_BOOTSTRAP_REPLICATES,
204 bootstrap_seed: CODING_PAIRED_BOOTSTRAP_SEED,
205 statistical_unit: "task".to_string(),
206 method: CODING_PAIRED_BOOTSTRAP_METHOD.to_string(),
207 algorithm: CODING_PAIRED_BOOTSTRAP_ALGORITHM.to_string(),
208 percentile_rule: CODING_PAIRED_BOOTSTRAP_PERCENTILE_RULE.to_string(),
209 }
210}
211
212fn compute_coding_paired_statistic(
213 report_path: &str,
214 outcomes: &[&CodingTaskOutcome],
215 comparison_id: &str,
216 treatment: &str,
217 control: &str,
218) -> CodingPairedStatistic {
219 let treatment_means = resolved_means_by_task(outcomes, treatment);
220 let control_means = resolved_means_by_task(outcomes, control);
221 let mut treatment_rates = Vec::new();
222 let mut control_rates = Vec::new();
223 let mut paired_effects = Vec::new();
224
225 for task_id in matrix::CLAIM_BEARING_TASK_IDS {
226 let treatment_rate = treatment_means[task_id];
227 let control_rate = control_means[task_id];
228 treatment_rates.push(treatment_rate);
229 control_rates.push(control_rate);
230 paired_effects.push(treatment_rate - control_rate);
231 }
232
233 let effect_pp = mean_required(&paired_effects) * 100.0;
234 let (ci_lower_pp, ci_upper_pp) = bootstrap_paired_ci(&paired_effects);
235
236 CodingPairedStatistic {
237 comparison_id: comparison_id.to_string(),
238 treatment: treatment.to_string(),
239 control: control.to_string(),
240 metric: "resolved_rate".to_string(),
241 report_path: Some(report_path.to_string()),
242 status: "computed".to_string(),
243 insufficient_reason: None,
244 tasks: matrix::CLAIM_BEARING_TASK_IDS.len(),
245 runs_per_task: matrix::REGISTERED_RUN_INDICES.len(),
246 treatment_resolved_rate: Some(mean_required(&treatment_rates)),
247 control_resolved_rate: Some(mean_required(&control_rates)),
248 effect_pp: Some(effect_pp),
249 ci_level: CODING_PAIRED_BOOTSTRAP_CI_LEVEL,
250 ci_lower_pp: Some(ci_lower_pp),
251 ci_upper_pp: Some(ci_upper_pp),
252 bootstrap_replicates: CODING_PAIRED_BOOTSTRAP_REPLICATES,
253 bootstrap_seed: CODING_PAIRED_BOOTSTRAP_SEED,
254 statistical_unit: "task".to_string(),
255 method: CODING_PAIRED_BOOTSTRAP_METHOD.to_string(),
256 algorithm: CODING_PAIRED_BOOTSTRAP_ALGORITHM.to_string(),
257 percentile_rule: CODING_PAIRED_BOOTSTRAP_PERCENTILE_RULE.to_string(),
258 }
259}
260
261fn report_outcomes_structurally_complete(outcomes: &[&CodingTaskOutcome]) -> bool {
262 if outcomes.len()
263 != matrix::CLAIM_BEARING_CODING_CONDITIONS.len()
264 * matrix::CLAIM_BEARING_TASK_IDS.len()
265 * matrix::REGISTERED_RUN_INDICES.len()
266 {
267 return false;
268 }
269 if !outcomes.iter().all(|outcome| {
270 outcome.benchmark_id == matrix::REGISTERED_BENCHMARK_ID
271 && outcome.benchmark_version == matrix::REGISTERED_BENCHMARK_VERSION
272 && outcome.run_phase == matrix::REGISTERED_RUN_PHASE
273 && outcome.matrix_namespace == matrix::REGISTERED_MATRIX_NAMESPACE
274 }) {
275 return false;
276 }
277
278 let mut conditions: BTreeMap<&str, BTreeMap<&str, BTreeSet<u32>>> = BTreeMap::new();
279 for outcome in outcomes {
280 conditions
281 .entry(&outcome.condition)
282 .or_default()
283 .entry(&outcome.task_id)
284 .or_default()
285 .insert(outcome.run_index);
286 }
287 let condition_names = conditions
288 .keys()
289 .map(|condition| (*condition).to_string())
290 .collect();
291 if !matrix::has_claim_bearing_coding_conditions(&condition_names) {
292 return false;
293 }
294
295 let registered_tasks = BTreeSet::from(matrix::CLAIM_BEARING_TASK_IDS);
296 let registered_indices = BTreeSet::from(matrix::REGISTERED_RUN_INDICES);
297 matrix::CLAIM_BEARING_CODING_CONDITIONS
298 .iter()
299 .all(|condition| {
300 let task_runs = &conditions[*condition];
301 task_runs.keys().copied().collect::<BTreeSet<_>>() == registered_tasks
302 && task_runs
303 .values()
304 .all(|run_indices| run_indices == ®istered_indices)
305 })
306}
307
308fn resolved_means_by_task<'a>(
309 outcomes: &'a [&'a CodingTaskOutcome],
310 condition: &str,
311) -> BTreeMap<&'a str, f64> {
312 let mut grouped: BTreeMap<&str, Vec<bool>> = BTreeMap::new();
313 for outcome in outcomes
314 .iter()
315 .copied()
316 .filter(|outcome| outcome.condition == condition)
317 {
318 grouped
319 .entry(&outcome.task_id)
320 .or_default()
321 .push(outcome.resolved);
322 }
323 grouped
324 .into_iter()
325 .map(|(task_id, runs)| {
326 let resolved = runs.iter().filter(|resolved| **resolved).count();
327 (task_id, resolved as f64 / runs.len() as f64)
328 })
329 .collect()
330}
331
332fn bootstrap_paired_ci(task_effects: &[f64]) -> (f64, f64) {
333 let mut rng = SplitMix64::new(CODING_PAIRED_BOOTSTRAP_SEED);
334 let task_count = task_effects.len();
335 let mut samples = Vec::with_capacity(CODING_PAIRED_BOOTSTRAP_REPLICATES);
336 for _ in 0..CODING_PAIRED_BOOTSTRAP_REPLICATES {
337 let mut sum = 0.0;
338 for _ in 0..task_count {
339 sum += task_effects[rng.next_index(task_count)];
340 }
341 samples.push(sum / task_count as f64 * 100.0);
342 }
343 samples.sort_by(f64::total_cmp);
344
345 let alpha = 1.0 - CODING_PAIRED_BOOTSTRAP_CI_LEVEL;
346 let lower_index = ((alpha / 2.0) * samples.len() as f64).floor() as usize;
347 let upper_index = ((1.0 - alpha / 2.0) * samples.len() as f64).ceil() as usize;
348 let upper_index = upper_index.saturating_sub(1).min(samples.len() - 1);
349 (samples[lower_index], samples[upper_index])
350}
351
352fn mean(values: &[f64]) -> Option<f64> {
353 (!values.is_empty()).then(|| mean_required(values))
354}
355
356fn mean_required(values: &[f64]) -> f64 {
357 values.iter().sum::<f64>() / values.len() as f64
358}
359
360fn sample_variance(values: &[f64]) -> Option<f64> {
361 if values.len() < 2 {
362 return None;
363 }
364 let average = mean(values)?;
365 Some(
366 values
367 .iter()
368 .map(|value| {
369 let delta = value - average;
370 delta * delta
371 })
372 .sum::<f64>()
373 / (values.len() - 1) as f64,
374 )
375}
376
377struct SplitMix64 {
378 state: u64,
379}
380
381impl SplitMix64 {
382 fn new(seed: u64) -> Self {
383 Self { state: seed }
384 }
385
386 fn next_u64(&mut self) -> u64 {
387 self.state = self.state.wrapping_add(0x9e37_79b9_7f4a_7c15);
388 let mut value = self.state;
389 value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
390 value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
391 value ^ (value >> 31)
392 }
393
394 fn next_index(&mut self, len: usize) -> usize {
395 (self.next_u64() % len as u64) as usize
396 }
397}