navi_core/
benchmark.rs

1//! Agentic benchmark schema and comparison helpers.
2//!
3//! Benchmarks differ from verifier-replay evals: each case is intended to run a
4//! real NAVI headless turn inside an isolated fixture, then validate the result
5//! with verifier commands.
6
7use crate::event::RuntimeEvent;
8use crate::verifier::VerifierResult;
9use crate::verifier::VerifierSpec;
10use anyhow::{Context, Result, bail};
11use serde::{Deserialize, Serialize};
12use std::collections::{BTreeMap, BTreeSet};
13use std::path::{Path, PathBuf};
14
15#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
16#[serde(default)]
17pub struct BenchCase {
18    pub version: u32,
19    pub id: String,
20    pub title: String,
21    pub category: String,
22    pub fixture: PathBuf,
23    pub task: String,
24    pub max_turns: Option<u32>,
25    pub max_tool_calls: Option<u32>,
26    pub timeout_ms: Option<u64>,
27    pub agent: BenchAgentConfig,
28    pub setup: Vec<VerifierSpec>,
29    pub verifiers: Vec<VerifierSpec>,
30    pub tags: Vec<String>,
31    pub notes: Option<String>,
32    pub metadata: BTreeMap<String, String>,
33}
34
35impl BenchCase {
36    pub const CURRENT_VERSION: u32 = 1;
37}
38
39impl Default for BenchCase {
40    fn default() -> Self {
41        Self {
42            version: Self::CURRENT_VERSION,
43            id: String::new(),
44            title: String::new(),
45            category: "agentic_repo_task".to_string(),
46            fixture: PathBuf::new(),
47            task: String::new(),
48            max_turns: None,
49            max_tool_calls: None,
50            timeout_ms: None,
51            agent: BenchAgentConfig::default(),
52            setup: Vec::new(),
53            verifiers: Vec::new(),
54            tags: Vec::new(),
55            notes: None,
56            metadata: BTreeMap::new(),
57        }
58    }
59}
60
61#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
62#[serde(default, rename_all = "kebab-case")]
63pub struct BenchAgentConfig {
64    pub mode: String,
65    pub profile: Option<String>,
66    pub provider: Option<String>,
67    pub model: Option<String>,
68    pub active_skills: Vec<String>,
69}
70
71impl Default for BenchAgentConfig {
72    fn default() -> Self {
73        Self {
74            mode: "parity".to_string(),
75            profile: None,
76            provider: None,
77            model: None,
78            active_skills: Vec::new(),
79        }
80    }
81}
82
83#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
84pub struct BenchSuite {
85    pub name: String,
86    pub cases: Vec<BenchCase>,
87}
88
89impl BenchSuite {
90    pub fn load(path: &Path) -> Result<Self> {
91        if path.is_file() {
92            let case = load_case(path)?;
93            return Ok(Self {
94                name: path
95                    .file_stem()
96                    .and_then(|value| value.to_str())
97                    .unwrap_or("bench")
98                    .to_string(),
99                cases: vec![case],
100            });
101        }
102
103        if !path.is_dir() {
104            bail!("benchmark path does not exist: {}", path.display());
105        }
106
107        let mut files = Vec::new();
108        collect_bench_case_files(path, &mut files)
109            .with_context(|| format!("failed to read benchmark suite {}", path.display()))?;
110        files.sort();
111
112        let mut cases = Vec::new();
113        for file in files {
114            cases.push(load_case(&file)?);
115        }
116
117        if cases.is_empty() {
118            bail!(
119                "benchmark suite has no .json or .toml cases: {}",
120                path.display()
121            );
122        }
123        validate_unique_case_ids(&cases, path)?;
124
125        Ok(Self {
126            name: path
127                .file_name()
128                .and_then(|value| value.to_str())
129                .unwrap_or("bench-suite")
130                .to_string(),
131            cases,
132        })
133    }
134}
135
136#[derive(Debug, Clone, Serialize, Deserialize)]
137pub struct BenchRun {
138    pub version: u32,
139    pub run_id: String,
140    pub suite_name: String,
141    #[serde(default, skip_serializing_if = "Option::is_none")]
142    pub provider: Option<String>,
143    #[serde(default, skip_serializing_if = "Option::is_none")]
144    pub model: Option<String>,
145    pub started_at: u64,
146    pub ended_at: u64,
147    pub project_root: PathBuf,
148    pub metrics: BenchRunMetrics,
149    pub results: Vec<BenchCaseResult>,
150}
151
152impl BenchRun {
153    pub const CURRENT_VERSION: u32 = 1;
154}
155
156#[derive(Debug, Clone, Serialize, Deserialize)]
157pub struct BenchCaseResult {
158    pub case_id: String,
159    pub title: String,
160    pub category: String,
161    pub passed: bool,
162    pub workspace: PathBuf,
163    pub assistant_text: String,
164    pub setup_results: Vec<VerifierResult>,
165    pub verifier_results: Vec<VerifierResult>,
166    pub metrics: BenchCaseMetrics,
167    #[serde(default, skip_serializing_if = "Vec::is_empty")]
168    pub events: Vec<RuntimeEvent>,
169    #[serde(default, skip_serializing_if = "Option::is_none")]
170    pub error: Option<String>,
171}
172
173#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
174pub struct BenchCaseMetrics {
175    pub input_tokens: u64,
176    pub output_tokens: u64,
177    pub total_tokens: u64,
178    /// Prompt-cache read tokens reported by the provider (0 if unknown).
179    #[serde(default)]
180    pub cache_read_tokens: u64,
181    /// Prompt-cache write/creation tokens reported by the provider (0 if unknown).
182    #[serde(default)]
183    pub cache_write_tokens: u64,
184    pub turn_count: usize,
185    pub tool_calls: usize,
186    pub failed_tool_calls: usize,
187    pub verifier_count: usize,
188    pub verifier_pass_count: usize,
189    pub wall_time_ms: u64,
190    pub files_changed: usize,
191    pub diff_lines_added: u64,
192    pub diff_lines_removed: u64,
193}
194
195#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
196pub struct BenchRunMetrics {
197    pub total_cases: usize,
198    pub passed_cases: usize,
199    pub failed_cases: usize,
200    pub verified_success_rate: f64,
201    pub tokens_per_success: Option<f64>,
202    pub tool_calls_per_success: Option<f64>,
203    pub wall_time_ms: u64,
204    pub total_tokens: u64,
205    #[serde(default)]
206    pub cache_read_tokens: u64,
207    #[serde(default)]
208    pub cache_write_tokens: u64,
209    /// cache_read / max(input, 1) when any usage reported; None if no inputs.
210    #[serde(default, skip_serializing_if = "Option::is_none")]
211    pub cache_hit_rate: Option<f64>,
212    pub tool_calls: usize,
213    pub failed_tool_calls: usize,
214    pub files_changed: usize,
215    pub diff_lines_added: u64,
216    pub diff_lines_removed: u64,
217}
218
219#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
220pub struct BenchComparison {
221    pub passed: bool,
222    pub baseline_success_rate: f64,
223    pub candidate_success_rate: f64,
224    pub success_rate_delta: f64,
225    pub baseline_tokens_per_success: Option<f64>,
226    pub candidate_tokens_per_success: Option<f64>,
227    pub baseline_tool_calls_per_success: Option<f64>,
228    pub candidate_tool_calls_per_success: Option<f64>,
229    pub failures: Vec<String>,
230}
231
232#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
233pub struct BenchCompareConfig {
234    pub min_success_rate: f64,
235    pub max_success_rate_drop: f64,
236    pub require_token_improvement: bool,
237    pub require_tool_call_improvement: bool,
238}
239
240impl Default for BenchCompareConfig {
241    fn default() -> Self {
242        Self {
243            min_success_rate: 1.0,
244            max_success_rate_drop: 0.0,
245            require_token_improvement: false,
246            require_tool_call_improvement: false,
247        }
248    }
249}
250
251pub fn aggregate_bench_metrics(results: &[BenchCaseResult], wall_time_ms: u64) -> BenchRunMetrics {
252    let total_cases = results.len();
253    let passed_cases = results.iter().filter(|result| result.passed).count();
254    let failed_cases = total_cases.saturating_sub(passed_cases);
255    let total_tokens = results
256        .iter()
257        .map(|result| result.metrics.total_tokens)
258        .sum();
259    let input_tokens: u64 = results.iter().map(|r| r.metrics.input_tokens).sum();
260    let cache_read_tokens: u64 = results.iter().map(|r| r.metrics.cache_read_tokens).sum();
261    let cache_write_tokens: u64 = results.iter().map(|r| r.metrics.cache_write_tokens).sum();
262    let tool_calls = results.iter().map(|result| result.metrics.tool_calls).sum();
263    let failed_tool_calls = results
264        .iter()
265        .map(|result| result.metrics.failed_tool_calls)
266        .sum();
267    let files_changed = results
268        .iter()
269        .map(|result| result.metrics.files_changed)
270        .sum();
271    let diff_lines_added = results
272        .iter()
273        .map(|result| result.metrics.diff_lines_added)
274        .sum();
275    let diff_lines_removed = results
276        .iter()
277        .map(|result| result.metrics.diff_lines_removed)
278        .sum();
279    let cache_hit_rate = (input_tokens > 0).then(|| cache_read_tokens as f64 / input_tokens as f64);
280
281    BenchRunMetrics {
282        total_cases,
283        passed_cases,
284        failed_cases,
285        verified_success_rate: ratio(passed_cases, total_cases),
286        tokens_per_success: (passed_cases > 0 && total_tokens > 0)
287            .then(|| total_tokens as f64 / passed_cases as f64),
288        tool_calls_per_success: (passed_cases > 0 && tool_calls > 0)
289            .then(|| tool_calls as f64 / passed_cases as f64),
290        wall_time_ms,
291        total_tokens,
292        cache_read_tokens,
293        cache_write_tokens,
294        cache_hit_rate,
295        tool_calls,
296        failed_tool_calls,
297        files_changed,
298        diff_lines_added,
299        diff_lines_removed,
300    }
301}
302
303pub fn compare_bench_runs(
304    baseline: Option<&BenchRun>,
305    candidate: &BenchRun,
306    config: BenchCompareConfig,
307) -> BenchComparison {
308    let baseline_success_rate = baseline
309        .map(|run| run.metrics.verified_success_rate)
310        .unwrap_or(0.0);
311    let candidate_success_rate = candidate.metrics.verified_success_rate;
312    let mut failures = Vec::new();
313
314    if candidate_success_rate < config.min_success_rate {
315        failures.push(format!(
316            "candidate success rate {:.3} is below minimum {:.3}",
317            candidate_success_rate, config.min_success_rate
318        ));
319    }
320    if let Some(baseline) = baseline {
321        let allowed = baseline
322            .metrics
323            .verified_success_rate
324            .saturating_sub(config.max_success_rate_drop);
325        if candidate_success_rate < allowed {
326            failures.push(format!(
327                "candidate success rate {:.3} dropped below allowed {:.3}",
328                candidate_success_rate, allowed
329            ));
330        }
331        if config.require_token_improvement
332            && let (Some(base), Some(candidate)) = (
333                baseline.metrics.tokens_per_success,
334                candidate.metrics.tokens_per_success,
335            )
336            && candidate > base
337        {
338            failures.push(format!(
339                "candidate tokens_per_success {:.3} is worse than baseline {:.3}",
340                candidate, base
341            ));
342        }
343        if config.require_tool_call_improvement
344            && let (Some(base), Some(candidate)) = (
345                baseline.metrics.tool_calls_per_success,
346                candidate.metrics.tool_calls_per_success,
347            )
348            && candidate > base
349        {
350            failures.push(format!(
351                "candidate tool_calls_per_success {:.3} is worse than baseline {:.3}",
352                candidate, base
353            ));
354        }
355    }
356
357    BenchComparison {
358        passed: failures.is_empty(),
359        baseline_success_rate,
360        candidate_success_rate,
361        success_rate_delta: candidate_success_rate - baseline_success_rate,
362        baseline_tokens_per_success: baseline.and_then(|run| run.metrics.tokens_per_success),
363        candidate_tokens_per_success: candidate.metrics.tokens_per_success,
364        baseline_tool_calls_per_success: baseline
365            .and_then(|run| run.metrics.tool_calls_per_success),
366        candidate_tool_calls_per_success: candidate.metrics.tool_calls_per_success,
367        failures,
368    }
369}
370
371trait SaturatingSubF64 {
372    fn saturating_sub(self, rhs: f64) -> f64;
373}
374
375impl SaturatingSubF64 for f64 {
376    fn saturating_sub(self, rhs: f64) -> f64 {
377        (self - rhs).max(0.0)
378    }
379}
380
381fn load_case(path: &Path) -> Result<BenchCase> {
382    let content = std::fs::read_to_string(path)
383        .with_context(|| format!("failed to read benchmark case {}", path.display()))?;
384    let case = match path.extension().and_then(|ext| ext.to_str()) {
385        Some("json") => serde_json::from_str(&content)
386            .with_context(|| format!("failed to parse JSON benchmark case {}", path.display()))?,
387        Some("toml") => toml::from_str(&content)
388            .with_context(|| format!("failed to parse TOML benchmark case {}", path.display()))?,
389        _ => bail!("unsupported benchmark case format: {}", path.display()),
390    };
391    validate_case(case, path)
392}
393
394fn validate_case(case: BenchCase, path: &Path) -> Result<BenchCase> {
395    if case.version != BenchCase::CURRENT_VERSION {
396        bail!(
397            "unsupported benchmark case version {} in {}",
398            case.version,
399            path.display()
400        );
401    }
402    if case.id.trim().is_empty() {
403        bail!("benchmark case missing id: {}", path.display());
404    }
405    if case.title.trim().is_empty() {
406        bail!("benchmark case missing title: {}", path.display());
407    }
408    if case.category.trim().is_empty() {
409        bail!("benchmark case missing category: {}", path.display());
410    }
411    if case.fixture.as_os_str().is_empty() {
412        bail!("benchmark case missing fixture: {}", path.display());
413    }
414    if case.task.trim().is_empty() {
415        bail!("benchmark case missing task: {}", path.display());
416    }
417    if case.verifiers.is_empty() {
418        bail!(
419            "benchmark case must define at least one verifier: {}",
420            path.display()
421        );
422    }
423    for spec in case.setup.iter().chain(&case.verifiers) {
424        if spec.command.trim().is_empty() {
425            bail!(
426                "benchmark case has an empty verifier command: {}",
427                path.display()
428            );
429        }
430    }
431    Ok(case)
432}
433
434fn validate_unique_case_ids(cases: &[BenchCase], suite_path: &Path) -> Result<()> {
435    let mut seen = BTreeSet::new();
436    for case in cases {
437        if !seen.insert(case.id.as_str()) {
438            bail!(
439                "duplicate benchmark case id `{}` in suite {}",
440                case.id,
441                suite_path.display()
442            );
443        }
444    }
445    Ok(())
446}
447
448fn is_bench_case_file(path: &Path) -> bool {
449    path.is_file()
450        && matches!(
451            path.extension().and_then(|ext| ext.to_str()),
452            Some("json" | "toml")
453        )
454}
455
456fn collect_bench_case_files(path: &Path, files: &mut Vec<PathBuf>) -> Result<()> {
457    for entry in std::fs::read_dir(path)? {
458        let entry = entry?;
459        let entry_path = entry.path();
460        if entry_path.is_dir() {
461            collect_bench_case_files(&entry_path, files)?;
462        } else if is_bench_case_file(&entry_path) {
463            files.push(entry_path);
464        }
465    }
466    Ok(())
467}
468
469fn ratio(numerator: usize, denominator: usize) -> f64 {
470    if denominator == 0 {
471        0.0
472    } else {
473        numerator as f64 / denominator as f64
474    }
475}
476
477#[cfg(test)]
478mod tests {
479    use super::*;
480
481    fn verifier(command: &str) -> VerifierSpec {
482        VerifierSpec {
483            verifier_type: "command".to_string(),
484            command: command.to_string(),
485            cwd: None,
486            timeout_ms: None,
487            required: true,
488        }
489    }
490
491    fn result(id: &str, passed: bool, tokens: u64, tool_calls: usize) -> BenchCaseResult {
492        BenchCaseResult {
493            case_id: id.to_string(),
494            title: id.to_string(),
495            category: "test".to_string(),
496            passed,
497            workspace: PathBuf::from("/tmp/workspace"),
498            assistant_text: String::new(),
499            setup_results: Vec::new(),
500            verifier_results: Vec::new(),
501            metrics: BenchCaseMetrics {
502                total_tokens: tokens,
503                tool_calls,
504                ..BenchCaseMetrics::default()
505            },
506            events: Vec::new(),
507            error: None,
508        }
509    }
510
511    #[test]
512    fn loads_benchmark_suite_from_toml_directory() {
513        let dir = tempfile::tempdir().unwrap();
514        std::fs::write(
515            dir.path().join("case.toml"),
516            r#"
517version = 1
518id = "bench-case"
519title = "Bench case"
520category = "repo"
521fixture = "benchmarks/fixtures/example"
522task = "Fix the repo."
523
524[[verifiers]]
525verifier_type = "command"
526command = "test -f Cargo.toml"
527required = true
528"#,
529        )
530        .unwrap();
531
532        let suite = BenchSuite::load(dir.path()).unwrap();
533
534        assert_eq!(
535            suite.name,
536            dir.path().file_name().unwrap().to_string_lossy()
537        );
538        assert_eq!(suite.cases[0].id, "bench-case");
539        assert_eq!(suite.cases[0].agent.mode, "parity");
540    }
541
542    #[test]
543    fn rejects_duplicate_benchmark_case_ids() {
544        let dir = tempfile::tempdir().unwrap();
545        let case = r#"
546version = 1
547id = "duplicate"
548title = "Bench case"
549category = "repo"
550fixture = "benchmarks/fixtures/example"
551task = "Fix the repo."
552
553[[verifiers]]
554verifier_type = "command"
555command = "true"
556required = true
557"#;
558        std::fs::write(dir.path().join("a.toml"), case).unwrap();
559        std::fs::write(dir.path().join("b.toml"), case).unwrap();
560
561        let error = BenchSuite::load(dir.path()).expect_err("duplicates must fail");
562
563        assert!(error.to_string().contains("duplicate benchmark case id"));
564    }
565
566    #[test]
567    fn loads_benchmark_suite_recursively() {
568        let dir = tempfile::tempdir().unwrap();
569        let nested = dir.path().join("agentic");
570        std::fs::create_dir_all(&nested).unwrap();
571        std::fs::write(
572            nested.join("case.toml"),
573            r#"
574version = 1
575id = "nested-case"
576title = "Nested case"
577category = "repo"
578fixture = "benchmarks/fixtures/example"
579task = "Fix the repo."
580
581[[verifiers]]
582verifier_type = "command"
583command = "true"
584required = true
585"#,
586        )
587        .unwrap();
588
589        let suite = BenchSuite::load(dir.path()).unwrap();
590
591        assert_eq!(suite.cases.len(), 1);
592        assert_eq!(suite.cases[0].id, "nested-case");
593    }
594
595    #[test]
596    fn aggregates_metrics_for_success_rates_and_efficiency() {
597        let metrics =
598            aggregate_bench_metrics(&[result("a", true, 100, 3), result("b", false, 50, 2)], 250);
599
600        assert_eq!(metrics.total_cases, 2);
601        assert_eq!(metrics.passed_cases, 1);
602        assert_eq!(metrics.verified_success_rate, 0.5);
603        assert_eq!(metrics.tokens_per_success, Some(150.0));
604        assert_eq!(metrics.tool_calls_per_success, Some(5.0));
605        assert_eq!(metrics.wall_time_ms, 250);
606    }
607
608    #[test]
609    fn compare_reports_success_rate_regression() {
610        let baseline = BenchRun {
611            version: 1,
612            run_id: "base".to_string(),
613            suite_name: "suite".to_string(),
614            provider: None,
615            model: None,
616            started_at: 1,
617            ended_at: 2,
618            project_root: PathBuf::from("."),
619            metrics: aggregate_bench_metrics(&[result("a", true, 100, 3)], 10),
620            results: Vec::new(),
621        };
622        let candidate = BenchRun {
623            version: 1,
624            run_id: "candidate".to_string(),
625            suite_name: "suite".to_string(),
626            provider: None,
627            model: None,
628            started_at: 1,
629            ended_at: 2,
630            project_root: PathBuf::from("."),
631            metrics: aggregate_bench_metrics(&[result("a", false, 100, 3)], 10),
632            results: Vec::new(),
633        };
634
635        let comparison = compare_bench_runs(Some(&baseline), &candidate, Default::default());
636
637        assert!(!comparison.passed);
638        assert!(
639            comparison
640                .failures
641                .iter()
642                .any(|failure| failure.contains("success rate"))
643        );
644    }
645
646    #[test]
647    fn verifier_helper_keeps_required_default_explicit() {
648        assert!(verifier("true").required);
649    }
650}
navi_core/benchmark.rs

navi_core/
benchmark.rs
