aurum_core/eval/
perf.rs

1//! Named-hardware end-to-end performance programme (JOE-2218).
2//!
3//! Versioned reports, scenario catalogue, percentile helpers, and fail-closed
4//! regression budgets. Download/network time is never mixed into local
5//! inference budgets. Reports retain scenario IDs and timings only — no
6//! transcripts, audio, or secrets.
7
8use crate::error::{Result, UserError};
9use serde::{Deserialize, Serialize};
10use std::collections::BTreeMap;
11use std::fs;
12use std::path::Path;
13
14/// Performance report schema version.
15pub const PERF_SCHEMA_VERSION: u32 = 2;
16
17/// Evidence / programme version.
18pub const PERF_EVIDENCE_VERSION: &str = "0.0.22-perf-v1";
19
20// ---------------------------------------------------------------------------
21// Hardware identity (coarse product specs only)
22// ---------------------------------------------------------------------------
23
24/// Tier A family for release-gated local performance.
25#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)]
26#[serde(rename_all = "snake_case")]
27pub enum HardwareTier {
28    /// macOS arm64 (Apple Silicon).
29    MacosArm64,
30    /// Linux x86_64 GNU.
31    LinuxX86_64Gnu,
32    /// Windows x86_64 MSVC.
33    WindowsX86_64Msvc,
34}
35
36impl HardwareTier {
37    pub fn as_str(self) -> &'static str {
38        match self {
39            Self::MacosArm64 => "macos_arm64",
40            Self::LinuxX86_64Gnu => "linux_x86_64_gnu",
41            Self::WindowsX86_64Msvc => "windows_x86_64_msvc",
42        }
43    }
44
45    pub fn all() -> &'static [HardwareTier] {
46        &[
47            Self::MacosArm64,
48            Self::LinuxX86_64Gnu,
49            Self::WindowsX86_64Msvc,
50        ]
51    }
52}
53
54/// Coarse named-hardware profile (no serials / usernames).
55#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
56pub struct NamedHardwareProfile {
57    pub profile_id: String,
58    pub tier: HardwareTier,
59    /// e.g. "Apple M2 Pro", "AMD EPYC …"
60    pub cpu_label: String,
61    pub core_count: u32,
62    pub memory_gib: u32,
63    /// e.g. "macOS 15.x", "Ubuntu 24.04 kernel 6.x", "Windows 11 build …"
64    pub os_label: String,
65    #[serde(default, skip_serializing_if = "Option::is_none")]
66    pub power_mode: Option<String>,
67}
68
69// ---------------------------------------------------------------------------
70// Scenarios
71// ---------------------------------------------------------------------------
72
73/// Stable scenario catalogue entry.
74#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
75pub struct PerfScenario {
76    pub id: String,
77    /// stt_local | tts_local | workflow | remote_info | governor
78    pub kind: String,
79    pub description: String,
80    /// Whether this scenario is release-gated on the same named machine.
81    pub release_gated: bool,
82    /// Minimum measured repetitions (warmups excluded).
83    pub min_repetitions: u32,
84    /// Warmup runs excluded from samples.
85    #[serde(default)]
86    pub warmups: u32,
87}
88
89/// Built-in scenario catalogue required by JOE-2218.
90pub fn perf_scenario_catalogue() -> Vec<PerfScenario> {
91    let mut v = Vec::new();
92    let stt_models = [
93        "tiny-q5_1",
94        "base",
95        "large-v3-turbo",
96        "profile_speed",
97        "profile_balance",
98        "profile_quality",
99    ];
100    let durations = [("30s", 5), ("5m", 5), ("long_form", 5)];
101    for m in stt_models {
102        for (d, reps) in durations {
103            v.push(PerfScenario {
104                id: format!("stt_local/{m}/{d}/warm"),
105                kind: "stt_local".into(),
106                description: format!("Local STT {m} {d} warm"),
107                release_gated: m == "tiny-q5_1" || m == "base",
108                min_repetitions: reps,
109                warmups: 1,
110            });
111        }
112        v.push(PerfScenario {
113            id: format!("stt_local/{m}/cold_load"),
114            kind: "stt_local".into(),
115            description: format!("Cold process model load {m}"),
116            release_gated: m == "tiny-q5_1",
117            min_repetitions: 5,
118            warmups: 0,
119        });
120        for conc in [1u32, 2, 4] {
121            v.push(PerfScenario {
122                id: format!("stt_local/{m}/concurrency_{conc}"),
123                kind: "governor".into(),
124                description: format!("STT concurrency {conc} with governor ({m})"),
125                release_gated: m == "tiny-q5_1" && conc <= 2,
126                min_repetitions: 5,
127                warmups: 1,
128            });
129        }
130    }
131    for (model, voice) in [("kitten-nano-int8", "Luna"), ("kokoro-82m-int8", "default")] {
132        for phrase in ["short", "paragraph", "multi_chunk"] {
133            v.push(PerfScenario {
134                id: format!("tts_local/{model}/{voice}/{phrase}"),
135                kind: "tts_local".into(),
136                description: format!("Local TTS {model}/{voice} {phrase}"),
137                release_gated: model == "kitten-nano-int8" && phrase == "short",
138                min_repetitions: 5,
139                warmups: 1,
140            });
141        }
142    }
143    for (id, desc, gated, reps) in [
144        (
145            "workflow/cli_stt_one_file",
146            "One-file CLI STT decode+commit",
147            true,
148            20u32,
149        ),
150        (
151            "workflow/batch_20_small",
152            "Resumable batch of ≥20 small files",
153            true,
154            5,
155        ),
156        (
157            "workflow/doctor_startup",
158            "doctor / cache status startup",
159            true,
160            20,
161        ),
162        (
163            "workflow/c_abi_job_overhead",
164            "C ABI job start/poll/take overhead",
165            false,
166            20,
167        ),
168        (
169            "workflow/long_form_mock_remote",
170            "Long-form chunk orchestration with mock provider",
171            false,
172            5,
173        ),
174        (
175            "remote_info/stt_upload_latency",
176            "Remote STT informational latency",
177            false,
178            5,
179        ),
180    ] {
181        v.push(PerfScenario {
182            id: id.into(),
183            kind: if id.starts_with("remote") {
184                "remote_info".into()
185            } else {
186                "workflow".into()
187            },
188            description: desc.into(),
189            release_gated: gated,
190            min_repetitions: reps,
191            warmups: if reps >= 20 { 2 } else { 1 },
192        });
193    }
194    v
195}
196
197// ---------------------------------------------------------------------------
198// Samples & report
199// ---------------------------------------------------------------------------
200
201/// One measured scenario result (warmups already excluded from samples).
202#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
203pub struct PerfScenarioResult {
204    pub scenario_id: String,
205    /// Wall times in milliseconds (measured samples only).
206    pub samples_ms: Vec<f64>,
207    pub p50_ms: f64,
208    pub p95_ms: f64,
209    pub mean_ms: f64,
210    #[serde(default, skip_serializing_if = "Option::is_none")]
211    pub audio_or_synth_duration_ms: Option<f64>,
212    #[serde(default, skip_serializing_if = "Option::is_none")]
213    pub rtf_p50: Option<f64>,
214    #[serde(default, skip_serializing_if = "Option::is_none")]
215    pub peak_rss_bytes: Option<u64>,
216    #[serde(default, skip_serializing_if = "Option::is_none")]
217    pub steady_rss_bytes: Option<u64>,
218    #[serde(default, skip_serializing_if = "Option::is_none")]
219    pub queue_wait_p50_ms: Option<f64>,
220    #[serde(default, skip_serializing_if = "Option::is_none")]
221    pub inference_p50_ms: Option<f64>,
222    #[serde(default, skip_serializing_if = "Option::is_none")]
223    pub throughput_ops_per_s: Option<f64>,
224    #[serde(default)]
225    pub concurrency: u32,
226    #[serde(default)]
227    pub warm: bool,
228    /// Separated download/network ms (must not enter local inference budgets).
229    #[serde(default, skip_serializing_if = "Option::is_none")]
230    pub download_ms: Option<f64>,
231    #[serde(default)]
232    pub release_gated: bool,
233}
234
235impl PerfScenarioResult {
236    pub fn from_samples(
237        scenario_id: &str,
238        mut samples_ms: Vec<f64>,
239        audio_or_synth_duration_ms: Option<f64>,
240        concurrency: u32,
241        warm: bool,
242        release_gated: bool,
243    ) -> Result<Self> {
244        if samples_ms.is_empty() {
245            return Err(UserError::Other {
246                message: format!("scenario '{scenario_id}' has no samples"),
247            }
248            .into());
249        }
250        for s in &samples_ms {
251            if !s.is_finite() || *s < 0.0 {
252                return Err(UserError::Other {
253                    message: format!("scenario '{scenario_id}' has invalid sample {s}"),
254                }
255                .into());
256            }
257        }
258        samples_ms.sort_by(|a, b| a.partial_cmp(b).unwrap());
259        let p50 = percentile_sorted(&samples_ms, 0.50);
260        let p95 = percentile_sorted(&samples_ms, 0.95);
261        let mean = samples_ms.iter().sum::<f64>() / samples_ms.len() as f64;
262        let rtf_p50 = audio_or_synth_duration_ms.map(|d| if d <= 0.0 { 0.0 } else { p50 / d });
263        Ok(Self {
264            scenario_id: scenario_id.into(),
265            samples_ms,
266            p50_ms: p50,
267            p95_ms: p95,
268            mean_ms: mean,
269            audio_or_synth_duration_ms,
270            rtf_p50,
271            peak_rss_bytes: None,
272            steady_rss_bytes: None,
273            queue_wait_p50_ms: None,
274            inference_p50_ms: None,
275            throughput_ops_per_s: None,
276            concurrency,
277            warm,
278            download_ms: None,
279            release_gated,
280        })
281    }
282}
283
284/// Percentile on a **sorted** sample slice. `p` in [0, 1].
285pub fn percentile_sorted(sorted: &[f64], p: f64) -> f64 {
286    if sorted.is_empty() {
287        return 0.0;
288    }
289    if sorted.len() == 1 {
290        return sorted[0];
291    }
292    let p = p.clamp(0.0, 1.0);
293    let rank = p * (sorted.len() as f64 - 1.0);
294    let lo = rank.floor() as usize;
295    let hi = rank.ceil() as usize;
296    if lo == hi {
297        sorted[lo]
298    } else {
299        let w = rank - lo as f64;
300        sorted[lo] * (1.0 - w) + sorted[hi] * w
301    }
302}
303
304/// Full performance report for one named machine run.
305#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
306pub struct PerfReport {
307    pub schema_version: u32,
308    pub evidence_version: String,
309    pub hardware: NamedHardwareProfile,
310    #[serde(default, skip_serializing_if = "Option::is_none")]
311    pub aurum_version: Option<String>,
312    #[serde(default, skip_serializing_if = "Option::is_none")]
313    pub commit: Option<String>,
314    #[serde(default, skip_serializing_if = "Option::is_none")]
315    pub rustc: Option<String>,
316    #[serde(default, skip_serializing_if = "Option::is_none")]
317    pub target_triple: Option<String>,
318    #[serde(default, skip_serializing_if = "Option::is_none")]
319    pub build_profile: Option<String>,
320    #[serde(default, skip_serializing_if = "Option::is_none")]
321    pub features: Option<String>,
322    #[serde(default, skip_serializing_if = "Option::is_none")]
323    pub whisper_cpp_version: Option<String>,
324    #[serde(default, skip_serializing_if = "Option::is_none")]
325    pub onnx_runtime_version: Option<String>,
326    /// model_id → sha256
327    #[serde(default)]
328    pub model_digests: BTreeMap<String, String>,
329    #[serde(default, skip_serializing_if = "Option::is_none")]
330    pub cache_state: Option<String>,
331    pub scenarios: Vec<PerfScenarioResult>,
332    #[serde(default, skip_serializing_if = "Option::is_none")]
333    pub notes: Option<String>,
334}
335
336impl PerfReport {
337    pub fn new(hardware: NamedHardwareProfile) -> Self {
338        Self {
339            schema_version: PERF_SCHEMA_VERSION,
340            evidence_version: PERF_EVIDENCE_VERSION.into(),
341            hardware,
342            aurum_version: Some(env!("CARGO_PKG_VERSION").into()),
343            commit: std::env::var("GITHUB_SHA")
344                .or_else(|_| std::env::var("AURUM_BENCH_COMMIT"))
345                .ok(),
346            rustc: None,
347            target_triple: Some(format!(
348                "{}-{}",
349                std::env::consts::ARCH,
350                std::env::consts::OS
351            )),
352            build_profile: None,
353            features: None,
354            whisper_cpp_version: None,
355            onnx_runtime_version: None,
356            model_digests: BTreeMap::new(),
357            cache_state: None,
358            scenarios: Vec::new(),
359            notes: None,
360        }
361    }
362
363    pub fn load(path: &Path) -> Result<Self> {
364        let data = fs::read_to_string(path).map_err(|e| UserError::Other {
365            message: format!("read perf report {}: {e}", path.display()),
366        })?;
367        serde_json::from_str(&data).map_err(|e| {
368            UserError::Other {
369                message: format!("parse perf report: {e}"),
370            }
371            .into()
372        })
373    }
374
375    pub fn to_json_pretty(&self) -> Result<String> {
376        serde_json::to_string_pretty(self).map_err(|e| {
377            UserError::Other {
378                message: format!("serialize perf report: {e}"),
379            }
380            .into()
381        })
382    }
383
384    pub fn to_markdown(&self) -> String {
385        let mut out = String::new();
386        out.push_str("# Performance report\n\n");
387        out.push_str(&format!(
388            "- **Evidence:** {}\n- **Profile:** `{}` ({})\n- **CPU:** {} ({} cores)\n- **Memory:** {} GiB\n- **OS:** {}\n",
389            self.evidence_version,
390            self.hardware.profile_id,
391            self.hardware.tier.as_str(),
392            self.hardware.cpu_label,
393            self.hardware.core_count,
394            self.hardware.memory_gib,
395            self.hardware.os_label
396        ));
397        if let Some(ref c) = self.commit {
398            out.push_str(&format!("- **Commit:** `{c}`\n"));
399        }
400        out.push_str(
401            "\n| Scenario | p50 ms | p95 ms | RTF p50 | RSS | conc | gated |\n|----------|--------|--------|---------|-----|------|-------|\n",
402        );
403        let mut scenarios = self.scenarios.clone();
404        scenarios.sort_by(|a, b| a.scenario_id.cmp(&b.scenario_id));
405        for s in &scenarios {
406            out.push_str(&format!(
407                "| {} | {:.1} | {:.1} | {} | {} | {} | {} |\n",
408                s.scenario_id,
409                s.p50_ms,
410                s.p95_ms,
411                s.rtf_p50
412                    .map(|r| format!("{r:.3}"))
413                    .unwrap_or_else(|| "—".into()),
414                s.peak_rss_bytes
415                    .map(|b| format!("{:.0} MiB", b as f64 / (1024.0 * 1024.0)))
416                    .unwrap_or_else(|| "—".into()),
417                s.concurrency,
418                s.release_gated
419            ));
420        }
421        out.push('\n');
422        out
423    }
424}
425
426// ---------------------------------------------------------------------------
427// Budgets
428// ---------------------------------------------------------------------------
429
430/// Per-scenario baseline budget on one named machine + model digest.
431#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
432pub struct PerfScenarioBudget {
433    pub scenario_id: String,
434    pub baseline_p50_ms: f64,
435    pub baseline_p95_ms: f64,
436    #[serde(default, skip_serializing_if = "Option::is_none")]
437    pub baseline_rtf_p50: Option<f64>,
438    #[serde(default, skip_serializing_if = "Option::is_none")]
439    pub baseline_peak_rss_bytes: Option<u64>,
440    #[serde(default, skip_serializing_if = "Option::is_none")]
441    pub baseline_throughput_ops_per_s: Option<f64>,
442    /// p50 wall/RTF regression fraction → warning (default 0.10).
443    #[serde(default = "default_p50_warn")]
444    pub max_p50_relative_warn: f64,
445    /// p95 regression fraction → fail (default 0.15).
446    #[serde(default = "default_p95_fail")]
447    pub max_p95_relative_fail: f64,
448    /// Peak RSS relative or absolute 256 MiB, whichever larger.
449    #[serde(default = "default_rss_rel")]
450    pub max_rss_relative_fail: f64,
451    #[serde(default = "default_rss_abs")]
452    pub max_rss_absolute_bytes: u64,
453    #[serde(default = "default_tp_rel")]
454    pub max_throughput_relative_drop: f64,
455}
456
457fn default_p50_warn() -> f64 {
458    0.10
459}
460fn default_p95_fail() -> f64 {
461    0.15
462}
463fn default_rss_rel() -> f64 {
464    0.15
465}
466fn default_rss_abs() -> u64 {
467    256 * 1024 * 1024
468}
469fn default_tp_rel() -> f64 {
470    0.15
471}
472
473/// Committed performance budget file for one hardware profile.
474#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
475pub struct PerfBudget {
476    pub schema_version: u32,
477    pub evidence_version: String,
478    pub hardware_profile_id: String,
479    #[serde(default, skip_serializing_if = "Option::is_none")]
480    pub model_digest_pin: Option<String>,
481    pub scenarios: Vec<PerfScenarioBudget>,
482    #[serde(default, skip_serializing_if = "Option::is_none")]
483    pub notes: Option<String>,
484}
485
486impl PerfBudget {
487    pub fn load(path: &Path) -> Result<Self> {
488        let data = fs::read_to_string(path).map_err(|e| UserError::Other {
489            message: format!("read perf budget {}: {e}", path.display()),
490        })?;
491        serde_json::from_str(&data).map_err(|e| {
492            UserError::Other {
493                message: format!("parse perf budget: {e}"),
494            }
495            .into()
496        })
497    }
498}
499
500#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
501#[serde(rename_all = "snake_case")]
502pub enum PerfSeverity {
503    Pass,
504    Warn,
505    Fail,
506}
507
508#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
509pub struct PerfFinding {
510    pub severity: PerfSeverity,
511    pub scenario_id: String,
512    pub code: String,
513    pub message: String,
514}
515
516#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
517pub struct PerfComparison {
518    pub hardware_profile_id: String,
519    pub passed: bool,
520    pub findings: Vec<PerfFinding>,
521}
522
523/// Compare candidate report to budget (same machine). Cross-machine never used to hide regressions.
524pub fn compare_perf_budget(report: &PerfReport, budget: &PerfBudget) -> PerfComparison {
525    let mut findings = Vec::new();
526
527    if report.hardware.profile_id != budget.hardware_profile_id {
528        findings.push(PerfFinding {
529            severity: PerfSeverity::Fail,
530            scenario_id: "*".into(),
531            code: "hardware_mismatch".into(),
532            message: format!(
533                "report profile '{}' != budget '{}'",
534                report.hardware.profile_id, budget.hardware_profile_id
535            ),
536        });
537    }
538
539    if let Some(ref pin) = budget.model_digest_pin {
540        let any = report.model_digests.values().any(|d| d == pin);
541        if !report.model_digests.is_empty() && !any {
542            findings.push(PerfFinding {
543                severity: PerfSeverity::Fail,
544                scenario_id: "*".into(),
545                code: "model_digest_mismatch".into(),
546                message: format!("budget model_digest_pin {pin} not present in report digests"),
547            });
548        }
549    }
550
551    let by_id: BTreeMap<_, _> = report
552        .scenarios
553        .iter()
554        .map(|s| (s.scenario_id.as_str(), s))
555        .collect();
556
557    for b in &budget.scenarios {
558        let Some(cand) = by_id.get(b.scenario_id.as_str()) else {
559            findings.push(PerfFinding {
560                severity: PerfSeverity::Fail,
561                scenario_id: b.scenario_id.clone(),
562                code: "missing_scenario".into(),
563                message: "budget scenario missing from candidate report".into(),
564            });
565            continue;
566        };
567
568        // p50 warn
569        let p50_lim = b.baseline_p50_ms * (1.0 + b.max_p50_relative_warn);
570        if cand.p50_ms > p50_lim + f64::EPSILON {
571            findings.push(PerfFinding {
572                severity: PerfSeverity::Warn,
573                scenario_id: b.scenario_id.clone(),
574                code: "p50_regression".into(),
575                message: format!(
576                    "p50 {:.1}ms exceeds warn {:.1}ms (baseline {:.1}, +{:.0}%)",
577                    cand.p50_ms,
578                    p50_lim,
579                    b.baseline_p50_ms,
580                    b.max_p50_relative_warn * 100.0
581                ),
582            });
583        }
584
585        // p95 fail
586        let p95_lim = b.baseline_p95_ms * (1.0 + b.max_p95_relative_fail);
587        if cand.p95_ms > p95_lim + f64::EPSILON {
588            findings.push(PerfFinding {
589                severity: PerfSeverity::Fail,
590                scenario_id: b.scenario_id.clone(),
591                code: "p95_regression".into(),
592                message: format!(
593                    "p95 {:.1}ms exceeds fail {:.1}ms (baseline {:.1}, +{:.0}%)",
594                    cand.p95_ms,
595                    p95_lim,
596                    b.baseline_p95_ms,
597                    b.max_p95_relative_fail * 100.0
598                ),
599            });
600        }
601
602        // RTF p50 warn (same relative as p50)
603        if let (Some(base_rtf), Some(cand_rtf)) = (b.baseline_rtf_p50, cand.rtf_p50) {
604            let lim = base_rtf * (1.0 + b.max_p50_relative_warn);
605            if cand_rtf > lim + f64::EPSILON {
606                findings.push(PerfFinding {
607                    severity: PerfSeverity::Warn,
608                    scenario_id: b.scenario_id.clone(),
609                    code: "rtf_p50_regression".into(),
610                    message: format!(
611                        "RTF p50 {cand_rtf:.4} exceeds warn {lim:.4} (baseline {base_rtf:.4})"
612                    ),
613                });
614            }
615        }
616
617        // Peak RSS fail
618        if let (Some(base_rss), Some(cand_rss)) = (b.baseline_peak_rss_bytes, cand.peak_rss_bytes) {
619            let rel_lim = (base_rss as f64 * (1.0 + b.max_rss_relative_fail)) as u64;
620            let abs_lim = base_rss.saturating_add(b.max_rss_absolute_bytes);
621            let lim = rel_lim.max(abs_lim);
622            if cand_rss > lim {
623                findings.push(PerfFinding {
624                    severity: PerfSeverity::Fail,
625                    scenario_id: b.scenario_id.clone(),
626                    code: "rss_regression".into(),
627                    message: format!(
628                        "peak RSS {cand_rss} exceeds limit {lim} (baseline {base_rss})"
629                    ),
630                });
631            }
632        }
633
634        // Throughput drop fail
635        if let (Some(base_tp), Some(cand_tp)) =
636            (b.baseline_throughput_ops_per_s, cand.throughput_ops_per_s)
637        {
638            let floor = base_tp * (1.0 - b.max_throughput_relative_drop);
639            if cand_tp + f64::EPSILON < floor {
640                findings.push(PerfFinding {
641                    severity: PerfSeverity::Fail,
642                    scenario_id: b.scenario_id.clone(),
643                    code: "throughput_regression".into(),
644                    message: format!(
645                        "throughput {cand_tp:.3} ops/s below floor {floor:.3} (baseline {base_tp:.3})"
646                    ),
647                });
648            }
649        }
650    }
651
652    if findings.is_empty() {
653        findings.push(PerfFinding {
654            severity: PerfSeverity::Pass,
655            scenario_id: "*".into(),
656            code: "ok".into(),
657            message: "all performance budget checks passed".into(),
658        });
659    }
660
661    let passed = findings.iter().all(|f| f.severity != PerfSeverity::Fail);
662    PerfComparison {
663        hardware_profile_id: budget.hardware_profile_id.clone(),
664        passed,
665        findings,
666    }
667}
668
669pub fn perf_budget_exit_code(cmp: &PerfComparison) -> i32 {
670    if cmp.passed {
671        0
672    } else {
673        1
674    }
675}
676
677/// Documented Tier A profile placeholders (exact machines filled by operators).
678pub fn tier_a_profile_templates() -> Vec<NamedHardwareProfile> {
679    vec![
680        NamedHardwareProfile {
681            profile_id: "tier_a_macos_arm64".into(),
682            tier: HardwareTier::MacosArm64,
683            cpu_label: "Apple Silicon (exact chip recorded per run)".into(),
684            core_count: 0,
685            memory_gib: 0,
686            os_label: "macOS (version recorded per run)".into(),
687            power_mode: Some("performance_or_default".into()),
688        },
689        NamedHardwareProfile {
690            profile_id: "tier_a_linux_x86_64_gnu".into(),
691            tier: HardwareTier::LinuxX86_64Gnu,
692            cpu_label: "x86_64 (exact CPU recorded per run)".into(),
693            core_count: 0,
694            memory_gib: 0,
695            os_label: "Linux GNU (distro/kernel recorded per run)".into(),
696            power_mode: None,
697        },
698        NamedHardwareProfile {
699            profile_id: "tier_a_windows_x86_64_msvc".into(),
700            tier: HardwareTier::WindowsX86_64Msvc,
701            cpu_label: "x86_64 (exact CPU recorded per run)".into(),
702            core_count: 0,
703            memory_gib: 0,
704            os_label: "Windows MSVC (build recorded per run)".into(),
705            power_mode: None,
706        },
707    ]
708}
709
710#[cfg(test)]
711mod tests {
712    use super::*;
713
714    fn sample_hardware() -> NamedHardwareProfile {
715        NamedHardwareProfile {
716            profile_id: "tier_a_macos_arm64".into(),
717            tier: HardwareTier::MacosArm64,
718            cpu_label: "Apple M2".into(),
719            core_count: 8,
720            memory_gib: 16,
721            os_label: "macOS 15.0".into(),
722            power_mode: None,
723        }
724    }
725
726    #[test]
727    fn percentiles() {
728        let s = [1.0, 2.0, 3.0, 4.0, 5.0];
729        assert!((percentile_sorted(&s, 0.50) - 3.0).abs() < 1e-9);
730        assert!(percentile_sorted(&s, 0.95) > 4.0);
731        assert_eq!(percentile_sorted(&[42.0], 0.95), 42.0);
732    }
733
734    #[test]
735    fn catalogue_has_tier_a_scenarios() {
736        let c = perf_scenario_catalogue();
737        assert!(c.len() > 20);
738        assert!(c.iter().any(|s| s.id.contains("tiny-q5_1")));
739        assert!(c.iter().any(|s| s.id.contains("tts_local")));
740        assert!(c.iter().any(|s| s.release_gated));
741        assert!(c.iter().any(|s| !s.release_gated));
742    }
743
744    #[test]
745    fn budget_pass_and_p95_fail() {
746        let mut report = PerfReport::new(sample_hardware());
747        let ok = PerfScenarioResult::from_samples(
748            "workflow/cli_stt_one_file",
749            vec![100.0, 102.0, 101.0, 99.0, 100.0],
750            Some(5000.0),
751            1,
752            true,
753            true,
754        )
755        .unwrap();
756        report.scenarios.push(ok);
757
758        let budget = PerfBudget {
759            schema_version: PERF_SCHEMA_VERSION,
760            evidence_version: PERF_EVIDENCE_VERSION.into(),
761            hardware_profile_id: "tier_a_macos_arm64".into(),
762            model_digest_pin: None,
763            scenarios: vec![PerfScenarioBudget {
764                scenario_id: "workflow/cli_stt_one_file".into(),
765                baseline_p50_ms: 100.0,
766                baseline_p95_ms: 110.0,
767                baseline_rtf_p50: Some(0.02),
768                baseline_peak_rss_bytes: None,
769                baseline_throughput_ops_per_s: None,
770                max_p50_relative_warn: 0.10,
771                max_p95_relative_fail: 0.15,
772                max_rss_relative_fail: 0.15,
773                max_rss_absolute_bytes: 256 * 1024 * 1024,
774                max_throughput_relative_drop: 0.15,
775            }],
776            notes: None,
777        };
778        let cmp = compare_perf_budget(&report, &budget);
779        assert!(cmp.passed, "{:?}", cmp.findings);
780
781        // Inject regression
782        report.scenarios[0] = PerfScenarioResult::from_samples(
783            "workflow/cli_stt_one_file",
784            vec![200.0, 210.0, 220.0, 230.0, 250.0],
785            Some(5000.0),
786            1,
787            true,
788            true,
789        )
790        .unwrap();
791        let cmp2 = compare_perf_budget(&report, &budget);
792        assert!(!cmp2.passed);
793        assert_eq!(perf_budget_exit_code(&cmp2), 1);
794        assert!(cmp2.findings.iter().any(|f| f.code == "p95_regression"));
795    }
796
797    #[test]
798    fn hardware_mismatch_fails() {
799        let report = PerfReport::new(sample_hardware());
800        let budget = PerfBudget {
801            schema_version: PERF_SCHEMA_VERSION,
802            evidence_version: PERF_EVIDENCE_VERSION.into(),
803            hardware_profile_id: "tier_a_linux_x86_64_gnu".into(),
804            model_digest_pin: None,
805            scenarios: vec![],
806            notes: None,
807        };
808        let cmp = compare_perf_budget(&report, &budget);
809        assert!(!cmp.passed);
810        assert!(cmp.findings.iter().any(|f| f.code == "hardware_mismatch"));
811    }
812
813    #[test]
814    fn markdown_deterministic_order() {
815        let mut report = PerfReport::new(sample_hardware());
816        report
817            .scenarios
818            .push(PerfScenarioResult::from_samples("b", vec![2.0], None, 1, true, false).unwrap());
819        report
820            .scenarios
821            .push(PerfScenarioResult::from_samples("a", vec![1.0], None, 1, true, false).unwrap());
822        let md = report.to_markdown();
823        let ia = md.find("| a |").unwrap();
824        let ib = md.find("| b |").unwrap();
825        assert!(ia < ib);
826    }
827}
aurum_core/eval/perf.rs

aurum_core/eval/
perf.rs
