aurum_core/eval/
mod.rs

1//! Versioned quality evaluation helpers (JOE-1607 / JOE-2216).
2//!
3//! Offline, deterministic metrics for STT (WER/CER) and simple TTS objective
4//! checks. Fixture corpora live under `evals/` at the repo root.
5//!
6//! The production STT quality observatory (schemas, budgets, scorecards) lives
7//! in [`observatory`].
8
9pub mod observatory;
10pub mod perf;
11pub mod tts_listening;
12
13pub use observatory::{
14    allowed_mean_wer, budget_exit_code, compare_stt_budget, observatory_core_budget_tiny,
15    observatory_core_corpus, score_observatory_fixture, AssetResolution, BudgetComparison,
16    BudgetFinding, BudgetSeverity, CorpusCoverage, ObservatoryCorpus, ObservatoryFixture,
17    ObservatoryFixtureScore, ObservatoryReport, ObservatoryScoreExtras, RunIdentity, SttBudget,
18    NORMALIZATION_POLICY_VERSION, OBSERVATORY_SCHEMA_VERSION, STT_OBSERVATORY_EVIDENCE_VERSION,
19};
20pub use perf::{
21    compare_perf_budget, percentile_sorted, perf_budget_exit_code, perf_scenario_catalogue,
22    tier_a_profile_templates, HardwareTier, NamedHardwareProfile, PerfBudget, PerfComparison,
23    PerfFinding, PerfReport, PerfScenario, PerfScenarioBudget, PerfScenarioResult, PerfSeverity,
24    PERF_EVIDENCE_VERSION, PERF_SCHEMA_VERSION,
25};
26pub use tts_listening::{
27    aggregate_listening, evaluate_support_tier, join_discontinuity_score, score_tts_pcm,
28    tts_local_matrix, tts_production_pack, ListeningAggregate, ListeningRating, ListeningReport,
29    SupportTierDecision, TtsEvalFixture, TtsEvalPack, TtsEvalParticipation, TtsObjectiveReport,
30    TtsObjectiveScore, TtsObjectiveThresholds, TtsRunIdentity, TTS_EVAL_SCHEMA_VERSION,
31    TTS_EVIDENCE_VERSION,
32};
33
34use serde::{Deserialize, Serialize};
35
36/// One STT reference fixture (text-only; audio path is optional for offline text scoring).
37#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
38pub struct SttFixture {
39    pub id: String,
40    pub language: String,
41    /// Normalized reference transcript.
42    pub reference: String,
43    /// Relative path to audio under the corpus root (optional for pure text tests).
44    #[serde(default, skip_serializing_if = "Option::is_none")]
45    pub audio: Option<String>,
46    /// Tags: clean, noisy, short, long, silence, …
47    #[serde(default)]
48    pub tags: Vec<String>,
49    /// When true, timestamps are expected to be reliable (ASR, not LLM-assisted).
50    #[serde(default = "default_true")]
51    pub timestamps_expected_reliable: bool,
52    /// Provenance / licensing note.
53    #[serde(default)]
54    pub license: String,
55}
56
57fn default_true() -> bool {
58    true
59}
60
61/// TTS text fixture for pronunciation / chunk-join checks.
62#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
63pub struct TtsFixture {
64    pub id: String,
65    pub text: String,
66    #[serde(default)]
67    pub tags: Vec<String>,
68    /// Expected approximate duration bounds (ms), if known.
69    #[serde(default, skip_serializing_if = "Option::is_none")]
70    pub duration_ms_min: Option<u64>,
71    #[serde(default, skip_serializing_if = "Option::is_none")]
72    pub duration_ms_max: Option<u64>,
73    #[serde(default)]
74    pub license: String,
75}
76
77/// Corpus manifest (versioned).
78#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
79pub struct EvalCorpus {
80    pub version: u32,
81    pub name: String,
82    #[serde(default)]
83    pub stt: Vec<SttFixture>,
84    #[serde(default)]
85    pub tts: Vec<TtsFixture>,
86}
87
88/// STT scoring result for one hypothesis.
89#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
90pub struct SttScore {
91    pub fixture_id: String,
92    pub wer: f64,
93    pub cer: f64,
94    pub empty_hypothesis: bool,
95    pub ref_words: usize,
96    pub hyp_words: usize,
97    /// True when backend timestamps are claimed reliable (host-supplied).
98    pub timestamps_reliable: bool,
99    /// Non-empty hypothesis when the reference is empty (silence false positive).
100    #[serde(default)]
101    pub silence_false_positive: bool,
102    /// Rough degeneration proxy: max run of identical tokens / hyp length.
103    #[serde(default)]
104    pub repetition_ratio: f64,
105}
106
107/// Aggregate report (machine-readable).
108#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
109pub struct EvalReport {
110    pub corpus_version: u32,
111    pub corpus_name: String,
112    pub model: String,
113    pub backend_kind: String,
114    pub stt_scores: Vec<SttScore>,
115    pub mean_wer: f64,
116    pub mean_cer: f64,
117    /// Count of silence false positives in this report.
118    #[serde(default)]
119    pub silence_false_positives: u32,
120    /// Mean repetition ratio across non-empty hypotheses.
121    #[serde(default)]
122    pub mean_repetition_ratio: f64,
123    /// Optional hardware / run metadata for release retention.
124    #[serde(default, skip_serializing_if = "Option::is_none")]
125    pub hardware_profile: Option<String>,
126    #[serde(default, skip_serializing_if = "Option::is_none")]
127    pub notes: Option<String>,
128}
129
130/// Normalize text for WER: lowercase, strip punctuation, collapse whitespace.
131pub fn normalize_transcript(s: &str) -> String {
132    let mut out = String::with_capacity(s.len());
133    let mut last_space = true;
134    for ch in s.chars() {
135        if ch.is_alphanumeric() {
136            for c in ch.to_lowercase() {
137                out.push(c);
138            }
139            last_space = false;
140        } else if ch.is_whitespace() && !last_space {
141            out.push(' ');
142            last_space = true;
143        }
144        // drop punctuation
145    }
146    out.trim().to_string()
147}
148
149fn words(s: &str) -> Vec<&str> {
150    s.split_whitespace().filter(|w| !w.is_empty()).collect()
151}
152
153/// Word error rate via classic Levenshtein on word tokens.
154pub fn word_error_rate(reference: &str, hypothesis: &str) -> f64 {
155    let r = normalize_transcript(reference);
156    let h = normalize_transcript(hypothesis);
157    let rw = words(&r);
158    let hw = words(&h);
159    if rw.is_empty() {
160        return if hw.is_empty() { 0.0 } else { 1.0 };
161    }
162    let dist = levenshtein(&rw, &hw);
163    dist as f64 / rw.len() as f64
164}
165
166/// Character error rate on normalized strings (spaces kept as characters).
167pub fn char_error_rate(reference: &str, hypothesis: &str) -> f64 {
168    let r: Vec<char> = normalize_transcript(reference).chars().collect();
169    let h: Vec<char> = normalize_transcript(hypothesis).chars().collect();
170    if r.is_empty() {
171        return if h.is_empty() { 0.0 } else { 1.0 };
172    }
173    let dist = levenshtein(&r, &h);
174    dist as f64 / r.len() as f64
175}
176
177fn levenshtein<T: PartialEq>(a: &[T], b: &[T]) -> usize {
178    let n = a.len();
179    let m = b.len();
180    if n == 0 {
181        return m;
182    }
183    if m == 0 {
184        return n;
185    }
186    let mut prev: Vec<usize> = (0..=m).collect();
187    let mut curr = vec![0; m + 1];
188    for i in 1..=n {
189        curr[0] = i;
190        for j in 1..=m {
191            let cost = if a[i - 1] == b[j - 1] { 0 } else { 1 };
192            curr[j] = (prev[j] + 1) // deletion
193                .min(curr[j - 1] + 1) // insertion
194                .min(prev[j - 1] + cost); // substitution
195        }
196        std::mem::swap(&mut prev, &mut curr);
197    }
198    prev[m]
199}
200
201/// Silence false positive: non-empty hypothesis for an empty reference.
202pub fn silence_false_positive(reference: &str, hypothesis: &str) -> bool {
203    normalize_transcript(reference).is_empty() && !normalize_transcript(hypothesis).is_empty()
204}
205
206/// Degeneration proxy: longest run of identical tokens over hypothesis length.
207pub fn repetition_ratio(hypothesis: &str) -> f64 {
208    let normalized = normalize_transcript(hypothesis);
209    let hw = words(&normalized);
210    if hw.is_empty() {
211        return 0.0;
212    }
213    let mut best = 1usize;
214    let mut run = 1usize;
215    for w in hw.windows(2) {
216        if w[0] == w[1] {
217            run += 1;
218            best = best.max(run);
219        } else {
220            run = 1;
221        }
222    }
223    best as f64 / hw.len() as f64
224}
225
226/// Score one STT hypothesis against a fixture.
227pub fn score_stt(fixture: &SttFixture, hypothesis: &str, timestamps_reliable: bool) -> SttScore {
228    let wer = word_error_rate(&fixture.reference, hypothesis);
229    let cer = char_error_rate(&fixture.reference, hypothesis);
230    let hyp_n = words(&normalize_transcript(hypothesis)).len();
231    let ref_n = words(&normalize_transcript(&fixture.reference)).len();
232    SttScore {
233        fixture_id: fixture.id.clone(),
234        wer,
235        cer,
236        empty_hypothesis: hypothesis.trim().is_empty(),
237        ref_words: ref_n,
238        hyp_words: hyp_n,
239        timestamps_reliable,
240        silence_false_positive: silence_false_positive(&fixture.reference, hypothesis),
241        repetition_ratio: repetition_ratio(hypothesis),
242    }
243}
244
245/// Build an aggregate report from scores.
246pub fn build_report(
247    corpus: &EvalCorpus,
248    model: &str,
249    backend_kind: &str,
250    scores: Vec<SttScore>,
251) -> EvalReport {
252    let n = scores.len().max(1) as f64;
253    let mean_wer = scores.iter().map(|s| s.wer).sum::<f64>() / n;
254    let mean_cer = scores.iter().map(|s| s.cer).sum::<f64>() / n;
255    let silence_false_positives = scores.iter().filter(|s| s.silence_false_positive).count() as u32;
256    let rep_scores: Vec<f64> = scores
257        .iter()
258        .filter(|s| s.hyp_words > 0)
259        .map(|s| s.repetition_ratio)
260        .collect();
261    let mean_repetition_ratio = if rep_scores.is_empty() {
262        0.0
263    } else {
264        rep_scores.iter().sum::<f64>() / rep_scores.len() as f64
265    };
266    EvalReport {
267        corpus_version: corpus.version,
268        corpus_name: corpus.name.clone(),
269        model: model.into(),
270        backend_kind: backend_kind.into(),
271        stt_scores: scores,
272        mean_wer,
273        mean_cer,
274        silence_false_positives,
275        mean_repetition_ratio,
276        hardware_profile: None,
277        notes: None,
278    }
279}
280
281/// Built-in smoke corpus (synthetic text + optional synthetic audio paths).
282///
283/// Audio under `evals/audio/` is generated CC0 PCM (silence / tone), not speech.
284/// Real multi-accent speech corpora remain external/private with the same schema.
285pub fn smoke_corpus() -> EvalCorpus {
286    EvalCorpus {
287        version: 2,
288        name: "aurum-smoke-v2".into(),
289        stt: vec![
290            SttFixture {
291                id: "clean_short_en".into(),
292                language: "en".into(),
293                reference: "hello world".into(),
294                audio: None,
295                tags: vec!["clean".into(), "short".into()],
296                timestamps_expected_reliable: true,
297                license: "synthetic CC0".into(),
298            },
299            SttFixture {
300                id: "numbers_en".into(),
301                language: "en".into(),
302                reference: "the meeting is at 3 30 pm".into(),
303                audio: None,
304                tags: vec!["numbers".into(), "punctuation".into()],
305                timestamps_expected_reliable: true,
306                license: "synthetic CC0".into(),
307            },
308            SttFixture {
309                id: "silence_empty".into(),
310                language: "en".into(),
311                reference: "".into(),
312                audio: Some("audio/silence_1s.wav".into()),
313                tags: vec!["silence".into()],
314                timestamps_expected_reliable: true,
315                license: "synthetic CC0".into(),
316            },
317            SttFixture {
318                id: "tone_non_speech".into(),
319                language: "en".into(),
320                reference: "".into(),
321                audio: Some("audio/tone_440_1s.wav".into()),
322                tags: vec!["noise".into(), "music".into(), "non_speech".into()],
323                timestamps_expected_reliable: true,
324                license: "synthetic CC0".into(),
325            },
326            SttFixture {
327                id: "long_phrase_en".into(),
328                language: "en".into(),
329                reference: "the quick brown fox jumps over the lazy dog near the river bank"
330                    .into(),
331                audio: None,
332                tags: vec!["clean".into(), "long".into()],
333                timestamps_expected_reliable: true,
334                license: "synthetic CC0".into(),
335            },
336            SttFixture {
337                id: "accent_placeholder_en".into(),
338                language: "en".into(),
339                reference: "schedule the call for tomorrow morning".into(),
340                audio: None,
341                tags: vec!["accent".into(), "placeholder".into()],
342                timestamps_expected_reliable: true,
343                license: "synthetic CC0 — replace with licensed multi-accent speech".into(),
344            },
345        ],
346        tts: vec![
347            TtsFixture {
348                id: "tts_short".into(),
349                text: "Hello from Aurum.".into(),
350                tags: vec!["short".into()],
351                duration_ms_min: Some(200),
352                duration_ms_max: Some(5_000),
353                license: "synthetic CC0".into(),
354            },
355            TtsFixture {
356                id: "tts_numbers".into(),
357                text: "Call me at extension 42.".into(),
358                tags: vec!["numbers".into(), "abbreviations".into()],
359                duration_ms_min: Some(300),
360                duration_ms_max: Some(8_000),
361                license: "synthetic CC0".into(),
362            },
363            TtsFixture {
364                id: "tts_long_join".into(),
365                text: "First sentence ends here. Second sentence starts now and continues for a bit longer.".into(),
366                tags: vec!["long".into(), "join".into()],
367                duration_ms_min: Some(500),
368                duration_ms_max: Some(20_000),
369                license: "synthetic CC0".into(),
370            },
371        ],
372    }
373}
374
375/// Objective TTS duration check (not MOS).
376pub fn tts_duration_in_range(fixture: &TtsFixture, duration_ms: u64) -> bool {
377    if let Some(min) = fixture.duration_ms_min {
378        if duration_ms < min {
379            return false;
380        }
381    }
382    if let Some(max) = fixture.duration_ms_max {
383        if duration_ms > max {
384            return false;
385        }
386    }
387    true
388}
389
390#[cfg(test)]
391mod tests {
392    use super::*;
393
394    #[test]
395    fn perfect_match_zero_wer() {
396        assert_eq!(word_error_rate("Hello, world!", "hello world"), 0.0);
397    }
398
399    #[test]
400    fn one_sub_wer() {
401        let wer = word_error_rate("a b c", "a x c");
402        assert!((wer - 1.0 / 3.0).abs() < 1e-9);
403    }
404
405    #[test]
406    fn cer_basic() {
407        let cer = char_error_rate("abc", "axc");
408        assert!((cer - 1.0 / 3.0).abs() < 1e-9);
409    }
410
411    #[test]
412    fn smoke_corpus_scores() {
413        let c = smoke_corpus();
414        let scores: Vec<_> = c
415            .stt
416            .iter()
417            .map(|f| score_stt(f, &f.reference, true))
418            .collect();
419        let report = build_report(&c, "tiny-q5_1", "asr", scores);
420        assert_eq!(report.mean_wer, 0.0);
421        assert_eq!(report.corpus_version, 2);
422        assert_eq!(report.silence_false_positives, 0);
423        let json = serde_json::to_string_pretty(&report).unwrap();
424        assert!(json.contains("mean_wer"));
425        assert!(json.contains("silence_false_positives"));
426    }
427
428    #[test]
429    fn silence_fp_and_repetition() {
430        assert!(silence_false_positive("", "hello hello hello"));
431        assert!(!silence_false_positive("", ""));
432        let r = repetition_ratio("yes yes yes yes no");
433        assert!(r >= 0.5);
434    }
435
436    #[test]
437    fn empty_ref_empty_hyp() {
438        assert_eq!(word_error_rate("", ""), 0.0);
439        assert_eq!(word_error_rate("", "hi"), 1.0);
440    }
441}
aurum_core/eval/mod.rs

aurum_core/eval/
mod.rs
