aurum_core/eval/
mod.rs

1//! Versioned quality evaluation helpers (JOE-1607).
2//!
3//! Offline, deterministic metrics for STT (WER/CER) and simple TTS objective
4//! checks. Fixture corpora live under `evals/` at the repo root.
5
6use serde::{Deserialize, Serialize};
7
8/// One STT reference fixture (text-only; audio path is optional for offline text scoring).
9#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
10pub struct SttFixture {
11    pub id: String,
12    pub language: String,
13    /// Normalized reference transcript.
14    pub reference: String,
15    /// Relative path to audio under the corpus root (optional for pure text tests).
16    #[serde(default, skip_serializing_if = "Option::is_none")]
17    pub audio: Option<String>,
18    /// Tags: clean, noisy, short, long, silence, …
19    #[serde(default)]
20    pub tags: Vec<String>,
21    /// When true, timestamps are expected to be reliable (ASR, not LLM-assisted).
22    #[serde(default = "default_true")]
23    pub timestamps_expected_reliable: bool,
24    /// Provenance / licensing note.
25    #[serde(default)]
26    pub license: String,
27}
28
29fn default_true() -> bool {
30    true
31}
32
33/// TTS text fixture for pronunciation / chunk-join checks.
34#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
35pub struct TtsFixture {
36    pub id: String,
37    pub text: String,
38    #[serde(default)]
39    pub tags: Vec<String>,
40    /// Expected approximate duration bounds (ms), if known.
41    #[serde(default, skip_serializing_if = "Option::is_none")]
42    pub duration_ms_min: Option<u64>,
43    #[serde(default, skip_serializing_if = "Option::is_none")]
44    pub duration_ms_max: Option<u64>,
45    #[serde(default)]
46    pub license: String,
47}
48
49/// Corpus manifest (versioned).
50#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
51pub struct EvalCorpus {
52    pub version: u32,
53    pub name: String,
54    #[serde(default)]
55    pub stt: Vec<SttFixture>,
56    #[serde(default)]
57    pub tts: Vec<TtsFixture>,
58}
59
60/// STT scoring result for one hypothesis.
61#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
62pub struct SttScore {
63    pub fixture_id: String,
64    pub wer: f64,
65    pub cer: f64,
66    pub empty_hypothesis: bool,
67    pub ref_words: usize,
68    pub hyp_words: usize,
69    /// True when backend timestamps are claimed reliable (host-supplied).
70    pub timestamps_reliable: bool,
71    /// Non-empty hypothesis when the reference is empty (silence false positive).
72    #[serde(default)]
73    pub silence_false_positive: bool,
74    /// Rough degeneration proxy: max run of identical tokens / hyp length.
75    #[serde(default)]
76    pub repetition_ratio: f64,
77}
78
79/// Aggregate report (machine-readable).
80#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
81pub struct EvalReport {
82    pub corpus_version: u32,
83    pub corpus_name: String,
84    pub model: String,
85    pub backend_kind: String,
86    pub stt_scores: Vec<SttScore>,
87    pub mean_wer: f64,
88    pub mean_cer: f64,
89    /// Count of silence false positives in this report.
90    #[serde(default)]
91    pub silence_false_positives: u32,
92    /// Mean repetition ratio across non-empty hypotheses.
93    #[serde(default)]
94    pub mean_repetition_ratio: f64,
95    /// Optional hardware / run metadata for release retention.
96    #[serde(default, skip_serializing_if = "Option::is_none")]
97    pub hardware_profile: Option<String>,
98    #[serde(default, skip_serializing_if = "Option::is_none")]
99    pub notes: Option<String>,
100}
101
102/// Normalize text for WER: lowercase, strip punctuation, collapse whitespace.
103pub fn normalize_transcript(s: &str) -> String {
104    let mut out = String::with_capacity(s.len());
105    let mut last_space = true;
106    for ch in s.chars() {
107        if ch.is_alphanumeric() {
108            for c in ch.to_lowercase() {
109                out.push(c);
110            }
111            last_space = false;
112        } else if ch.is_whitespace() && !last_space {
113            out.push(' ');
114            last_space = true;
115        }
116        // drop punctuation
117    }
118    out.trim().to_string()
119}
120
121fn words(s: &str) -> Vec<&str> {
122    s.split_whitespace().filter(|w| !w.is_empty()).collect()
123}
124
125/// Word error rate via classic Levenshtein on word tokens.
126pub fn word_error_rate(reference: &str, hypothesis: &str) -> f64 {
127    let r = normalize_transcript(reference);
128    let h = normalize_transcript(hypothesis);
129    let rw = words(&r);
130    let hw = words(&h);
131    if rw.is_empty() {
132        return if hw.is_empty() { 0.0 } else { 1.0 };
133    }
134    let dist = levenshtein(&rw, &hw);
135    dist as f64 / rw.len() as f64
136}
137
138/// Character error rate on normalized strings (spaces kept as characters).
139pub fn char_error_rate(reference: &str, hypothesis: &str) -> f64 {
140    let r: Vec<char> = normalize_transcript(reference).chars().collect();
141    let h: Vec<char> = normalize_transcript(hypothesis).chars().collect();
142    if r.is_empty() {
143        return if h.is_empty() { 0.0 } else { 1.0 };
144    }
145    let dist = levenshtein(&r, &h);
146    dist as f64 / r.len() as f64
147}
148
149fn levenshtein<T: PartialEq>(a: &[T], b: &[T]) -> usize {
150    let n = a.len();
151    let m = b.len();
152    if n == 0 {
153        return m;
154    }
155    if m == 0 {
156        return n;
157    }
158    let mut prev: Vec<usize> = (0..=m).collect();
159    let mut curr = vec![0; m + 1];
160    for i in 1..=n {
161        curr[0] = i;
162        for j in 1..=m {
163            let cost = if a[i - 1] == b[j - 1] { 0 } else { 1 };
164            curr[j] = (prev[j] + 1) // deletion
165                .min(curr[j - 1] + 1) // insertion
166                .min(prev[j - 1] + cost); // substitution
167        }
168        std::mem::swap(&mut prev, &mut curr);
169    }
170    prev[m]
171}
172
173/// Silence false positive: non-empty hypothesis for an empty reference.
174pub fn silence_false_positive(reference: &str, hypothesis: &str) -> bool {
175    normalize_transcript(reference).is_empty() && !normalize_transcript(hypothesis).is_empty()
176}
177
178/// Degeneration proxy: longest run of identical tokens over hypothesis length.
179pub fn repetition_ratio(hypothesis: &str) -> f64 {
180    let normalized = normalize_transcript(hypothesis);
181    let hw = words(&normalized);
182    if hw.is_empty() {
183        return 0.0;
184    }
185    let mut best = 1usize;
186    let mut run = 1usize;
187    for w in hw.windows(2) {
188        if w[0] == w[1] {
189            run += 1;
190            best = best.max(run);
191        } else {
192            run = 1;
193        }
194    }
195    best as f64 / hw.len() as f64
196}
197
198/// Score one STT hypothesis against a fixture.
199pub fn score_stt(fixture: &SttFixture, hypothesis: &str, timestamps_reliable: bool) -> SttScore {
200    let wer = word_error_rate(&fixture.reference, hypothesis);
201    let cer = char_error_rate(&fixture.reference, hypothesis);
202    let hyp_n = words(&normalize_transcript(hypothesis)).len();
203    let ref_n = words(&normalize_transcript(&fixture.reference)).len();
204    SttScore {
205        fixture_id: fixture.id.clone(),
206        wer,
207        cer,
208        empty_hypothesis: hypothesis.trim().is_empty(),
209        ref_words: ref_n,
210        hyp_words: hyp_n,
211        timestamps_reliable,
212        silence_false_positive: silence_false_positive(&fixture.reference, hypothesis),
213        repetition_ratio: repetition_ratio(hypothesis),
214    }
215}
216
217/// Build an aggregate report from scores.
218pub fn build_report(
219    corpus: &EvalCorpus,
220    model: &str,
221    backend_kind: &str,
222    scores: Vec<SttScore>,
223) -> EvalReport {
224    let n = scores.len().max(1) as f64;
225    let mean_wer = scores.iter().map(|s| s.wer).sum::<f64>() / n;
226    let mean_cer = scores.iter().map(|s| s.cer).sum::<f64>() / n;
227    let silence_false_positives = scores.iter().filter(|s| s.silence_false_positive).count() as u32;
228    let rep_scores: Vec<f64> = scores
229        .iter()
230        .filter(|s| s.hyp_words > 0)
231        .map(|s| s.repetition_ratio)
232        .collect();
233    let mean_repetition_ratio = if rep_scores.is_empty() {
234        0.0
235    } else {
236        rep_scores.iter().sum::<f64>() / rep_scores.len() as f64
237    };
238    EvalReport {
239        corpus_version: corpus.version,
240        corpus_name: corpus.name.clone(),
241        model: model.into(),
242        backend_kind: backend_kind.into(),
243        stt_scores: scores,
244        mean_wer,
245        mean_cer,
246        silence_false_positives,
247        mean_repetition_ratio,
248        hardware_profile: None,
249        notes: None,
250    }
251}
252
253/// Built-in smoke corpus (synthetic text + optional synthetic audio paths).
254///
255/// Audio under `evals/audio/` is generated CC0 PCM (silence / tone), not speech.
256/// Real multi-accent speech corpora remain external/private with the same schema.
257pub fn smoke_corpus() -> EvalCorpus {
258    EvalCorpus {
259        version: 2,
260        name: "aurum-smoke-v2".into(),
261        stt: vec![
262            SttFixture {
263                id: "clean_short_en".into(),
264                language: "en".into(),
265                reference: "hello world".into(),
266                audio: None,
267                tags: vec!["clean".into(), "short".into()],
268                timestamps_expected_reliable: true,
269                license: "synthetic CC0".into(),
270            },
271            SttFixture {
272                id: "numbers_en".into(),
273                language: "en".into(),
274                reference: "the meeting is at 3 30 pm".into(),
275                audio: None,
276                tags: vec!["numbers".into(), "punctuation".into()],
277                timestamps_expected_reliable: true,
278                license: "synthetic CC0".into(),
279            },
280            SttFixture {
281                id: "silence_empty".into(),
282                language: "en".into(),
283                reference: "".into(),
284                audio: Some("audio/silence_1s.wav".into()),
285                tags: vec!["silence".into()],
286                timestamps_expected_reliable: true,
287                license: "synthetic CC0".into(),
288            },
289            SttFixture {
290                id: "tone_non_speech".into(),
291                language: "en".into(),
292                reference: "".into(),
293                audio: Some("audio/tone_440_1s.wav".into()),
294                tags: vec!["noise".into(), "music".into(), "non_speech".into()],
295                timestamps_expected_reliable: true,
296                license: "synthetic CC0".into(),
297            },
298            SttFixture {
299                id: "long_phrase_en".into(),
300                language: "en".into(),
301                reference: "the quick brown fox jumps over the lazy dog near the river bank"
302                    .into(),
303                audio: None,
304                tags: vec!["clean".into(), "long".into()],
305                timestamps_expected_reliable: true,
306                license: "synthetic CC0".into(),
307            },
308            SttFixture {
309                id: "accent_placeholder_en".into(),
310                language: "en".into(),
311                reference: "schedule the call for tomorrow morning".into(),
312                audio: None,
313                tags: vec!["accent".into(), "placeholder".into()],
314                timestamps_expected_reliable: true,
315                license: "synthetic CC0 — replace with licensed multi-accent speech".into(),
316            },
317        ],
318        tts: vec![
319            TtsFixture {
320                id: "tts_short".into(),
321                text: "Hello from Aurum.".into(),
322                tags: vec!["short".into()],
323                duration_ms_min: Some(200),
324                duration_ms_max: Some(5_000),
325                license: "synthetic CC0".into(),
326            },
327            TtsFixture {
328                id: "tts_numbers".into(),
329                text: "Call me at extension 42.".into(),
330                tags: vec!["numbers".into(), "abbreviations".into()],
331                duration_ms_min: Some(300),
332                duration_ms_max: Some(8_000),
333                license: "synthetic CC0".into(),
334            },
335            TtsFixture {
336                id: "tts_long_join".into(),
337                text: "First sentence ends here. Second sentence starts now and continues for a bit longer.".into(),
338                tags: vec!["long".into(), "join".into()],
339                duration_ms_min: Some(500),
340                duration_ms_max: Some(20_000),
341                license: "synthetic CC0".into(),
342            },
343        ],
344    }
345}
346
347/// Objective TTS duration check (not MOS).
348pub fn tts_duration_in_range(fixture: &TtsFixture, duration_ms: u64) -> bool {
349    if let Some(min) = fixture.duration_ms_min {
350        if duration_ms < min {
351            return false;
352        }
353    }
354    if let Some(max) = fixture.duration_ms_max {
355        if duration_ms > max {
356            return false;
357        }
358    }
359    true
360}
361
362#[cfg(test)]
363mod tests {
364    use super::*;
365
366    #[test]
367    fn perfect_match_zero_wer() {
368        assert_eq!(word_error_rate("Hello, world!", "hello world"), 0.0);
369    }
370
371    #[test]
372    fn one_sub_wer() {
373        let wer = word_error_rate("a b c", "a x c");
374        assert!((wer - 1.0 / 3.0).abs() < 1e-9);
375    }
376
377    #[test]
378    fn cer_basic() {
379        let cer = char_error_rate("abc", "axc");
380        assert!((cer - 1.0 / 3.0).abs() < 1e-9);
381    }
382
383    #[test]
384    fn smoke_corpus_scores() {
385        let c = smoke_corpus();
386        let scores: Vec<_> = c
387            .stt
388            .iter()
389            .map(|f| score_stt(f, &f.reference, true))
390            .collect();
391        let report = build_report(&c, "tiny-q5_1", "asr", scores);
392        assert_eq!(report.mean_wer, 0.0);
393        assert_eq!(report.corpus_version, 2);
394        assert_eq!(report.silence_false_positives, 0);
395        let json = serde_json::to_string_pretty(&report).unwrap();
396        assert!(json.contains("mean_wer"));
397        assert!(json.contains("silence_false_positives"));
398    }
399
400    #[test]
401    fn silence_fp_and_repetition() {
402        assert!(silence_false_positive("", "hello hello hello"));
403        assert!(!silence_false_positive("", ""));
404        let r = repetition_ratio("yes yes yes yes no");
405        assert!(r >= 0.5);
406    }
407
408    #[test]
409    fn empty_ref_empty_hyp() {
410        assert_eq!(word_error_rate("", ""), 0.0);
411        assert_eq!(word_error_rate("", "hi"), 1.0);
412    }
413}
aurum_core/eval/mod.rs

aurum_core/eval/
mod.rs
