aurum_core/eval/
mod.rs

1//! Versioned quality evaluation helpers (JOE-1607).
2//!
3//! Offline, deterministic metrics for STT (WER/CER) and simple TTS objective
4//! checks. Fixture corpora live under `evals/` at the repo root.
5
6use serde::{Deserialize, Serialize};
7
8/// One STT reference fixture (text-only; audio path is optional for offline text scoring).
9#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
10pub struct SttFixture {
11    pub id: String,
12    pub language: String,
13    /// Normalized reference transcript.
14    pub reference: String,
15    /// Relative path to audio under the corpus root (optional for pure text tests).
16    #[serde(default, skip_serializing_if = "Option::is_none")]
17    pub audio: Option<String>,
18    /// Tags: clean, noisy, short, long, silence, …
19    #[serde(default)]
20    pub tags: Vec<String>,
21    /// When true, timestamps are expected to be reliable (ASR, not LLM-assisted).
22    #[serde(default = "default_true")]
23    pub timestamps_expected_reliable: bool,
24    /// Provenance / licensing note.
25    #[serde(default)]
26    pub license: String,
27}
28
29fn default_true() -> bool {
30    true
31}
32
33/// TTS text fixture for pronunciation / chunk-join checks.
34#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
35pub struct TtsFixture {
36    pub id: String,
37    pub text: String,
38    #[serde(default)]
39    pub tags: Vec<String>,
40    /// Expected approximate duration bounds (ms), if known.
41    #[serde(default, skip_serializing_if = "Option::is_none")]
42    pub duration_ms_min: Option<u64>,
43    #[serde(default, skip_serializing_if = "Option::is_none")]
44    pub duration_ms_max: Option<u64>,
45    #[serde(default)]
46    pub license: String,
47}
48
49/// Corpus manifest (versioned).
50#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
51pub struct EvalCorpus {
52    pub version: u32,
53    pub name: String,
54    #[serde(default)]
55    pub stt: Vec<SttFixture>,
56    #[serde(default)]
57    pub tts: Vec<TtsFixture>,
58}
59
60/// STT scoring result for one hypothesis.
61#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
62pub struct SttScore {
63    pub fixture_id: String,
64    pub wer: f64,
65    pub cer: f64,
66    pub empty_hypothesis: bool,
67    pub ref_words: usize,
68    pub hyp_words: usize,
69    /// True when backend timestamps are claimed reliable (host-supplied).
70    pub timestamps_reliable: bool,
71}
72
73/// Aggregate report (machine-readable).
74#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
75pub struct EvalReport {
76    pub corpus_version: u32,
77    pub corpus_name: String,
78    pub model: String,
79    pub backend_kind: String,
80    pub stt_scores: Vec<SttScore>,
81    pub mean_wer: f64,
82    pub mean_cer: f64,
83}
84
85/// Normalize text for WER: lowercase, strip punctuation, collapse whitespace.
86pub fn normalize_transcript(s: &str) -> String {
87    let mut out = String::with_capacity(s.len());
88    let mut last_space = true;
89    for ch in s.chars() {
90        if ch.is_alphanumeric() {
91            for c in ch.to_lowercase() {
92                out.push(c);
93            }
94            last_space = false;
95        } else if ch.is_whitespace() && !last_space {
96            out.push(' ');
97            last_space = true;
98        }
99        // drop punctuation
100    }
101    out.trim().to_string()
102}
103
104fn words(s: &str) -> Vec<&str> {
105    s.split_whitespace().filter(|w| !w.is_empty()).collect()
106}
107
108/// Word error rate via classic Levenshtein on word tokens.
109pub fn word_error_rate(reference: &str, hypothesis: &str) -> f64 {
110    let r = normalize_transcript(reference);
111    let h = normalize_transcript(hypothesis);
112    let rw = words(&r);
113    let hw = words(&h);
114    if rw.is_empty() {
115        return if hw.is_empty() { 0.0 } else { 1.0 };
116    }
117    let dist = levenshtein(&rw, &hw);
118    dist as f64 / rw.len() as f64
119}
120
121/// Character error rate on normalized strings (spaces kept as characters).
122pub fn char_error_rate(reference: &str, hypothesis: &str) -> f64 {
123    let r: Vec<char> = normalize_transcript(reference).chars().collect();
124    let h: Vec<char> = normalize_transcript(hypothesis).chars().collect();
125    if r.is_empty() {
126        return if h.is_empty() { 0.0 } else { 1.0 };
127    }
128    let dist = levenshtein(&r, &h);
129    dist as f64 / r.len() as f64
130}
131
132fn levenshtein<T: PartialEq>(a: &[T], b: &[T]) -> usize {
133    let n = a.len();
134    let m = b.len();
135    if n == 0 {
136        return m;
137    }
138    if m == 0 {
139        return n;
140    }
141    let mut prev: Vec<usize> = (0..=m).collect();
142    let mut curr = vec![0; m + 1];
143    for i in 1..=n {
144        curr[0] = i;
145        for j in 1..=m {
146            let cost = if a[i - 1] == b[j - 1] { 0 } else { 1 };
147            curr[j] = (prev[j] + 1) // deletion
148                .min(curr[j - 1] + 1) // insertion
149                .min(prev[j - 1] + cost); // substitution
150        }
151        std::mem::swap(&mut prev, &mut curr);
152    }
153    prev[m]
154}
155
156/// Score one STT hypothesis against a fixture.
157pub fn score_stt(fixture: &SttFixture, hypothesis: &str, timestamps_reliable: bool) -> SttScore {
158    let wer = word_error_rate(&fixture.reference, hypothesis);
159    let cer = char_error_rate(&fixture.reference, hypothesis);
160    let hyp_n = words(&normalize_transcript(hypothesis)).len();
161    let ref_n = words(&normalize_transcript(&fixture.reference)).len();
162    SttScore {
163        fixture_id: fixture.id.clone(),
164        wer,
165        cer,
166        empty_hypothesis: hypothesis.trim().is_empty(),
167        ref_words: ref_n,
168        hyp_words: hyp_n,
169        timestamps_reliable,
170    }
171}
172
173/// Build an aggregate report from scores.
174pub fn build_report(
175    corpus: &EvalCorpus,
176    model: &str,
177    backend_kind: &str,
178    scores: Vec<SttScore>,
179) -> EvalReport {
180    let n = scores.len().max(1) as f64;
181    let mean_wer = scores.iter().map(|s| s.wer).sum::<f64>() / n;
182    let mean_cer = scores.iter().map(|s| s.cer).sum::<f64>() / n;
183    EvalReport {
184        corpus_version: corpus.version,
185        corpus_name: corpus.name.clone(),
186        model: model.into(),
187        backend_kind: backend_kind.into(),
188        stt_scores: scores,
189        mean_wer,
190        mean_cer,
191    }
192}
193
194/// Built-in smoke corpus (no binary audio assets — text scoring only).
195pub fn smoke_corpus() -> EvalCorpus {
196    EvalCorpus {
197        version: 1,
198        name: "aurum-smoke-v1".into(),
199        stt: vec![
200            SttFixture {
201                id: "clean_short_en".into(),
202                language: "en".into(),
203                reference: "hello world".into(),
204                audio: None,
205                tags: vec!["clean".into(), "short".into()],
206                timestamps_expected_reliable: true,
207                license: "synthetic CC0".into(),
208            },
209            SttFixture {
210                id: "numbers_en".into(),
211                language: "en".into(),
212                reference: "the meeting is at 3 30 pm".into(),
213                audio: None,
214                tags: vec!["numbers".into()],
215                timestamps_expected_reliable: true,
216                license: "synthetic CC0".into(),
217            },
218            SttFixture {
219                id: "silence_empty".into(),
220                language: "en".into(),
221                reference: "".into(),
222                audio: None,
223                tags: vec!["silence".into()],
224                timestamps_expected_reliable: true,
225                license: "synthetic CC0".into(),
226            },
227        ],
228        tts: vec![
229            TtsFixture {
230                id: "tts_short".into(),
231                text: "Hello from Aurum.".into(),
232                tags: vec!["short".into()],
233                duration_ms_min: Some(200),
234                duration_ms_max: Some(5_000),
235                license: "synthetic CC0".into(),
236            },
237            TtsFixture {
238                id: "tts_numbers".into(),
239                text: "Call me at extension 42.".into(),
240                tags: vec!["numbers".into()],
241                duration_ms_min: Some(300),
242                duration_ms_max: Some(8_000),
243                license: "synthetic CC0".into(),
244            },
245        ],
246    }
247}
248
249/// Objective TTS duration check (not MOS).
250pub fn tts_duration_in_range(fixture: &TtsFixture, duration_ms: u64) -> bool {
251    if let Some(min) = fixture.duration_ms_min {
252        if duration_ms < min {
253            return false;
254        }
255    }
256    if let Some(max) = fixture.duration_ms_max {
257        if duration_ms > max {
258            return false;
259        }
260    }
261    true
262}
263
264#[cfg(test)]
265mod tests {
266    use super::*;
267
268    #[test]
269    fn perfect_match_zero_wer() {
270        assert_eq!(word_error_rate("Hello, world!", "hello world"), 0.0);
271    }
272
273    #[test]
274    fn one_sub_wer() {
275        let wer = word_error_rate("a b c", "a x c");
276        assert!((wer - 1.0 / 3.0).abs() < 1e-9);
277    }
278
279    #[test]
280    fn cer_basic() {
281        let cer = char_error_rate("abc", "axc");
282        assert!((cer - 1.0 / 3.0).abs() < 1e-9);
283    }
284
285    #[test]
286    fn smoke_corpus_scores() {
287        let c = smoke_corpus();
288        let scores: Vec<_> = c
289            .stt
290            .iter()
291            .map(|f| score_stt(f, &f.reference, true))
292            .collect();
293        let report = build_report(&c, "tiny-q5_1", "asr", scores);
294        assert_eq!(report.mean_wer, 0.0);
295        assert_eq!(report.corpus_version, 1);
296        let json = serde_json::to_string_pretty(&report).unwrap();
297        assert!(json.contains("mean_wer"));
298    }
299
300    #[test]
301    fn empty_ref_empty_hyp() {
302        assert_eq!(word_error_rate("", ""), 0.0);
303        assert_eq!(word_error_rate("", "hi"), 1.0);
304    }
305}
aurum_core/eval/mod.rs

aurum_core/eval/
mod.rs
