pub fn normalize_transcript(text: &str) -> Vec<String> {
text.split_whitespace()
.map(|word| {
word.chars()
.filter(|c| c.is_alphanumeric() || *c == '\'')
.flat_map(char::to_lowercase)
.collect::<String>()
})
.filter(|word| !word.is_empty())
.collect()
}
pub fn word_errors(
hypothesis: &[String],
reference: &[String],
) -> usize {
let mut prev: Vec<usize> = (0..=hypothesis.len()).collect();
let mut cur = vec![0usize; hypothesis.len() + 1];
for (r, refr) in reference.iter().enumerate() {
cur[0] = r + 1;
for (h, hyp) in hypothesis.iter().enumerate() {
let cost = usize::from(refr != hyp);
cur[h + 1] = (prev[h] + cost).min(prev[h + 1] + 1).min(cur[h] + 1);
}
std::mem::swap(&mut prev, &mut cur);
}
prev[hypothesis.len()]
}
pub fn word_error_rate(
hypothesis: &[String],
reference: &[String],
) -> f64 {
if reference.is_empty() {
return if hypothesis.is_empty() { 0.0 } else { 1.0 };
}
word_errors(hypothesis, reference) as f64 / reference.len() as f64
}
pub fn text_error_rate(
hypothesis: &str,
reference: &str,
) -> f64 {
word_error_rate(
&normalize_transcript(hypothesis),
&normalize_transcript(reference),
)
}
pub fn cp_word_error_rate(
reference: &[(String, String)],
hypothesis: &[(String, String)],
) -> f64 {
let by_speaker = |turns: &[(String, String)]| -> Vec<Vec<String>> {
let mut names: Vec<&str> = Vec::new();
let mut words: Vec<Vec<String>> = Vec::new();
for (speaker, text) in turns {
let k = match names.iter().position(|n| n == speaker) {
Some(k) => k,
None => {
names.push(speaker);
words.push(Vec::new());
names.len() - 1
}
};
words[k].extend(normalize_transcript(text));
}
words
};
let refs = by_speaker(reference);
let hyps = by_speaker(hypothesis);
let total: usize = refs.iter().map(Vec::len).sum();
let errors: Vec<Vec<usize>> = refs
.iter()
.map(|r| hyps.iter().map(|h| word_errors(h, r)).collect())
.collect();
fn best(
i: usize,
used: &mut Vec<bool>,
refs: &[Vec<String>],
hyps: &[Vec<String>],
errors: &[Vec<usize>],
) -> usize {
if i == refs.len() {
return hyps
.iter()
.zip(used.iter())
.filter(|(_, u)| !**u)
.map(|(h, _)| h.len())
.sum();
}
let mut least = refs[i].len() + best(i + 1, used, refs, hyps, errors);
for j in 0..hyps.len() {
if !used[j] {
used[j] = true;
least = least.min(errors[i][j] + best(i + 1, used, refs, hyps, errors));
used[j] = false;
}
}
least
}
let errors = best(0, &mut vec![false; hyps.len()], &refs, &hyps, &errors);
if total == 0 {
return if errors == 0 { 0.0 } else { f64::INFINITY };
}
errors as f64 / total as f64
}
#[cfg(test)]
mod tests {
use super::*;
fn words(s: &str) -> Vec<String> {
normalize_transcript(s)
}
#[test]
fn test_normalize_strips_case_and_punctuation() {
assert_eq!(
words(" The Moon?? Why -- choose this, as our goal! "),
["the", "moon", "why", "choose", "this", "as", "our", "goal"],
);
assert_eq!(words("we're"), ["we're"]);
}
#[test]
fn test_wer_counts_each_edit_once() {
let refr = words("we choose to go to the moon");
assert_eq!(word_error_rate(&refr, &refr), 0.0);
let sub = words("we choose to go to the sun");
assert!((word_error_rate(&sub, &refr) - 1.0 / 7.0).abs() < 1e-12);
let del = words("we choose to go to moon");
assert!((word_error_rate(&del, &refr) - 1.0 / 7.0).abs() < 1e-12);
let ins = words("we choose to go to the big moon");
assert!((word_error_rate(&ins, &refr) - 1.0 / 7.0).abs() < 1e-12);
}
#[test]
fn test_wer_edge_cases() {
assert_eq!(word_error_rate(&[], &[]), 0.0);
assert_eq!(word_error_rate(&words("anything"), &[]), 1.0);
assert_eq!(word_error_rate(&[], &words("we choose to go")), 1.0);
}
#[test]
fn test_cp_word_error_rate() {
let t = |s: &str, x: &str| (s.to_string(), x.to_string());
let reference = [t("A", "we choose"), t("B", "to go"), t("A", "to the moon")];
let renamed = [t("x", "we choose"), t("y", "to go"), t("x", "to the moon")];
assert_eq!(cp_word_error_rate(&reference, &renamed), 0.0);
let merged = [t("x", "we choose to go to the moon")];
assert!((cp_word_error_rate(&reference, &merged) - 4.0 / 7.0).abs() < 1e-12);
let extra = [
t("x", "we choose to the moon"),
t("y", "to go"),
t("z", "applause"),
];
assert!((cp_word_error_rate(&reference, &extra) - 1.0 / 7.0).abs() < 1e-12);
assert_eq!(cp_word_error_rate(&[], &[]), 0.0);
}
}