rust_metrics 0.1.1

Incremental evaluation metrics for classification and text generation pipelines.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

use crate::core::{Metric, MetricError};
use crate::utils::levenshtein_distance;

/// Reduction strategy used by [`EditDistance`].
#[derive(Debug, Clone, PartialEq)]
pub enum Reduction {
    /// Return the sum of edit distances in the buffer.
    Sum,
    /// Return the mean edit distance over observed pairs.
    Mean,
}

/// Streaming Levenshtein distance.
///
/// ```
/// use rust_metrics::{EditDistance, Metric};
///
/// let preds = ["kitten"];
/// let targets = ["sitting"];
/// let mut edit = EditDistance::default();
/// edit.update((&preds, &targets)).unwrap();
/// assert_eq!(edit.compute(), Some(3.0));
/// ```
#[derive(Debug, Clone)]
pub struct EditDistance {
    reduction: Reduction,
    edit_scores: Vec<f64>,
    total: usize,
}

impl Default for EditDistance {
    fn default() -> Self {
        Self::new(Reduction::Mean)
    }
}

impl EditDistance {
    pub fn new(reduction: Reduction) -> Self {
        Self {
            reduction,
            edit_scores: vec![],
            total: 0,
        }
    }
}

impl Metric<(&[&str], &[&str])> for EditDistance {
    type Output = f64;

    fn update(&mut self, (predictions, targets): (&[&str], &[&str])) -> Result<(), MetricError> {
        if predictions.len() != targets.len() {
            return Err(MetricError::LengthMismatch {
                predictions: predictions.len(),
                targets: targets.len(),
            });
        }
        self.total += predictions.len();
        for (&prediction, &target) in predictions.iter().zip(targets.iter()) {
            let edit_distance = levenshtein_distance(prediction, target) as f64;
            self.edit_scores.push(edit_distance);
        }
        Ok(())
    }

    fn reset(&mut self) {
        self.total = 0;
        self.edit_scores.clear();
    }

    fn compute(&self) -> Option<Self::Output> {
        if self.total == 0 {
            return None;
        }
        if self.reduction == Reduction::Sum {
            return Some(self.edit_scores.iter().sum::<f64>());
        }
        Some(self.edit_scores.iter().sum::<f64>() / self.total as f64)
    }
}

#[cfg(test)]
mod tests {
    use super::EditDistance;
    use crate::core::Metric;

    #[test]
    fn edit_over_batches() {
        let mut edit_distance = EditDistance::default();

        let preds = vec!["the cat is on the mat"];
        let targets = vec!["the cat is on the mat"];

        edit_distance.update((&preds, &targets)).unwrap();
        let score = edit_distance.compute().unwrap();
        assert_eq!(score, 0.0);

        edit_distance.reset();
        let score = edit_distance.compute();
        assert_eq!(score, None);

        let mut edit_distance = EditDistance::default();
        let preds = vec!["the cat is on the bath"];
        let targets = vec!["the cat is on the mat"];
        edit_distance.update((&preds, &targets)).unwrap();
        let score = edit_distance.compute().unwrap();
        assert_eq!(score, 2.0);

        let preds = vec!["the cat is on the mat"];
        let targets = vec!["the cat is on the mat"];
        edit_distance.update((&preds, &targets)).unwrap();
        let score = edit_distance.compute().unwrap();
        assert_eq!(score, 1.0);
    }
}