reproducible 0.2.0

Utilities for reproducible accuracy and benchmark reporting in Rust projects
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

use crate::metrics::{ErrorMetric, MetricValue};
use serde::{Deserialize, Serialize};

/// Statistical aggregator to use for a column.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum ColumnStat {
    Mean,
    Median,
    Max,
    P99,
    P95,
    Variance,
}

/// A column definition for a report.
pub enum Column<T = f64> {
    /// An accuracy column that evaluates error metrics on test cases.
    Accuracy(AccuracyColumn<T>),
    /// A performance column that reads benchmark data from Criterion.
    Performance(PerformanceColumn),
    /// A custom column with a user-defined renderer.
    Custom(
        String,
        Box<dyn Fn(&crate::stats::Stats) -> String + Send + Sync>,
    ),
}

impl<T: std::fmt::Debug> std::fmt::Debug for Column<T> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Column::Accuracy(c) => f.debug_tuple("Accuracy").field(c).finish(),
            Column::Performance(c) => f.debug_tuple("Performance").field(c).finish(),
            Column::Custom(name, _) => f.debug_tuple("Custom").field(name).finish(),
        }
    }
}

impl<T> Column<T> {
    /// Create a new accuracy column.
    ///
    /// # Example
    ///
    /// ```
    /// # use reproducible::columns::Column;
    /// let col = Column::<f64>::accuracy("Mean L2");
    /// ```
    pub fn accuracy(name: impl Into<String>) -> AccuracyColumn<T> {
        AccuracyColumn {
            name: name.into(),
            metric: None,
            target_stat: ColumnStat::Mean,
        }
    }

    /// Create a new performance column.
    pub fn perf(name: impl Into<String>) -> PerformanceColumn {
        PerformanceColumn {
            name: name.into(),
            target_stat: ColumnStat::Median,
        }
    }
}

pub struct AccuracyColumn<T = f64> {
    pub name: String,
    pub metric: Option<Box<ErrorMetric<T>>>,
    pub target_stat: ColumnStat,
}

impl<T> std::fmt::Debug for AccuracyColumn<T> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("AccuracyColumn")
            .field("name", &self.name)
            .field("target_stat", &self.target_stat)
            .finish()
    }
}

impl<T> AccuracyColumn<T> {
    /// Set a custom error metric for this column.
    ///
    /// # Example
    ///
    /// ```
    /// # use reproducible::columns::Column;
    /// # use reproducible::metrics::MetricValue;
    /// let col = Column::<f64>::accuracy("Abs Err")
    ///     .with_metric(|a, e| MetricValue::Numerical((a[0] - e[0]).abs()));
    /// ```
    pub fn with_metric<F>(mut self, metric: F) -> Self
    where
        F: Fn(&[T], &[T]) -> MetricValue + Send + Sync + 'static,
    {
        self.metric = Some(Box::new(metric));
        self
    }

    /// Set the statistical aggregator for this column.
    pub fn with_stat(mut self, stat: ColumnStat) -> Self {
        self.target_stat = stat;
        self
    }
}

impl<T> From<AccuracyColumn<T>> for Column<T> {
    fn from(c: AccuracyColumn<T>) -> Self {
        Column::Accuracy(c)
    }
}

#[derive(Debug)]
pub struct PerformanceColumn {
    pub name: String,
    pub target_stat: ColumnStat,
}

impl PerformanceColumn {
    pub fn with_stat(mut self, stat: ColumnStat) -> Self {
        self.target_stat = stat;
        self
    }
}

impl From<PerformanceColumn> for Column {
    fn from(c: PerformanceColumn) -> Self {
        Column::Performance(c)
    }
}