eredu_evaluation/
evidence.rs

1//! General evaluation evidence and performance summaries.
2
3use std::collections::BTreeMap;
4
5use eredu_core::{
6    ObservationSet, ObservationValue, RealtimeOutputFrame, TensorObservation, TensorObservationData,
7};
8use eredu_nn::Tensor;
9use serde::{Deserialize, Serialize};
10
11/// Portable evidence emitted by an evaluator, backend probe, or reference.
12#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
13pub struct EvaluationEvidence {
14    /// Evidence schema version.
15    pub format_version: u32,
16    /// Extensible evaluation kind, such as `text_checkpoint` or `realtime_audio`.
17    pub kind: String,
18    /// Stable producer and environment fields.
19    pub provenance: BTreeMap<String, String>,
20    /// Path-addressed values available for comparison or inspection.
21    pub observations: ObservationSet,
22}
23
24impl EvaluationEvidence {
25    /// Creates version-one evidence with no provenance fields.
26    pub fn new(kind: impl Into<String>, observations: ObservationSet) -> Self {
27        Self {
28            format_version: 1,
29            kind: kind.into(),
30            provenance: BTreeMap::new(),
31            observations,
32        }
33    }
34
35    /// Adds one stable provenance field.
36    pub fn with_provenance(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
37        self.provenance.insert(key.into(), value.into());
38        self
39    }
40}
41
42/// Materializes a neutral tensor as F32 evaluation evidence.
43pub fn observe_f32_tensor<T: Tensor>(
44    tensor: &T,
45    context: &T::Context,
46) -> Result<TensorObservation, EvidenceError> {
47    let shape = observation_shape(tensor.shape())?;
48    let values = tensor.to_f32_vec(context)?;
49    TensorObservation::new(shape, TensorObservationData::F32(values)).map_err(Into::into)
50}
51
52/// Materializes a neutral tensor as signed integer evaluation evidence.
53pub fn observe_i32_tensor<T: Tensor>(
54    tensor: &T,
55    context: &T::Context,
56) -> Result<TensorObservation, EvidenceError> {
57    let shape = observation_shape(tensor.shape())?;
58    let values = tensor
59        .to_i32_vec(context)?
60        .into_iter()
61        .map(i64::from)
62        .collect();
63    TensorObservation::new(shape, TensorObservationData::I64(values)).map_err(Into::into)
64}
65
66fn observation_shape(shape: &[i32]) -> Result<Vec<usize>, EvidenceError> {
67    shape
68        .iter()
69        .map(|dimension| {
70            usize::try_from(*dimension).map_err(|_| EvidenceError::NegativeDimension(*dimension))
71        })
72        .collect()
73}
74
75/// Converts a completed portable realtime frame into general observations.
76pub fn observe_realtime_frame(
77    frame: &RealtimeOutputFrame,
78) -> Result<ObservationSet, EvidenceError> {
79    let mut observations = ObservationSet::new();
80    insert_realtime_tokens(
81        &mut observations,
82        "tokens.text",
83        frame.batch(),
84        frame.text_tokens(),
85    )?;
86    insert_realtime_tokens(
87        &mut observations,
88        "tokens.audio_decisions",
89        frame.batch(),
90        frame.decision_audio_tokens(),
91    )?;
92    insert_realtime_tokens(
93        &mut observations,
94        "tokens.audio_sampled",
95        frame.batch(),
96        frame.sampled_audio_tokens(),
97    )?;
98    if let Some(tokens) = frame.output_audio_tokens() {
99        insert_realtime_tokens(
100            &mut observations,
101            "tokens.audio_output",
102            frame.batch(),
103            tokens,
104        )?;
105    }
106    for diagnostic in frame.diagnostics() {
107        observations.insert(
108            format!("decisions.{}.logits", diagnostic.prediction()),
109            ObservationValue::Tensor(diagnostic.tensor().clone()),
110        )?;
111    }
112    Ok(observations)
113}
114
115fn insert_realtime_tokens(
116    observations: &mut ObservationSet,
117    path: &str,
118    batch: usize,
119    tokens: &[i32],
120) -> Result<(), EvidenceError> {
121    if batch == 0 || !tokens.len().is_multiple_of(batch) {
122        return Err(EvidenceError::RealtimeTokenShape {
123            path: path.into(),
124            batch,
125            values: tokens.len(),
126        });
127    }
128    observations.insert(
129        path,
130        ObservationValue::Tensor(TensorObservation::new(
131            vec![batch, tokens.len() / batch],
132            TensorObservationData::I64(tokens.iter().copied().map(i64::from).collect()),
133        )?),
134    )?;
135    Ok(())
136}
137
138/// Summary of repeated operation latencies.
139#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
140pub struct LatencySummary {
141    /// Number of samples.
142    pub samples: usize,
143    /// Arithmetic mean in milliseconds.
144    pub mean_ms: f64,
145    /// Median in milliseconds.
146    pub p50_ms: f64,
147    /// 95th percentile in milliseconds.
148    pub p95_ms: f64,
149    /// Largest latency in milliseconds.
150    pub max_ms: f64,
151    /// Optional deadline in milliseconds.
152    pub deadline_ms: Option<f64>,
153    /// Samples strictly exceeding the deadline.
154    pub deadline_misses: usize,
155}
156
157/// Summarizes finite, nonnegative latency samples.
158pub fn summarize_latencies(
159    samples_ms: &[f64],
160    deadline_ms: Option<f64>,
161) -> Result<LatencySummary, EvidenceError> {
162    if samples_ms.is_empty() {
163        return Err(EvidenceError::EmptyLatencies);
164    }
165    if samples_ms
166        .iter()
167        .any(|value| !value.is_finite() || *value < 0.0)
168    {
169        return Err(EvidenceError::InvalidLatency);
170    }
171    if deadline_ms.is_some_and(|value| !value.is_finite() || value < 0.0) {
172        return Err(EvidenceError::InvalidDeadline);
173    }
174    let mut ordered = samples_ms.to_vec();
175    ordered.sort_by(f64::total_cmp);
176    let percentile = |fraction: f64| {
177        let index = ((ordered.len() - 1) as f64 * fraction).ceil() as usize;
178        ordered[index]
179    };
180    Ok(LatencySummary {
181        samples: ordered.len(),
182        mean_ms: ordered.iter().sum::<f64>() / ordered.len() as f64,
183        p50_ms: percentile(0.50),
184        p95_ms: percentile(0.95),
185        max_ms: *ordered.last().expect("samples are nonempty"),
186        deadline_ms,
187        deadline_misses: deadline_ms.map_or(0, |deadline| {
188            ordered.iter().filter(|value| **value > deadline).count()
189        }),
190    })
191}
192
193/// Invalid evaluation evidence.
194#[derive(Debug, thiserror::Error)]
195pub enum EvidenceError {
196    /// Backend-neutral tensor materialization failed.
197    #[error(transparent)]
198    Tensor(#[from] eredu_nn::Error),
199    /// Portable observation construction failed.
200    #[error(transparent)]
201    Observation(#[from] eredu_core::ObservationError),
202    /// Tensor shapes must not contain negative dimensions.
203    #[error("observed tensor has negative dimension {0}")]
204    NegativeDimension(i32),
205    /// A realtime token vector cannot be represented with its declared batch.
206    #[error("realtime observation {path:?} has {values} values for batch {batch}")]
207    RealtimeTokenShape {
208        /// Observation path.
209        path: String,
210        /// Declared batch.
211        batch: usize,
212        /// Token count.
213        values: usize,
214    },
215    /// At least one latency is required.
216    #[error("latency summary requires at least one sample")]
217    EmptyLatencies,
218    /// Latencies must be finite and nonnegative.
219    #[error("latency samples must be finite and nonnegative")]
220    InvalidLatency,
221    /// A deadline must be finite and nonnegative.
222    #[error("latency deadline must be finite and nonnegative")]
223    InvalidDeadline,
224}
225
226#[cfg(test)]
227mod tests {
228    use super::*;
229
230    #[test]
231    fn latency_summary_is_deterministic() {
232        let summary = summarize_latencies(&[4.0, 1.0, 3.0, 2.0], Some(2.5)).unwrap();
233        assert_eq!(summary.mean_ms, 2.5);
234        assert_eq!(summary.p50_ms, 3.0);
235        assert_eq!(summary.p95_ms, 4.0);
236        assert_eq!(summary.deadline_misses, 2);
237    }
238
239    #[test]
240    fn realtime_frames_use_the_general_observation_schema() {
241        let diagnostic =
242            eredu_core::RealtimeDecisionDiagnostics::new(0, vec![1, 3], vec![0.0, 2.0, 1.0])
243                .unwrap();
244        let frame = RealtimeOutputFrame::new(
245            1,
246            vec![7],
247            vec![8, 9],
248            vec![8],
249            Some(vec![6]),
250            vec![diagnostic],
251        );
252        let observations = observe_realtime_frame(&frame).unwrap();
253        assert!(observations.get("tokens.text").is_some());
254        assert!(observations.get("decisions.0.logits").is_some());
255    }
256}
eredu_evaluation/evidence.rs

eredu_evaluation/
evidence.rs
