persistant_blob/
stats.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//! Stage 2 of backend benchmarking for the blob kind: **precision and per-operation statistics**.
3//! The same shape as `persistant_kv::stats`. No timing (stage 3) and no ranking.
4//!
5//! [`measure`] runs each leaf's test `runs` times, one attempt each, and reports how often it passed,
6//! how often the backend broke the contract and how often it errored, with the first failure's text.
7//! [`Instrumented`] counts, per operation, the calls and the errors by kind. A leaf is **exact** only
8//! if every run passed; a fraction is never rounded up, and the contract (stage 1) still decides
9//! eligibility.
10
11use crate::conformance::{Failure, Leaf, run_leaf};
12use crate::{BlobBackend, BlobError, Entry, Page};
13use std::collections::BTreeMap;
14use std::sync::Mutex;
15
16/// What one operation did while a test ran: calls, successes and errors by kind.
17#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
18pub struct OpCount {
19    /// Times it was called.
20    pub calls: u64,
21    /// Times it returned `Ok`.
22    pub ok: u64,
23    /// `BlobError::Backend` results.
24    pub backend: u64,
25    /// `BlobError::Unsupported` results.
26    pub unsupported: u64,
27}
28
29impl OpCount {
30    /// Every error, whatever its kind.
31    pub fn errors(&self) -> u64 {
32        self.backend + self.unsupported
33    }
34}
35
36/// A [`BlobBackend`] that counts what the wrapped backend does, per operation. It changes nothing.
37#[derive(Debug)]
38pub struct Instrumented<'a, B> {
39    inner: &'a B,
40    counts: Mutex<BTreeMap<&'static str, OpCount>>,
41}
42
43impl<'a, B: BlobBackend> Instrumented<'a, B> {
44    /// Wrap `inner`.
45    pub fn new(inner: &'a B) -> Self {
46        Instrumented {
47            inner,
48            counts: Mutex::new(BTreeMap::new()),
49        }
50    }
51
52    /// The counts so far, by operation name (`read`, `write`, `append`, ...).
53    pub fn counts(&self) -> BTreeMap<&'static str, OpCount> {
54        self.counts.lock().map(|c| c.clone()).unwrap_or_default()
55    }
56
57    fn note<T>(&self, op: &'static str, r: Result<T, BlobError>) -> Result<T, BlobError> {
58        if let Ok(mut c) = self.counts.lock() {
59            let e = c.entry(op).or_default();
60            e.calls += 1;
61            match &r {
62                Ok(_) => e.ok += 1,
63                Err(BlobError::Backend(_)) => e.backend += 1,
64                Err(BlobError::Unsupported(_)) => e.unsupported += 1,
65            }
66        }
67        r
68    }
69}
70
71impl<B: BlobBackend> BlobBackend for Instrumented<'_, B> {
72    fn backend(&self) -> &str {
73        self.inner.backend()
74    }
75
76    fn version(&self) -> &str {
77        self.inner.version()
78    }
79
80    async fn read(&self, key: &str) -> Result<Option<Vec<u8>>, BlobError> {
81        self.note("read", self.inner.read(key).await)
82    }
83
84    async fn write(&self, key: &str, bytes: Vec<u8>) -> Result<(), BlobError> {
85        self.note("write", self.inner.write(key, bytes).await)
86    }
87
88    async fn delete(&self, key: &str) -> Result<(), BlobError> {
89        self.note("delete", self.inner.delete(key).await)
90    }
91
92    async fn stat(&self, key: &str) -> Result<Option<Entry>, BlobError> {
93        self.note("stat", self.inner.stat(key).await)
94    }
95
96    async fn list(
97        &self,
98        prefix: &str,
99        after: Option<&str>,
100        limit: usize,
101    ) -> Result<Page, BlobError> {
102        self.note("list", self.inner.list(prefix, after, limit).await)
103    }
104
105    async fn create_exclusive(&self, key: &str, bytes: Vec<u8>) -> Result<bool, BlobError> {
106        self.note(
107            "create_exclusive",
108            self.inner.create_exclusive(key, bytes).await,
109        )
110    }
111
112    async fn replace_atomic(&self, key: &str, bytes: Vec<u8>) -> Result<(), BlobError> {
113        self.note(
114            "replace_atomic",
115            self.inner.replace_atomic(key, bytes).await,
116        )
117    }
118
119    async fn append(&self, key: &str, bytes: Vec<u8>) -> Result<(), BlobError> {
120        self.note("append", self.inner.append(key, bytes).await)
121    }
122}
123
124/// The result of running one leaf's test `runs` times on a target.
125#[derive(Debug, Clone, PartialEq, Eq)]
126pub struct LeafStats {
127    /// The leaf.
128    pub leaf: Leaf,
129    /// Times the test ran.
130    pub runs: u32,
131    /// Runs that passed.
132    pub passed: u32,
133    /// Runs where the backend returned something the contract forbids.
134    pub wrong: u32,
135    /// Runs where the backend errored (including `Unsupported`) before the test could decide.
136    pub errored: u32,
137    /// The first failing run's text, if any.
138    pub first_failure: Option<String>,
139    /// What each operation did across all the runs.
140    pub ops: BTreeMap<&'static str, OpCount>,
141}
142
143impl LeafStats {
144    /// `true` only if every run passed. One failure in a thousand is not exact.
145    pub fn is_exact(&self) -> bool {
146        self.runs > 0 && self.passed == self.runs
147    }
148
149    /// Passed runs over runs, `0.0` for no runs. For reading; use [`is_exact`](Self::is_exact) to
150    /// decide.
151    pub fn precision(&self) -> f64 {
152        if self.runs == 0 {
153            0.0
154        } else {
155            f64::from(self.passed) / f64::from(self.runs)
156        }
157    }
158}
159
160/// Run each leaf's test `runs` times on `target` (one attempt per run) and count.
161pub async fn measure<B: BlobBackend>(target: &B, leaves: &[Leaf], runs: u32) -> Vec<LeafStats> {
162    let mut out = Vec::with_capacity(leaves.len());
163    for leaf in leaves.iter().copied() {
164        let counted = Instrumented::new(target);
165        let (mut passed, mut wrong, mut errored) = (0, 0, 0);
166        let mut first_failure = None;
167        for _ in 0..runs {
168            match run_leaf(leaf, &counted).await {
169                Ok(()) => passed += 1,
170                Err(Failure::Wrong(m)) => {
171                    wrong += 1;
172                    first_failure.get_or_insert(m);
173                }
174                Err(Failure::Backend(e)) => {
175                    errored += 1;
176                    first_failure.get_or_insert(e.to_string());
177                }
178            }
179        }
180        out.push(LeafStats {
181            leaf,
182            runs,
183            passed,
184            wrong,
185            errored,
186            first_failure,
187            ops: counted.counts(),
188        });
189    }
190    out
191}
192
193#[cfg(test)]
194mod tests {
195    use super::*;
196    use crate::{Defect, MemoryBlob};
197
198    #[tokio::test]
199    async fn a_correct_backend_is_exact_on_every_run_and_the_counts_add_up() {
200        let s = measure(&MemoryBlob::new(), &[Leaf::Read, Leaf::CreateExclusive], 3).await;
201        for l in &s {
202            assert!(l.is_exact(), "{l:?}");
203            assert_eq!(l.precision(), 1.0);
204            assert_eq!((l.wrong, l.errored), (0, 0));
205        }
206        let w = &s[0].ops["write"];
207        assert_eq!(w.calls, w.ok + w.errors());
208        assert!(w.calls >= 3, "the test writes on every run");
209    }
210
211    #[tokio::test]
212    async fn a_defect_is_counted_as_wrong_not_rounded_to_a_pass() {
213        let b = MemoryBlob::with_defect(Defect::DeleteIsIgnored);
214        let s = measure(&b, &[Leaf::Delete], 4).await;
215        assert_eq!((s[0].passed, s[0].wrong, s[0].errored), (0, 4, 0));
216        assert!(!s[0].is_exact());
217        assert!(s[0].first_failure.is_some());
218    }
219
220    #[tokio::test]
221    async fn zero_runs_is_not_exact() {
222        let s = measure(&MemoryBlob::new(), &[Leaf::Read], 0).await;
223        assert!(!s[0].is_exact());
224        assert_eq!(s[0].precision(), 0.0);
225    }
226}
persistant_blob/stats.rs

persistant_blob/
stats.rs
