Skip to main content

sensor_db/
run.rs

1//! Run-encoded entries: many evenly spaced samples under one entry header.
2//!
3//! A single `f32` sample costs 20 bytes of entry overhead plus a 1-byte schema
4//! tag to store 4 bytes of data — **84% overhead**. For a burst of samples taken
5//! at a fixed cadence, almost all of that is repetition: the same sensor id, and
6//! a timestamp the previous one already implies.
7//!
8//! A *run* stores the burst as one entry:
9//!
10//! ```text
11//! payload = [ schema_version : u8 ] [ interval : u64 LE ] [ count : u16 LE ] [ T ; count ]
12//! ```
13//!
14//! with the entry header's `timestamp` holding the first sample's, so sample `i`
15//! is at `base + i * interval`. Thirty-two `f32` samples cost 148 bytes instead
16//! of 800.
17//!
18//! # A run is invisible to readers
19//!
20//! Every read path expands a run back into the samples it holds, so
21//! `query_range`, `latest`, `oldest` and `count` deliver **exactly what
22//! [`append_sensor`](crate::LogSinkExt::append_sensor) of the same samples would
23//! have delivered** — same headers, same `[version][value]` payloads, same
24//! order. Only [`read_from`](crate::SensorDb::read_from) sees the run as one
25//! entry, because replication copies entries and a replica should keep the
26//! amortisation rather than unpack it.
27//!
28//! That transparency is what makes runs a pure storage optimisation: a caller
29//! can start using them without changing a single read.
30//!
31//! # Why the marker and not the payload
32//!
33//! An entry is a run because its header carries
34//! [`ENTRY_MARKER_RUN`](crate::entry::ENTRY_MARKER_RUN) instead of
35//! [`ENTRY_MARKER`](crate::entry::ENTRY_MARKER) — a field the entry header CRC
36//! covers and the raw [`append`](crate::LogSink::append) path cannot write.
37//!
38//! The alternative — flagging a run in the payload, say with the high bit of the
39//! schema tag — was tried and reverted. It cannot work, because a raw payload is
40//! *arbitrary bytes*: any payload whose first byte happened to have bit 7 set
41//! would read as a run, and the sector index would inherit its invented time
42//! span. `tests/model.rs` caught exactly that, on a workload containing no typed
43//! data at all. A marker in the header is structural: the raw API always writes
44//! `0xAA`, so a run is a run.
45//!
46//! # Runs are a typed-API feature
47//!
48//! [`Run::of`] takes `&[T]` for a [`SensorData`] type, because the frame carries
49//! a schema tag and expansion produces `[version][value]` payloads — which is
50//! precisely what the typed single-sample path writes. There is deliberately no
51//! untyped constructor: an untagged run would expand into payloads that no read
52//! path could interpret.
53
54use zerocopy::IntoBytes;
55
56use crate::entry::{EntryHeader, EntryKind};
57use crate::error::Error;
58use crate::schema::SCHEMA_TAG_SIZE;
59use crate::typed::SensorData;
60
61/// Bytes a run frame prepends to its samples: the schema tag, the interval, and
62/// the sample count.
63///
64/// A run of `n` values of `T` therefore occupies
65/// `ENTRY_OVERHEAD + RUN_FRAME_SIZE + n * size_of::<T>()` bytes of flash,
66/// against `n * (ENTRY_OVERHEAD + SCHEMA_TAG_SIZE + size_of::<T>())` for the
67/// same samples appended one at a time.
68pub const RUN_FRAME_SIZE: usize = SCHEMA_TAG_SIZE + 8 + 2;
69
70/// Offset of the `interval` field within a run payload.
71const INTERVAL_AT: usize = SCHEMA_TAG_SIZE;
72
73/// Offset of the `count` field within a run payload.
74const COUNT_AT: usize = INTERVAL_AT + 8;
75
76/// Evenly spaced samples of one sensor, ready to be appended as a single entry.
77///
78/// Construct with [`Run::of`], which validates everything an appender would
79/// otherwise have to re-check, and hand it to
80/// [`append_run`](crate::LogSink::append_run):
81///
82/// ```ignore
83/// use sensor_db::{LogSink, Run};
84///
85/// let burst: [Temperature; 32] = collect_burst();
86/// db.append_run(&Run::of(t0, 10_000, &burst)?)?;
87/// ```
88///
89/// The samples are borrowed, not copied: the driver writes the frame header and
90/// then the sample bytes straight into its page buffer.
91#[derive(Debug, Clone, Copy)]
92pub struct Run<'a> {
93    sensor_id: u16,
94    base_timestamp: u64,
95    interval: u64,
96    schema_version: u8,
97    count: u16,
98    samples: &'a [u8],
99}
100
101impl<'a> Run<'a> {
102    /// Describe a run of `T` values, the first at `base_timestamp` and each
103    /// subsequent one `interval` later.
104    ///
105    /// # Errors
106    ///
107    /// [`Error::InvalidRun`] if `samples` is empty or longer than `u16::MAX`,
108    /// if `T` is zero-sized, if the last sample's timestamp would overflow
109    /// `u64`, or if the frame would exceed the `u16` payload length the entry
110    /// header can carry. Whether it fits a *page* is the appending log's
111    /// question, and it answers [`Error::PayloadTooLarge`].
112    pub fn of<T: SensorData>(
113        base_timestamp: u64,
114        interval: u64,
115        samples: &'a [T],
116    ) -> Result<Self, Error> {
117        let stride = core::mem::size_of::<T>();
118        if stride == 0 {
119            return Err(Error::InvalidRun);
120        }
121        let count = u16::try_from(samples.len())
122            .ok()
123            .filter(|&n| n > 0)
124            .ok_or(Error::InvalidRun)?;
125
126        // The run is unreadable if its own last timestamp is not representable,
127        // so refuse to write it rather than storing samples that read back
128        // clamped. Checked here, once, which is what lets every reader use
129        // saturating arithmetic without hiding a defect.
130        interval
131            .checked_mul(u64::from(count - 1))
132            .and_then(|span| base_timestamp.checked_add(span))
133            .ok_or(Error::InvalidRun)?;
134
135        let bytes = samples.as_bytes();
136        if RUN_FRAME_SIZE + bytes.len() > u16::MAX as usize {
137            return Err(Error::InvalidRun);
138        }
139
140        Ok(Self {
141            sensor_id: T::SENSOR_ID,
142            base_timestamp,
143            interval,
144            schema_version: T::SCHEMA_VERSION,
145            count,
146            samples: bytes,
147        })
148    }
149
150    /// The sensor id the samples are stored under.
151    pub fn sensor_id(&self) -> u16 {
152        self.sensor_id
153    }
154
155    /// Timestamp of the first sample, which is the entry header's timestamp.
156    pub fn base_timestamp(&self) -> u64 {
157        self.base_timestamp
158    }
159
160    /// Timestamp step between consecutive samples.
161    pub fn interval(&self) -> u64 {
162        self.interval
163    }
164
165    /// How many samples the run holds; never zero.
166    pub fn count(&self) -> u32 {
167        u32::from(self.count)
168    }
169
170    /// Timestamp of the last sample.
171    pub fn span_end(&self) -> u64 {
172        // Cannot overflow: `of` rejected a run whose end is not representable.
173        self.base_timestamp + self.interval * u64::from(self.count - 1)
174    }
175
176    /// Bytes the run occupies as an entry payload.
177    pub fn payload_len(&self) -> usize {
178        RUN_FRAME_SIZE + self.samples.len()
179    }
180
181    /// The sample bytes, `count * size_of::<T>()` of them.
182    pub(crate) fn samples(&self) -> &'a [u8] {
183        self.samples
184    }
185
186    /// Write the frame header — everything before the samples — into `buf`.
187    ///
188    /// # Panics
189    ///
190    /// If `buf` is shorter than [`RUN_FRAME_SIZE`]. Callers size it from
191    /// [`payload_len`](Self::payload_len).
192    pub(crate) fn write_frame(&self, buf: &mut [u8]) {
193        buf[0] = self.schema_version;
194        buf[INTERVAL_AT..INTERVAL_AT + 8].copy_from_slice(&self.interval.to_le_bytes());
195        buf[COUNT_AT..COUNT_AT + 2].copy_from_slice(&self.count.to_le_bytes());
196    }
197}
198
199/// The frame of a stored run, read back from the payload of an entry carrying
200/// [`ENTRY_MARKER_RUN`](crate::entry::ENTRY_MARKER_RUN).
201///
202/// Borrows nothing: it is the four numbers that describe the samples, so a
203/// caller can hold it while still handing the payload buffer out mutably to
204/// [`tagged_sample`](Self::tagged_sample).
205#[derive(Debug, Clone, Copy, PartialEq, Eq)]
206pub(crate) struct RunFrame {
207    schema_version: u8,
208    interval: u64,
209    count: u32,
210    stride: usize,
211}
212
213impl RunFrame {
214    /// Read the frame out of a run's payload, or `None` if the payload is not a
215    /// well-formed run.
216    ///
217    /// The marker already says the writer meant a run, and the entry CRC says
218    /// these are the bytes it wrote — so this rejects only what no writer of
219    /// this crate can produce: no samples, a zero count, or a sample region that
220    /// does not divide evenly into `count` samples. Callers treat `None` exactly
221    /// as they treat a CRC failure, which keeps every scan walking a sector the
222    /// same way.
223    pub(crate) fn parse(payload: &[u8]) -> Option<Self> {
224        // Strictly greater: a run with no room for even one sample is malformed.
225        if payload.len() <= RUN_FRAME_SIZE {
226            return None;
227        }
228
229        let mut interval = [0u8; 8];
230        interval.copy_from_slice(&payload[INTERVAL_AT..INTERVAL_AT + 8]);
231        let count = u32::from(u16::from_le_bytes([
232            payload[COUNT_AT],
233            payload[COUNT_AT + 1],
234        ]));
235        if count == 0 {
236            return None;
237        }
238
239        let body = payload.len() - RUN_FRAME_SIZE;
240        if !body.is_multiple_of(count as usize) {
241            return None;
242        }
243
244        Some(Self {
245            schema_version: payload[0],
246            interval: u64::from_le_bytes(interval),
247            count,
248            // Non-zero: `body >= 1` and `count` divides it, so `count <= body`.
249            stride: body / count as usize,
250        })
251    }
252
253    /// How many samples the run holds; never zero.
254    pub(crate) fn count(&self) -> u32 {
255        self.count
256    }
257
258    /// Timestamp of sample `index`, counting from the entry header's `base`.
259    ///
260    /// Saturating, which cannot bite a run this crate wrote: [`Run::of`] refuses
261    /// one whose end is not representable.
262    pub(crate) fn timestamp_at(&self, base: u64, index: u32) -> u64 {
263        base.saturating_add(self.interval.saturating_mul(u64::from(index)))
264    }
265
266    /// Timestamp of the last sample — how far past the entry header's timestamp
267    /// this entry reaches.
268    pub(crate) fn span_end(&self, base: u64) -> u64 {
269        self.timestamp_at(base, self.count - 1)
270    }
271
272    /// The header sample `index` would have carried had it been appended alone.
273    pub(crate) fn header_at(&self, sensor_id: u16, base: u64, index: u32) -> EntryHeader {
274        EntryHeader {
275            sensor_id,
276            timestamp: self.timestamp_at(base, index),
277            // Always representable: the whole run fit a `u16` payload length.
278            payload_len: (SCHEMA_TAG_SIZE + self.stride) as u16,
279            // An expanded sample is indistinguishable from one appended alone,
280            // and that includes its kind.
281            kind: EntryKind::Single,
282        }
283    }
284
285    /// Sample `index` as a schema-tagged payload — byte for byte what
286    /// `append_sensor` would have stored for that sample alone.
287    ///
288    /// # The tag byte is written in place
289    ///
290    /// A tagged payload is `[version][value]`, and the run stores the version
291    /// *once*, at the front. Rather than copy the sample into a scratch buffer
292    /// to put a tag in front of it, this writes the version into the byte
293    /// immediately preceding sample `index` and hands out the two together.
294    ///
295    /// That byte is the last byte of sample `index - 1` — already delivered, in
296    /// a forward walk — or, for `index == 0`, the high byte of `count`, which
297    /// this frame has already read. Nothing that is still needed is overwritten,
298    /// and the buffer holds a copy of flash, so the medium is untouched.
299    ///
300    /// # Panics
301    ///
302    /// If `index >= count` or `payload` is not the run's whole payload.
303    pub(crate) fn tagged_sample<'a>(&self, payload: &'a mut [u8], index: u32) -> &'a [u8] {
304        debug_assert!(index < self.count);
305        let start = RUN_FRAME_SIZE + index as usize * self.stride;
306        payload[start - SCHEMA_TAG_SIZE] = self.schema_version;
307        &payload[start - SCHEMA_TAG_SIZE..start + self.stride]
308    }
309
310    /// Copy sample `index` into `out` as a schema-tagged payload, returning its
311    /// length, or `None` if `out` cannot hold it.
312    ///
313    /// The copying twin of [`tagged_sample`](Self::tagged_sample), for the
314    /// readers whose payload is behind a shared reference — the page buffer,
315    /// and a replication stream's callback — or who are filling a caller's
316    /// output buffer anyway and would gain nothing from writing in place.
317    pub(crate) fn write_tagged_sample(
318        &self,
319        payload: &[u8],
320        index: u32,
321        out: &mut [u8],
322    ) -> Option<usize> {
323        debug_assert!(index < self.count);
324        let len = SCHEMA_TAG_SIZE + self.stride;
325        if out.len() < len {
326            return None;
327        }
328        let start = RUN_FRAME_SIZE + index as usize * self.stride;
329        out[0] = self.schema_version;
330        out[SCHEMA_TAG_SIZE..len].copy_from_slice(&payload[start..start + self.stride]);
331        Some(len)
332    }
333}
334
335/// Deliver the samples an entry holds whose timestamps fall in `[t_start, t_end]`.
336///
337/// One callback per *sample*, which is what makes a run indistinguishable from
338/// the same samples appended one at a time — the transparency the whole feature
339/// rests on. A single-sample entry is delivered as itself, unfiltered by any
340/// framing, so the raw API is untouched.
341///
342/// `payload` is written through: see [`RunFrame::tagged_sample`]. It must be a
343/// buffer holding a *copy* of the entry, which every caller already has, since
344/// payloads reach them through a read into scratch.
345///
346/// Returns `true` if the callback asked to stop.
347#[allow(clippy::too_many_arguments)]
348pub(crate) fn deliver_samples(
349    is_run: bool,
350    sensor_id: u16,
351    base: u64,
352    payload: &mut [u8],
353    t_start: u64,
354    t_end: u64,
355    count: &mut u32,
356    callback: &mut dyn FnMut(&EntryHeader, &[u8]) -> bool,
357) -> bool {
358    let frame = match is_run {
359        true => match RunFrame::parse(payload) {
360            Some(frame) => frame,
361            // Rejected upstream by the same test that rejects a bad CRC; this
362            // is the belt to that braces.
363            None => return false,
364        },
365        false => {
366            if base < t_start || base > t_end {
367                return false;
368            }
369            *count += 1;
370            let header = EntryHeader {
371                sensor_id,
372                timestamp: base,
373                payload_len: payload.len() as u16,
374                kind: EntryKind::Single,
375            };
376            return !callback(&header, payload);
377        }
378    };
379
380    for index in 0..frame.count() {
381        let timestamp = frame.timestamp_at(base, index);
382        // Ascending and unsigned, so once past the end nothing later qualifies.
383        if timestamp > t_end {
384            break;
385        }
386        if timestamp < t_start {
387            continue;
388        }
389        *count += 1;
390        let header = frame.header_at(sensor_id, base, index);
391        if !callback(&header, frame.tagged_sample(payload, index)) {
392            return true;
393        }
394    }
395
396    false
397}
398
399/// How many samples an entry delivers: one, unless it is a well-formed run.
400///
401/// The single place that answers "what does this entry count as", so the index,
402/// `count` and `query_range` cannot drift apart.
403pub(crate) fn samples_in(is_run: bool, payload: &[u8]) -> u32 {
404    if !is_run {
405        return 1;
406    }
407    RunFrame::parse(payload).map_or(0, |frame| frame.count())
408}
409
410#[cfg(test)]
411mod tests {
412    use super::*;
413    use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
414
415    #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone, Copy, PartialEq)]
416    #[repr(C)]
417    struct Reading {
418        raw: u32,
419    }
420
421    impl SensorData for Reading {
422        const SENSOR_ID: u16 = 0x42;
423        const SCHEMA_VERSION: u8 = 3;
424    }
425
426    /// Lay a `Run` out the way an appending driver does.
427    fn encode(run: &Run<'_>) -> heapless::Vec<u8, 512> {
428        let mut buf = heapless::Vec::new();
429        buf.resize(run.payload_len(), 0).unwrap();
430        run.write_frame(&mut buf[..RUN_FRAME_SIZE]);
431        buf[RUN_FRAME_SIZE..].copy_from_slice(run.samples());
432        buf
433    }
434
435    #[test]
436    fn a_run_round_trips_through_its_own_frame() {
437        let samples = [Reading { raw: 1 }, Reading { raw: 2 }, Reading { raw: 3 }];
438        let run = Run::of(1_000, 25, &samples).unwrap();
439
440        assert_eq!(run.sensor_id(), 0x42);
441        assert_eq!(run.count(), 3);
442        assert_eq!(run.span_end(), 1_050);
443        assert_eq!(run.payload_len(), RUN_FRAME_SIZE + 12);
444
445        let mut payload = encode(&run);
446        let frame = RunFrame::parse(&payload).unwrap();
447
448        assert_eq!(frame.count(), 3);
449        assert_eq!(frame.span_end(1_000), 1_050);
450        for (i, sample) in samples.iter().enumerate() {
451            let i = i as u32;
452            assert_eq!(frame.timestamp_at(1_000, i), 1_000 + 25 * u64::from(i));
453            let tagged = frame.tagged_sample(&mut payload, i);
454            assert_eq!(tagged[0], Reading::SCHEMA_VERSION);
455            assert_eq!(&tagged[1..], sample.as_bytes());
456        }
457    }
458
459    #[test]
460    fn an_expanded_sample_is_byte_for_byte_a_single_append() {
461        // The transparency invariant, at the framing level: what `tagged_sample`
462        // hands out must equal what the single-sample path stores.
463        let samples = [Reading { raw: 0xDEAD_BEEF }, Reading { raw: 7 }];
464        let run = Run::of(0, 1, &samples).unwrap();
465        let mut payload = encode(&run);
466        let frame = frame_of(&payload);
467
468        for (i, sample) in samples.iter().enumerate() {
469            let single = crate::schema::tag(sample);
470            assert_eq!(frame.tagged_sample(&mut payload, i as u32), single.bytes());
471        }
472    }
473
474    fn frame_of(payload: &[u8]) -> RunFrame {
475        RunFrame::parse(payload).unwrap()
476    }
477
478    #[test]
479    fn the_headers_a_run_expands_to_are_the_ones_single_appends_would_carry() {
480        let samples = [Reading { raw: 1 }; 4];
481        let run = Run::of(500, 10, &samples).unwrap();
482        let frame = frame_of(&encode(&run));
483
484        for i in 0..4u32 {
485            assert_eq!(
486                frame.header_at(Reading::SENSOR_ID, 500, i),
487                EntryHeader {
488                    sensor_id: Reading::SENSOR_ID,
489                    timestamp: 500 + 10 * u64::from(i),
490                    payload_len: (SCHEMA_TAG_SIZE + 4) as u16,
491                    kind: EntryKind::Single,
492                }
493            );
494        }
495    }
496
497    #[test]
498    fn a_single_sample_run_is_legal_and_spans_nothing() {
499        let run = Run::of(9, 1_000, &[Reading { raw: 1 }]).unwrap();
500        assert_eq!(run.count(), 1);
501        assert_eq!(run.span_end(), 9);
502
503        let frame = frame_of(&encode(&run));
504        assert_eq!(frame.span_end(9), 9);
505    }
506
507    #[test]
508    fn a_zero_interval_run_stacks_every_sample_on_one_timestamp() {
509        let run = Run::of(4, 0, &[Reading { raw: 1 }; 3]).unwrap();
510        assert_eq!(run.span_end(), 4);
511
512        let frame = frame_of(&encode(&run));
513        assert_eq!(frame.timestamp_at(4, 2), 4);
514    }
515
516    #[test]
517    fn an_empty_run_is_rejected_rather_than_stored_as_nothing() {
518        assert_eq!(
519            Run::of::<Reading>(0, 1, &[]).unwrap_err(),
520            Error::InvalidRun
521        );
522    }
523
524    #[test]
525    fn a_run_whose_last_timestamp_would_overflow_is_refused() {
526        // The check is on the *last* sample, not the first: this base is fine.
527        assert!(Run::of(u64::MAX, 0, &[Reading { raw: 1 }; 4]).is_ok());
528        assert_eq!(
529            Run::of(u64::MAX - 1, 1, &[Reading { raw: 1 }; 4]).unwrap_err(),
530            Error::InvalidRun
531        );
532        assert_eq!(
533            Run::of(0, u64::MAX, &[Reading { raw: 1 }; 3]).unwrap_err(),
534            Error::InvalidRun
535        );
536    }
537
538    #[test]
539    fn a_payload_that_is_not_a_well_formed_run_parses_to_nothing() {
540        // Too short to hold a sample.
541        assert!(RunFrame::parse(&[0u8; RUN_FRAME_SIZE]).is_none());
542        assert!(RunFrame::parse(&[]).is_none());
543
544        // A zero count would divide by zero.
545        let mut payload = [0u8; RUN_FRAME_SIZE + 4];
546        assert!(RunFrame::parse(&payload).is_none());
547
548        // A count that does not divide the sample region evenly.
549        payload[COUNT_AT] = 3;
550        assert!(RunFrame::parse(&payload).is_none());
551
552        payload[COUNT_AT] = 4;
553        assert_eq!(RunFrame::parse(&payload).unwrap().stride, 1);
554    }
555
556    #[test]
557    fn samples_in_answers_for_both_entry_kinds() {
558        let run = Run::of(0, 1, &[Reading { raw: 1 }; 5]).unwrap();
559        assert_eq!(samples_in(true, &encode(&run)), 5);
560        assert_eq!(samples_in(false, &encode(&run)), 1);
561        // A malformed run contributes nothing, matching the scans that skip it.
562        assert_eq!(samples_in(true, &[0u8; 4]), 0);
563    }
564}