sensor_db/run.rs
1//! Run-encoded entries: many evenly spaced samples under one entry header.
2//!
3//! A single `f32` sample costs 20 bytes of entry overhead plus a 1-byte schema
4//! tag to store 4 bytes of data — **84% overhead**. For a burst of samples taken
5//! at a fixed cadence, almost all of that is repetition: the same sensor id, and
6//! a timestamp the previous one already implies.
7//!
8//! A *run* stores the burst as one entry:
9//!
10//! ```text
11//! payload = [ schema_version : u8 ] [ interval : u64 LE ] [ count : u16 LE ] [ T ; count ]
12//! ```
13//!
14//! with the entry header's `timestamp` holding the first sample's, so sample `i`
15//! is at `base + i * interval`. Thirty-two `f32` samples cost 148 bytes instead
16//! of 800.
17//!
18//! # A run is invisible to readers
19//!
20//! Every read path expands a run back into the samples it holds, so
21//! `query_range`, `latest`, `oldest` and `count` deliver **exactly what
22//! [`append_sensor`](crate::LogSinkExt::append_sensor) of the same samples would
23//! have delivered** — same headers, same `[version][value]` payloads, same
24//! order. Only [`read_from`](crate::SensorDb::read_from) sees the run as one
25//! entry, because replication copies entries and a replica should keep the
26//! amortisation rather than unpack it.
27//!
28//! That transparency is what makes runs a pure storage optimisation: a caller
29//! can start using them without changing a single read.
30//!
31//! # Why the marker and not the payload
32//!
33//! An entry is a run because its header carries
34//! [`ENTRY_MARKER_RUN`](crate::entry::ENTRY_MARKER_RUN) instead of
35//! [`ENTRY_MARKER`](crate::entry::ENTRY_MARKER) — a field the entry header CRC
36//! covers and the raw [`append`](crate::LogSink::append) path cannot write.
37//!
38//! The alternative — flagging a run in the payload, say with the high bit of the
39//! schema tag — was tried and reverted. It cannot work, because a raw payload is
40//! *arbitrary bytes*: any payload whose first byte happened to have bit 7 set
41//! would read as a run, and the sector index would inherit its invented time
42//! span. `tests/model.rs` caught exactly that, on a workload containing no typed
43//! data at all. A marker in the header is structural: the raw API always writes
44//! `0xAA`, so a run is a run.
45//!
46//! # Runs are a typed-API feature
47//!
48//! [`Run::of`] takes `&[T]` for a [`SensorData`] type, because the frame carries
49//! a schema tag and expansion produces `[version][value]` payloads — which is
50//! precisely what the typed single-sample path writes. There is deliberately no
51//! untyped constructor: an untagged run would expand into payloads that no read
52//! path could interpret.
53
54use zerocopy::IntoBytes;
55
56use crate::entry::{EntryHeader, EntryKind};
57use crate::error::Error;
58use crate::schema::SCHEMA_TAG_SIZE;
59use crate::typed::SensorData;
60
61/// Bytes a run frame prepends to its samples: the schema tag, the interval, and
62/// the sample count.
63///
64/// A run of `n` values of `T` therefore occupies
65/// `ENTRY_OVERHEAD + RUN_FRAME_SIZE + n * size_of::<T>()` bytes of flash,
66/// against `n * (ENTRY_OVERHEAD + SCHEMA_TAG_SIZE + size_of::<T>())` for the
67/// same samples appended one at a time.
68pub const RUN_FRAME_SIZE: usize = SCHEMA_TAG_SIZE + 8 + 2;
69
70/// Offset of the `interval` field within a run payload.
71const INTERVAL_AT: usize = SCHEMA_TAG_SIZE;
72
73/// Offset of the `count` field within a run payload.
74const COUNT_AT: usize = INTERVAL_AT + 8;
75
76/// Evenly spaced samples of one sensor, ready to be appended as a single entry.
77///
78/// Construct with [`Run::of`], which validates everything an appender would
79/// otherwise have to re-check, and hand it to
80/// [`append_run`](crate::LogSink::append_run):
81///
82/// ```ignore
83/// use sensor_db::{LogSink, Run};
84///
85/// let burst: [Temperature; 32] = collect_burst();
86/// db.append_run(&Run::of(t0, 10_000, &burst)?)?;
87/// ```
88///
89/// The samples are borrowed, not copied: the driver writes the frame header and
90/// then the sample bytes straight into its page buffer.
91#[derive(Debug, Clone, Copy)]
92pub struct Run<'a> {
93 sensor_id: u16,
94 base_timestamp: u64,
95 interval: u64,
96 schema_version: u8,
97 count: u16,
98 samples: &'a [u8],
99}
100
101impl<'a> Run<'a> {
102 /// Describe a run of `T` values, the first at `base_timestamp` and each
103 /// subsequent one `interval` later.
104 ///
105 /// # Errors
106 ///
107 /// [`Error::InvalidRun`] if `samples` is empty or longer than `u16::MAX`,
108 /// if `T` is zero-sized, if the last sample's timestamp would overflow
109 /// `u64`, or if the frame would exceed the `u16` payload length the entry
110 /// header can carry. Whether it fits a *page* is the appending log's
111 /// question, and it answers [`Error::PayloadTooLarge`].
112 pub fn of<T: SensorData>(
113 base_timestamp: u64,
114 interval: u64,
115 samples: &'a [T],
116 ) -> Result<Self, Error> {
117 let stride = core::mem::size_of::<T>();
118 if stride == 0 {
119 return Err(Error::InvalidRun);
120 }
121 let count = u16::try_from(samples.len())
122 .ok()
123 .filter(|&n| n > 0)
124 .ok_or(Error::InvalidRun)?;
125
126 // The run is unreadable if its own last timestamp is not representable,
127 // so refuse to write it rather than storing samples that read back
128 // clamped. Checked here, once, which is what lets every reader use
129 // saturating arithmetic without hiding a defect.
130 interval
131 .checked_mul(u64::from(count - 1))
132 .and_then(|span| base_timestamp.checked_add(span))
133 .ok_or(Error::InvalidRun)?;
134
135 let bytes = samples.as_bytes();
136 if RUN_FRAME_SIZE + bytes.len() > u16::MAX as usize {
137 return Err(Error::InvalidRun);
138 }
139
140 Ok(Self {
141 sensor_id: T::SENSOR_ID,
142 base_timestamp,
143 interval,
144 schema_version: T::SCHEMA_VERSION,
145 count,
146 samples: bytes,
147 })
148 }
149
150 /// The sensor id the samples are stored under.
151 pub fn sensor_id(&self) -> u16 {
152 self.sensor_id
153 }
154
155 /// Timestamp of the first sample, which is the entry header's timestamp.
156 pub fn base_timestamp(&self) -> u64 {
157 self.base_timestamp
158 }
159
160 /// Timestamp step between consecutive samples.
161 pub fn interval(&self) -> u64 {
162 self.interval
163 }
164
165 /// How many samples the run holds; never zero.
166 pub fn count(&self) -> u32 {
167 u32::from(self.count)
168 }
169
170 /// Timestamp of the last sample.
171 pub fn span_end(&self) -> u64 {
172 // Cannot overflow: `of` rejected a run whose end is not representable.
173 self.base_timestamp + self.interval * u64::from(self.count - 1)
174 }
175
176 /// Bytes the run occupies as an entry payload.
177 pub fn payload_len(&self) -> usize {
178 RUN_FRAME_SIZE + self.samples.len()
179 }
180
181 /// The sample bytes, `count * size_of::<T>()` of them.
182 pub(crate) fn samples(&self) -> &'a [u8] {
183 self.samples
184 }
185
186 /// Write the frame header — everything before the samples — into `buf`.
187 ///
188 /// # Panics
189 ///
190 /// If `buf` is shorter than [`RUN_FRAME_SIZE`]. Callers size it from
191 /// [`payload_len`](Self::payload_len).
192 pub(crate) fn write_frame(&self, buf: &mut [u8]) {
193 buf[0] = self.schema_version;
194 buf[INTERVAL_AT..INTERVAL_AT + 8].copy_from_slice(&self.interval.to_le_bytes());
195 buf[COUNT_AT..COUNT_AT + 2].copy_from_slice(&self.count.to_le_bytes());
196 }
197}
198
199/// The frame of a stored run, read back from the payload of an entry carrying
200/// [`ENTRY_MARKER_RUN`](crate::entry::ENTRY_MARKER_RUN).
201///
202/// Borrows nothing: it is the four numbers that describe the samples, so a
203/// caller can hold it while still handing the payload buffer out mutably to
204/// [`tagged_sample`](Self::tagged_sample).
205#[derive(Debug, Clone, Copy, PartialEq, Eq)]
206pub(crate) struct RunFrame {
207 schema_version: u8,
208 interval: u64,
209 count: u32,
210 stride: usize,
211}
212
213impl RunFrame {
214 /// Read the frame out of a run's payload, or `None` if the payload is not a
215 /// well-formed run.
216 ///
217 /// The marker already says the writer meant a run, and the entry CRC says
218 /// these are the bytes it wrote — so this rejects only what no writer of
219 /// this crate can produce: no samples, a zero count, or a sample region that
220 /// does not divide evenly into `count` samples. Callers treat `None` exactly
221 /// as they treat a CRC failure, which keeps every scan walking a sector the
222 /// same way.
223 pub(crate) fn parse(payload: &[u8]) -> Option<Self> {
224 // Strictly greater: a run with no room for even one sample is malformed.
225 if payload.len() <= RUN_FRAME_SIZE {
226 return None;
227 }
228
229 let mut interval = [0u8; 8];
230 interval.copy_from_slice(&payload[INTERVAL_AT..INTERVAL_AT + 8]);
231 let count = u32::from(u16::from_le_bytes([
232 payload[COUNT_AT],
233 payload[COUNT_AT + 1],
234 ]));
235 if count == 0 {
236 return None;
237 }
238
239 let body = payload.len() - RUN_FRAME_SIZE;
240 if !body.is_multiple_of(count as usize) {
241 return None;
242 }
243
244 Some(Self {
245 schema_version: payload[0],
246 interval: u64::from_le_bytes(interval),
247 count,
248 // Non-zero: `body >= 1` and `count` divides it, so `count <= body`.
249 stride: body / count as usize,
250 })
251 }
252
253 /// How many samples the run holds; never zero.
254 pub(crate) fn count(&self) -> u32 {
255 self.count
256 }
257
258 /// Timestamp of sample `index`, counting from the entry header's `base`.
259 ///
260 /// Saturating, which cannot bite a run this crate wrote: [`Run::of`] refuses
261 /// one whose end is not representable.
262 pub(crate) fn timestamp_at(&self, base: u64, index: u32) -> u64 {
263 base.saturating_add(self.interval.saturating_mul(u64::from(index)))
264 }
265
266 /// Timestamp of the last sample — how far past the entry header's timestamp
267 /// this entry reaches.
268 pub(crate) fn span_end(&self, base: u64) -> u64 {
269 self.timestamp_at(base, self.count - 1)
270 }
271
272 /// The header sample `index` would have carried had it been appended alone.
273 pub(crate) fn header_at(&self, sensor_id: u16, base: u64, index: u32) -> EntryHeader {
274 EntryHeader {
275 sensor_id,
276 timestamp: self.timestamp_at(base, index),
277 // Always representable: the whole run fit a `u16` payload length.
278 payload_len: (SCHEMA_TAG_SIZE + self.stride) as u16,
279 // An expanded sample is indistinguishable from one appended alone,
280 // and that includes its kind.
281 kind: EntryKind::Single,
282 }
283 }
284
285 /// Sample `index` as a schema-tagged payload — byte for byte what
286 /// `append_sensor` would have stored for that sample alone.
287 ///
288 /// # The tag byte is written in place
289 ///
290 /// A tagged payload is `[version][value]`, and the run stores the version
291 /// *once*, at the front. Rather than copy the sample into a scratch buffer
292 /// to put a tag in front of it, this writes the version into the byte
293 /// immediately preceding sample `index` and hands out the two together.
294 ///
295 /// That byte is the last byte of sample `index - 1` — already delivered, in
296 /// a forward walk — or, for `index == 0`, the high byte of `count`, which
297 /// this frame has already read. Nothing that is still needed is overwritten,
298 /// and the buffer holds a copy of flash, so the medium is untouched.
299 ///
300 /// # Panics
301 ///
302 /// If `index >= count` or `payload` is not the run's whole payload.
303 pub(crate) fn tagged_sample<'a>(&self, payload: &'a mut [u8], index: u32) -> &'a [u8] {
304 debug_assert!(index < self.count);
305 let start = RUN_FRAME_SIZE + index as usize * self.stride;
306 payload[start - SCHEMA_TAG_SIZE] = self.schema_version;
307 &payload[start - SCHEMA_TAG_SIZE..start + self.stride]
308 }
309
310 /// Copy sample `index` into `out` as a schema-tagged payload, returning its
311 /// length, or `None` if `out` cannot hold it.
312 ///
313 /// The copying twin of [`tagged_sample`](Self::tagged_sample), for the
314 /// readers whose payload is behind a shared reference — the page buffer,
315 /// and a replication stream's callback — or who are filling a caller's
316 /// output buffer anyway and would gain nothing from writing in place.
317 pub(crate) fn write_tagged_sample(
318 &self,
319 payload: &[u8],
320 index: u32,
321 out: &mut [u8],
322 ) -> Option<usize> {
323 debug_assert!(index < self.count);
324 let len = SCHEMA_TAG_SIZE + self.stride;
325 if out.len() < len {
326 return None;
327 }
328 let start = RUN_FRAME_SIZE + index as usize * self.stride;
329 out[0] = self.schema_version;
330 out[SCHEMA_TAG_SIZE..len].copy_from_slice(&payload[start..start + self.stride]);
331 Some(len)
332 }
333}
334
335/// Deliver the samples an entry holds whose timestamps fall in `[t_start, t_end]`.
336///
337/// One callback per *sample*, which is what makes a run indistinguishable from
338/// the same samples appended one at a time — the transparency the whole feature
339/// rests on. A single-sample entry is delivered as itself, unfiltered by any
340/// framing, so the raw API is untouched.
341///
342/// `payload` is written through: see [`RunFrame::tagged_sample`]. It must be a
343/// buffer holding a *copy* of the entry, which every caller already has, since
344/// payloads reach them through a read into scratch.
345///
346/// Returns `true` if the callback asked to stop.
347#[allow(clippy::too_many_arguments)]
348pub(crate) fn deliver_samples(
349 is_run: bool,
350 sensor_id: u16,
351 base: u64,
352 payload: &mut [u8],
353 t_start: u64,
354 t_end: u64,
355 count: &mut u32,
356 callback: &mut dyn FnMut(&EntryHeader, &[u8]) -> bool,
357) -> bool {
358 let frame = match is_run {
359 true => match RunFrame::parse(payload) {
360 Some(frame) => frame,
361 // Rejected upstream by the same test that rejects a bad CRC; this
362 // is the belt to that braces.
363 None => return false,
364 },
365 false => {
366 if base < t_start || base > t_end {
367 return false;
368 }
369 *count += 1;
370 let header = EntryHeader {
371 sensor_id,
372 timestamp: base,
373 payload_len: payload.len() as u16,
374 kind: EntryKind::Single,
375 };
376 return !callback(&header, payload);
377 }
378 };
379
380 for index in 0..frame.count() {
381 let timestamp = frame.timestamp_at(base, index);
382 // Ascending and unsigned, so once past the end nothing later qualifies.
383 if timestamp > t_end {
384 break;
385 }
386 if timestamp < t_start {
387 continue;
388 }
389 *count += 1;
390 let header = frame.header_at(sensor_id, base, index);
391 if !callback(&header, frame.tagged_sample(payload, index)) {
392 return true;
393 }
394 }
395
396 false
397}
398
399/// How many samples an entry delivers: one, unless it is a well-formed run.
400///
401/// The single place that answers "what does this entry count as", so the index,
402/// `count` and `query_range` cannot drift apart.
403pub(crate) fn samples_in(is_run: bool, payload: &[u8]) -> u32 {
404 if !is_run {
405 return 1;
406 }
407 RunFrame::parse(payload).map_or(0, |frame| frame.count())
408}
409
410#[cfg(test)]
411mod tests {
412 use super::*;
413 use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
414
415 #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone, Copy, PartialEq)]
416 #[repr(C)]
417 struct Reading {
418 raw: u32,
419 }
420
421 impl SensorData for Reading {
422 const SENSOR_ID: u16 = 0x42;
423 const SCHEMA_VERSION: u8 = 3;
424 }
425
426 /// Lay a `Run` out the way an appending driver does.
427 fn encode(run: &Run<'_>) -> heapless::Vec<u8, 512> {
428 let mut buf = heapless::Vec::new();
429 buf.resize(run.payload_len(), 0).unwrap();
430 run.write_frame(&mut buf[..RUN_FRAME_SIZE]);
431 buf[RUN_FRAME_SIZE..].copy_from_slice(run.samples());
432 buf
433 }
434
435 #[test]
436 fn a_run_round_trips_through_its_own_frame() {
437 let samples = [Reading { raw: 1 }, Reading { raw: 2 }, Reading { raw: 3 }];
438 let run = Run::of(1_000, 25, &samples).unwrap();
439
440 assert_eq!(run.sensor_id(), 0x42);
441 assert_eq!(run.count(), 3);
442 assert_eq!(run.span_end(), 1_050);
443 assert_eq!(run.payload_len(), RUN_FRAME_SIZE + 12);
444
445 let mut payload = encode(&run);
446 let frame = RunFrame::parse(&payload).unwrap();
447
448 assert_eq!(frame.count(), 3);
449 assert_eq!(frame.span_end(1_000), 1_050);
450 for (i, sample) in samples.iter().enumerate() {
451 let i = i as u32;
452 assert_eq!(frame.timestamp_at(1_000, i), 1_000 + 25 * u64::from(i));
453 let tagged = frame.tagged_sample(&mut payload, i);
454 assert_eq!(tagged[0], Reading::SCHEMA_VERSION);
455 assert_eq!(&tagged[1..], sample.as_bytes());
456 }
457 }
458
459 #[test]
460 fn an_expanded_sample_is_byte_for_byte_a_single_append() {
461 // The transparency invariant, at the framing level: what `tagged_sample`
462 // hands out must equal what the single-sample path stores.
463 let samples = [Reading { raw: 0xDEAD_BEEF }, Reading { raw: 7 }];
464 let run = Run::of(0, 1, &samples).unwrap();
465 let mut payload = encode(&run);
466 let frame = frame_of(&payload);
467
468 for (i, sample) in samples.iter().enumerate() {
469 let single = crate::schema::tag(sample);
470 assert_eq!(frame.tagged_sample(&mut payload, i as u32), single.bytes());
471 }
472 }
473
474 fn frame_of(payload: &[u8]) -> RunFrame {
475 RunFrame::parse(payload).unwrap()
476 }
477
478 #[test]
479 fn the_headers_a_run_expands_to_are_the_ones_single_appends_would_carry() {
480 let samples = [Reading { raw: 1 }; 4];
481 let run = Run::of(500, 10, &samples).unwrap();
482 let frame = frame_of(&encode(&run));
483
484 for i in 0..4u32 {
485 assert_eq!(
486 frame.header_at(Reading::SENSOR_ID, 500, i),
487 EntryHeader {
488 sensor_id: Reading::SENSOR_ID,
489 timestamp: 500 + 10 * u64::from(i),
490 payload_len: (SCHEMA_TAG_SIZE + 4) as u16,
491 kind: EntryKind::Single,
492 }
493 );
494 }
495 }
496
497 #[test]
498 fn a_single_sample_run_is_legal_and_spans_nothing() {
499 let run = Run::of(9, 1_000, &[Reading { raw: 1 }]).unwrap();
500 assert_eq!(run.count(), 1);
501 assert_eq!(run.span_end(), 9);
502
503 let frame = frame_of(&encode(&run));
504 assert_eq!(frame.span_end(9), 9);
505 }
506
507 #[test]
508 fn a_zero_interval_run_stacks_every_sample_on_one_timestamp() {
509 let run = Run::of(4, 0, &[Reading { raw: 1 }; 3]).unwrap();
510 assert_eq!(run.span_end(), 4);
511
512 let frame = frame_of(&encode(&run));
513 assert_eq!(frame.timestamp_at(4, 2), 4);
514 }
515
516 #[test]
517 fn an_empty_run_is_rejected_rather_than_stored_as_nothing() {
518 assert_eq!(
519 Run::of::<Reading>(0, 1, &[]).unwrap_err(),
520 Error::InvalidRun
521 );
522 }
523
524 #[test]
525 fn a_run_whose_last_timestamp_would_overflow_is_refused() {
526 // The check is on the *last* sample, not the first: this base is fine.
527 assert!(Run::of(u64::MAX, 0, &[Reading { raw: 1 }; 4]).is_ok());
528 assert_eq!(
529 Run::of(u64::MAX - 1, 1, &[Reading { raw: 1 }; 4]).unwrap_err(),
530 Error::InvalidRun
531 );
532 assert_eq!(
533 Run::of(0, u64::MAX, &[Reading { raw: 1 }; 3]).unwrap_err(),
534 Error::InvalidRun
535 );
536 }
537
538 #[test]
539 fn a_payload_that_is_not_a_well_formed_run_parses_to_nothing() {
540 // Too short to hold a sample.
541 assert!(RunFrame::parse(&[0u8; RUN_FRAME_SIZE]).is_none());
542 assert!(RunFrame::parse(&[]).is_none());
543
544 // A zero count would divide by zero.
545 let mut payload = [0u8; RUN_FRAME_SIZE + 4];
546 assert!(RunFrame::parse(&payload).is_none());
547
548 // A count that does not divide the sample region evenly.
549 payload[COUNT_AT] = 3;
550 assert!(RunFrame::parse(&payload).is_none());
551
552 payload[COUNT_AT] = 4;
553 assert_eq!(RunFrame::parse(&payload).unwrap().stride, 1);
554 }
555
556 #[test]
557 fn samples_in_answers_for_both_entry_kinds() {
558 let run = Run::of(0, 1, &[Reading { raw: 1 }; 5]).unwrap();
559 assert_eq!(samples_in(true, &encode(&run)), 5);
560 assert_eq!(samples_in(false, &encode(&run)), 1);
561 // A malformed run contributes nothing, matching the scans that skip it.
562 assert_eq!(samples_in(true, &[0u8; 4]), 0);
563 }
564}