use crate::entry::{
ENTRY_CRC_SIZE, ENTRY_HEADER_SIZE, ENTRY_OVERHEAD, EntryHeader, EntryHeaderWire, EntryKind,
FLAGS_COMMITTED, FLAGS_UNCOMMITTED, compute_entry_crc, is_entry_marker,
};
use crate::error::Error;
use crate::partition::Partition;
use crate::run::{RUN_FRAME_SIZE, Run, RunFrame, deliver_samples, samples_in};
use crate::schema;
use crate::sector::{
LogId, SECTOR_FOOTER_SIZE, SECTOR_HEADER_SIZE, SectorClass, SectorFooter, SectorHeader,
SectorState, SectorSummary, sensor_id_to_bit,
};
use crate::typed::SensorData;
use embedded_storage::nor_flash::{NorFlash, ReadNorFlash};
use heapless::Vec as HeaplessVec;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
pub(crate) fn sort_by_seq<const N: usize>(
vec: &mut HeaplessVec<(u16, u32), N>,
cmp: fn(&u32, &u32) -> core::cmp::Ordering,
) {
for i in 1..vec.len() {
let mut j = i;
while j > 0 && cmp(&vec[j - 1].1, &vec[j].1) == core::cmp::Ordering::Greater {
vec.swap(j - 1, j);
j -= 1;
}
}
}
#[derive(Debug, Clone, Copy, Default)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub struct DbStats {
pub total_entries: u32,
pub total_samples: u32,
pub used_sectors: u16,
pub free_sectors: u16,
pub reserved_sectors: u16,
pub min_erase_count: u32,
pub max_erase_count: u32,
pub total_erase_count: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub struct DbConfig {
base_offset: u32,
sector_count: u16,
reserved_sectors: u16,
log_id: Option<LogId>,
}
impl DbConfig {
pub fn new(base_offset: u32, sector_count: u16, reserved_sectors: u16) -> Result<Self, Error> {
if reserved_sectors >= sector_count {
return Err(Error::InvalidConfig);
}
Ok(Self {
base_offset,
sector_count,
reserved_sectors,
log_id: None,
})
}
pub fn with_log_id(mut self, log_id: LogId) -> Self {
self.log_id = Some(log_id);
self
}
pub fn base_offset(&self) -> u32 {
self.base_offset
}
pub fn sector_count(&self) -> u16 {
self.sector_count
}
pub fn reserved_sectors(&self) -> u16 {
self.reserved_sectors
}
pub fn log_id(&self) -> Option<LogId> {
self.log_id
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub enum CompactScope {
#[default]
All,
SpooledOnly,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub enum MountMode {
#[default]
Full,
Verify,
HeadersOnly,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub struct SpoolStats {
pub unspooled_entries: u32,
pub unspooled_samples: u32,
pub watermark: LogPosition,
pub head: LogPosition,
pub dropped_unspooled: u32,
pub data_lost: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub struct Replicated {
pub entries: u32,
pub samples: u32,
}
impl Replicated {
pub const NONE: Self = Self {
entries: 0,
samples: 0,
};
pub const fn singles(n: u32) -> Self {
Self {
entries: n,
samples: n,
}
}
pub fn push(&mut self, samples: u32) {
self.entries = self.entries.saturating_add(1);
self.samples = self.samples.saturating_add(samples);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default, Hash)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub struct LogPosition {
pub sequence: u32,
pub offset: u32,
}
impl LogPosition {
pub const BEGINNING: Self = Self {
sequence: 0,
offset: 0,
};
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub struct ReadCursor {
pub next: LogPosition,
pub entries: u32,
pub exhausted: bool,
pub gap: bool,
}
#[derive(Debug, Clone, Copy, Default)]
#[cfg_attr(feature = "defmt", derive(defmt::Format))]
pub struct MountStats {
pub total_entries: u32,
pub total_samples: u32,
pub sealed_sectors: u16,
pub corrupt_entries: u32,
pub scanned_sectors: u16,
}
pub(crate) struct ScannedEntry {
pub(crate) header: EntryHeaderWire,
pub(crate) offset_in_sector: u32,
pub(crate) payload_len: usize,
pub(crate) entry_size: u32,
}
impl ScannedEntry {
pub(crate) fn is_run(&self) -> bool {
self.header.is_run()
}
pub(crate) fn sensor_id(&self) -> u16 {
self.header.sensor_id.get()
}
pub(crate) fn timestamp(&self) -> u64 {
self.header.timestamp.get()
}
}
struct BufferedEntry {
header: EntryHeaderWire,
payload_start: usize,
payload_len: usize,
}
#[derive(Clone, Copy)]
pub(crate) enum Edge {
First,
Last,
}
#[derive(Clone, Copy)]
struct BufferedSample {
payload_start: usize,
payload_len: usize,
run: Option<(RunFrame, u32)>,
}
impl BufferedEntry {
fn is_committed(&self) -> bool {
self.header.flags == FLAGS_COMMITTED
}
fn is_run(&self) -> bool {
self.header.is_run()
}
fn sensor_id(&self) -> u16 {
self.header.sensor_id.get()
}
fn timestamp(&self) -> u64 {
self.header.timestamp.get()
}
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct ScannedSector {
pub(crate) entries: u32,
pub(crate) samples: u32,
pub(crate) write_offset: u32,
pub(crate) min_timestamp: u64,
pub(crate) max_timestamp: u64,
pub(crate) sensor_bloom: u64,
}
impl ScannedSector {
pub(crate) const EMPTY: Self = Self {
entries: 0,
samples: 0,
write_offset: 0,
min_timestamp: u64::MAX,
max_timestamp: 0,
sensor_bloom: 0,
};
pub(crate) fn record(&mut self, entry: &ScannedEntry, payload: &[u8]) {
let base = entry.timestamp();
let span_end = match RunFrame::parse(payload).filter(|_| entry.is_run()) {
Some(frame) => {
self.samples += frame.count();
frame.span_end(base)
}
None => {
self.samples += 1;
base
}
};
self.entries += 1;
self.min_timestamp = self.min_timestamp.min(base);
self.max_timestamp = self.max_timestamp.max(span_end);
self.sensor_bloom |= sensor_id_to_bit(entry.sensor_id());
}
pub(crate) fn apply_to(&self, summary: &mut SectorSummary) {
summary.entry_count = self.entries;
summary.sample_count = self.samples;
summary.write_offset = self.write_offset;
summary.min_timestamp = self.min_timestamp;
summary.max_timestamp = self.max_timestamp;
summary.sensor_bloom = self.sensor_bloom;
}
}
pub(crate) enum Scan {
Read(u32),
Found(ScannedEntry),
Done,
}
pub(crate) struct SectorScanner {
sector_offset: u32,
offset: u32,
limit: u32,
pub(crate) corrupt_count: u32,
}
impl SectorScanner {
pub(crate) fn new(sector_offset: u32, entry_start: u32, limit: u32) -> Self {
Self {
sector_offset,
offset: entry_start,
limit,
corrupt_count: 0,
}
}
pub(crate) fn offset(&self) -> u32 {
self.offset
}
pub(crate) fn corrupt_count(&self) -> u32 {
self.corrupt_count
}
pub(crate) fn skip(&mut self, entry_size: u32) {
self.offset += entry_size;
}
pub(crate) fn begin(&self) -> Scan {
if self.offset + ENTRY_OVERHEAD as u32 > self.limit {
return Scan::Done;
}
Scan::Read(self.sector_offset + self.offset)
}
pub(crate) fn feed(&mut self, header_buf: &[u8; ENTRY_HEADER_SIZE], page_size: usize) -> Scan {
if header_buf[0] == 0xFF {
let next_page = ((self.offset as usize / page_size) + 1) * page_size;
if next_page as u32 + ENTRY_OVERHEAD as u32 > self.limit {
return Scan::Done;
}
self.offset = next_page as u32;
return self.begin();
}
if !is_entry_marker(header_buf[0]) {
return self.resync();
}
let Ok(header) = EntryHeaderWire::read_from_bytes(&header_buf[..]) else {
return self.resync();
};
let payload_len = header.payload_len.get() as usize;
let max_payload = page_size.saturating_sub(ENTRY_OVERHEAD);
if (header.flags != FLAGS_COMMITTED && header.flags != FLAGS_UNCOMMITTED)
|| payload_len > max_payload
|| !header.header_crc_valid()
{
return self.resync();
}
let entry_size = (ENTRY_OVERHEAD + payload_len) as u32;
if self.offset + entry_size > self.limit {
return Scan::Done;
}
if header.flags != FLAGS_COMMITTED {
self.offset += entry_size;
return self.begin();
}
Scan::Found(ScannedEntry {
header,
offset_in_sector: self.offset,
payload_len,
entry_size,
})
}
fn resync(&mut self) -> Scan {
self.corrupt_count += 1;
self.offset += 1;
self.begin()
}
fn next_entry<F: ReadNorFlash>(
&mut self,
flash: &mut F,
header_buf: &mut [u8; ENTRY_HEADER_SIZE],
page_size: usize,
) -> Result<Option<ScannedEntry>, Error> {
let mut step = self.begin();
loop {
match step {
Scan::Done => return Ok(None),
Scan::Found(entry) => return Ok(Some(entry)),
Scan::Read(addr) => {
flash.read(addr, &mut header_buf[..])?;
step = self.feed(header_buf, page_size);
}
}
}
}
}
impl ScannedEntry {
pub(crate) fn edge<'a>(&self, payload: &'a mut [u8], edge: Edge) -> (EntryHeader, &'a [u8]) {
match RunFrame::parse(payload).filter(|_| self.is_run()) {
Some(frame) => {
let index = match edge {
Edge::First => 0,
Edge::Last => frame.count() - 1,
};
(
frame.header_at(self.sensor_id(), self.timestamp(), index),
frame.tagged_sample(payload, index),
)
}
None => (self.header.to_header(), payload),
}
}
pub(crate) fn payload_read(&self, sector_offset: u32) -> (u32, usize) {
(
sector_offset + self.offset_in_sector + ENTRY_HEADER_SIZE as u32,
self.payload_len + ENTRY_CRC_SIZE,
)
}
pub(crate) fn accept(&self, buf: &[u8]) -> bool {
let stored_crc = u32::from_le_bytes([
buf[self.payload_len],
buf[self.payload_len + 1],
buf[self.payload_len + 2],
buf[self.payload_len + 3],
]);
let computed_crc = compute_entry_crc(
&self.header,
&buf[..self.payload_len],
self.offset_in_sector,
);
if stored_crc != computed_crc {
return false;
}
!self.is_run() || RunFrame::parse(&buf[..self.payload_len]).is_some()
}
}
fn accept_entry<F: ReadNorFlash>(
flash: &mut F,
sector_offset: u32,
entry: &ScannedEntry,
buf: &mut [u8],
) -> Result<bool, Error> {
let (addr, read_len) = entry.payload_read(sector_offset);
if read_len > buf.len() {
return Ok(false);
}
flash.read(addr, &mut buf[..read_len])?;
Ok(entry.accept(buf))
}
pub(crate) struct SealPlan {
pub(crate) new_active: u16,
pub(crate) erase_count: u32,
victim: SectorSummary,
}
pub(crate) enum AppendStep {
Stage,
FlushPage,
Seal,
}
pub(crate) struct SensorDbState<const PAGE_SIZE: usize, const MAX_SECTORS: usize> {
pub(crate) base_offset: u32,
pub(crate) sector_count: u16,
pub(crate) reserved_sectors: u16,
pub(crate) sector_size: u32,
pub(crate) sectors: [SectorSummary; MAX_SECTORS],
pub(crate) active_sector: Option<u16>,
pub(crate) page_buf: [u8; PAGE_SIZE],
pub(crate) page_buf_offset: usize,
pub(crate) page_buf_durable: usize,
pub(crate) page_flash_offset: u32,
pub(crate) mounted: bool,
pub(crate) next_sequence: u32,
pub(crate) total_entries: u32,
pub(crate) total_samples: u32,
pub(crate) log_id: Option<LogId>,
pub(crate) epoch: u32,
pub(crate) epoch_pending: bool,
pub(crate) mount_mode: MountMode,
pub(crate) spool_watermark: LogPosition,
pub(crate) spooled_entries: u32,
pub(crate) spooled_samples: u32,
pub(crate) dropped_unspooled: u32,
pub(crate) sectors_erased: u32,
}
pub(crate) struct Geometry<F: ?Sized, const PAGE_SIZE: usize>(core::marker::PhantomData<F>);
impl<F: NorFlash + ?Sized, const PAGE_SIZE: usize> Geometry<F, PAGE_SIZE> {
pub(crate) const PAGE_IS_WRITABLE: () = assert!(
PAGE_SIZE.is_multiple_of(F::WRITE_SIZE),
"PAGE_SIZE must be a multiple of the flash driver's WRITE_SIZE",
);
}
impl<const PAGE_SIZE: usize, const MAX_SECTORS: usize> SensorDbState<PAGE_SIZE, MAX_SECTORS> {
pub(crate) fn new(config: DbConfig, sector_size: u32) -> Result<Self, Error> {
if config.sector_count as usize > MAX_SECTORS {
return Err(Error::TooManySectors);
}
if sector_size < 3 * PAGE_SIZE as u32 {
return Err(Error::InvalidConfig);
}
let total_size = (config.sector_count as u32)
.checked_mul(sector_size)
.ok_or(Error::Overflow)?;
config
.base_offset
.checked_add(total_size)
.ok_or(Error::Overflow)?;
Ok(Self {
base_offset: config.base_offset,
sector_count: config.sector_count,
reserved_sectors: config.reserved_sectors,
sector_size,
sectors: [SectorSummary::default(); MAX_SECTORS],
active_sector: None,
page_buf: [0xFF; PAGE_SIZE],
page_buf_offset: 0,
page_buf_durable: 0,
page_flash_offset: 0,
mounted: false,
next_sequence: 1,
total_entries: 0,
total_samples: 0,
log_id: config.log_id,
epoch: 1,
epoch_pending: false,
mount_mode: MountMode::Full,
spool_watermark: LogPosition::BEGINNING,
spooled_entries: 0,
spooled_samples: 0,
dropped_unspooled: 0,
sectors_erased: 0,
})
}
pub(crate) fn log_id_raw(&self) -> u32 {
self.log_id.map_or(0, LogId::get)
}
pub(crate) fn require_reads(&self) -> Result<(), Error> {
if !self.mounted {
return Err(Error::NotMounted);
}
if self.mount_mode == MountMode::HeadersOnly {
return Err(Error::ReadsUnavailable);
}
Ok(())
}
pub(crate) fn sector_header(&self, sequence: u32, erase_count: u32) -> SectorHeader {
SectorHeader::new(self.log_id, self.epoch, sequence, erase_count)
}
pub(crate) fn is_fully_spooled(&self, summary: &SectorSummary) -> bool {
summary.sequence < self.spool_watermark.sequence
}
pub(crate) fn account_eviction(&mut self, victim: &SectorSummary) {
let entries = victim.entry_count;
let samples = victim.sample_count;
self.total_entries = self.total_entries.saturating_sub(entries);
self.total_samples = self.total_samples.saturating_sub(samples);
if self.is_fully_spooled(victim) {
self.spooled_entries = self.spooled_entries.saturating_sub(entries);
self.spooled_samples = self.spooled_samples.saturating_sub(samples);
return;
}
let spooled_elsewhere = self.spooled_in_whole_sectors();
let victim_share = self.spooled_entries.saturating_sub(spooled_elsewhere);
self.spooled_entries = spooled_elsewhere;
self.dropped_unspooled = self
.dropped_unspooled
.saturating_add(entries.saturating_sub(victim_share));
let samples_elsewhere = self.spooled_samples_in_whole_sectors();
self.spooled_samples = samples_elsewhere;
}
fn spooled_in_whole_sectors(&self) -> u32 {
self.fold_whole_spooled_sectors(|s| s.entry_count)
}
fn spooled_samples_in_whole_sectors(&self) -> u32 {
self.fold_whole_spooled_sectors(|s| s.sample_count)
}
fn fold_whole_spooled_sectors(&self, of: impl Fn(&SectorSummary) -> u32) -> u32 {
(0..self.sector_count)
.map(|i| &self.sectors[i as usize])
.filter(|s| s.state != SectorState::Erased && self.is_fully_spooled(s))
.fold(0u32, |acc, s| acc.saturating_add(of(s)))
}
}
pub struct SensorDb<F, const PAGE_SIZE: usize, const MAX_SECTORS: usize> {
flash: F,
state: SensorDbState<PAGE_SIZE, MAX_SECTORS>,
}
impl<F, const PAGE_SIZE: usize, const MAX_SECTORS: usize> core::fmt::Debug
for SensorDb<F, PAGE_SIZE, MAX_SECTORS>
{
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("SensorDb")
.field("base_offset", &self.state.base_offset)
.field("sector_count", &self.state.sector_count)
.field("sector_size", &self.state.sector_size)
.field("mounted", &self.state.mounted)
.field("active_sector", &self.state.active_sector)
.field("total_entries", &self.state.total_entries)
.finish_non_exhaustive()
}
}
impl<F, const PAGE_SIZE: usize, const MAX_SECTORS: usize> SensorDb<F, PAGE_SIZE, MAX_SECTORS>
where
F: NorFlash + ReadNorFlash,
{
pub fn new(
flash: F,
base_offset: u32,
sector_count: u16,
reserved_sectors: u16,
) -> Result<Self, Error> {
Self::with_config(
flash,
DbConfig::new(base_offset, sector_count, reserved_sectors)?,
)
}
pub fn with_config(flash: F, config: DbConfig) -> Result<Self, Error> {
let () = Geometry::<F, PAGE_SIZE>::PAGE_IS_WRITABLE;
Ok(Self {
flash,
state: SensorDbState::new(config, F::ERASE_SIZE as u32)?,
})
}
pub fn format(&mut self) -> Result<(), Error> {
let flash = &mut self.flash;
for i in 0..self.state.sector_count {
self.state.erase_sector(flash, i)?;
self.state.sectors[i as usize] = SectorSummary::erased();
self.state.sectors[i as usize].erase_count = 1;
}
self.state.epoch = 1;
self.state.epoch_pending = false;
self.state.init_sector(flash, 0, 1, 1)?;
self.state.sectors[0].state = SectorState::Active;
self.state.sectors[0].sequence = 1;
self.state.sectors[0].write_offset = SECTOR_HEADER_SIZE as u32;
self.state.sectors[0].erase_count = 1;
self.state.active_sector = Some(0);
self.state.next_sequence = 2;
self.state.total_entries = 0;
self.state.total_samples = 0;
self.state.mounted = true;
self.state.mount_mode = MountMode::Full;
self.state.spool_watermark = LogPosition::BEGINNING;
self.state.spooled_entries = 0;
self.state.spooled_samples = 0;
self.state.dropped_unspooled = 0;
self.state.page_flash_offset = self.state.base_offset;
self.state.page_buf_offset = SECTOR_HEADER_SIZE;
self.state.page_buf_durable = SECTOR_HEADER_SIZE;
self.state.page_buf.fill(0xFF);
let header = self.state.sector_header(1, 1);
self.state.page_buf[..SECTOR_HEADER_SIZE].copy_from_slice(header.as_bytes());
Ok(())
}
pub fn mount(&mut self) -> Result<MountStats, Error> {
self.mount_with(MountMode::Full)
}
pub fn mount_with(&mut self, mode: MountMode) -> Result<MountStats, Error> {
if self.state.mounted {
return Err(Error::AlreadyMounted);
}
let flash = &mut self.flash;
let mut stats = MountStats::default();
let mut max_sequence: u32 = 0;
let mut max_epoch: u32 = 0;
let mut valid: HeaplessVec<u16, MAX_SECTORS> = HeaplessVec::new();
for i in 0..self.state.sector_count {
let sector_offset = self.state.sector_offset(i);
let mut header_buf = [0u8; SECTOR_HEADER_SIZE];
flash.read(sector_offset, &mut header_buf)?;
let header =
SectorHeader::read_from_bytes(&header_buf).map_err(|_| Error::CorruptEntry)?;
match header.classify(self.state.log_id) {
SectorClass::Valid => {
let seq = header.sequence.get();
let epoch = header.epoch.get();
self.state.sectors[i as usize].sequence = seq;
self.state.sectors[i as usize].erase_count = header.erase_count.get();
max_sequence = max_sequence.max(seq);
max_epoch = max_epoch.max(epoch);
let _ = valid.push(i);
}
SectorClass::Free => {
self.state.sectors[i as usize] = SectorSummary::erased();
}
SectorClass::Foreign { found } => {
return Err(Error::ForeignVolume {
expected: self.state.log_id_raw(),
found,
});
}
SectorClass::Unsupported { found } => {
return Err(Error::UnsupportedVersion { found });
}
}
}
let newest = valid
.iter()
.copied()
.max_by_key(|i| self.state.sectors[*i as usize].sequence);
let mut newest_sealed = false;
for i in valid.iter().copied() {
let is_newest = Some(i) == newest;
let footer = if self.state.wants_footer(mode, is_newest) {
self.state.read_sector_footer(flash, i)?
} else {
None
};
if is_newest {
newest_sealed = footer.is_some();
}
if let Some(footer) = footer.filter(|_| mode != MountMode::Verify) {
self.state.adopt_footer(i, &footer, &mut stats);
} else if mode != MountMode::HeadersOnly || is_newest {
let (scanned, corrupt) = self.state.scan_sector_entries(flash, i)?;
self.state.adopt_scan(i, &scanned, corrupt, &mut stats);
} else {
self.state.adopt_unindexed(i);
}
self.state.sectors[i as usize].state = SectorState::Sealed;
stats.sealed_sectors += 1;
}
self.state
.finish_mount(mode, max_sequence, max_epoch, &stats);
if let Some(idx) = newest {
self.state.sectors[idx as usize].state = SectorState::Active;
self.state.active_sector = Some(idx);
stats.sealed_sectors = stats.sealed_sectors.saturating_sub(1);
self.state.epoch_pending = true;
let sector_offset = self.state.sector_offset(idx);
let resume = if newest_sealed {
self.state.footer_offset()
} else {
self.state.sectors[idx as usize].write_offset
};
let page_start = (resume / PAGE_SIZE as u32) * PAGE_SIZE as u32;
self.state.page_flash_offset = sector_offset + page_start;
self.state.page_buf_offset = (resume % PAGE_SIZE as u32) as usize;
self.state.page_buf_durable = self.state.page_buf_offset;
self.state.page_buf.fill(0xFF);
let mut last_dirty: Option<u32> = None;
let mut probe = page_start;
while probe < self.state.footer_offset() {
flash.read(sector_offset + probe, &mut self.state.page_buf)?;
let from = if probe == page_start {
self.state.page_buf_offset
} else {
0
};
if self.state.page_buf[from..].iter().any(|b| *b != 0xFF) {
last_dirty = Some(probe);
}
probe += PAGE_SIZE as u32;
}
match last_dirty {
None if resume < self.state.footer_offset() => {
flash.read(self.state.page_flash_offset, &mut self.state.page_buf)?;
}
None => {}
Some(dirty) => {
self.state.page_flash_offset = sector_offset + dirty + PAGE_SIZE as u32;
self.state.page_buf_offset = 0;
self.state.page_buf_durable = 0;
self.state.page_buf.fill(0xFF);
self.state.sectors[idx as usize].write_offset = dirty + PAGE_SIZE as u32;
}
}
} else {
self.state.erase_sector(flash, 0)?;
self.state
.init_sector(flash, 0, self.state.next_sequence, 1)?;
self.state.epoch_pending = false;
self.state.sectors[0].state = SectorState::Active;
self.state.sectors[0].sequence = self.state.next_sequence;
self.state.sectors[0].write_offset = SECTOR_HEADER_SIZE as u32;
self.state.sectors[0].erase_count = 1;
self.state.active_sector = Some(0);
self.state.next_sequence += 1;
self.state.page_flash_offset = self.state.base_offset;
self.state.page_buf_offset = SECTOR_HEADER_SIZE;
self.state.page_buf_durable = SECTOR_HEADER_SIZE;
self.state.page_buf.fill(0xFF);
let header = self.state.sector_header(self.state.sectors[0].sequence, 1);
self.state.page_buf[..SECTOR_HEADER_SIZE].copy_from_slice(header.as_bytes());
}
self.state.mounted = true;
Ok(stats)
}
pub fn append(&mut self, sensor_id: u16, timestamp: u64, payload: &[u8]) -> Result<(), Error> {
self.append_entry(
EntryKind::Single,
sensor_id,
timestamp,
timestamp,
1,
&[],
payload,
)
}
pub fn append_run(&mut self, run: &Run<'_>) -> Result<(), Error> {
let mut frame = [0u8; RUN_FRAME_SIZE];
run.write_frame(&mut frame);
self.append_entry(
EntryKind::Run,
run.sensor_id(),
run.base_timestamp(),
run.span_end(),
run.count(),
&frame,
run.samples(),
)
}
pub fn append_framed(&mut self, header: &EntryHeader, payload: &[u8]) -> Result<(), Error> {
match header.kind {
EntryKind::Single => self.append(header.sensor_id, header.timestamp, payload),
EntryKind::Run => {
let frame = RunFrame::parse(payload).ok_or(Error::CorruptEntry)?;
self.append_entry(
EntryKind::Run,
header.sensor_id,
header.timestamp,
frame.span_end(header.timestamp),
frame.count(),
&[],
payload,
)
}
}
}
#[allow(clippy::too_many_arguments)]
fn append_entry(
&mut self,
kind: EntryKind,
sensor_id: u16,
timestamp: u64,
span_end: u64,
samples: u32,
head: &[u8],
tail: &[u8],
) -> Result<(), Error> {
if !self.state.mounted {
return Err(Error::NotMounted);
}
let payload_len = head.len() + tail.len();
let entry_size = ENTRY_OVERHEAD + payload_len;
if payload_len > self.max_payload_size() {
return Err(Error::PayloadTooLarge);
}
let flash = &mut self.flash;
if self.state.epoch_pending {
match self.state.seal_and_advance(flash) {
Ok(()) => {}
Err(Error::NoSpace) => self.state.epoch_pending = false,
Err(e) => return Err(e),
}
}
let mut sealed_once = false;
loop {
match self.state.plan_append(entry_size)? {
AppendStep::Stage => break,
AppendStep::FlushPage => self.state.flush_page(flash)?,
AppendStep::Seal => {
if sealed_once {
return Err(Error::PayloadTooLarge);
}
sealed_once = true;
self.state.seal_and_advance(flash)?;
}
}
}
self.state
.stage_entry(kind, sensor_id, timestamp, span_end, samples, head, tail)
}
pub fn sync(&mut self) -> Result<(), Error> {
if !self.state.mounted {
return Err(Error::NotMounted);
}
self.state.flush_page(&mut self.flash)?;
Ok(())
}
pub fn latest(
&mut self,
sensor_id: u16,
buf: &mut [u8],
) -> Result<Option<(EntryHeader, usize)>, Error> {
self.state.require_reads()?;
let flash = &mut self.flash;
let mut sector_order: HeaplessVec<(u16, u32), MAX_SECTORS> = HeaplessVec::new();
for i in 0..self.state.sector_count {
let summary = &self.state.sectors[i as usize];
if summary.state != SectorState::Erased && summary.might_contain_sensor(sensor_id) {
let _ = sector_order.push((i, summary.sequence));
}
}
sort_by_seq(&mut sector_order, |a, b| b.cmp(a));
let mut best: Option<(EntryHeader, usize)> = None;
for (sector_idx, _) in sector_order {
if let Some(result) =
self.state
.find_latest_in_sector(&mut *flash, sector_idx, sensor_id, buf)?
{
best = Some(result);
break;
}
}
let min_ts = best.as_ref().map_or(0, |(h, _)| h.timestamp);
if let Some((hdr, len)) = self
.state
.scan_page_buf_for_latest(sensor_id, min_ts, buf)?
{
best = Some((hdr, len));
}
Ok(best)
}
pub fn query_range<C>(
&mut self,
sensor_id: Option<u16>,
t_start: u64,
t_end: u64,
mut callback: C,
) -> Result<u32, Error>
where
C: FnMut(&EntryHeader, &[u8]) -> bool,
{
self.state.require_reads()?;
let flash = &mut self.flash;
let mut count = 0u32;
let mut sector_order: HeaplessVec<(u16, u32), MAX_SECTORS> = HeaplessVec::new();
for i in 0..self.state.sector_count {
let summary = &self.state.sectors[i as usize];
if summary.state != SectorState::Erased
&& summary.entry_count > 0
&& summary.overlaps_time_range(t_start, t_end)
&& sensor_id.is_none_or(|id| summary.might_contain_sensor(id))
{
let _ = sector_order.push((i, summary.sequence));
}
}
sort_by_seq(&mut sector_order, u32::cmp);
let mut entry_buf = [0u8; PAGE_SIZE];
let mut stopped = false;
for (sector_idx, _) in sector_order {
stopped = self.state.scan_sector_for_query(
flash,
sector_idx,
sensor_id,
t_start,
t_end,
&mut entry_buf,
&mut count,
&mut callback,
)?;
if stopped {
break;
}
}
if !stopped {
self.state.scan_page_buf_for_query(
sensor_id,
t_start,
t_end,
&mut entry_buf,
&mut count,
&mut callback,
);
}
Ok(count)
}
pub fn query_all<C>(&mut self, sensor_id: Option<u16>, callback: C) -> Result<u32, Error>
where
C: FnMut(&EntryHeader, &[u8]) -> bool,
{
self.query_range(sensor_id, 0, u64::MAX, callback)
}
pub fn oldest(
&mut self,
sensor_id: u16,
buf: &mut [u8],
) -> Result<Option<(EntryHeader, usize)>, Error> {
self.state.require_reads()?;
let flash = &mut self.flash;
let mut sector_order: HeaplessVec<(u16, u32), MAX_SECTORS> = HeaplessVec::new();
for i in 0..self.state.sector_count {
let summary = &self.state.sectors[i as usize];
if summary.state != SectorState::Erased && summary.might_contain_sensor(sensor_id) {
let _ = sector_order.push((i, summary.sequence));
}
}
sort_by_seq(&mut sector_order, u32::cmp);
for (sector_idx, _) in sector_order {
if let Some((header, payload_len)) =
self.state
.find_oldest_in_sector(&mut *flash, sector_idx, sensor_id, buf)?
{
return Ok(Some((header, payload_len)));
}
}
if let Some(result) = self.state.scan_page_buf_for_oldest(sensor_id, buf) {
return Ok(Some(result));
}
Ok(None)
}
pub fn count(&mut self, sensor_id: Option<u16>) -> Result<u32, Error> {
self.state.require_reads()?;
let Some(target) = sensor_id else {
return Ok(self.state.total_samples);
};
let flash = &mut self.flash;
let mut count = 0u32;
for i in 0..self.state.sector_count {
let summary = &self.state.sectors[i as usize];
if summary.state != SectorState::Erased && summary.might_contain_sensor(target) {
count += self.state.count_sensor_in_sector(&mut *flash, i, target)?;
}
}
count += self.state.count_sensor_in_page_buf(target);
Ok(count)
}
pub fn compact(&mut self, count: u16, scope: CompactScope) -> Result<u16, Error> {
if !self.state.mounted {
return Err(Error::NotMounted);
}
let mut sealed: HeaplessVec<(u16, u32), MAX_SECTORS> = HeaplessVec::new();
for i in 0..self.state.sector_count {
let summary = &self.state.sectors[i as usize];
if summary.state != SectorState::Sealed {
continue;
}
if scope == CompactScope::SpooledOnly && !self.state.is_fully_spooled(summary) {
continue;
}
let _ = sealed.push((i, summary.sequence));
}
sort_by_seq(&mut sealed, u32::cmp);
let flash = &mut self.flash;
let mut erased = 0u16;
for (sector_idx, _) in sealed.iter().take(count as usize) {
let victim = self.state.sectors[*sector_idx as usize];
self.state.erase_sector(flash, *sector_idx)?;
self.state.account_eviction(&victim);
self.state.sectors[*sector_idx as usize] = SectorSummary::erased();
self.state.sectors[*sector_idx as usize].erase_count = victim.erase_count;
erased += 1;
}
Ok(erased)
}
pub fn stats(&self) -> DbStats {
self.state.stats()
}
pub fn sectors_erased(&self) -> u32 {
self.state.sectors_erased
}
pub fn is_mounted(&self) -> bool {
self.state.mounted
}
pub fn max_payload_size(&self) -> usize {
PAGE_SIZE - ENTRY_OVERHEAD
}
pub fn append_value<T: IntoBytes + Immutable>(
&mut self,
sensor_id: u16,
timestamp: u64,
data: &T,
) -> Result<(), Error> {
self.append(sensor_id, timestamp, data.as_bytes())
}
pub fn latest_value<T: FromBytes + KnownLayout>(
&mut self,
sensor_id: u16,
) -> Result<Option<(EntryHeader, T)>, Error> {
let expected_size = core::mem::size_of::<T>();
let mut buf = [0u8; PAGE_SIZE];
match self.latest(sensor_id, &mut buf)? {
Some((header, payload_len)) => {
if payload_len != expected_size {
return Err(Error::SizeMismatch {
expected: expected_size,
actual: payload_len,
});
}
let value =
T::read_from_bytes(&buf[..payload_len]).map_err(|_| Error::InvalidData)?;
Ok(Some((header, value)))
}
None => Ok(None),
}
}
pub fn append_sensor<T: SensorData>(&mut self, timestamp: u64, data: &T) -> Result<(), Error> {
let framed = schema::tag(data);
self.append(T::SENSOR_ID, timestamp, framed.bytes())
}
pub fn latest_sensor<T: SensorData>(&mut self) -> Result<Option<(EntryHeader, T)>, Error> {
schema::read_tagged(|buf| self.latest(T::SENSOR_ID, buf))
}
pub fn query_sensor<T: SensorData, C>(
&mut self,
t_start: u64,
t_end: u64,
mut callback: C,
) -> Result<u32, Error>
where
C: FnMut(&EntryHeader, &T) -> bool,
{
let mut count = 0u32;
self.query_range(Some(T::SENSOR_ID), t_start, t_end, |header, payload| {
let Some(value) = schema::untag_payload::<T>(payload) else {
return true;
};
count += 1;
callback(header, &value)
})?;
Ok(count)
}
pub fn query_all_sensor<T: SensorData, C>(&mut self, callback: C) -> Result<u32, Error>
where
C: FnMut(&EntryHeader, &T) -> bool,
{
self.query_sensor(0, u64::MAX, callback)
}
pub fn oldest_sensor<T: SensorData>(&mut self) -> Result<Option<(EntryHeader, T)>, Error> {
schema::read_tagged(|buf| self.oldest(T::SENSOR_ID, buf))
}
pub fn count_sensor<T: SensorData>(&mut self) -> Result<u32, Error> {
self.count(Some(T::SENSOR_ID))
}
pub fn read_from<C>(
&mut self,
from: LogPosition,
max_entries: u32,
mut callback: C,
) -> Result<ReadCursor, Error>
where
C: FnMut(LogPosition, &EntryHeader, &[u8]) -> Result<bool, Error>,
{
if !self.state.mounted {
return Err(Error::NotMounted);
}
let mut order: HeaplessVec<(u16, u32), MAX_SECTORS> = HeaplessVec::new();
let mut oldest_seq = u32::MAX;
for i in 0..self.state.sector_count {
let summary = &self.state.sectors[i as usize];
if summary.state == SectorState::Erased {
continue;
}
if summary.sequence < oldest_seq {
oldest_seq = summary.sequence;
}
if summary.sequence >= from.sequence {
let _ = order.push((i, summary.sequence));
}
}
sort_by_seq(&mut order, u32::cmp);
let gap =
from != LogPosition::BEGINNING && oldest_seq != u32::MAX && from.sequence < oldest_seq;
let mut cursor = ReadCursor {
next: from,
entries: 0,
exhausted: false,
gap,
};
if max_entries == 0 {
return Ok(cursor);
}
let flash = &mut self.flash;
let mut header_buf = [0u8; ENTRY_HEADER_SIZE];
let mut buf = [0u8; PAGE_SIZE];
let mut stopped = false;
for (sector_idx, seq) in order {
let sector_offset = self.state.sector_offset(sector_idx);
let limit = self.state.sectors[sector_idx as usize].write_offset;
let start = if seq == from.sequence {
from.offset.max(SECTOR_HEADER_SIZE as u32)
} else {
SECTOR_HEADER_SIZE as u32
};
if start >= limit {
continue;
}
let mut scanner = SectorScanner::new(sector_offset, start, limit);
while let Some(entry) = scanner.next_entry(flash, &mut header_buf, PAGE_SIZE)? {
if !accept_entry(flash, sector_offset, &entry, &mut buf)? {
scanner.skip(1);
continue;
}
let position = LogPosition {
sequence: seq,
offset: entry.offset_in_sector,
};
let header = entry.header.to_header();
if !callback(position, &header, &buf[..entry.payload_len])? {
stopped = true;
break;
}
scanner.skip(entry.entry_size);
cursor.entries += 1;
cursor.next = LogPosition {
sequence: seq,
offset: scanner.offset(),
};
if cursor.entries >= max_entries {
stopped = true;
break;
}
}
if stopped {
break;
}
}
cursor.exhausted = !stopped;
Ok(cursor)
}
pub fn drain_unflushed<C>(&mut self, callback: C) -> Result<u32, Error>
where
C: FnMut(&EntryHeader, &[u8]) -> Result<(), Error>,
{
self.state.drain_unflushed(callback)
}
pub fn peek_unflushed<C>(&self, callback: C) -> Result<u32, Error>
where
C: FnMut(&EntryHeader, &[u8]) -> Result<(), Error>,
{
self.state.peek_unflushed(callback)
}
pub fn sector_epoch_at(&mut self, position: LogPosition) -> Result<Option<u32>, Error> {
if !self.state.mounted {
return Err(Error::NotMounted);
}
let Some(idx) = self.state.sector_at_sequence(position.sequence) else {
return Ok(None);
};
let offset = self.state.sector_offset(idx);
let mut buf = [0u8; SECTOR_HEADER_SIZE];
self.flash.read(offset, &mut buf)?;
let header = SectorHeader::read_from_bytes(&buf).map_err(|_| Error::CorruptEntry)?;
Ok(header.is_valid().then(|| header.epoch.get()))
}
pub fn head_position(&self) -> LogPosition {
self.state.head_position()
}
pub fn tail_position(&self) -> LogPosition {
self.state.tail_position()
}
pub fn with_region<R>(
&mut self,
base: u32,
len: u32,
f: impl FnOnce(&mut Partition<&mut F>) -> R,
) -> Result<R, Error> {
self.state.check_disjoint(base, len)?;
let mut partition = Partition::new(&mut self.flash, base, len)?;
Ok(f(&mut partition))
}
pub fn driver_mut(&mut self) -> &mut F {
&mut self.flash
}
pub fn note_spooled(&mut self, upto: LogPosition, moved: Replicated) {
self.state.note_spooled(upto, moved);
}
pub fn restore_spool_watermark(&mut self, upto: LogPosition) -> Result<(), Error> {
let Some(boundary) = self.state.restore_spool_watermark(upto) else {
return Ok(());
};
let counted = self
.state
.count_entries_before(&mut self.flash, boundary, upto.offset)?;
self.state.add_spooled(counted);
Ok(())
}
pub fn spool_stats(&self) -> SpoolStats {
self.state.spool_stats()
}
pub fn log_id(&self) -> Option<LogId> {
self.state.log_id
}
pub fn epoch(&self) -> u32 {
self.state.epoch
}
pub fn mount_mode(&self) -> MountMode {
self.state.mount_mode
}
pub fn mark_as_replica(&mut self) {
self.state.epoch_pending = false;
}
pub fn into_inner(self) -> F {
self.flash
}
}
impl<const PAGE_SIZE: usize, const MAX_SECTORS: usize> SensorDbState<PAGE_SIZE, MAX_SECTORS> {
pub(crate) fn sector_offset(&self, sector_idx: u16) -> u32 {
self.base_offset + sector_idx as u32 * self.sector_size
}
pub(crate) fn footer_offset(&self) -> u32 {
self.sector_size - PAGE_SIZE as u32
}
pub(crate) fn region_end(&self) -> u32 {
self.base_offset + self.sector_count as u32 * self.sector_size
}
pub(crate) fn check_disjoint(&self, base: u32, len: u32) -> Result<(), Error> {
let end = (base as u64)
.checked_add(len as u64)
.ok_or(Error::Overflow)?;
let (db_base, db_end) = (self.base_offset, self.region_end());
if (base as u64) < db_end as u64 && (db_base as u64) < end {
return Err(Error::RegionOverlap { db_base, db_end });
}
Ok(())
}
pub(crate) fn find_next_sector(&mut self) -> Result<u16, Error> {
let start = self
.active_sector
.map_or(0, |a| (a + 1) % self.sector_count);
let mut best_idx: Option<u16> = None;
let mut best_erase = u32::MAX;
for offset in 0..self.sector_count {
let i = (start + offset) % self.sector_count;
let summary = &self.sectors[i as usize];
if summary.state == SectorState::Erased && summary.erase_count < best_erase {
best_erase = summary.erase_count;
best_idx = Some(i);
}
}
if let Some(idx) = best_idx {
return Ok(idx);
}
let sealed_count: u16 = self.sectors[..self.sector_count as usize]
.iter()
.filter(|s| s.state == SectorState::Sealed)
.count() as u16;
if sealed_count <= self.reserved_sectors {
return Err(Error::NoSpace);
}
self.oldest_sealed(true)
.or_else(|| self.oldest_sealed(false))
.ok_or(Error::NoSpace)
}
pub(crate) fn head_position(&self) -> LogPosition {
match self.active_sector {
Some(idx) => LogPosition {
sequence: self.sectors[idx as usize].sequence,
offset: self.sectors[idx as usize].write_offset,
},
None => LogPosition::BEGINNING,
}
}
pub(crate) fn tail_position(&self) -> LogPosition {
let mut oldest: Option<u32> = None;
for i in 0..self.sector_count {
let summary = &self.sectors[i as usize];
if summary.state != SectorState::Erased
&& oldest.is_none_or(|seq| summary.sequence < seq)
{
oldest = Some(summary.sequence);
}
}
match oldest {
Some(sequence) => LogPosition {
sequence,
offset: SECTOR_HEADER_SIZE as u32,
},
None => LogPosition::BEGINNING,
}
}
pub(crate) fn note_spooled(&mut self, upto: LogPosition, moved: Replicated) {
if upto <= self.spool_watermark {
return;
}
self.spool_watermark = upto;
self.add_spooled(moved);
}
pub(crate) fn sector_at_sequence(&self, sequence: u32) -> Option<u16> {
(0..self.sector_count).find(|i| {
let summary = &self.sectors[*i as usize];
summary.state != SectorState::Erased && summary.sequence == sequence
})
}
pub(crate) fn restore_spool_watermark(&mut self, upto: LogPosition) -> Option<u16> {
if upto <= self.spool_watermark {
return None;
}
self.spool_watermark = upto;
self.spooled_entries = self.spooled_in_whole_sectors().min(self.total_entries);
self.spooled_samples = self
.spooled_samples_in_whole_sectors()
.min(self.total_samples);
self.sector_at_sequence(upto.sequence).filter(|i| {
upto.offset > SECTOR_HEADER_SIZE as u32 && self.sectors[*i as usize].entry_count > 0
})
}
pub(crate) fn add_spooled(&mut self, moved: Replicated) {
self.spooled_entries = self
.spooled_entries
.saturating_add(moved.entries)
.min(self.total_entries);
self.spooled_samples = self
.spooled_samples
.saturating_add(moved.samples)
.min(self.total_samples);
}
pub(crate) fn spool_stats(&self) -> SpoolStats {
SpoolStats {
unspooled_entries: self.total_entries.saturating_sub(self.spooled_entries),
unspooled_samples: self.total_samples.saturating_sub(self.spooled_samples),
watermark: self.spool_watermark,
head: self.head_position(),
dropped_unspooled: self.dropped_unspooled,
data_lost: self.dropped_unspooled > 0,
}
}
fn unflushed_start(&self) -> usize {
self.page_buf_scan_start().max(self.page_buf_durable)
}
fn walk_unflushed<C>(&self, mut callback: C) -> Result<Replicated, Error>
where
C: FnMut(&EntryHeader, &[u8]) -> Result<(), Error>,
{
let end = self.page_buf_offset;
let mut seen = Replicated::NONE;
let mut pos = self.unflushed_start();
while let Some(entry) = self.next_buffered(&mut pos, end) {
let payload = self.buffered_payload(&entry);
callback(&entry.header.to_header(), payload)?;
seen.push(samples_in(entry.is_run(), payload));
}
Ok(seen)
}
pub(crate) fn peek_unflushed<C>(&self, callback: C) -> Result<u32, Error>
where
C: FnMut(&EntryHeader, &[u8]) -> Result<(), Error>,
{
if !self.mounted {
return Err(Error::NotMounted);
}
Ok(self.walk_unflushed(callback)?.entries)
}
pub(crate) fn drain_unflushed<C>(&mut self, callback: C) -> Result<u32, Error>
where
C: FnMut(&EntryHeader, &[u8]) -> Result<(), Error>,
{
if !self.mounted {
return Err(Error::NotMounted);
}
let scan_start = self.unflushed_start();
let end = self.page_buf_offset;
if scan_start >= end {
return Ok(0);
}
let drained = self.walk_unflushed(callback)?;
if drained.entries == 0 {
return Ok(0);
}
let truncate_to = scan_start;
let bytes_dropped = (end - truncate_to) as u32;
self.page_buf[truncate_to..].fill(0xFF);
self.page_buf_offset = truncate_to;
if let Some(active_idx) = self.active_sector {
let summary = &mut self.sectors[active_idx as usize];
summary.entry_count = summary.entry_count.saturating_sub(drained.entries);
summary.sample_count = summary.sample_count.saturating_sub(drained.samples);
summary.write_offset = summary.write_offset.saturating_sub(bytes_dropped);
}
self.total_entries = self.total_entries.saturating_sub(drained.entries);
self.total_samples = self.total_samples.saturating_sub(drained.samples);
Ok(drained.entries)
}
fn oldest_sealed(&self, require_spooled: bool) -> Option<u16> {
let mut oldest_idx = None;
let mut oldest_seq = u32::MAX;
for i in 0..self.sector_count {
let summary = &self.sectors[i as usize];
if summary.state != SectorState::Sealed || summary.sequence >= oldest_seq {
continue;
}
if require_spooled && !self.is_fully_spooled(summary) {
continue;
}
oldest_seq = summary.sequence;
oldest_idx = Some(i);
}
oldest_idx
}
pub(crate) fn page_buf_scan_start(&self) -> usize {
let active_idx = match self.active_sector {
Some(idx) => idx,
None => return self.page_buf_offset, };
if self.page_flash_offset == self.sector_offset(active_idx) {
SECTOR_HEADER_SIZE
} else {
0
}
}
fn next_buffered(&self, pos: &mut usize, end: usize) -> Option<BufferedEntry> {
if *pos + ENTRY_HEADER_SIZE > end || !is_entry_marker(self.page_buf[*pos]) {
return None;
}
let header =
EntryHeaderWire::read_from_bytes(&self.page_buf[*pos..*pos + ENTRY_HEADER_SIZE])
.ok()?;
let payload_len = header.payload_len.get() as usize;
if *pos + ENTRY_OVERHEAD + payload_len > end {
return None;
}
let payload_start = *pos + ENTRY_HEADER_SIZE;
*pos += ENTRY_OVERHEAD + payload_len;
Some(BufferedEntry {
header,
payload_start,
payload_len,
})
}
fn buffered_payload(&self, entry: &BufferedEntry) -> &[u8] {
&self.page_buf[entry.payload_start..entry.payload_start + entry.payload_len]
}
fn buffered_sample(&self, entry: &BufferedEntry, edge: Edge) -> (EntryHeader, BufferedSample) {
let located = |run| BufferedSample {
payload_start: entry.payload_start,
payload_len: entry.payload_len,
run,
};
match RunFrame::parse(self.buffered_payload(entry)).filter(|_| entry.is_run()) {
Some(frame) => {
let index = match edge {
Edge::First => 0,
Edge::Last => frame.count() - 1,
};
(
frame.header_at(entry.sensor_id(), entry.timestamp(), index),
located(Some((frame, index))),
)
}
None => (entry.header.to_header(), located(None)),
}
}
fn copy_buffered_sample(
&self,
sample: &BufferedSample,
out: &mut [u8],
) -> Result<usize, Error> {
let payload =
&self.page_buf[sample.payload_start..sample.payload_start + sample.payload_len];
match sample.run {
Some((frame, index)) => frame
.write_tagged_sample(payload, index, out)
.ok_or(Error::BufferTooSmall),
None => {
if payload.len() > out.len() {
return Err(Error::BufferTooSmall);
}
out[..payload.len()].copy_from_slice(payload);
Ok(payload.len())
}
}
}
pub(crate) fn scan_page_buf_for_latest(
&self,
sensor_id: u16,
min_ts: u64,
buf: &mut [u8],
) -> Result<Option<(EntryHeader, usize)>, Error> {
let scan_start = self.page_buf_scan_start();
if scan_start >= self.page_buf_offset {
return Ok(None);
}
let mut best: Option<(EntryHeader, BufferedSample)> = None;
let mut best_ts: u64 = min_ts;
let mut pos = scan_start;
while let Some(entry) = self.next_buffered(&mut pos, self.page_buf_offset) {
if entry.is_committed() || entry.sensor_id() != sensor_id {
continue;
}
let (header, sample) = self.buffered_sample(&entry, Edge::Last);
if header.timestamp >= best_ts {
best_ts = header.timestamp;
best = Some((header, sample));
}
}
let Some((header, sample)) = best else {
return Ok(None);
};
let len = self.copy_buffered_sample(&sample, buf)?;
Ok(Some((header, len)))
}
pub(crate) fn scan_page_buf_for_query<C>(
&self,
sensor_filter: Option<u16>,
t_start: u64,
t_end: u64,
scratch: &mut [u8],
count: &mut u32,
callback: &mut C,
) -> bool
where
C: FnMut(&EntryHeader, &[u8]) -> bool,
{
let mut pos = self.page_buf_scan_start();
while let Some(entry) = self.next_buffered(&mut pos, self.page_buf_offset) {
let sensor = entry.sensor_id();
if entry.is_committed() || sensor_filter.is_some_and(|f| f != sensor) {
continue;
}
let payload = self.buffered_payload(&entry);
let Some(dst) = scratch.get_mut(..payload.len()) else {
continue;
};
dst.copy_from_slice(payload);
if deliver_samples(
entry.is_run(),
sensor,
entry.timestamp(),
dst,
t_start,
t_end,
count,
callback,
) {
return true; }
}
false
}
pub(crate) fn scan_page_buf_for_oldest(
&self,
sensor_id: u16,
buf: &mut [u8],
) -> Option<(EntryHeader, usize)> {
let mut pos = self.page_buf_scan_start();
while let Some(entry) = self.next_buffered(&mut pos, self.page_buf_offset) {
if entry.is_committed() || entry.sensor_id() != sensor_id {
continue;
}
let (header, sample) = self.buffered_sample(&entry, Edge::First);
if let Ok(len) = self.copy_buffered_sample(&sample, buf) {
return Some((header, len));
}
}
None
}
pub(crate) fn count_sensor_in_page_buf(&self, sensor_id: u16) -> u32 {
let mut count = 0u32;
let mut pos = self.page_buf_scan_start();
while let Some(entry) = self.next_buffered(&mut pos, self.page_buf_offset) {
if entry.is_committed() || entry.sensor_id() != sensor_id {
continue;
}
count += samples_in(entry.is_run(), self.buffered_payload(&entry));
}
count
}
fn erase_sector<F: NorFlash>(&mut self, flash: &mut F, sector_idx: u16) -> Result<(), Error> {
let offset = self.sector_offset(sector_idx);
flash.erase(offset, offset + self.sector_size)?;
self.sectors_erased = self.sectors_erased.saturating_add(1);
Ok(())
}
fn init_sector<F: NorFlash>(
&self,
flash: &mut F,
sector_idx: u16,
sequence: u32,
erase_count: u32,
) -> Result<(), Error> {
let header = self.sector_header(sequence, erase_count);
let offset = self.sector_offset(sector_idx);
let mut page = [0xFFu8; PAGE_SIZE];
page[..SECTOR_HEADER_SIZE].copy_from_slice(header.as_bytes());
flash.write(offset, &page)?;
Ok(())
}
fn read_sector_footer<F: ReadNorFlash>(
&self,
flash: &mut F,
sector_idx: u16,
) -> Result<Option<SectorFooter>, Error> {
let offset = self.sector_offset(sector_idx) + self.footer_offset();
let mut buf = [0u8; SECTOR_FOOTER_SIZE];
flash.read(offset, &mut buf)?;
let footer = SectorFooter::read_from_bytes(&buf).map_err(|_| Error::CorruptEntry)?;
let sequence = self.sectors[sector_idx as usize].sequence;
Ok(footer.describes(sequence).then_some(footer))
}
fn write_sector_footer<F: NorFlash>(
&self,
flash: &mut F,
sector_idx: u16,
) -> Result<(), Error> {
debug_assert_eq!(
self.page_buf_offset, self.page_buf_durable,
"a sector is sealed only with a page buffer that flash already has"
);
let footer = SectorFooter::new(&self.sectors[sector_idx as usize]);
let mut page = [0xFFu8; PAGE_SIZE];
page[..SECTOR_FOOTER_SIZE].copy_from_slice(footer.as_bytes());
flash.write(self.sector_offset(sector_idx) + self.footer_offset(), &page)?;
Ok(())
}
pub(crate) fn plan_append(&self, entry_size: usize) -> Result<AppendStep, Error> {
let active = self.active_sector.ok_or(Error::NoSpace)?;
let used = self.page_flash_offset - self.sector_offset(active);
let sector_remaining = self.footer_offset().saturating_sub(used) as usize;
if sector_remaining < entry_size || sector_remaining < ENTRY_OVERHEAD {
return Ok(AppendStep::Seal);
}
if entry_size > PAGE_SIZE - self.page_buf_offset {
return Ok(AppendStep::FlushPage);
}
Ok(AppendStep::Stage)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn stage_entry(
&mut self,
kind: EntryKind,
sensor_id: u16,
timestamp: u64,
span_end: u64,
samples: u32,
head: &[u8],
tail: &[u8],
) -> Result<(), Error> {
let payload_len = head.len() + tail.len();
let entry_size = ENTRY_OVERHEAD + payload_len;
let header = match kind {
EntryKind::Single => EntryHeaderWire::new(sensor_id, timestamp, payload_len as u16),
EntryKind::Run => EntryHeaderWire::new_run(sensor_id, timestamp, payload_len as u16),
};
let active_idx = self.active_sector.ok_or(Error::NoSpace)?;
let sector_offset = self.sector_offset(active_idx);
let offset_in_sector =
(self.page_flash_offset - sector_offset) + self.page_buf_offset as u32;
let offset = self.page_buf_offset;
let body = offset + ENTRY_HEADER_SIZE;
self.page_buf[offset..body].copy_from_slice(header.as_bytes());
self.page_buf[body..body + head.len()].copy_from_slice(head);
self.page_buf[body + head.len()..body + payload_len].copy_from_slice(tail);
let crc = compute_entry_crc(
&header,
&self.page_buf[body..body + payload_len],
offset_in_sector,
);
self.page_buf[body + payload_len..offset + entry_size].copy_from_slice(&crc.to_le_bytes());
self.page_buf_offset += entry_size;
let summary = &mut self.sectors[active_idx as usize];
summary.entry_count += 1;
summary.sample_count += samples;
summary.write_offset += entry_size as u32;
summary.record_span(timestamp, span_end);
summary.record_sensor(sensor_id);
self.total_entries += 1;
self.total_samples += samples;
Ok(())
}
pub(crate) fn commit_target(&self) -> Option<u32> {
(self.page_buf_offset > 0).then_some(self.page_flash_offset)
}
pub(crate) fn mark_page_committed(&mut self) {
let scan_start =
if self.page_flash_offset == self.sector_offset(self.active_sector.unwrap_or(0)) {
SECTOR_HEADER_SIZE
} else {
0
};
let mut pos = scan_start;
while pos + ENTRY_HEADER_SIZE <= self.page_buf_offset {
if is_entry_marker(self.page_buf[pos]) {
self.page_buf[pos + 1] = FLAGS_COMMITTED;
let payload_len =
u16::from_le_bytes([self.page_buf[pos + 12], self.page_buf[pos + 13]]) as usize;
let entry_size = ENTRY_OVERHEAD + payload_len;
pos += entry_size;
} else {
break;
}
}
}
pub(crate) fn advance_page(&mut self) {
self.page_flash_offset += PAGE_SIZE as u32;
self.page_buf.fill(0xFF);
self.page_buf_offset = 0;
self.page_buf_durable = 0;
if let Some(active_idx) = self.active_sector {
let sector_offset = self.sector_offset(active_idx);
self.sectors[active_idx as usize].write_offset = self.page_flash_offset - sector_offset;
}
}
fn flush_page<F: NorFlash>(&mut self, flash: &mut F) -> Result<(), Error> {
let Some(addr) = self.commit_target() else {
return Ok(());
};
flash.write(addr, &self.page_buf)?;
self.mark_page_committed();
flash.write(addr, &self.page_buf)?;
self.advance_page();
Ok(())
}
pub(crate) fn stats(&self) -> DbStats {
let mut used = 0u16;
let mut free = 0u16;
let mut min_erase = u32::MAX;
let mut max_erase = 0u32;
let mut total_erase = 0u64;
for i in 0..self.sector_count as usize {
let ec = self.sectors[i].erase_count;
if ec < min_erase {
min_erase = ec;
}
if ec > max_erase {
max_erase = ec;
}
total_erase += ec as u64;
match self.sectors[i].state {
SectorState::Erased => free += 1,
SectorState::Active | SectorState::Sealed => used += 1,
}
}
if self.sector_count == 0 {
min_erase = 0;
}
DbStats {
total_entries: self.total_entries,
total_samples: self.total_samples,
used_sectors: used,
free_sectors: free,
reserved_sectors: self.reserved_sectors,
min_erase_count: min_erase,
max_erase_count: max_erase,
total_erase_count: total_erase,
}
}
pub(crate) fn wants_footer(&self, mode: MountMode, is_newest: bool) -> bool {
mode == MountMode::Full || is_newest
}
pub(crate) fn adopt_footer(&mut self, i: u16, footer: &SectorFooter, stats: &mut MountStats) {
footer.apply_to(&mut self.sectors[i as usize]);
stats.total_entries += footer.entry_count.get();
stats.total_samples += footer.sample_count.get();
}
pub(crate) fn adopt_scan(
&mut self,
i: u16,
scanned: &ScannedSector,
corrupt: u32,
stats: &mut MountStats,
) {
scanned.apply_to(&mut self.sectors[i as usize]);
stats.total_entries += scanned.entries;
stats.total_samples += scanned.samples;
stats.corrupt_entries += corrupt;
stats.scanned_sectors += 1;
}
pub(crate) fn adopt_unindexed(&mut self, i: u16) {
let data_limit = self.footer_offset();
let summary = &mut self.sectors[i as usize];
summary.entry_count = 0;
summary.sample_count = 0;
summary.write_offset = data_limit;
summary.min_timestamp = 0;
summary.max_timestamp = u64::MAX;
summary.sensor_bloom = u64::MAX;
}
pub(crate) fn finish_mount(
&mut self,
mode: MountMode,
max_sequence: u32,
max_epoch: u32,
stats: &MountStats,
) {
self.total_entries = stats.total_entries;
self.total_samples = stats.total_samples;
self.next_sequence = max_sequence + 1;
self.epoch = max_epoch.saturating_add(1);
self.mount_mode = mode;
self.spool_watermark = LogPosition::BEGINNING;
self.spooled_entries = 0;
self.spooled_samples = 0;
self.dropped_unspooled = 0;
}
pub(crate) fn sealing(&self) -> Result<u16, Error> {
self.active_sector.ok_or(Error::NoSpace)
}
pub(crate) fn begin_seal(&mut self, old_active: u16) -> Result<SealPlan, Error> {
self.sectors[old_active as usize].state = SectorState::Sealed;
let new_active = self.find_next_sector()?;
let victim = self.sectors[new_active as usize];
let erase_count = victim.erase_count.saturating_add(1);
Ok(SealPlan {
new_active,
erase_count,
victim,
})
}
pub(crate) fn account_seal(&mut self, plan: &SealPlan) {
if plan.victim.state == SectorState::Sealed {
self.account_eviction(&plan.victim);
}
}
pub(crate) fn finish_seal(&mut self, plan: &SealPlan) {
self.epoch_pending = false;
self.sectors[plan.new_active as usize] = SectorSummary {
state: SectorState::Active,
sequence: self.next_sequence,
entry_count: 0,
sample_count: 0,
write_offset: SECTOR_HEADER_SIZE as u32,
min_timestamp: u64::MAX,
max_timestamp: 0,
sensor_bloom: 0,
erase_count: plan.erase_count,
};
self.active_sector = Some(plan.new_active);
self.next_sequence += 1;
let sector_offset = self.sector_offset(plan.new_active);
self.page_flash_offset = sector_offset;
self.page_buf.fill(0xFF);
let header = self.sector_header(
self.sectors[plan.new_active as usize].sequence,
plan.erase_count,
);
self.page_buf[..SECTOR_HEADER_SIZE].copy_from_slice(header.as_bytes());
self.page_buf_offset = SECTOR_HEADER_SIZE;
self.page_buf_durable = SECTOR_HEADER_SIZE;
}
fn seal_and_advance<F: NorFlash>(&mut self, flash: &mut F) -> Result<(), Error> {
let old_active = self.sealing()?;
let _ = self.write_sector_footer(flash, old_active);
let plan = self.begin_seal(old_active)?;
self.erase_sector(flash, plan.new_active)?;
self.account_seal(&plan);
self.init_sector(flash, plan.new_active, self.next_sequence, plan.erase_count)?;
self.finish_seal(&plan);
Ok(())
}
fn scan_sector_entries<F: ReadNorFlash>(
&mut self,
flash: &mut F,
sector_idx: u16,
) -> Result<(ScannedSector, u32), Error> {
let sector_offset = self.sector_offset(sector_idx);
let entry_start = SECTOR_HEADER_SIZE as u32;
let mut scanner = SectorScanner::new(sector_offset, entry_start, self.footer_offset());
let mut header_buf = [0u8; ENTRY_HEADER_SIZE];
let mut buf = [0u8; PAGE_SIZE];
let mut found = ScannedSector {
write_offset: entry_start,
..ScannedSector::EMPTY
};
while let Some(entry) = scanner.next_entry(flash, &mut header_buf, PAGE_SIZE)? {
if !accept_entry(flash, sector_offset, &entry, &mut buf)? {
scanner.corrupt_count += 1;
scanner.skip(1);
continue;
}
found.record(&entry, &buf[..entry.payload_len]);
scanner.skip(entry.entry_size);
found.write_offset = scanner.offset();
}
Ok((found, scanner.corrupt_count()))
}
fn find_latest_in_sector<F: ReadNorFlash>(
&self,
flash: &mut F,
sector_idx: u16,
target_sensor: u16,
buf: &mut [u8],
) -> Result<Option<(EntryHeader, usize)>, Error> {
let sector_offset = self.sector_offset(sector_idx);
let write_offset = self.sectors[sector_idx as usize].write_offset;
let entry_start = SECTOR_HEADER_SIZE as u32;
let mut scanner = SectorScanner::new(sector_offset, entry_start, write_offset);
let mut header_buf = [0u8; ENTRY_HEADER_SIZE];
let mut payload_buf = [0u8; PAGE_SIZE];
let mut latest: Option<(EntryHeader, usize)> = None;
let mut latest_ts = 0u64;
while let Some(entry) = scanner.next_entry(flash, &mut header_buf, PAGE_SIZE)? {
if !accept_entry(flash, sector_offset, &entry, &mut payload_buf)? {
scanner.skip(1);
continue;
}
if entry.sensor_id() == target_sensor {
let (header, payload) =
entry.edge(&mut payload_buf[..entry.payload_len], Edge::Last);
if header.timestamp >= latest_ts {
let len = payload.len();
if len > buf.len() {
return Err(Error::BufferTooSmall);
}
buf[..len].copy_from_slice(payload);
latest_ts = header.timestamp;
latest = Some((header, len));
}
}
scanner.skip(entry.entry_size);
}
Ok(latest)
}
#[allow(clippy::too_many_arguments)]
fn scan_sector_for_query<F: ReadNorFlash, C>(
&self,
flash: &mut F,
sector_idx: u16,
sensor_filter: Option<u16>,
t_start: u64,
t_end: u64,
buf: &mut [u8],
count: &mut u32,
callback: &mut C,
) -> Result<bool, Error>
where
C: FnMut(&EntryHeader, &[u8]) -> bool,
{
let sector_offset = self.sector_offset(sector_idx);
let write_offset = self.sectors[sector_idx as usize].write_offset;
let entry_start = SECTOR_HEADER_SIZE as u32;
let mut scanner = SectorScanner::new(sector_offset, entry_start, write_offset);
let mut header_buf = [0u8; ENTRY_HEADER_SIZE];
while let Some(entry) = scanner.next_entry(flash, &mut header_buf, PAGE_SIZE)? {
if !accept_entry(flash, sector_offset, &entry, buf)? {
scanner.skip(1);
continue;
}
let sensor = entry.sensor_id();
let base = entry.timestamp();
if sensor_filter.is_none_or(|f| f == sensor)
&& deliver_samples(
entry.is_run(),
sensor,
base,
&mut buf[..entry.payload_len],
t_start,
t_end,
count,
callback,
)
{
return Ok(true); }
scanner.skip(entry.entry_size);
}
Ok(false)
}
fn find_oldest_in_sector<F: ReadNorFlash>(
&self,
flash: &mut F,
sector_idx: u16,
target_sensor: u16,
buf: &mut [u8],
) -> Result<Option<(EntryHeader, usize)>, Error> {
let sector_offset = self.sector_offset(sector_idx);
let write_offset = self.sectors[sector_idx as usize].write_offset;
let entry_start = SECTOR_HEADER_SIZE as u32;
let mut scanner = SectorScanner::new(sector_offset, entry_start, write_offset);
let mut header_buf = [0u8; ENTRY_HEADER_SIZE];
let mut payload_buf = [0u8; PAGE_SIZE];
while let Some(entry) = scanner.next_entry(flash, &mut header_buf, PAGE_SIZE)? {
if !accept_entry(flash, sector_offset, &entry, &mut payload_buf)? {
scanner.skip(1);
continue;
}
let sensor = entry.sensor_id();
if sensor == target_sensor {
let (header, payload) =
entry.edge(&mut payload_buf[..entry.payload_len], Edge::First);
if payload.len() > buf.len() {
return Err(Error::BufferTooSmall);
}
buf[..payload.len()].copy_from_slice(payload);
return Ok(Some((header, payload.len())));
}
scanner.skip(entry.entry_size);
}
Ok(None)
}
pub(crate) fn count_entries_before<F: ReadNorFlash>(
&self,
flash: &mut F,
sector_idx: u16,
offset_limit: u32,
) -> Result<Replicated, Error> {
let summary = &self.sectors[sector_idx as usize];
let sector_offset = self.sector_offset(sector_idx);
let entry_start = SECTOR_HEADER_SIZE as u32;
let limit = offset_limit.min(summary.write_offset);
if limit <= entry_start {
return Ok(Replicated::NONE);
}
let mut scanner = SectorScanner::new(sector_offset, entry_start, limit);
let mut header_buf = [0u8; ENTRY_HEADER_SIZE];
let mut buf = [0u8; PAGE_SIZE];
let mut found = Replicated::NONE;
while let Some(entry) = scanner.next_entry(flash, &mut header_buf, PAGE_SIZE)? {
if !accept_entry(flash, sector_offset, &entry, &mut buf)? {
scanner.skip(1);
continue;
}
found.push(samples_in(entry.is_run(), &buf[..entry.payload_len]));
scanner.skip(entry.entry_size);
}
Ok(Replicated {
entries: found.entries.min(summary.entry_count),
samples: found.samples.min(summary.sample_count),
})
}
fn count_sensor_in_sector<F: ReadNorFlash>(
&self,
flash: &mut F,
sector_idx: u16,
target_sensor: u16,
) -> Result<u32, Error> {
let sector_offset = self.sector_offset(sector_idx);
let write_offset = self.sectors[sector_idx as usize].write_offset;
let entry_start = SECTOR_HEADER_SIZE as u32;
let mut scanner = SectorScanner::new(sector_offset, entry_start, write_offset);
let mut header_buf = [0u8; ENTRY_HEADER_SIZE];
let mut payload_buf = [0u8; PAGE_SIZE];
let mut count = 0u32;
while let Some(entry) = scanner.next_entry(flash, &mut header_buf, PAGE_SIZE)? {
if !accept_entry(flash, sector_offset, &entry, &mut payload_buf)? {
scanner.skip(1);
continue;
}
if entry.sensor_id() == target_sensor {
count += samples_in(entry.is_run(), &payload_buf[..entry.payload_len]);
}
scanner.skip(entry.entry_size);
}
Ok(count)
}
}
impl<F, const PAGE_SIZE: usize, const MAX_SECTORS: usize> crate::port::LogSink
for SensorDb<F, PAGE_SIZE, MAX_SECTORS>
where
F: NorFlash + ReadNorFlash,
{
fn append(&mut self, sensor_id: u16, timestamp: u64, payload: &[u8]) -> Result<(), Error> {
SensorDb::append(self, sensor_id, timestamp, payload)
}
fn append_run(&mut self, run: &Run<'_>) -> Result<(), Error> {
SensorDb::append_run(self, run)
}
fn sync(&mut self) -> Result<(), Error> {
SensorDb::sync(self)
}
}
impl<F, const PAGE_SIZE: usize, const MAX_SECTORS: usize> crate::port::LogSource
for SensorDb<F, PAGE_SIZE, MAX_SECTORS>
where
F: NorFlash + ReadNorFlash,
{
fn latest(
&mut self,
sensor_id: u16,
buf: &mut [u8],
) -> Result<Option<(EntryHeader, usize)>, Error> {
SensorDb::latest(self, sensor_id, buf)
}
fn oldest(
&mut self,
sensor_id: u16,
buf: &mut [u8],
) -> Result<Option<(EntryHeader, usize)>, Error> {
SensorDb::oldest(self, sensor_id, buf)
}
fn count(&mut self, sensor_id: Option<u16>) -> Result<u32, Error> {
SensorDb::count(self, sensor_id)
}
fn query_range(
&mut self,
sensor_id: Option<u16>,
t_start: u64,
t_end: u64,
callback: &mut dyn FnMut(&EntryHeader, &[u8]) -> bool,
) -> Result<u32, Error> {
SensorDb::query_range(self, sensor_id, t_start, t_end, callback)
}
}
#[cfg(all(test, feature = "std"))]
mod tests {
extern crate alloc;
use alloc::vec;
use super::*;
use crate::mock::MockFlash;
const PAGE: usize = 256;
const SECTOR: usize = 4096;
const SECTORS: usize = 8;
fn make_db() -> SensorDb<MockFlash<PAGE, SECTOR>, PAGE, SECTORS> {
let flash = MockFlash::<PAGE, SECTOR>::new(SECTORS * SECTOR);
let mut db = SensorDb::new(flash, 0, SECTORS as u16, 1).unwrap();
db.format().unwrap();
db
}
#[test]
fn format_and_mount() {
let flash = MockFlash::<PAGE, SECTOR>::new(SECTORS * SECTOR);
let mut db: SensorDb<_, PAGE, SECTORS> =
SensorDb::new(flash, 0, SECTORS as u16, 1).unwrap();
db.format().unwrap();
assert!(db.is_mounted());
assert_eq!(db.stats().total_entries, 0);
}
#[test]
fn append_and_latest() {
let mut db = make_db();
let payload = [0xDE, 0xAD, 0xBE, 0xEF];
db.append(1, 1000, &payload).unwrap();
db.append(2, 1001, &[0x42]).unwrap();
db.append(1, 1002, &[0x01, 0x02]).unwrap();
let mut buf = [0u8; 16];
let (hdr, n) = db.latest(1, &mut buf).unwrap().unwrap();
assert_eq!(hdr.sensor_id, 1);
assert_eq!(hdr.timestamp, 1002);
assert_eq!(&buf[..n], &[0x01, 0x02]);
let (hdr2, n2) = db.latest(2, &mut buf).unwrap().unwrap();
assert_eq!(hdr2.sensor_id, 2);
assert_eq!(&buf[..n2], &[0x42]);
}
#[test]
fn query_range_filters_correctly() {
let mut db = make_db();
db.append(1, 100, &[0x01]).unwrap();
db.append(1, 200, &[0x02]).unwrap();
db.append(2, 150, &[0x0A]).unwrap();
db.append(1, 300, &[0x03]).unwrap();
db.sync().unwrap();
let mut results = alloc::vec::Vec::new();
let count = db
.query_range(Some(1), 100, 250, |hdr, payload| {
results.push((hdr.timestamp, payload[0]));
true
})
.unwrap();
assert_eq!(count, 2);
assert_eq!(results, vec![(100, 0x01), (200, 0x02)]);
}
#[test]
fn survives_remount() {
let flash = MockFlash::<PAGE, SECTOR>::new(SECTORS * SECTOR);
let mut db: SensorDb<_, PAGE, SECTORS> =
SensorDb::new(flash, 0, SECTORS as u16, 1).unwrap();
db.format().unwrap();
db.append(5, 42, &[0xAA, 0xBB]).unwrap();
db.append(5, 43, &[0xCC]).unwrap();
db.sync().unwrap();
let _stats_before = db.stats();
let flash = db.into_inner();
let mut db2: SensorDb<_, PAGE, SECTORS> =
SensorDb::new(flash, 0, SECTORS as u16, 1).unwrap();
let mount_stats = db2.mount().unwrap();
assert_eq!(mount_stats.total_entries, 2);
let mut buf = [0u8; 16];
let (hdr, n) = db2.latest(5, &mut buf).unwrap().unwrap();
assert_eq!(hdr.timestamp, 43);
assert_eq!(&buf[..n], &[0xCC]);
}
#[test]
fn circular_wrap_reclaims_old_sectors() {
const SMALL: usize = 4;
let flash = MockFlash::<PAGE, SECTOR>::new(SMALL * SECTOR);
let mut db: SensorDb<_, PAGE, SMALL> = SensorDb::new(flash, 0, SMALL as u16, 1).unwrap();
db.format().unwrap();
for i in 0..800u64 {
let payload = (i as u32).to_le_bytes();
db.append(1, i * 100, &payload).unwrap();
}
let stats = db.stats();
assert!(stats.total_entries > 0);
assert!(stats.used_sectors <= SMALL as u16);
let mut buf = [0u8; 4];
let (hdr, _) = db.latest(1, &mut buf).unwrap().unwrap();
assert_eq!(hdr.timestamp, 799 * 100);
}
#[test]
fn compact_frees_sectors() {
let mut db = make_db();
for i in 0..50u64 {
db.append(1, i, &[i as u8]).unwrap();
}
db.sync().unwrap();
let before = db.stats();
let freed = db.compact(1, CompactScope::All).unwrap();
let after = db.stats();
if before.used_sectors > 1 {
assert!(freed > 0);
assert!(after.free_sectors > before.free_sectors);
}
}
#[test]
fn max_payload_works() {
let mut db = make_db();
let max = db.max_payload_size();
let payload = vec![0x42u8; max];
db.append(1, 999, &payload).unwrap();
db.sync().unwrap();
let mut buf = vec![0u8; max];
let (hdr, n) = db.latest(1, &mut buf).unwrap().unwrap();
assert_eq!(n, max);
assert_eq!(hdr.payload_len as usize, max);
}
#[test]
fn rejects_oversized_payload() {
let mut db = make_db();
let too_big = vec![0u8; db.max_payload_size() + 1];
let result = db.append(1, 100, &too_big);
assert!(result.is_err());
}
mod typed_api {
use super::*;
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
#[derive(Debug, Clone, Copy, PartialEq, FromBytes, IntoBytes, KnownLayout, Immutable)]
#[repr(C)]
struct Temperature {
celsius: f32,
}
impl SensorData for Temperature {
const SENSOR_ID: u16 = 0x01;
}
#[derive(Debug, Clone, Copy, PartialEq, FromBytes, IntoBytes, KnownLayout, Immutable)]
#[repr(C)]
struct Accelerometer {
x: i16,
y: i16,
z: i16,
}
impl SensorData for Accelerometer {
const SENSOR_ID: u16 = 0x02;
}
#[test]
fn typed_roundtrip_layer1() {
let mut db = make_db();
let temp = Temperature { celsius: 25.5 };
db.append_value(0x01, 1000, &temp).unwrap();
let (hdr, retrieved) = db.latest_value::<Temperature>(0x01).unwrap().unwrap();
assert_eq!(hdr.sensor_id, 0x01);
assert_eq!(hdr.timestamp, 1000);
assert_eq!(retrieved.celsius, 25.5);
}
#[test]
fn typed_roundtrip_layer2() {
let mut db = make_db();
let temp = Temperature { celsius: -10.0 };
db.append_sensor(2000, &temp).unwrap();
let (hdr, retrieved) = db.latest_sensor::<Temperature>().unwrap().unwrap();
assert_eq!(hdr.sensor_id, Temperature::SENSOR_ID);
assert_eq!(hdr.timestamp, 2000);
assert_eq!(retrieved.celsius, -10.0);
}
#[test]
fn typed_multiple_sensors() {
let mut db = make_db();
let temp = Temperature { celsius: 22.5 };
let accel = Accelerometer {
x: 100,
y: -50,
z: 980,
};
db.append_sensor(1000, &temp).unwrap();
db.append_sensor(1001, &accel).unwrap();
let (_, t) = db.latest_sensor::<Temperature>().unwrap().unwrap();
let (_, a) = db.latest_sensor::<Accelerometer>().unwrap().unwrap();
assert_eq!(t.celsius, 22.5);
assert_eq!(a.x, 100);
assert_eq!(a.y, -50);
assert_eq!(a.z, 980);
}
#[test]
fn cross_api_raw_write_typed_read() {
let mut db = make_db();
db.append(Temperature::SENSOR_ID, 3000, &30.0f32.to_ne_bytes())
.unwrap();
assert_eq!(
db.latest_sensor::<Temperature>(),
Err(Error::SizeMismatch {
expected: 5,
actual: 4
})
);
let mut framed = [0u8; 5];
framed[0] = Temperature::SCHEMA_VERSION;
framed[1..].copy_from_slice(&31.0f32.to_ne_bytes());
db.append(Temperature::SENSOR_ID, 3100, &framed).unwrap();
let (hdr, temp) = db.latest_sensor::<Temperature>().unwrap().unwrap();
assert_eq!(hdr.timestamp, 3100);
assert_eq!(temp.celsius, 31.0);
}
#[test]
fn cross_api_typed_write_raw_read() {
let mut db = make_db();
let temp = Temperature { celsius: 42.0 };
db.append_sensor(4000, &temp).unwrap();
let mut buf = [0u8; 16];
let (hdr, n) = db
.latest(Temperature::SENSOR_ID, &mut buf)
.unwrap()
.unwrap();
assert_eq!(hdr.timestamp, 4000);
assert_eq!(n, crate::SCHEMA_TAG_SIZE + 4);
assert_eq!(hdr.payload_len as usize, n);
assert_eq!(buf[0], Temperature::SCHEMA_VERSION);
let celsius = f32::from_ne_bytes([buf[1], buf[2], buf[3], buf[4]]);
assert_eq!(celsius, 42.0);
}
#[test]
fn size_mismatch_detection() {
let mut db = make_db();
db.append(Temperature::SENSOR_ID, 5000, &[Temperature::SCHEMA_VERSION])
.unwrap();
let result = db.latest_sensor::<Temperature>();
match result {
Err(Error::SizeMismatch { expected, actual }) => {
assert_eq!(expected, 5);
assert_eq!(actual, 1);
}
_ => panic!("Expected SizeMismatch error"),
}
}
#[test]
fn a_foreign_first_byte_is_reported_as_the_version_it_claims() {
let mut db = make_db();
db.append(Temperature::SENSOR_ID, 5000, &[0x42]).unwrap();
assert_eq!(
db.latest_sensor::<Temperature>(),
Err(Error::SchemaMismatch {
expected: 0,
found: 0x42
})
);
}
#[test]
fn an_entry_of_another_schema_version_is_named_not_misread() {
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, PartialEq)]
#[repr(C)]
struct Fahrenheit {
degrees: f32,
}
impl SensorData for Fahrenheit {
const SENSOR_ID: u16 = Temperature::SENSOR_ID;
const SCHEMA_VERSION: u8 = 1;
}
let mut db = make_db();
db.append_sensor(1000, &Temperature { celsius: 100.0 })
.unwrap();
assert_eq!(
db.latest_sensor::<Fahrenheit>(),
Err(Error::SchemaMismatch {
expected: 1,
found: 0
})
);
db.append_sensor(2000, &Fahrenheit { degrees: 212.0 })
.unwrap();
let mut celsius = alloc::vec::Vec::new();
db.query_all_sensor::<Temperature, _>(|_, t| {
celsius.push(t.celsius);
true
})
.unwrap();
let mut degrees = alloc::vec::Vec::new();
db.query_all_sensor::<Fahrenheit, _>(|_, f| {
degrees.push(f.degrees);
true
})
.unwrap();
assert_eq!(celsius, [100.0]);
assert_eq!(degrees, [212.0]);
let (_, f) = db.latest_sensor::<Fahrenheit>().unwrap().unwrap();
assert_eq!(f.degrees, 212.0);
}
#[test]
fn query_sensor_typed() {
let mut db = make_db();
for i in 0..5 {
let temp = Temperature {
celsius: 20.0 + i as f32,
};
db.append_sensor(100 + i * 10, &temp).unwrap();
}
db.sync().unwrap();
let mut readings = alloc::vec::Vec::new();
let count = db
.query_sensor::<Temperature, _>(110, 130, |hdr, temp| {
readings.push((hdr.timestamp, temp.celsius));
true
})
.unwrap();
assert_eq!(count, 3); assert_eq!(readings.len(), 3);
assert_eq!(readings[0], (110, 21.0));
assert_eq!(readings[1], (120, 22.0));
assert_eq!(readings[2], (130, 23.0));
}
#[test]
fn query_sensor_skips_wrong_size() {
let mut db = make_db();
let temp = Temperature { celsius: 25.0 };
db.append_sensor(1000, &temp).unwrap();
db.append(Temperature::SENSOR_ID, 2000, &[0x01, 0x02])
.unwrap();
let temp2 = Temperature { celsius: 30.0 };
db.append_sensor(3000, &temp2).unwrap();
db.sync().unwrap();
let mut readings = alloc::vec::Vec::new();
let count = db
.query_sensor::<Temperature, _>(0, 10000, |hdr, temp| {
readings.push((hdr.timestamp, temp.celsius));
true
})
.unwrap();
assert_eq!(count, 2);
assert_eq!(readings[0], (1000, 25.0));
assert_eq!(readings[1], (3000, 30.0));
}
#[test]
fn typed_survives_remount() {
let flash = MockFlash::<PAGE, SECTOR>::new(SECTORS * SECTOR);
let mut db: SensorDb<_, PAGE, SECTORS> =
SensorDb::new(flash, 0, SECTORS as u16, 1).unwrap();
db.format().unwrap();
let temp = Temperature { celsius: 99.9 };
db.append_sensor(6000, &temp).unwrap();
db.sync().unwrap();
let flash = db.into_inner();
let mut db2: SensorDb<_, PAGE, SECTORS> =
SensorDb::new(flash, 0, SECTORS as u16, 1).unwrap();
db2.mount().unwrap();
let (hdr, retrieved) = db2.latest_sensor::<Temperature>().unwrap().unwrap();
assert_eq!(hdr.timestamp, 6000);
assert_eq!(retrieved.celsius, 99.9);
}
#[test]
fn latest_returns_most_recent() {
let mut db = make_db();
db.append_sensor(100, &Temperature { celsius: 10.0 })
.unwrap();
db.append_sensor(200, &Temperature { celsius: 20.0 })
.unwrap();
db.append_sensor(300, &Temperature { celsius: 30.0 })
.unwrap();
let (hdr, temp) = db.latest_sensor::<Temperature>().unwrap().unwrap();
assert_eq!(hdr.timestamp, 300);
assert_eq!(temp.celsius, 30.0);
}
#[test]
fn no_entries_returns_none() {
let mut db = make_db();
let result = db.latest_sensor::<Temperature>().unwrap();
assert!(result.is_none());
}
#[test]
fn oldest_returns_first_entry() {
let mut db = make_db();
db.append_sensor(100, &Temperature { celsius: 10.0 })
.unwrap();
db.append_sensor(200, &Temperature { celsius: 20.0 })
.unwrap();
db.append_sensor(300, &Temperature { celsius: 30.0 })
.unwrap();
let (hdr, temp) = db.oldest_sensor::<Temperature>().unwrap().unwrap();
assert_eq!(hdr.timestamp, 100);
assert_eq!(temp.celsius, 10.0);
}
#[test]
fn oldest_returns_none_when_empty() {
let mut db = make_db();
let result = db.oldest_sensor::<Temperature>().unwrap();
assert!(result.is_none());
}
#[test]
fn count_sensor_counts_correctly() {
let mut db = make_db();
for i in 0..5 {
db.append_sensor(i * 100, &Temperature { celsius: i as f32 })
.unwrap();
}
for i in 0..3 {
db.append_sensor(
i * 100 + 50,
&Accelerometer {
x: i as i16,
y: 0,
z: 0,
},
)
.unwrap();
}
db.sync().unwrap();
let temp_count = db.count_sensor::<Temperature>().unwrap();
let accel_count = db.count_sensor::<Accelerometer>().unwrap();
let total = db.count(None).unwrap();
assert_eq!(temp_count, 5);
assert_eq!(accel_count, 3);
assert_eq!(total, 8);
}
#[test]
fn query_all_sensor_gets_all_entries() {
let mut db = make_db();
for i in 0..5 {
db.append_sensor(
i * 100,
&Temperature {
celsius: i as f32 * 10.0,
},
)
.unwrap();
}
db.sync().unwrap();
let mut readings = alloc::vec::Vec::new();
let count = db
.query_all_sensor::<Temperature, _>(|hdr, temp| {
readings.push((hdr.timestamp, temp.celsius));
true
})
.unwrap();
assert_eq!(count, 5);
assert_eq!(readings.len(), 5);
assert_eq!(readings[0], (0, 0.0));
assert_eq!(readings[4], (400, 40.0));
}
}
#[test]
fn query_all_gets_all_entries() {
let mut db = make_db();
db.append(1, 100, &[0x01]).unwrap();
db.append(1, 200, &[0x02]).unwrap();
db.append(2, 150, &[0x0A]).unwrap();
db.sync().unwrap();
let mut results = alloc::vec::Vec::new();
let count = db
.query_all(Some(1), |hdr, payload| {
results.push((hdr.timestamp, payload[0]));
true
})
.unwrap();
assert_eq!(count, 2);
assert_eq!(results, vec![(100, 0x01), (200, 0x02)]);
let mut all_results = alloc::vec::Vec::new();
let total = db
.query_all(None, |hdr, _| {
all_results.push(hdr.timestamp);
true
})
.unwrap();
assert_eq!(total, 3);
}
#[test]
fn oldest_gets_first_entry() {
let mut db = make_db();
db.append(1, 300, &[0x03]).unwrap();
db.append(1, 100, &[0x01]).unwrap();
db.append(1, 200, &[0x02]).unwrap();
db.sync().unwrap();
let mut buf = [0u8; 16];
let (hdr, n) = db.oldest(1, &mut buf).unwrap().unwrap();
assert_eq!(hdr.timestamp, 300); assert_eq!(&buf[..n], &[0x03]);
}
#[test]
fn oldest_returns_none_for_missing_sensor() {
let mut db = make_db();
db.append(1, 100, &[0x01]).unwrap();
db.sync().unwrap();
let mut buf = [0u8; 16];
let result = db.oldest(99, &mut buf).unwrap();
assert!(result.is_none());
}
#[test]
fn count_works_correctly() {
let mut db = make_db();
db.append(1, 100, &[0x01]).unwrap();
db.append(1, 200, &[0x02]).unwrap();
db.append(2, 150, &[0x0A]).unwrap();
db.sync().unwrap();
assert_eq!(db.count(Some(1)).unwrap(), 2);
assert_eq!(db.count(Some(2)).unwrap(), 1);
assert_eq!(db.count(Some(99)).unwrap(), 0);
assert_eq!(db.count(None).unwrap(), 3);
}
#[test]
fn count_uses_cached_total_for_none() {
let mut db = make_db();
for i in 0..10 {
db.append(1, i, &[i as u8]).unwrap();
}
assert_eq!(db.count(None).unwrap(), 10);
assert_eq!(db.stats().total_entries, 10);
}
#[test]
fn a_long_run_of_appends_rotates_sectors() {
const SMALL: usize = 4;
let flash = MockFlash::<PAGE, SECTOR>::new(SMALL * SECTOR);
let mut db: SensorDb<_, PAGE, SMALL> = SensorDb::new(flash, 0, SMALL as u16, 1).unwrap();
db.format().unwrap();
let payload = [0x42u8; 4];
for i in 0..500u64 {
db.append(1, i * 10, &payload).unwrap();
}
db.sync().unwrap();
let stats = db.stats();
assert!(stats.total_entries > 0);
assert!(stats.used_sectors > 1);
let mut buf = [0u8; 4];
let (hdr, _) = db.latest(1, &mut buf).unwrap().unwrap();
assert_eq!(hdr.timestamp, 499 * 10);
}
#[test]
fn a_database_is_sync_when_its_driver_is() {
fn assert_sync<T: Sync>() {}
fn assert_send<T: Send>() {}
assert_sync::<SensorDb<MockFlash<PAGE, SECTOR>, PAGE, SECTORS>>();
assert_send::<SensorDb<MockFlash<PAGE, SECTOR>, PAGE, SECTORS>>();
}
#[test]
fn read_without_sync_sees_buffered_entries() {
let mut db = make_db();
db.append(1, 100, &[0x01]).unwrap();
db.append(1, 200, &[0x02]).unwrap();
db.append(2, 150, &[0x0A]).unwrap();
let mut buf = [0u8; 16];
let (hdr, n) = db.latest(1, &mut buf).unwrap().unwrap();
assert_eq!(hdr.timestamp, 200);
assert_eq!(&buf[..n], &[0x02]);
let (hdr, n) = db.oldest(1, &mut buf).unwrap().unwrap();
assert_eq!(hdr.timestamp, 100);
assert_eq!(&buf[..n], &[0x01]);
assert_eq!(db.count(Some(1)).unwrap(), 2);
assert_eq!(db.count(Some(2)).unwrap(), 1);
assert_eq!(db.count(None).unwrap(), 3);
let mut results = alloc::vec::Vec::new();
let count = db
.query_range(Some(1), 0, u64::MAX, |hdr, payload| {
results.push((hdr.timestamp, payload[0]));
true
})
.unwrap();
assert_eq!(count, 2);
assert_eq!(results, vec![(100, 0x01), (200, 0x02)]);
}
#[test]
fn read_after_sync_sees_committed_and_buffered() {
let mut db = make_db();
db.append(1, 100, &[0x01]).unwrap();
db.append(1, 200, &[0x02]).unwrap();
db.sync().unwrap();
db.append(1, 300, &[0x03]).unwrap();
db.append(2, 250, &[0x0B]).unwrap();
let mut buf = [0u8; 16];
let (hdr, n) = db.latest(1, &mut buf).unwrap().unwrap();
assert_eq!(hdr.timestamp, 300);
assert_eq!(&buf[..n], &[0x03]);
let (hdr, n) = db.oldest(1, &mut buf).unwrap().unwrap();
assert_eq!(hdr.timestamp, 100);
assert_eq!(&buf[..n], &[0x01]);
assert_eq!(db.count(Some(1)).unwrap(), 3);
assert_eq!(db.count(Some(2)).unwrap(), 1);
assert_eq!(db.count(None).unwrap(), 4);
let mut results = alloc::vec::Vec::new();
db.query_range(Some(1), 0, u64::MAX, |hdr, payload| {
results.push((hdr.timestamp, payload[0]));
true
})
.unwrap();
assert_eq!(results, vec![(100, 0x01), (200, 0x02), (300, 0x03)]);
}
#[test]
fn query_range_order_with_buffered() {
let mut db = make_db();
db.append(1, 100, &[0x01]).unwrap();
db.append(1, 200, &[0x02]).unwrap();
db.sync().unwrap();
db.append(1, 300, &[0x03]).unwrap();
db.append(1, 400, &[0x04]).unwrap();
let mut timestamps = alloc::vec::Vec::new();
db.query_range(Some(1), 0, u64::MAX, |hdr, _| {
timestamps.push(hdr.timestamp);
true
})
.unwrap();
assert_eq!(timestamps, vec![100, 200, 300, 400]);
}
#[test]
fn page_buf_empty_after_sync() {
let mut db = make_db();
db.append(1, 100, &[0x01]).unwrap();
db.append(1, 200, &[0x02]).unwrap();
db.sync().unwrap();
assert_eq!(db.count(Some(1)).unwrap(), 2);
let mut results = alloc::vec::Vec::new();
db.query_range(Some(1), 0, u64::MAX, |hdr, payload| {
results.push((hdr.timestamp, payload[0]));
true
})
.unwrap();
assert_eq!(results, vec![(100, 0x01), (200, 0x02)]);
}
}