modelvault-core 0.16.0

Core engine for ModelVault — application-focused embedded storage with model schemas, validation, and migrations.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

use crate::checksum::crc32c_append;
use crate::error::{DbError, FormatError};
use crate::file_format::check_segment_payload_len;
use crate::segments::header::SegmentType;
use crate::segments::header::{decode_segment_header, SegmentHeader, SEGMENT_HEADER_LEN};
use crate::storage::Store;

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SegmentMeta {
    pub offset: u64,
    pub header: SegmentHeader,
}

pub fn read_segment_header_at(
    store: &mut impl Store,
    offset: u64,
) -> Result<([u8; SEGMENT_HEADER_LEN], SegmentHeader), DbError> {
    let mut buf = [0u8; SEGMENT_HEADER_LEN];
    store.read_exact_at(offset, &mut buf)?;
    let header = decode_segment_header(&buf)?;
    Ok((buf, header))
}

pub fn scan_segments(store: &mut impl Store, start: u64) -> Result<Vec<SegmentMeta>, DbError> {
    let mut out = Vec::new();
    let mut offset = start;
    let file_len = store.len()?;

    while offset < file_len {
        if file_len - offset < SEGMENT_HEADER_LEN as u64 {
            return Err(DbError::Format(FormatError::TruncatedSegmentHeader {
                got: (file_len - offset) as usize,
                expected: SEGMENT_HEADER_LEN,
            }));
        }

        let (_, header) = read_segment_header_at(store, offset)?;
        let payload_start = offset + SEGMENT_HEADER_LEN as u64;
        let payload_end = payload_start + header.payload_len;
        if payload_end > file_len {
            return Err(DbError::Format(FormatError::SegmentPayloadPastEof));
        }

        // CRC check with bounded reads (no large allocations).
        let mut remaining = header.payload_len;
        let mut chunk = [0u8; 8192];
        let mut cursor = payload_start;
        let mut crc = 0u32;
        while remaining > 0 {
            let to_read = std::cmp::min(remaining as usize, chunk.len());
            store.read_exact_at(cursor, &mut chunk[..to_read])?;
            crc = crc32c_append(crc, &chunk[..to_read]);
            cursor += to_read as u64;
            remaining -= to_read as u64;
        }
        // Checkpoint/Temp payload corruption should not prevent opening and replaying the log.
        // (Checkpoint is validated when used; Temp is ephemeral and ignored by replay.)
        if header.segment_type != SegmentType::Checkpoint
            && header.segment_type != SegmentType::Temp
            && crc != header.payload_crc32c
        {
            return Err(DbError::Format(FormatError::BadSegmentPayloadChecksum));
        }

        out.push(SegmentMeta { offset, header });
        offset = payload_end;
    }

    Ok(out)
}

/// Read the payload bytes for a segment whose header was validated by [`scan_segments`].
pub fn read_segment_payload(
    store: &mut impl Store,
    meta: &SegmentMeta,
) -> Result<Vec<u8>, DbError> {
    check_segment_payload_len(meta.header.payload_len)?;
    let mut payload = vec![0u8; meta.header.payload_len as usize];
    let start = meta.offset + SEGMENT_HEADER_LEN as u64;
    store.read_exact_at(start, &mut payload)?;
    Ok(payload)
}