verit_core/
container.rs

1//! `.vertc` — the Veritate **container**: an at-rest file holding many messages
2//! for zero-copy random access, designed to be **`mmap`-ped and read in place**.
3//!
4//! A Veritate message is already zero-copy over a `&[u8]`, so a file of messages
5//! needs only enough structure to (a) find each record without scanning and (b)
6//! keep every record's internal 8-byte fields aligned once the file is mapped.
7//! The container does exactly that and nothing more:
8//!
9//! # Superseded
10//!
11//! **This module is superseded by [`crate::file`] (the `.verit` file) and is
12//! removed in 0.3.0.** See
13//! [ADR-0002](https://github.com/exavianlabs/veritate/blob/main/docs/decisions/ADR-0002%20-%20The%20verit%20File%20Format.md).
14//!
15//! A `.vertc` container is a record array: it has no schema section, so it is
16//! **not** interpretable without an out-of-band registry — the one property a
17//! file format exists to provide. `.verit` adds that section (each schema stored
18//! once, records hash-only), plus crash-safe appends, removal, and compaction.
19//! Migrate with [`FileBuilder`](crate::FileBuilder).
20//!
21//! ```text
22//! ┌─ header (32 B, 8-aligned) ─────────────────────────────────────────────┐
23//! │ 0  "VRTC"        magic                                                   │
24//! │ 4  u8  version=1                                                         │
25//! │ 5  u8  flags=0                                                           │
26//! │ 6  u16 reserved=0                                                        │
27//! │ 8  u32 record_count                                                      │
28//! │ 12 u32 reserved=0                                                        │
29//! │ 16 u64 index_offset   (absolute offset of the index)                    │
30//! │ 24 u64 file_len       (== total bytes; sanity)                          │
31//! ├─ records ───────────────────────────────────────────────────────────────┤
32//! │ each message's raw bytes, in order, every record 8-byte aligned          │
33//! ├─ index (at index_offset, 8-aligned) ─────────────────────────────────────┤
34//! │ record_count × { u64 offset, u64 len }   (offset absolute, len unpadded) │
35//! └──────────────────────────────────────────────────────────────────────────┘
36//! ```
37//!
38//! Because each record starts on an 8-byte boundary *within* the file and a
39//! memory map begins on a page boundary (a multiple of 8), every record's start
40//! address is 8-aligned in memory, so the message's own 8-aligned loads stay
41//! aligned — the point of the layout. The reader borrows the mapped bytes:
42//! [`Container::get`] returns a `&[u8]` sub-slice you hand straight to
43//! [`Message::parse`]. No `mmap` dependency lives here (the library stays
44//! zero-dep) — map the file with your platform's facility, or `std::fs::read`
45//! it, and pass the `&[u8]`.
46
47#![allow(deprecated)]
48
49use crate::error::{Error, Result};
50
51/// Container file magic: "VRTC" + this reader/writer implements version 1.
52pub const CONTAINER_MAGIC: &[u8; 4] = b"VRTC";
53/// Container format version this build reads and writes.
54pub const CONTAINER_VERSION: u8 = 1;
55/// Fixed header length, and the record alignment.
56pub const CONTAINER_HEADER_LEN: usize = 32;
57const ALIGN: usize = 8;
58const INDEX_ENTRY_LEN: usize = 16;
59
60#[inline]
61fn align_up(x: usize, a: usize) -> usize {
62    (x + a - 1) & !(a - 1)
63}
64
65/// Build a `.vertc` container from a sequence of Veritate messages. The output
66/// is a self-contained `Vec<u8>` you write to a file (and later `mmap`).
67///
68/// ```
69/// # #![allow(deprecated)]
70/// # use verit_core::container::{ContainerWriter, Container};
71/// let mut w = ContainerWriter::new();
72/// w.add(b"\x00message-a");
73/// w.add(b"\x01message-b");
74/// let file = w.finish();
75///
76/// let c = Container::parse(&file).unwrap();
77/// assert_eq!(c.len(), 2);
78/// assert_eq!(c.get(1).unwrap(), b"\x01message-b");
79/// ```
80#[derive(Default)]
81#[deprecated(
82    since = "0.2.0",
83    note = "superseded by the `.verit` file: use `FileBuilder` (in memory) or `FileWriter` (on disk). \
84            A `.vertc` container carries no schema section, so it cannot be read without an \
85            out-of-band registry. Removed in 0.3.0 — see ADR-0002."
86)]
87pub struct ContainerWriter {
88    buf: Vec<u8>,
89    // (absolute offset, unpadded length) per record.
90    index: Vec<(u64, u64)>,
91}
92
93impl ContainerWriter {
94    pub fn new() -> ContainerWriter {
95        ContainerWriter {
96            buf: vec![0; CONTAINER_HEADER_LEN], // header patched in `finish`
97            index: Vec::new(),
98        }
99    }
100
101    /// Append one message's bytes as the next record. Records are stored in the
102    /// order added and read back by that index.
103    pub fn add(&mut self, message: &[u8]) -> &mut Self {
104        let pad = align_up(self.buf.len(), ALIGN) - self.buf.len();
105        self.buf.resize(self.buf.len() + pad, 0);
106        let offset = self.buf.len() as u64;
107        self.buf.extend_from_slice(message);
108        self.index.push((offset, message.len() as u64));
109        self
110    }
111
112    /// Number of records added so far.
113    pub fn len(&self) -> usize {
114        self.index.len()
115    }
116
117    pub fn is_empty(&self) -> bool {
118        self.index.is_empty()
119    }
120
121    /// Finish the container and return its bytes: pad to the index alignment,
122    /// append the index, then patch the header.
123    pub fn finish(mut self) -> Vec<u8> {
124        let pad = align_up(self.buf.len(), ALIGN) - self.buf.len();
125        self.buf.resize(self.buf.len() + pad, 0);
126        let index_offset = self.buf.len() as u64;
127        for (off, len) in &self.index {
128            self.buf.extend_from_slice(&off.to_le_bytes());
129            self.buf.extend_from_slice(&len.to_le_bytes());
130        }
131        let file_len = self.buf.len() as u64;
132
133        self.buf[0..4].copy_from_slice(CONTAINER_MAGIC);
134        self.buf[4] = CONTAINER_VERSION;
135        // buf[5] flags, buf[6..8] reserved already zero.
136        self.buf[8..12].copy_from_slice(&(self.index.len() as u32).to_le_bytes());
137        // buf[12..16] reserved already zero.
138        self.buf[16..24].copy_from_slice(&index_offset.to_le_bytes());
139        self.buf[24..32].copy_from_slice(&file_len.to_le_bytes());
140        self.buf
141    }
142}
143
144/// A read-only view over a `.vertc` container's bytes (e.g. an `mmap`). Parsing
145/// validates the header and the whole index up front, so every later
146/// [`get`](Container::get) is a bounds-free slice.
147#[derive(Clone, Debug)]
148#[deprecated(
149    since = "0.2.0",
150    note = "superseded by the `.verit` file: use `FileView`. A `.vertc` container carries no \
151            schema section, so it cannot be read without an out-of-band registry. \
152            Removed in 0.3.0 — see ADR-0002."
153)]
154pub struct Container<'a> {
155    buf: &'a [u8],
156    index_offset: usize,
157    count: usize,
158}
159
160impl<'a> Container<'a> {
161    /// Validate the header and index of a container image. Rejects bad
162    /// magic/version, an out-of-range or unaligned index, and any record whose
163    /// extent falls outside the record region or is misaligned.
164    pub fn parse(buf: &'a [u8]) -> Result<Container<'a>> {
165        if buf.len() < CONTAINER_HEADER_LEN {
166            return Err(Error::Truncated);
167        }
168        if &buf[0..4] != CONTAINER_MAGIC {
169            return Err(Error::BadContainer("bad magic"));
170        }
171        if buf[4] != CONTAINER_VERSION {
172            return Err(Error::BadContainer("unsupported container version"));
173        }
174        if buf[5] != 0 || u16::from_le_bytes(buf[6..8].try_into().unwrap()) != 0 {
175            return Err(Error::BadContainer("nonzero reserved header field"));
176        }
177        let count = u32::from_le_bytes(buf[8..12].try_into().unwrap()) as usize;
178        let index_offset = u64::from_le_bytes(buf[16..24].try_into().unwrap());
179        let file_len = u64::from_le_bytes(buf[24..32].try_into().unwrap());
180        if file_len as usize != buf.len() {
181            return Err(Error::BadContainer("file length mismatch"));
182        }
183        let index_offset = usize::try_from(index_offset)
184            .map_err(|_| Error::BadContainer("index offset overflow"))?;
185        if index_offset % ALIGN != 0 || index_offset < CONTAINER_HEADER_LEN {
186            return Err(Error::BadContainer("misaligned index offset"));
187        }
188        // The index must fit exactly between its start and end of file.
189        let index_bytes = count
190            .checked_mul(INDEX_ENTRY_LEN)
191            .ok_or(Error::BadContainer("index size overflow"))?;
192        let index_end = index_offset
193            .checked_add(index_bytes)
194            .ok_or(Error::BadContainer("index end overflow"))?;
195        if index_end > buf.len() {
196            return Err(Error::BadContainer("index out of bounds"));
197        }
198        let container = Container {
199            buf,
200            index_offset,
201            count,
202        };
203        // Validate every entry once so `get` is infallible on bounds.
204        for i in 0..count {
205            let (off, len) = container.raw_entry(i);
206            let off =
207                usize::try_from(off).map_err(|_| Error::BadContainer("record offset overflow"))?;
208            let len =
209                usize::try_from(len).map_err(|_| Error::BadContainer("record length overflow"))?;
210            if off % ALIGN != 0 {
211                return Err(Error::BadContainer("misaligned record"));
212            }
213            let end = off
214                .checked_add(len)
215                .ok_or(Error::BadContainer("record extent overflow"))?;
216            // Records live strictly in [header, index_offset).
217            if off < CONTAINER_HEADER_LEN || end > index_offset {
218                return Err(Error::BadContainer("record outside record region"));
219            }
220        }
221        Ok(container)
222    }
223
224    #[inline]
225    fn raw_entry(&self, i: usize) -> (u64, u64) {
226        let base = self.index_offset + i * INDEX_ENTRY_LEN;
227        let off = u64::from_le_bytes(self.buf[base..base + 8].try_into().unwrap());
228        let len = u64::from_le_bytes(self.buf[base + 8..base + 16].try_into().unwrap());
229        (off, len)
230    }
231
232    /// Number of records.
233    pub fn len(&self) -> usize {
234        self.count
235    }
236
237    pub fn is_empty(&self) -> bool {
238        self.count == 0
239    }
240
241    /// The raw message bytes of record `i`, borrowing the container image.
242    /// Hand the result straight to [`Message::parse`](crate::Message::parse).
243    pub fn get(&self, i: usize) -> Result<&'a [u8]> {
244        if i >= self.count {
245            return Err(Error::IndexOutOfBounds);
246        }
247        let (off, len) = self.raw_entry(i);
248        // Validated in `parse`; this cannot go out of bounds.
249        Ok(&self.buf[off as usize..(off + len) as usize])
250    }
251
252    /// Iterate over each record's bytes in order.
253    pub fn iter(&self) -> impl Iterator<Item = &'a [u8]> + '_ {
254        (0..self.count).map(move |i| self.get(i).expect("index validated in parse"))
255    }
256}
257
258#[cfg(test)]
259mod tests {
260    use super::*;
261
262    #[test]
263    fn round_trips_messages() {
264        let msgs: Vec<Vec<u8>> = vec![
265            b"a".to_vec(),
266            b"".to_vec(),
267            (0..100u8).collect(),
268            b"the last one".to_vec(),
269        ];
270        let mut w = ContainerWriter::new();
271        for m in &msgs {
272            w.add(m);
273        }
274        assert_eq!(w.len(), 4);
275        let file = w.finish();
276
277        let c = Container::parse(&file).unwrap();
278        assert_eq!(c.len(), 4);
279        for (i, m) in msgs.iter().enumerate() {
280            assert_eq!(c.get(i).unwrap(), &m[..]);
281        }
282        let collected: Vec<&[u8]> = c.iter().collect();
283        assert_eq!(collected.len(), 4);
284        assert!(c.get(4).is_err());
285    }
286
287    #[test]
288    fn records_are_eight_byte_aligned() {
289        let mut w = ContainerWriter::new();
290        w.add(b"odd-length-7").add(b"x"); // force padding between records
291        let file = w.finish();
292        let c = Container::parse(&file).unwrap();
293        for i in 0..c.len() {
294            let (off, _) = c.raw_entry(i);
295            assert_eq!(off % 8, 0, "record {i} not 8-aligned");
296        }
297    }
298
299    #[test]
300    fn empty_container_is_valid() {
301        let file = ContainerWriter::new().finish();
302        let c = Container::parse(&file).unwrap();
303        assert_eq!(c.len(), 0);
304        assert!(c.is_empty());
305    }
306
307    #[test]
308    fn rejects_corruption() {
309        let mut file = {
310            let mut w = ContainerWriter::new();
311            w.add(b"hello");
312            w.finish()
313        };
314        assert!(Container::parse(&file[..10]).is_err(), "truncated");
315
316        let mut bad_magic = file.clone();
317        bad_magic[0] = b'X';
318        assert!(matches!(
319            Container::parse(&bad_magic),
320            Err(Error::BadContainer(_))
321        ));
322
323        let mut bad_ver = file.clone();
324        bad_ver[4] = 2;
325        assert!(matches!(
326            Container::parse(&bad_ver),
327            Err(Error::BadContainer(_))
328        ));
329
330        // Corrupt a record offset in the index to point past the record region.
331        let idx_off = u64::from_le_bytes(file[16..24].try_into().unwrap()) as usize;
332        file[idx_off..idx_off + 8].copy_from_slice(&u64::MAX.to_le_bytes());
333        assert!(matches!(
334            Container::parse(&file),
335            Err(Error::BadContainer(_))
336        ));
337    }
338}
verit_core/container.rs

verit_core/
container.rs
