verit_core/
container.rs

1//! `.vertc` — the Veritate **container**: an at-rest file holding many messages
2//! for zero-copy random access, designed to be **`mmap`-ped and read in place**.
3//!
4//! A Veritate message is already zero-copy over a `&[u8]`, so a file of messages
5//! needs only enough structure to (a) find each record without scanning and (b)
6//! keep every record's internal 8-byte fields aligned once the file is mapped.
7//! The container does exactly that and nothing more:
8//!
9//! ```text
10//! ┌─ header (32 B, 8-aligned) ─────────────────────────────────────────────┐
11//! │ 0  "VRTC"        magic                                                   │
12//! │ 4  u8  version=1                                                         │
13//! │ 5  u8  flags=0                                                           │
14//! │ 6  u16 reserved=0                                                        │
15//! │ 8  u32 record_count                                                      │
16//! │ 12 u32 reserved=0                                                        │
17//! │ 16 u64 index_offset   (absolute offset of the index)                    │
18//! │ 24 u64 file_len       (== total bytes; sanity)                          │
19//! ├─ records ───────────────────────────────────────────────────────────────┤
20//! │ each message's raw bytes, in order, every record 8-byte aligned          │
21//! ├─ index (at index_offset, 8-aligned) ─────────────────────────────────────┤
22//! │ record_count × { u64 offset, u64 len }   (offset absolute, len unpadded) │
23//! └──────────────────────────────────────────────────────────────────────────┘
24//! ```
25//!
26//! Because each record starts on an 8-byte boundary *within* the file and a
27//! memory map begins on a page boundary (a multiple of 8), every record's start
28//! address is 8-aligned in memory, so the message's own 8-aligned loads stay
29//! aligned — the point of the layout. The reader borrows the mapped bytes:
30//! [`Container::get`] returns a `&[u8]` sub-slice you hand straight to
31//! [`Message::parse`]. No `mmap` dependency lives here (the library stays
32//! zero-dep) — map the file with your platform's facility, or `std::fs::read`
33//! it, and pass the `&[u8]`.
34
35use crate::error::{Error, Result};
36
37/// Container file magic: "VRTC" + this reader/writer implements version 1.
38pub const CONTAINER_MAGIC: &[u8; 4] = b"VRTC";
39/// Container format version this build reads and writes.
40pub const CONTAINER_VERSION: u8 = 1;
41/// Fixed header length, and the record alignment.
42pub const CONTAINER_HEADER_LEN: usize = 32;
43const ALIGN: usize = 8;
44const INDEX_ENTRY_LEN: usize = 16;
45
46#[inline]
47fn align_up(x: usize, a: usize) -> usize {
48    (x + a - 1) & !(a - 1)
49}
50
51/// Build a `.vertc` container from a sequence of Veritate messages. The output
52/// is a self-contained `Vec<u8>` you write to a file (and later `mmap`).
53///
54/// ```
55/// # use verit_core::container::{ContainerWriter, Container};
56/// let mut w = ContainerWriter::new();
57/// w.add(b"\x00message-a");
58/// w.add(b"\x01message-b");
59/// let file = w.finish();
60///
61/// let c = Container::parse(&file).unwrap();
62/// assert_eq!(c.len(), 2);
63/// assert_eq!(c.get(1).unwrap(), b"\x01message-b");
64/// ```
65#[derive(Default)]
66pub struct ContainerWriter {
67    buf: Vec<u8>,
68    // (absolute offset, unpadded length) per record.
69    index: Vec<(u64, u64)>,
70}
71
72impl ContainerWriter {
73    pub fn new() -> ContainerWriter {
74        ContainerWriter {
75            buf: vec![0; CONTAINER_HEADER_LEN], // header patched in `finish`
76            index: Vec::new(),
77        }
78    }
79
80    /// Append one message's bytes as the next record. Records are stored in the
81    /// order added and read back by that index.
82    pub fn add(&mut self, message: &[u8]) -> &mut Self {
83        let pad = align_up(self.buf.len(), ALIGN) - self.buf.len();
84        self.buf.resize(self.buf.len() + pad, 0);
85        let offset = self.buf.len() as u64;
86        self.buf.extend_from_slice(message);
87        self.index.push((offset, message.len() as u64));
88        self
89    }
90
91    /// Number of records added so far.
92    pub fn len(&self) -> usize {
93        self.index.len()
94    }
95
96    pub fn is_empty(&self) -> bool {
97        self.index.is_empty()
98    }
99
100    /// Finish the container and return its bytes: pad to the index alignment,
101    /// append the index, then patch the header.
102    pub fn finish(mut self) -> Vec<u8> {
103        let pad = align_up(self.buf.len(), ALIGN) - self.buf.len();
104        self.buf.resize(self.buf.len() + pad, 0);
105        let index_offset = self.buf.len() as u64;
106        for (off, len) in &self.index {
107            self.buf.extend_from_slice(&off.to_le_bytes());
108            self.buf.extend_from_slice(&len.to_le_bytes());
109        }
110        let file_len = self.buf.len() as u64;
111
112        self.buf[0..4].copy_from_slice(CONTAINER_MAGIC);
113        self.buf[4] = CONTAINER_VERSION;
114        // buf[5] flags, buf[6..8] reserved already zero.
115        self.buf[8..12].copy_from_slice(&(self.index.len() as u32).to_le_bytes());
116        // buf[12..16] reserved already zero.
117        self.buf[16..24].copy_from_slice(&index_offset.to_le_bytes());
118        self.buf[24..32].copy_from_slice(&file_len.to_le_bytes());
119        self.buf
120    }
121}
122
123/// A read-only view over a `.vertc` container's bytes (e.g. an `mmap`). Parsing
124/// validates the header and the whole index up front, so every later
125/// [`get`](Container::get) is a bounds-free slice.
126#[derive(Clone, Debug)]
127pub struct Container<'a> {
128    buf: &'a [u8],
129    index_offset: usize,
130    count: usize,
131}
132
133impl<'a> Container<'a> {
134    /// Validate the header and index of a container image. Rejects bad
135    /// magic/version, an out-of-range or unaligned index, and any record whose
136    /// extent falls outside the record region or is misaligned.
137    pub fn parse(buf: &'a [u8]) -> Result<Container<'a>> {
138        if buf.len() < CONTAINER_HEADER_LEN {
139            return Err(Error::Truncated);
140        }
141        if &buf[0..4] != CONTAINER_MAGIC {
142            return Err(Error::BadContainer("bad magic"));
143        }
144        if buf[4] != CONTAINER_VERSION {
145            return Err(Error::BadContainer("unsupported container version"));
146        }
147        if buf[5] != 0 || u16::from_le_bytes(buf[6..8].try_into().unwrap()) != 0 {
148            return Err(Error::BadContainer("nonzero reserved header field"));
149        }
150        let count = u32::from_le_bytes(buf[8..12].try_into().unwrap()) as usize;
151        let index_offset = u64::from_le_bytes(buf[16..24].try_into().unwrap());
152        let file_len = u64::from_le_bytes(buf[24..32].try_into().unwrap());
153        if file_len as usize != buf.len() {
154            return Err(Error::BadContainer("file length mismatch"));
155        }
156        let index_offset = usize::try_from(index_offset)
157            .map_err(|_| Error::BadContainer("index offset overflow"))?;
158        if index_offset % ALIGN != 0 || index_offset < CONTAINER_HEADER_LEN {
159            return Err(Error::BadContainer("misaligned index offset"));
160        }
161        // The index must fit exactly between its start and end of file.
162        let index_bytes = count
163            .checked_mul(INDEX_ENTRY_LEN)
164            .ok_or(Error::BadContainer("index size overflow"))?;
165        let index_end = index_offset
166            .checked_add(index_bytes)
167            .ok_or(Error::BadContainer("index end overflow"))?;
168        if index_end > buf.len() {
169            return Err(Error::BadContainer("index out of bounds"));
170        }
171        let container = Container {
172            buf,
173            index_offset,
174            count,
175        };
176        // Validate every entry once so `get` is infallible on bounds.
177        for i in 0..count {
178            let (off, len) = container.raw_entry(i);
179            let off =
180                usize::try_from(off).map_err(|_| Error::BadContainer("record offset overflow"))?;
181            let len =
182                usize::try_from(len).map_err(|_| Error::BadContainer("record length overflow"))?;
183            if off % ALIGN != 0 {
184                return Err(Error::BadContainer("misaligned record"));
185            }
186            let end = off
187                .checked_add(len)
188                .ok_or(Error::BadContainer("record extent overflow"))?;
189            // Records live strictly in [header, index_offset).
190            if off < CONTAINER_HEADER_LEN || end > index_offset {
191                return Err(Error::BadContainer("record outside record region"));
192            }
193        }
194        Ok(container)
195    }
196
197    #[inline]
198    fn raw_entry(&self, i: usize) -> (u64, u64) {
199        let base = self.index_offset + i * INDEX_ENTRY_LEN;
200        let off = u64::from_le_bytes(self.buf[base..base + 8].try_into().unwrap());
201        let len = u64::from_le_bytes(self.buf[base + 8..base + 16].try_into().unwrap());
202        (off, len)
203    }
204
205    /// Number of records.
206    pub fn len(&self) -> usize {
207        self.count
208    }
209
210    pub fn is_empty(&self) -> bool {
211        self.count == 0
212    }
213
214    /// The raw message bytes of record `i`, borrowing the container image.
215    /// Hand the result straight to [`Message::parse`](crate::Message::parse).
216    pub fn get(&self, i: usize) -> Result<&'a [u8]> {
217        if i >= self.count {
218            return Err(Error::IndexOutOfBounds);
219        }
220        let (off, len) = self.raw_entry(i);
221        // Validated in `parse`; this cannot go out of bounds.
222        Ok(&self.buf[off as usize..(off + len) as usize])
223    }
224
225    /// Iterate over each record's bytes in order.
226    pub fn iter(&self) -> impl Iterator<Item = &'a [u8]> + '_ {
227        (0..self.count).map(move |i| self.get(i).expect("index validated in parse"))
228    }
229}
230
231#[cfg(test)]
232mod tests {
233    use super::*;
234
235    #[test]
236    fn round_trips_messages() {
237        let msgs: Vec<Vec<u8>> = vec![
238            b"a".to_vec(),
239            b"".to_vec(),
240            (0..100u8).collect(),
241            b"the last one".to_vec(),
242        ];
243        let mut w = ContainerWriter::new();
244        for m in &msgs {
245            w.add(m);
246        }
247        assert_eq!(w.len(), 4);
248        let file = w.finish();
249
250        let c = Container::parse(&file).unwrap();
251        assert_eq!(c.len(), 4);
252        for (i, m) in msgs.iter().enumerate() {
253            assert_eq!(c.get(i).unwrap(), &m[..]);
254        }
255        let collected: Vec<&[u8]> = c.iter().collect();
256        assert_eq!(collected.len(), 4);
257        assert!(c.get(4).is_err());
258    }
259
260    #[test]
261    fn records_are_eight_byte_aligned() {
262        let mut w = ContainerWriter::new();
263        w.add(b"odd-length-7").add(b"x"); // force padding between records
264        let file = w.finish();
265        let c = Container::parse(&file).unwrap();
266        for i in 0..c.len() {
267            let (off, _) = c.raw_entry(i);
268            assert_eq!(off % 8, 0, "record {i} not 8-aligned");
269        }
270    }
271
272    #[test]
273    fn empty_container_is_valid() {
274        let file = ContainerWriter::new().finish();
275        let c = Container::parse(&file).unwrap();
276        assert_eq!(c.len(), 0);
277        assert!(c.is_empty());
278    }
279
280    #[test]
281    fn rejects_corruption() {
282        let mut file = {
283            let mut w = ContainerWriter::new();
284            w.add(b"hello");
285            w.finish()
286        };
287        assert!(Container::parse(&file[..10]).is_err(), "truncated");
288
289        let mut bad_magic = file.clone();
290        bad_magic[0] = b'X';
291        assert!(matches!(
292            Container::parse(&bad_magic),
293            Err(Error::BadContainer(_))
294        ));
295
296        let mut bad_ver = file.clone();
297        bad_ver[4] = 2;
298        assert!(matches!(
299            Container::parse(&bad_ver),
300            Err(Error::BadContainer(_))
301        ));
302
303        // Corrupt a record offset in the index to point past the record region.
304        let idx_off = u64::from_le_bytes(file[16..24].try_into().unwrap()) as usize;
305        file[idx_off..idx_off + 8].copy_from_slice(&u64::MAX.to_le_bytes());
306        assert!(matches!(
307            Container::parse(&file),
308            Err(Error::BadContainer(_))
309        ));
310    }
311}
verit_core/container.rs

verit_core/
container.rs
