rac_engine/
index_format.rs

1//! Binary segment codec for the persistent index store (ADR-104).
2//!
3//! Byte-for-byte port of `services/index_format.py` per
4//! `rust/spec/index-store-format.md` §2. Fixed struct reads over a byte
5//! buffer — no code-bearing deserialisation; every read is bounds-checked
6//! and a segment's declared payload length must match its file exactly, so
7//! truncation or trailing garbage fails closed on open (a cache miss).
8
9use std::fmt;
10
11/// 8 magic bytes opening every segment file.
12pub const SEGMENT_MAGIC: &[u8; 8] = b"RACIDX01";
13/// The binary layout version (v4: tags tier, ADR-109).
14pub const SEGMENT_FORMAT_VERSION: u16 = 4;
15
16const HEADER_SIZE: usize = 8 + 2 + 8; // magic | version u16 LE | payload_len u64 LE
17
18/// A segment is corrupt, truncated, wrong-magic, or wrong-version. The store
19/// treats this as a cache miss and rebuilds; it never escapes to a caller.
20#[derive(Debug)]
21pub struct IndexFormatError(pub String);
22
23impl fmt::Display for IndexFormatError {
24    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
25        self.0.fmt(f)
26    }
27}
28
29impl std::error::Error for IndexFormatError {}
30
31fn err<T>(message: impl Into<String>) -> Result<T, IndexFormatError> {
32    Err(IndexFormatError(message.into()))
33}
34
35// ---------------------------------------------------------------------------
36// Writer — append-only encoder building one segment payload in memory.
37// ---------------------------------------------------------------------------
38
39#[derive(Default)]
40pub struct Writer {
41    buf: Vec<u8>,
42}
43
44impl Writer {
45    pub fn new() -> Self {
46        Self::default()
47    }
48
49    pub fn u32(&mut self, value: u64) -> Result<(), IndexFormatError> {
50        if value > u64::from(u32::MAX) {
51            return err(format!("u32 out of range: {value}"));
52        }
53        self.buf.extend_from_slice(&(value as u32).to_le_bytes());
54        Ok(())
55    }
56
57    pub fn u64(&mut self, value: u64) {
58        self.buf.extend_from_slice(&value.to_le_bytes());
59    }
60
61    pub fn raw(&mut self, data: &[u8]) {
62        self.buf.extend_from_slice(data);
63    }
64
65    pub fn blob(&mut self, data: &[u8]) -> Result<(), IndexFormatError> {
66        self.u32(data.len() as u64)?;
67        self.buf.extend_from_slice(data);
68        Ok(())
69    }
70
71    pub fn text(&mut self, value: &str) -> Result<(), IndexFormatError> {
72        self.blob(value.as_bytes())
73    }
74
75    /// A flag byte distinguishes `None` from the empty string.
76    pub fn opt_text(&mut self, value: Option<&str>) -> Result<(), IndexFormatError> {
77        match value {
78            None => {
79                self.buf.push(0);
80                Ok(())
81            }
82            Some(v) => {
83                self.buf.push(1);
84                self.text(v)
85            }
86        }
87    }
88
89    pub fn text_list<S: AsRef<str>>(&mut self, values: &[S]) -> Result<(), IndexFormatError> {
90        self.u32(values.len() as u64)?;
91        for value in values {
92            self.text(value.as_ref())?;
93        }
94        Ok(())
95    }
96
97    pub fn u32_list(&mut self, values: &[u32]) -> Result<(), IndexFormatError> {
98        self.u32(values.len() as u64)?;
99        for &value in values {
100            self.u32(u64::from(value))?;
101        }
102        Ok(())
103    }
104
105    pub fn payload(self) -> Vec<u8> {
106        self.buf
107    }
108}
109
110// ---------------------------------------------------------------------------
111// Reader — bounds-checked decoder over a mapped segment payload.
112// ---------------------------------------------------------------------------
113
114pub struct Reader<'a> {
115    view: &'a [u8],
116    pos: usize,
117}
118
119impl<'a> Reader<'a> {
120    pub fn new(view: &'a [u8]) -> Self {
121        Self { view, pos: 0 }
122    }
123
124    pub fn at(view: &'a [u8], offset: usize) -> Self {
125        Self { view, pos: offset }
126    }
127
128    fn require(&mut self, count: usize) -> Result<usize, IndexFormatError> {
129        let end = self.pos.checked_add(count);
130        match end {
131            Some(end) if end <= self.view.len() => {
132                let start = self.pos;
133                self.pos = end;
134                Ok(start)
135            }
136            _ => err("segment read past end (truncated or corrupt)"),
137        }
138    }
139
140    pub fn u32(&mut self) -> Result<u32, IndexFormatError> {
141        let start = self.require(4)?;
142        Ok(u32::from_le_bytes(
143            self.view[start..start + 4].try_into().expect("4 bytes"),
144        ))
145    }
146
147    pub fn u64(&mut self) -> Result<u64, IndexFormatError> {
148        let start = self.require(8)?;
149        Ok(u64::from_le_bytes(
150            self.view[start..start + 8].try_into().expect("8 bytes"),
151        ))
152    }
153
154    pub fn blob(&mut self) -> Result<&'a [u8], IndexFormatError> {
155        let length = self.u32()? as usize;
156        let start = self.require(length)?;
157        Ok(&self.view[start..start + length])
158    }
159
160    pub fn text(&mut self) -> Result<String, IndexFormatError> {
161        Ok(self.text_ref()?.to_string())
162    }
163
164    /// Bounds-checked UTF-8 text borrowed directly from the segment. Callers
165    /// must keep the value within the mapped reader's lifetime.
166    pub fn text_ref(&mut self) -> Result<&'a str, IndexFormatError> {
167        let raw = self.blob()?;
168        match std::str::from_utf8(raw) {
169            Ok(s) => Ok(s),
170            Err(_) => err("segment text is not valid UTF-8"),
171        }
172    }
173
174    pub fn opt_text(&mut self) -> Result<Option<String>, IndexFormatError> {
175        let start = self.require(1)?;
176        match self.view[start] {
177            0 => Ok(None),
178            1 => Ok(Some(self.text()?)),
179            flag => err(format!("bad optional flag: {flag}")),
180        }
181    }
182
183    pub fn text_list(&mut self) -> Result<Vec<String>, IndexFormatError> {
184        let count = self.u32()?;
185        let mut out = Vec::with_capacity(count.min(1 << 20) as usize);
186        for _ in 0..count {
187            out.push(self.text()?);
188        }
189        Ok(out)
190    }
191
192    pub fn u32_list(&mut self) -> Result<Vec<u32>, IndexFormatError> {
193        let count = self.u32()?;
194        let mut out = Vec::with_capacity(count.min(1 << 20) as usize);
195        for _ in 0..count {
196            out.push(self.u32()?);
197        }
198        Ok(out)
199    }
200}
201
202// ---------------------------------------------------------------------------
203// Framing
204// ---------------------------------------------------------------------------
205
206/// Frame a payload as a segment file's bytes: magic, version, length, payload.
207pub fn encode_segment(payload: &[u8]) -> Vec<u8> {
208    let mut out = Vec::with_capacity(HEADER_SIZE + payload.len());
209    out.extend_from_slice(SEGMENT_MAGIC);
210    out.extend_from_slice(&SEGMENT_FORMAT_VERSION.to_le_bytes());
211    out.extend_from_slice(&(payload.len() as u64).to_le_bytes());
212    out.extend_from_slice(payload);
213    out
214}
215
216/// Validate a mapped segment and return its payload slice (fail-closed).
217pub fn segment_payload(view: &[u8]) -> Result<&[u8], IndexFormatError> {
218    if view.len() < HEADER_SIZE {
219        return err("segment shorter than its header");
220    }
221    if &view[..8] != SEGMENT_MAGIC {
222        return err("bad segment magic (not an index-store segment)");
223    }
224    let version = u16::from_le_bytes(view[8..10].try_into().expect("2 bytes"));
225    if version != SEGMENT_FORMAT_VERSION {
226        return err(format!("unsupported segment format version: {version}"));
227    }
228    let payload_len = u64::from_le_bytes(view[10..18].try_into().expect("8 bytes"));
229    if (view.len() - HEADER_SIZE) as u64 != payload_len {
230        return err("segment length mismatch (truncated or corrupt)");
231    }
232    Ok(&view[HEADER_SIZE..])
233}
234
235/// Encode row blobs with a docid-indexed offset table for O(1) point access:
236/// `count(u32) | offsets(count × u64, relative to end of table) | rows`.
237pub fn write_indexed(rows: &[Vec<u8>]) -> Result<Vec<u8>, IndexFormatError> {
238    let mut writer = Writer::new();
239    writer.u32(rows.len() as u64)?;
240    let mut running: u64 = 0;
241    for row in rows {
242        writer.u64(running);
243        running += row.len() as u64;
244    }
245    for row in rows {
246        writer.raw(row);
247    }
248    Ok(writer.payload())
249}
250
251/// Reader over a `write_indexed` payload — random access by row index.
252pub struct IndexedSegment<'a> {
253    view: &'a [u8],
254    count: u32,
255    data_start: usize,
256}
257
258impl<'a> IndexedSegment<'a> {
259    pub fn new(view: &'a [u8]) -> Result<Self, IndexFormatError> {
260        let mut header = Reader::new(view);
261        let count = header.u32()?;
262        let data_start = 4usize.saturating_add((count as usize).saturating_mul(8));
263        if data_start > view.len() {
264            return err("indexed-segment offset table truncated");
265        }
266        Ok(Self {
267            view,
268            count,
269            data_start,
270        })
271    }
272
273    pub fn count(&self) -> u32 {
274        self.count
275    }
276
277    pub fn row(&self, index: u32) -> Result<Reader<'a>, IndexFormatError> {
278        if index >= self.count {
279            return err(format!("row index out of range: {index}"));
280        }
281        let table_at = 4 + 8 * index as usize;
282        let offset = u64::from_le_bytes(
283            self.view[table_at..table_at + 8].try_into().expect("8 bytes"),
284        );
285        let start = self.data_start.checked_add(offset as usize);
286        match start {
287            Some(start) if start <= self.view.len() => Ok(Reader::at(self.view, start)),
288            _ => err("indexed-segment row offset past end"),
289        }
290    }
291}
rac_engine/index_format.rs

rac_engine/
index_format.rs
