reddb_file/
native_index_artifact.rs

1//! Persisted native secondary-index artifact codecs (RDGA / RDFT / RDDP).
2//!
3//! These three server-defined payloads are written alongside native vector
4//! artifacts to describe graph adjacency, full-text postings, and document
5//! path/value projections. The engine owns tokenization, JSON path extraction,
6//! and the derived summary statistics; this module owns only the durable byte
7//! layout of each artifact.
8//!
9//! All three share a length-prefixed string encoding: a `u32` little-endian
10//! byte length followed by the UTF-8 bytes. All integers are little-endian.
11//! DO NOT change magic/order/width — these bytes live in existing artifacts.
12//!
13//! `RDDP` document `entity_id` is a fixed `u64` (the raw entity id), NOT a
14//! string — pinned by [`tests::rddp_entity_id_is_fixed_u64`].
15
16// ============================================================================
17// Shared helpers
18// ============================================================================
19
20/// Errors decoding a native index artifact.
21#[derive(Debug, Clone, PartialEq, Eq)]
22pub enum NativeArtifactFrameError {
23    TooShort {
24        artifact: &'static str,
25    },
26    InvalidMagic {
27        artifact: &'static str,
28    },
29    Truncated {
30        artifact: &'static str,
31        offset: usize,
32        reason: &'static str,
33    },
34    InvalidUtf8 {
35        offset: usize,
36    },
37    EntryCountMismatch {
38        expected: u64,
39        seen: u64,
40    },
41}
42
43impl std::fmt::Display for NativeArtifactFrameError {
44    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
45        match self {
46            Self::TooShort { artifact } => write!(f, "invalid {artifact} artifact"),
47            Self::InvalidMagic { artifact } => write!(f, "invalid {artifact} artifact"),
48            Self::Truncated {
49                artifact,
50                offset,
51                reason,
52            } => write!(
53                f,
54                "truncated {artifact} artifact at offset {offset}: {reason}"
55            ),
56            Self::InvalidUtf8 { offset } => {
57                write!(f, "invalid utf-8 in native artifact at offset {offset}")
58            }
59            Self::EntryCountMismatch { expected, seen } => write!(
60                f,
61                "document path/value artifact entry count mismatch: expected {expected}, saw {seen}"
62            ),
63        }
64    }
65}
66
67impl std::error::Error for NativeArtifactFrameError {}
68
69fn push_string(buf: &mut Vec<u8>, value: &str) {
70    let bytes = value.as_bytes();
71    buf.extend_from_slice(&(bytes.len() as u32).to_le_bytes());
72    buf.extend_from_slice(bytes);
73}
74
75fn read_string(
76    bytes: &[u8],
77    pos: &mut usize,
78    artifact: &'static str,
79) -> Result<String, NativeArtifactFrameError> {
80    let len = read_u32(bytes, pos, artifact, "string length")?;
81    let len = usize::try_from(len).map_err(|_| NativeArtifactFrameError::Truncated {
82        artifact,
83        offset: *pos,
84        reason: "string length",
85    })?;
86    let end = (*pos)
87        .checked_add(len)
88        .ok_or(NativeArtifactFrameError::Truncated {
89            artifact,
90            offset: *pos,
91            reason: "string content",
92        })?;
93    if end > bytes.len() {
94        return Err(NativeArtifactFrameError::Truncated {
95            artifact,
96            offset: *pos,
97            reason: "string content",
98        });
99    }
100    let value = std::str::from_utf8(&bytes[*pos..end])
101        .map_err(|_| NativeArtifactFrameError::InvalidUtf8 { offset: *pos })?
102        .to_string();
103    *pos = end;
104    Ok(value)
105}
106
107fn read_u32(
108    bytes: &[u8],
109    pos: &mut usize,
110    artifact: &'static str,
111    reason: &'static str,
112) -> Result<u32, NativeArtifactFrameError> {
113    if *pos + 4 > bytes.len() {
114        return Err(NativeArtifactFrameError::Truncated {
115            artifact,
116            offset: *pos,
117            reason,
118        });
119    }
120    let value = u32::from_le_bytes(
121        bytes[*pos..*pos + 4]
122            .try_into()
123            .expect("u32 length checked"),
124    );
125    *pos += 4;
126    Ok(value)
127}
128
129fn read_u64(
130    bytes: &[u8],
131    pos: &mut usize,
132    artifact: &'static str,
133    reason: &'static str,
134) -> Result<u64, NativeArtifactFrameError> {
135    if *pos + 8 > bytes.len() {
136        return Err(NativeArtifactFrameError::Truncated {
137            artifact,
138            offset: *pos,
139            reason,
140        });
141    }
142    let value = u64::from_le_bytes(
143        bytes[*pos..*pos + 8]
144            .try_into()
145            .expect("u64 length checked"),
146    );
147    *pos += 8;
148    Ok(value)
149}
150
151fn read_f32(
152    bytes: &[u8],
153    pos: &mut usize,
154    artifact: &'static str,
155    reason: &'static str,
156) -> Result<f32, NativeArtifactFrameError> {
157    if *pos + 4 > bytes.len() {
158        return Err(NativeArtifactFrameError::Truncated {
159            artifact,
160            offset: *pos,
161            reason,
162        });
163    }
164    let value = f32::from_le_bytes(
165        bytes[*pos..*pos + 4]
166            .try_into()
167            .expect("f32 length checked"),
168    );
169    *pos += 4;
170    Ok(value)
171}
172
173// ============================================================================
174// RDGA — graph adjacency
175// ============================================================================
176
177/// Magic prefix for a graph-adjacency artifact.
178pub const NATIVE_GRAPH_ADJACENCY_MAGIC: [u8; 4] = *b"RDGA";
179
180/// One decoded graph edge.
181#[derive(Debug, Clone, PartialEq)]
182pub struct NativeGraphEdge {
183    /// Raw entity id of the edge (`EntityId::raw()` in the engine).
184    pub edge_id: u64,
185    pub from_node: String,
186    pub to_node: String,
187    pub label: String,
188    pub weight: f32,
189}
190
191/// A decoded graph-adjacency (`RDGA`) artifact.
192#[derive(Debug, Clone, PartialEq)]
193pub struct NativeGraphAdjacencyFrame {
194    pub edges: Vec<NativeGraphEdge>,
195}
196
197/// Serialize a graph-adjacency artifact.
198pub fn encode_native_graph_adjacency_frame(frame: &NativeGraphAdjacencyFrame) -> Vec<u8> {
199    let mut data = Vec::with_capacity(32 + frame.edges.len() * 48);
200    data.extend_from_slice(&NATIVE_GRAPH_ADJACENCY_MAGIC);
201    data.extend_from_slice(&(frame.edges.len() as u32).to_le_bytes());
202    for edge in &frame.edges {
203        data.extend_from_slice(&edge.edge_id.to_le_bytes());
204        push_string(&mut data, &edge.from_node);
205        push_string(&mut data, &edge.to_node);
206        push_string(&mut data, &edge.label);
207        data.extend_from_slice(&edge.weight.to_le_bytes());
208    }
209    data
210}
211
212/// Deserialize a graph-adjacency artifact.
213pub fn decode_native_graph_adjacency_frame(
214    bytes: &[u8],
215) -> Result<NativeGraphAdjacencyFrame, NativeArtifactFrameError> {
216    const ARTIFACT: &str = "graph adjacency";
217    if bytes.len() < 8 || bytes[0..4] != NATIVE_GRAPH_ADJACENCY_MAGIC {
218        return Err(NativeArtifactFrameError::InvalidMagic { artifact: ARTIFACT });
219    }
220    let mut pos = 4usize;
221    let edge_count = read_u32(bytes, &mut pos, ARTIFACT, "edge count")?;
222    let mut edges = Vec::new();
223    for _ in 0..edge_count {
224        let edge_id = read_u64(bytes, &mut pos, ARTIFACT, "edge id")?;
225        let from_node = read_string(bytes, &mut pos, ARTIFACT)?;
226        let to_node = read_string(bytes, &mut pos, ARTIFACT)?;
227        let label = read_string(bytes, &mut pos, ARTIFACT)?;
228        let weight = read_f32(bytes, &mut pos, ARTIFACT, "edge weight")?;
229        edges.push(NativeGraphEdge {
230            edge_id,
231            from_node,
232            to_node,
233            label,
234            weight,
235        });
236    }
237    Ok(NativeGraphAdjacencyFrame { edges })
238}
239
240// ============================================================================
241// RDFT — full-text postings
242// ============================================================================
243
244/// Magic prefix for a full-text artifact.
245pub const NATIVE_FULLTEXT_MAGIC: [u8; 4] = *b"RDFT";
246
247/// One posting entry: an entity and its term frequency.
248#[derive(Debug, Clone, PartialEq, Eq)]
249pub struct NativeFulltextPosting {
250    /// Raw entity id (`EntityId::raw()` in the engine).
251    pub entity_id: u64,
252    pub term_count: u32,
253}
254
255/// A term and its posting list.
256#[derive(Debug, Clone, PartialEq, Eq)]
257pub struct NativeFulltextTerm {
258    pub term: String,
259    pub postings: Vec<NativeFulltextPosting>,
260}
261
262/// A decoded full-text (`RDFT`) artifact.
263#[derive(Debug, Clone, PartialEq, Eq)]
264pub struct NativeFulltextFrame {
265    pub collection: String,
266    /// Number of source documents (independent of the posting lists).
267    pub doc_count: u32,
268    pub terms: Vec<NativeFulltextTerm>,
269}
270
271/// Serialize a full-text artifact.
272pub fn encode_native_fulltext_frame(frame: &NativeFulltextFrame) -> Vec<u8> {
273    let mut data = Vec::with_capacity(64 + frame.terms.len() * 32);
274    data.extend_from_slice(&NATIVE_FULLTEXT_MAGIC);
275    push_string(&mut data, &frame.collection);
276    data.extend_from_slice(&frame.doc_count.to_le_bytes());
277    data.extend_from_slice(&(frame.terms.len() as u32).to_le_bytes());
278    for term in &frame.terms {
279        push_string(&mut data, &term.term);
280        data.extend_from_slice(&(term.postings.len() as u32).to_le_bytes());
281        for posting in &term.postings {
282            data.extend_from_slice(&posting.entity_id.to_le_bytes());
283            data.extend_from_slice(&posting.term_count.to_le_bytes());
284        }
285    }
286    data
287}
288
289/// Deserialize a full-text artifact.
290pub fn decode_native_fulltext_frame(
291    bytes: &[u8],
292) -> Result<NativeFulltextFrame, NativeArtifactFrameError> {
293    const ARTIFACT: &str = "fulltext";
294    if bytes.len() < 12 || bytes[0..4] != NATIVE_FULLTEXT_MAGIC {
295        return Err(NativeArtifactFrameError::InvalidMagic { artifact: ARTIFACT });
296    }
297    let mut pos = 4usize;
298    let collection = read_string(bytes, &mut pos, ARTIFACT)?;
299    let doc_count = read_u32(bytes, &mut pos, ARTIFACT, "doc count")?;
300    let term_count = read_u32(bytes, &mut pos, ARTIFACT, "term count")?;
301    let mut terms = Vec::new();
302    for _ in 0..term_count {
303        let term = read_string(bytes, &mut pos, ARTIFACT)?;
304        let entry_count = read_u32(bytes, &mut pos, ARTIFACT, "posting count")?;
305        let mut postings = Vec::new();
306        for _ in 0..entry_count {
307            let entity_id = read_u64(bytes, &mut pos, ARTIFACT, "posting entity id")?;
308            let term_count = read_u32(bytes, &mut pos, ARTIFACT, "posting term count")?;
309            postings.push(NativeFulltextPosting {
310                entity_id,
311                term_count,
312            });
313        }
314        terms.push(NativeFulltextTerm { term, postings });
315    }
316    Ok(NativeFulltextFrame {
317        collection,
318        doc_count,
319        terms,
320    })
321}
322
323// ============================================================================
324// RDDP — document path/value projection
325// ============================================================================
326
327/// Magic prefix for a document path/value artifact.
328pub const NATIVE_DOC_PATHVALUE_MAGIC: [u8; 4] = *b"RDDP";
329
330/// One path/value pair extracted from a document.
331#[derive(Debug, Clone, PartialEq, Eq)]
332pub struct NativeDocPathValueEntry {
333    pub path: String,
334    pub value: String,
335}
336
337/// One document and its extracted path/value entries.
338#[derive(Debug, Clone, PartialEq, Eq)]
339pub struct NativeDocPathValue {
340    /// Raw entity id — a fixed `u64`, NOT a string.
341    pub entity_id: u64,
342    pub entries: Vec<NativeDocPathValueEntry>,
343}
344
345/// A decoded document path/value (`RDDP`) artifact.
346#[derive(Debug, Clone, PartialEq, Eq)]
347pub struct NativeDocPathValueFrame {
348    pub collection: String,
349    pub docs: Vec<NativeDocPathValue>,
350}
351
352impl NativeDocPathValueFrame {
353    /// Total path/value entries across all documents — the `total_entries`
354    /// header word.
355    fn total_entries(&self) -> usize {
356        self.docs.iter().map(|doc| doc.entries.len()).sum()
357    }
358}
359
360/// Serialize a document path/value artifact.
361pub fn encode_native_doc_pathvalue_frame(frame: &NativeDocPathValueFrame) -> Vec<u8> {
362    let total_entries = frame.total_entries();
363    let mut data = Vec::with_capacity(64 + total_entries * 48);
364    data.extend_from_slice(&NATIVE_DOC_PATHVALUE_MAGIC);
365    push_string(&mut data, &frame.collection);
366    data.extend_from_slice(&(frame.docs.len() as u32).to_le_bytes());
367    data.extend_from_slice(&(total_entries as u32).to_le_bytes());
368    for doc in &frame.docs {
369        data.extend_from_slice(&doc.entity_id.to_le_bytes());
370        data.extend_from_slice(&(doc.entries.len() as u32).to_le_bytes());
371        for entry in &doc.entries {
372            push_string(&mut data, &entry.path);
373            push_string(&mut data, &entry.value);
374        }
375    }
376    data
377}
378
379/// Deserialize a document path/value artifact. Validates the on-disk
380/// `total_entries` header word against the documents actually parsed.
381pub fn decode_native_doc_pathvalue_frame(
382    bytes: &[u8],
383) -> Result<NativeDocPathValueFrame, NativeArtifactFrameError> {
384    const ARTIFACT: &str = "document path/value";
385    if bytes.len() < 12 || bytes[0..4] != NATIVE_DOC_PATHVALUE_MAGIC {
386        return Err(NativeArtifactFrameError::InvalidMagic { artifact: ARTIFACT });
387    }
388    let mut pos = 4usize;
389    let collection = read_string(bytes, &mut pos, ARTIFACT)?;
390    let doc_count = read_u32(bytes, &mut pos, ARTIFACT, "doc count")?;
391    let total_entries = read_u32(bytes, &mut pos, ARTIFACT, "total entries")? as u64;
392    let mut docs = Vec::new();
393    let mut seen_entries = 0u64;
394    for _ in 0..doc_count {
395        let entity_id = read_u64(bytes, &mut pos, ARTIFACT, "document entity id")?;
396        let entry_count = read_u32(bytes, &mut pos, ARTIFACT, "document entry count")?;
397        let mut entries = Vec::new();
398        for _ in 0..entry_count {
399            let path = read_string(bytes, &mut pos, ARTIFACT)?;
400            let value = read_string(bytes, &mut pos, ARTIFACT)?;
401            entries.push(NativeDocPathValueEntry { path, value });
402            seen_entries += 1;
403        }
404        docs.push(NativeDocPathValue { entity_id, entries });
405    }
406    if seen_entries != total_entries {
407        return Err(NativeArtifactFrameError::EntryCountMismatch {
408            expected: total_entries,
409            seen: seen_entries,
410        });
411    }
412    Ok(NativeDocPathValueFrame { collection, docs })
413}
414
415#[cfg(test)]
416mod tests {
417    use super::*;
418
419    #[test]
420    fn rdga_round_trips_and_pins_layout() {
421        let frame = NativeGraphAdjacencyFrame {
422            edges: vec![
423                NativeGraphEdge {
424                    edge_id: 42,
425                    from_node: "a".into(),
426                    to_node: "b".into(),
427                    label: "knows".into(),
428                    weight: 1.5,
429                },
430                NativeGraphEdge {
431                    edge_id: 7,
432                    from_node: "b".into(),
433                    to_node: "c".into(),
434                    label: "likes".into(),
435                    weight: -0.25,
436                },
437            ],
438        };
439        let encoded = encode_native_graph_adjacency_frame(&frame);
440        assert_eq!(&encoded[0..4], b"RDGA");
441        assert_eq!(&encoded[4..8], &2u32.to_le_bytes());
442        // First edge id is a fixed u64 immediately after the count.
443        assert_eq!(&encoded[8..16], &42u64.to_le_bytes());
444        let decoded = decode_native_graph_adjacency_frame(&encoded).unwrap();
445        assert_eq!(decoded, frame);
446        assert_eq!(encode_native_graph_adjacency_frame(&decoded), encoded);
447    }
448
449    #[test]
450    fn rdft_round_trips_and_pins_layout() {
451        let frame = NativeFulltextFrame {
452            collection: "docs".into(),
453            doc_count: 3,
454            terms: vec![
455                NativeFulltextTerm {
456                    term: "alpha".into(),
457                    postings: vec![
458                        NativeFulltextPosting {
459                            entity_id: 100,
460                            term_count: 2,
461                        },
462                        NativeFulltextPosting {
463                            entity_id: 101,
464                            term_count: 1,
465                        },
466                    ],
467                },
468                NativeFulltextTerm {
469                    term: "beta".into(),
470                    postings: vec![NativeFulltextPosting {
471                        entity_id: 100,
472                        term_count: 5,
473                    }],
474                },
475            ],
476        };
477        let encoded = encode_native_fulltext_frame(&frame);
478        assert_eq!(&encoded[0..4], b"RDFT");
479        let decoded = decode_native_fulltext_frame(&encoded).unwrap();
480        assert_eq!(decoded, frame);
481        assert_eq!(encode_native_fulltext_frame(&decoded), encoded);
482    }
483
484    #[test]
485    fn rddp_round_trips_and_validates_total() {
486        let frame = NativeDocPathValueFrame {
487            collection: "docs".into(),
488            docs: vec![
489                NativeDocPathValue {
490                    entity_id: 9,
491                    entries: vec![
492                        NativeDocPathValueEntry {
493                            path: "a.b".into(),
494                            value: "1".into(),
495                        },
496                        NativeDocPathValueEntry {
497                            path: "a.c".into(),
498                            value: "2".into(),
499                        },
500                    ],
501                },
502                NativeDocPathValue {
503                    entity_id: 10,
504                    entries: vec![NativeDocPathValueEntry {
505                        path: "x".into(),
506                        value: "y".into(),
507                    }],
508                },
509            ],
510        };
511        let encoded = encode_native_doc_pathvalue_frame(&frame);
512        assert_eq!(&encoded[0..4], b"RDDP");
513        let decoded = decode_native_doc_pathvalue_frame(&encoded).unwrap();
514        assert_eq!(decoded, frame);
515        assert_eq!(encode_native_doc_pathvalue_frame(&decoded), encoded);
516    }
517
518    #[test]
519    fn rddp_entity_id_is_fixed_u64() {
520        // A single document with no entries: after "RDDP", a 4-byte length-0
521        // collection string, doc_count=1, total_entries=0, the next 8 bytes are
522        // the entity id as a little-endian u64 — NOT a length-prefixed string.
523        let frame = NativeDocPathValueFrame {
524            collection: String::new(),
525            docs: vec![NativeDocPathValue {
526                entity_id: 0xDEAD_BEEF_CAFE_F00D,
527                entries: vec![],
528            }],
529        };
530        let encoded = encode_native_doc_pathvalue_frame(&frame);
531        // 4 magic + 4 collection-len(0) + 4 doc_count + 4 total_entries = 16
532        let id_off = 16;
533        assert_eq!(
534            &encoded[id_off..id_off + 8],
535            &0xDEAD_BEEF_CAFE_F00Du64.to_le_bytes()
536        );
537        let decoded = decode_native_doc_pathvalue_frame(&encoded).unwrap();
538        assert_eq!(decoded.docs[0].entity_id, 0xDEAD_BEEF_CAFE_F00D);
539    }
540
541    #[test]
542    fn native_artifacts_reject_bad_magic() {
543        assert!(matches!(
544            decode_native_graph_adjacency_frame(&[0u8; 8]),
545            Err(NativeArtifactFrameError::InvalidMagic { .. })
546        ));
547        assert!(matches!(
548            decode_native_fulltext_frame(&[0u8; 12]),
549            Err(NativeArtifactFrameError::InvalidMagic { .. })
550        ));
551        assert!(matches!(
552            decode_native_doc_pathvalue_frame(&[0u8; 12]),
553            Err(NativeArtifactFrameError::InvalidMagic { .. })
554        ));
555    }
556
557    #[test]
558    fn native_artifacts_do_not_preallocate_untrusted_counts() {
559        let mut encoded = Vec::new();
560        encoded.extend_from_slice(&NATIVE_GRAPH_ADJACENCY_MAGIC);
561        encoded.extend_from_slice(&u32::MAX.to_le_bytes());
562
563        assert!(matches!(
564            decode_native_graph_adjacency_frame(&encoded),
565            Err(NativeArtifactFrameError::Truncated { .. })
566        ));
567    }
568
569    #[test]
570    fn rddp_rejects_total_entries_mismatch() {
571        let frame = NativeDocPathValueFrame {
572            collection: "c".into(),
573            docs: vec![NativeDocPathValue {
574                entity_id: 1,
575                entries: vec![NativeDocPathValueEntry {
576                    path: "p".into(),
577                    value: "v".into(),
578                }],
579            }],
580        };
581        let mut encoded = encode_native_doc_pathvalue_frame(&frame);
582        // Corrupt the total_entries word (offset 4 magic + 4 collection-len + 1
583        // collection byte + 4 doc_count = 13).
584        let total_off = 4 + 4 + frame.collection.len() + 4;
585        encoded[total_off..total_off + 4].copy_from_slice(&9u32.to_le_bytes());
586        assert!(matches!(
587            decode_native_doc_pathvalue_frame(&encoded),
588            Err(NativeArtifactFrameError::EntryCountMismatch { .. })
589        ));
590    }
591}
reddb_file/native_index_artifact.rs

reddb_file/
native_index_artifact.rs
