urna_format/sections/
mod.rs

1//! Section payload formats (v1).
2//!
3//! Each non-binary section starts with a 12-byte header:
4//!
5//! ```text
6//! [0..4)   u32 version  (LE) — currently 1
7//! [4..12)  u64 count    (LE) — number of entries
8//! ```
9//!
10//! Then a payload that depends on the section. Embeddings use a different
11//! shape (no per-entry header — dim/count come from the file header).
12//!
13//! All multi-byte integers are little-endian. Strings are raw UTF-8 bytes
14//! prefixed by a u32 length (no NUL terminators).
15
16pub mod blob;
17mod canonical;
18mod chunk_ids;
19mod codec;
20mod contract;
21pub mod graph;
22mod provenance;
23mod space_table;
24mod spans;
25
26/// `intpack` (encoding id 4) repack kinds. the kind byte leads the
27/// packed payload so the wire codec can rebuild the exact canonical
28/// bytes of the section it was packed from, keeping `content_hash`
29/// stable. canonical sections are never version-bumped; this is a wire
30/// encoding, not a payload-format change.
31pub const REPACK_KIND_CHUNK_IDS: u8 = 0;
32pub const REPACK_KIND_SPANS: u8 = 1;
33
34pub use blob::{
35    BLOB_DATA_PAYLOAD_VERSION, BLOB_REF_NONE, BLOB_REFS_PAYLOAD_VERSION,
36    BLOB_SPAN_OVERLAY_PAYLOAD_VERSION, BlobDataTable, BlobRefRecord, BlobSpanEntry,
37    decode_blob_data_table, decode_blob_refs, decode_blob_span_overlay, encode_blob_data,
38    encode_blob_refs, encode_blob_span_overlay,
39};
40pub use canonical::{decode_chunks_canonical, encode_chunks_canonical};
41pub use chunk_ids::{
42    decode_chunk_ids, decode_chunk_ids_intpack, encode_chunk_ids, encode_chunk_ids_intpack,
43};
44pub use contract::{SearchContract, decode_search_contract, encode_search_contract};
45pub use provenance::{decode_provenance, encode_provenance};
46pub use space_table::{
47    SPACE_DTYPE_F16, SPACE_DTYPE_F32, SPACE_DTYPE_I4, SPACE_DTYPE_I8, SPACE_TABLE_PAYLOAD_VERSION,
48    SpaceEntry, decode_space_table, encode_space_table,
49};
50pub use spans::{
51    OriginalSpan, decode_chunks_original_spans, decode_chunks_original_spans_intpack,
52    encode_chunks_original_spans, encode_chunks_original_spans_intpack,
53};
54
55/// decode a `txt_streams` payload (the full wire bytes, including the
56/// leading kind/version byte) back to the canonical `chunks_canonical`
57/// section payload. byte-identical to [`encode_chunks_canonical`], so
58/// `content_hash` is unchanged. dispatched by `encoding::decode_payload`
59/// for encoding id 10 (parallel to [`decode_intpack_repack`]).
60pub fn decode_txt_streams(bytes: &[u8]) -> crate::Result<Vec<u8>> {
61    crate::encoding::decode_txt_streams_payload(bytes)
62}
63
64/// decode an `intpack` repack payload (the full wire bytes, including the
65/// leading kind byte) back to the canonical section payload it was packed
66/// from. dispatched by `encoding::decode_payload` for encoding id 4.
67pub fn decode_intpack_repack(bytes: &[u8]) -> crate::Result<Vec<u8>> {
68    let (kind, rest) =
69        bytes
70            .split_first()
71            .ok_or_else(|| crate::error::UrnaError::MalformedSectionPayload {
72                section_id: 0,
73                reason: "intpack repack: empty payload".into(),
74            })?;
75    match *kind {
76        REPACK_KIND_CHUNK_IDS => decode_chunk_ids_intpack(rest),
77        REPACK_KIND_SPANS => decode_chunks_original_spans_intpack(rest),
78        other => Err(crate::error::UrnaError::MalformedSectionPayload {
79            section_id: 0,
80            reason: format!("intpack repack: unknown kind {}", other),
81        }),
82    }
83}
urna_format/sections/mod.rs

urna_format/sections/
mod.rs
