urna_format/sections/blob/
data.rs

1//! blob_data (0x17): inlined media bytes for the self-contained twin of
2//! the 0x14 catalog.
3//!
4//! OPTIONAL and EXCLUDED from content_hash. The payload opens with an
5//! offset table PARALLEL to the blob_refs (0x14) record order - entry i
6//! describes where record i's bytes live - followed by the concatenated
7//! blob bytes. A record kept out-of-line (`inlined = false`) has a
8//! (0, 0) table entry. Offsets are relative to the first data byte, so
9//! the table can be decoded alone and the heavy bytes sliced lazily off
10//! the mmap; the section is never copied whole.
11//!
12//! wire encoding: `raw` (media bytes are already codec-compressed).
13//! all integers le.
14
15use crate::bytes::{le_u32, le_u64};
16use crate::error::UrnaError;
17use crate::layout::SECTION_BLOB_DATA;
18
19pub const BLOB_DATA_PAYLOAD_VERSION: u32 = 1;
20
21/// fixed prelude: u32 version + u64 entry count.
22const HEADER_SIZE: usize = 12;
23/// per-entry cost in the offset table: u64 offset + u64 len.
24const ENTRY_SIZE: usize = 16;
25
26/// decoded 0x17 offset table. `entries[i]` is `(offset, len)` of record
27/// i's bytes RELATIVE to `data_start`, which is the byte position of the
28/// first data byte within the section payload.
29#[derive(Clone, Debug, PartialEq, Eq)]
30pub struct BlobDataTable {
31    pub entries: Vec<(u64, u64)>,
32    pub data_start: usize,
33}
34
35fn malformed(reason: impl Into<String>) -> UrnaError {
36    UrnaError::MalformedSectionPayload {
37        section_id: SECTION_BLOB_DATA,
38        reason: reason.into(),
39    }
40}
41
42/// encode the 0x17 payload from per-record byte slices; `None` marks a
43/// record that stays out-of-line. deterministic: same blobs in the same
44/// order, same bytes.
45pub fn encode_blob_data(blobs: &[Option<&[u8]>]) -> Result<Vec<u8>, UrnaError> {
46    let data_len: usize = blobs.iter().flatten().map(|b| b.len()).sum();
47    let mut out = Vec::with_capacity(HEADER_SIZE + blobs.len() * ENTRY_SIZE + data_len);
48    out.extend_from_slice(&BLOB_DATA_PAYLOAD_VERSION.to_le_bytes());
49    out.extend_from_slice(&(blobs.len() as u64).to_le_bytes());
50    let mut offset = 0u64;
51    for b in blobs {
52        match b {
53            Some(bytes) => {
54                out.extend_from_slice(&offset.to_le_bytes());
55                out.extend_from_slice(&(bytes.len() as u64).to_le_bytes());
56                offset += bytes.len() as u64;
57            }
58            None => out.extend_from_slice(&[0u8; ENTRY_SIZE]),
59        }
60    }
61    for b in blobs.iter().flatten() {
62        out.extend_from_slice(b);
63    }
64    Ok(out)
65}
66
67/// decode ONLY the offset table off the section payload, bounds-checking
68/// every entry against the physical data region so a later lazy slice can
69/// never read past the section. typed errors, never panics.
70pub fn decode_blob_data_table(bytes: &[u8]) -> Result<BlobDataTable, UrnaError> {
71    if bytes.len() < HEADER_SIZE {
72        return Err(malformed("blob_data: payload shorter than header"));
73    }
74    let version = le_u32(&bytes[0..4])?;
75    if version != BLOB_DATA_PAYLOAD_VERSION {
76        return Err(UrnaError::UnsupportedSectionVersion {
77            section_id: SECTION_BLOB_DATA,
78            version,
79        });
80    }
81    let n = le_u64(&bytes[4..12])? as usize;
82    // bound the claim against the physical payload before allocating.
83    if n > (bytes.len() - HEADER_SIZE) / ENTRY_SIZE {
84        return Err(malformed("blob_data: entry count exceeds payload"));
85    }
86    let data_start = HEADER_SIZE + n * ENTRY_SIZE;
87    let data_len = (bytes.len() - data_start) as u64;
88    let mut entries = Vec::with_capacity(n);
89    for i in 0..n {
90        let at = HEADER_SIZE + i * ENTRY_SIZE;
91        let offset = le_u64(&bytes[at..at + 8])?;
92        let len = le_u64(&bytes[at + 8..at + 16])?;
93        let end = offset
94            .checked_add(len)
95            .ok_or_else(|| malformed(format!("blob_data: entry {} offset overflow", i)))?;
96        if end > data_len {
97            return Err(malformed(format!(
98                "blob_data: entry {} spans past payload ({} > {})",
99                i, end, data_len
100            )));
101        }
102        entries.push((offset, len));
103    }
104    Ok(BlobDataTable {
105        entries,
106        data_start,
107    })
108}
109
110#[cfg(test)]
111mod tests {
112    use super::*;
113
114    #[test]
115    fn roundtrip_with_gaps() {
116        let a = b"av1-shard-bytes".as_slice();
117        let b = b"x".as_slice();
118        let payload = encode_blob_data(&[Some(a), None, Some(b)]).unwrap();
119        let table = decode_blob_data_table(&payload).unwrap();
120        assert_eq!(table.entries, vec![(0, 15), (0, 0), (15, 1)]);
121        let data = &payload[table.data_start..];
122        assert_eq!(&data[0..15], a);
123        assert_eq!(&data[15..16], b);
124    }
125
126    #[test]
127    fn empty_table_roundtrips() {
128        let payload = encode_blob_data(&[]).unwrap();
129        let table = decode_blob_data_table(&payload).unwrap();
130        assert!(table.entries.is_empty());
131        assert_eq!(table.data_start, payload.len());
132    }
133
134    #[test]
135    fn hostile_count_is_rejected() {
136        let mut payload = encode_blob_data(&[Some(b"abc".as_slice())]).unwrap();
137        payload[4..12].copy_from_slice(&u64::MAX.to_le_bytes());
138        assert!(decode_blob_data_table(&payload).is_err());
139    }
140
141    #[test]
142    fn entry_past_payload_is_rejected() {
143        let mut payload = encode_blob_data(&[Some(b"abc".as_slice())]).unwrap();
144        // inflate the entry length past the physical data region.
145        payload[20..28].copy_from_slice(&1000u64.to_le_bytes());
146        assert!(decode_blob_data_table(&payload).is_err());
147    }
148
149    #[test]
150    fn bad_version_is_typed() {
151        let mut payload = encode_blob_data(&[]).unwrap();
152        payload[0..4].copy_from_slice(&9u32.to_le_bytes());
153        assert!(matches!(
154            decode_blob_data_table(&payload),
155            Err(UrnaError::UnsupportedSectionVersion { .. })
156        ));
157    }
158}
urna_format/sections/blob/data.rs

urna_format/sections/blob/
data.rs
