lumen/
container.rs

1//! The container: 32-byte file header, 32-byte chunk descriptors, the directory
2//! at the end of the file, and the 8-byte trailer ([`spec/02-file-structure.md`]).
3
4use crate::check::Check;
5use crate::error::{Error, Result};
6use crate::io::Reader;
7
8/// File magic, `LUMN`.
9pub const MAGIC: [u8; 4] = *b"LUMN";
10/// Trailer magic, `LEND`.
11pub const TRAILER_MAGIC: [u8; 4] = *b"LEND";
12/// Size of the fixed file header at offset 0.
13pub const FILE_HEADER_SIZE: usize = 32;
14/// Size of one chunk descriptor.
15pub const CHUNK_DESCRIPTOR_SIZE: usize = 32;
16/// Size of the directory trailer.
17pub const TRAILER_SIZE: usize = 8;
18
19/// `header.flags` bit 1: the file uses sector-based layer encoding.
20pub const FLAG_MULTI_SECTOR: u32 = 1 << 1;
21/// `header.flags` bit 3: an `AUTH` chunk is present and content chunks are sealed.
22pub const FLAG_ENCRYPTED: u32 = 1 << 3;
23/// Bits a conforming v1 writer must leave clear: 0, 2, 4 and 5-31.
24pub const FILE_HEADER_RESERVED_MASK: u32 = !(FLAG_MULTI_SECTOR | FLAG_ENCRYPTED);
25
26/// Chunk descriptor bit 4: this payload is sealed.
27pub const CHUNK_FLAG_ENCRYPTED: u32 = 1 << 4;
28
29/// CRC-32C (Castagnoli) as the container and the `LTBL`/trailer use it.
30pub fn crc32c(data: &[u8]) -> u32 {
31    crc32c::crc32c(data)
32}
33
34/// The check value the specification's generators self-test against.
35#[cfg(test)]
36const CRC32C_CHECK_VALUE: u32 = 0xE306_9283;
37
38#[cfg(test)]
39mod crc_tests {
40    use super::*;
41
42    #[test]
43    fn standard_check_value() {
44        assert_eq!(crc32c(b"123456789"), CRC32C_CHECK_VALUE);
45    }
46}
47
48/// A four-byte chunk type tag.
49#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
50pub struct ChunkType(pub [u8; 4]);
51
52impl ChunkType {
53    /// `HEAD` - display dimensions, layer count, encoder identity.
54    pub const HEAD: ChunkType = ChunkType(*b"HEAD");
55    /// `META` - print parameters as JSON.
56    pub const META: ChunkType = ChunkType(*b"META");
57    /// `PROF` - reusable named print profile.
58    pub const PROF: ChunkType = ChunkType(*b"PROF");
59    /// `AUTH` - encryption metadata.
60    pub const AUTH: ChunkType = ChunkType(*b"AUTH");
61    /// `LROV` - one (layer, sector)'s timing overrides.
62    pub const LROV: ChunkType = ChunkType(*b"LROV");
63    /// `PREV` - PNG preview image.
64    pub const PREV: ChunkType = ChunkType(*b"PREV");
65    /// `LTBL` - layer table, mapping (layer, sector) pairs to chunks.
66    pub const LTBL: ChunkType = ChunkType(*b"LTBL");
67    /// `ZDIC` - zstd dictionary shared by the `LAYR` frames.
68    pub const ZDIC: ChunkType = ChunkType(*b"ZDIC");
69    /// `LAYR` - one sector's mask data for one layer group, as a zstd frame.
70    pub const LAYR: ChunkType = ChunkType(*b"LAYR");
71    /// `LHAS` - per-layer hashes and Merkle root.
72    pub const LHAS: ChunkType = ChunkType(*b"LHAS");
73    /// `VOXL` - embedded scene, opaque to LUMEN.
74    pub const VOXL: ChunkType = ChunkType(*b"VOXL");
75    /// `EXTD` - vendor or future-standard extension.
76    pub const EXTD: ChunkType = ChunkType(*b"EXTD");
77
78    /// Every chunk type this specification defines.
79    pub const ALL: [ChunkType; 12] = [
80        ChunkType::HEAD,
81        ChunkType::META,
82        ChunkType::PROF,
83        ChunkType::AUTH,
84        ChunkType::LROV,
85        ChunkType::PREV,
86        ChunkType::LTBL,
87        ChunkType::ZDIC,
88        ChunkType::LAYR,
89        ChunkType::LHAS,
90        ChunkType::VOXL,
91        ChunkType::EXTD,
92    ];
93
94    /// The raw tag bytes.
95    pub fn as_bytes(&self) -> &[u8; 4] {
96        &self.0
97    }
98
99    /// Whether this is a chunk type v1 defines.
100    pub fn is_known(&self) -> bool {
101        ChunkType::ALL.contains(self)
102    }
103
104    /// Whether the payload carries a zstd frame, per the normative table in
105    /// [`spec/12-compression.md`] section 6.3.
106    ///
107    /// `LAYR` is excluded, and not because it is uncompressed: its payload is a
108    /// version field plus a frame the container does not itself decompress, in
109    /// the same way `EXTD`'s compression is per-extension and `PREV`'s is none.
110    /// `EXTD` is excluded too: the frame is opaque to this crate, so the caller
111    /// decides.
112    pub fn is_compressed(&self) -> bool {
113        matches!(
114            *self,
115            ChunkType::META | ChunkType::PROF | ChunkType::LROV | ChunkType::VOXL
116        )
117    }
118
119    /// The tag as a display string, with `HEAD` rendered as `HEAD`.
120    pub fn tag(&self) -> String {
121        let trimmed: Vec<u8> = self.0.iter().copied().take_while(|b| *b != 0).collect();
122        String::from_utf8_lossy(&trimmed).into_owned()
123    }
124}
125
126impl core::fmt::Display for ChunkType {
127    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
128        f.write_str(&self.tag())
129    }
130}
131
132/// The fixed 32-byte file header at offset 0.
133#[derive(Debug, Clone, Copy, PartialEq, Eq)]
134pub struct FileHeader {
135    /// Container format version. `1` for this specification.
136    pub version: u32,
137    /// Byte offset from the start of the file to the chunk directory.
138    pub dir_offset: u64,
139    /// Number of chunks, matching the directory entries.
140    pub chunk_count: u32,
141    /// Bitfield: bit 1 `MULTI_SECTOR`, bit 3 `ENCRYPTED`.
142    pub flags: u32,
143    /// Sum of uncompressed chunk sizes, or `0` when unknown.
144    pub total_uncompressed_size: u64,
145}
146
147impl FileHeader {
148    /// Parse the fixed header.
149    pub fn parse(buf: &[u8]) -> Result<FileHeader> {
150        let mut r = Reader::checked(buf, Check::HeaderMagic);
151        let magic = r.array::<4>()?;
152        if magic != MAGIC {
153            return Err(Error::new(
154                Check::HeaderMagic,
155                format!("expected LUMN, found {}", ChunkType(magic).tag()),
156            ));
157        }
158        let version = r.u32()?;
159        if version != 1 {
160            return Err(Error::new(
161                Check::HeaderVersion,
162                format!("unsupported container version {version}"),
163            ));
164        }
165        let dir_offset = r.u64()?;
166        let chunk_count = r.u32()?;
167        let flags = r.u32()?;
168        let total_uncompressed_size = r.u64()?;
169        Ok(FileHeader {
170            version,
171            dir_offset,
172            chunk_count,
173            flags,
174            total_uncompressed_size,
175        })
176    }
177
178    /// Serialize to exactly 32 bytes.
179    pub fn to_bytes(&self) -> [u8; FILE_HEADER_SIZE] {
180        let mut out = [0u8; FILE_HEADER_SIZE];
181        out[0..4].copy_from_slice(&MAGIC);
182        out[4..8].copy_from_slice(&self.version.to_le_bytes());
183        out[8..16].copy_from_slice(&self.dir_offset.to_le_bytes());
184        out[16..20].copy_from_slice(&self.chunk_count.to_le_bytes());
185        out[20..24].copy_from_slice(&self.flags.to_le_bytes());
186        out[24..32].copy_from_slice(&self.total_uncompressed_size.to_le_bytes());
187        out
188    }
189
190    /// Whether the file declares sector-based layer encoding.
191    pub fn multi_sector(&self) -> bool {
192        self.flags & FLAG_MULTI_SECTOR != 0
193    }
194
195    /// Whether the file declares an `AUTH` chunk and sealed content.
196    pub fn encrypted(&self) -> bool {
197        self.flags & FLAG_ENCRYPTED != 0
198    }
199
200    /// Flags a v1 writer must leave clear.
201    pub fn reserved_flags(&self) -> u32 {
202        self.flags & FILE_HEADER_RESERVED_MASK
203    }
204}
205
206/// One 32-byte chunk directory entry.
207#[derive(Debug, Clone, Copy, PartialEq, Eq)]
208pub struct ChunkDescriptor {
209    /// Four-letter chunk type tag.
210    pub chunk_type: ChunkType,
211    /// Absolute byte offset of the payload. `0` marks a null descriptor.
212    pub offset: u64,
213    /// Payload size after decompression.
214    pub size_uncompressed: u64,
215    /// Stored size including any AEAD framing, or `0` for a raw payload.
216    pub size_compressed: u64,
217    /// Chunk-specific flags: bit 4 is `ENCRYPTED`, and `PREV`/`EXTD` define more.
218    pub flags: u32,
219}
220
221impl ChunkDescriptor {
222    /// A descriptor occupying no space, as a writer pre-allocating directory
223    /// slots would leave it.
224    pub const NULL: ChunkDescriptor = ChunkDescriptor {
225        chunk_type: ChunkType([0, 0, 0, 0]),
226        offset: 0,
227        size_uncompressed: 0,
228        size_compressed: 0,
229        flags: 0,
230    };
231
232    /// Parse one descriptor.
233    pub fn parse(buf: &[u8]) -> Result<ChunkDescriptor> {
234        let mut r = Reader::checked(buf, Check::DirDescriptor);
235        let chunk_type = ChunkType(r.array::<4>()?);
236        let offset = r.u64()?;
237        let size_uncompressed = r.u64()?;
238        let size_compressed = r.u64()?;
239        let flags = r.u32()?;
240        Ok(ChunkDescriptor {
241            chunk_type,
242            offset,
243            size_uncompressed,
244            size_compressed,
245            flags,
246        })
247    }
248
249    /// Serialize to exactly 32 bytes.
250    pub fn to_bytes(&self) -> [u8; CHUNK_DESCRIPTOR_SIZE] {
251        let mut out = [0u8; CHUNK_DESCRIPTOR_SIZE];
252        out[0..4].copy_from_slice(&self.chunk_type.0);
253        out[4..12].copy_from_slice(&self.offset.to_le_bytes());
254        out[12..20].copy_from_slice(&self.size_uncompressed.to_le_bytes());
255        out[20..28].copy_from_slice(&self.size_compressed.to_le_bytes());
256        out[28..32].copy_from_slice(&self.flags.to_le_bytes());
257        out
258    }
259
260    /// Whether this slot holds no chunk.
261    pub fn is_null(&self) -> bool {
262        self.offset == 0
263    }
264
265    /// Whether this payload is sealed.
266    pub fn is_encrypted(&self) -> bool {
267        self.flags & CHUNK_FLAG_ENCRYPTED != 0
268    }
269
270    /// The on-disk byte length of this chunk: `size_uncompressed` for a raw
271    /// payload, `size_compressed` when the chunk is compressed or sealed.
272    ///
273    /// A `LAYR` chunk reads both ways round without ambiguity: its
274    /// `size_uncompressed` is the container's byte length - the version field
275    /// plus the frame - and `size_compressed` is zero while that frame is in the
276    /// clear and the same total once it is sealed.
277    pub fn stored_len(&self) -> u64 {
278        if self.size_compressed == 0 {
279            self.size_uncompressed
280        } else {
281            self.size_compressed
282        }
283    }
284
285    /// The half-open byte range this chunk occupies in the file.
286    pub fn extent(&self) -> (u64, u64) {
287        (self.offset, self.offset + self.stored_len())
288    }
289}
290
291/// Read and verify the directory trailer, returning its CRC-32C.
292pub fn parse_trailer(buf: &[u8]) -> Result<u32> {
293    if buf.len() < TRAILER_SIZE {
294        return Err(Error::new(
295            Check::TrailerMagic,
296            format!("file of {} bytes is too short for a trailer", buf.len()),
297        ));
298    }
299    let tail = &buf[buf.len() - TRAILER_SIZE..];
300    let magic = &tail[0..4];
301    if magic != TRAILER_MAGIC {
302        return Err(Error::new(
303            Check::TrailerMagic,
304            format!("expected LEND, found {}", ChunkType::from_tag_bytes(magic)),
305        ));
306    }
307    Ok(u32::from_le_bytes([tail[4], tail[5], tail[6], tail[7]]))
308}
309
310impl ChunkType {
311    fn from_tag_bytes(bytes: &[u8]) -> String {
312        let mut raw = [0u8; 4];
313        raw[..bytes.len().min(4)].copy_from_slice(&bytes[..bytes.len().min(4)]);
314        ChunkType(raw).tag()
315    }
316}
317
318/// Verify the trailer CRC-32C covers every preceding byte.
319pub fn verify_trailer_crc(buf: &[u8]) -> Result<()> {
320    let stored = parse_trailer(buf)?;
321    let computed = crc32c(&buf[..buf.len() - TRAILER_SIZE]);
322    if stored != computed {
323        return Err(Error::new(
324            Check::TrailerCrc32c,
325            format!("trailer CRC-32C {stored:#010x} does not match file bytes {computed:#010x}"),
326        ));
327    }
328    Ok(())
329}
330
331/// Parse the chunk directory.
332///
333/// Returns every descriptor in file order, including null ones; use
334/// [`Directory::entries`] to iterate only the present chunks.
335pub fn parse_directory(buf: &[u8], header: &FileHeader) -> Result<Directory> {
336    let dir_offset = header.dir_offset as usize;
337    let count = header.chunk_count as usize;
338    if dir_offset < FILE_HEADER_SIZE || dir_offset > buf.len() {
339        return Err(Error::new(
340            Check::HeaderDirOffset,
341            format!(
342                "directory offset {dir_offset} is outside the {} bytes of file",
343                buf.len()
344            ),
345        ));
346    }
347    let end = dir_offset
348        .checked_add(count * CHUNK_DESCRIPTOR_SIZE)
349        .ok_or_else(|| Error::new(Check::HeaderDirOffset, "directory length overflows"))?;
350    if end + TRAILER_SIZE > buf.len() {
351        return Err(Error::new(
352            Check::HeaderDirOffset,
353            format!("{count} descriptors at {dir_offset} do not fit before the trailer"),
354        ));
355    }
356    let mut descriptors = Vec::with_capacity(count);
357    let mut r = Reader::checked(&buf[dir_offset..end], Check::DirDescriptor);
358    for _ in 0..count {
359        descriptors.push(ChunkDescriptor::parse(r.bytes(CHUNK_DESCRIPTOR_SIZE)?)?);
360    }
361    Ok(Directory { descriptors })
362}
363
364/// The chunk directory: one descriptor per chunk, in file order.
365#[derive(Debug, Clone, PartialEq, Eq)]
366pub struct Directory {
367    /// Every descriptor, including null ones.
368    pub descriptors: Vec<ChunkDescriptor>,
369}
370
371impl Directory {
372    /// An empty directory.
373    pub fn new() -> Self {
374        Directory {
375            descriptors: Vec::new(),
376        }
377    }
378
379    /// Non-null chunks, in file order.
380    pub fn entries(&self) -> impl Iterator<Item = &ChunkDescriptor> {
381        self.descriptors.iter().filter(|d| !d.is_null())
382    }
383
384    /// Non-null chunks carrying `chunk_type`, in file order.
385    pub fn find_all(&self, chunk_type: ChunkType) -> impl Iterator<Item = &ChunkDescriptor> {
386        self.entries().filter(move |d| d.chunk_type == chunk_type)
387    }
388
389    /// The first non-null chunk of `chunk_type`.
390    ///
391    /// Where the specification says a chunk appears at most once (`PROF`,
392    /// `LROV`, `VOXL`, `ZDIC`), readers use the first.
393    pub fn find(&self, chunk_type: ChunkType) -> Option<&ChunkDescriptor> {
394        self.find_all(chunk_type).next()
395    }
396
397    /// Whether any non-null chunk carries `chunk_type`.
398    pub fn contains(&self, chunk_type: ChunkType) -> bool {
399        self.find(chunk_type).is_some()
400    }
401
402    /// The index of the first non-null descriptor, i.e. the first chunk.
403    pub fn first_entry_index(&self) -> Option<usize> {
404        self.descriptors.iter().position(|d| !d.is_null())
405    }
406
407    /// Push a descriptor.
408    pub fn push(&mut self, descriptor: ChunkDescriptor) {
409        self.descriptors.push(descriptor);
410    }
411
412    /// Serialize every descriptor, in order.
413    pub fn to_bytes(&self) -> Vec<u8> {
414        let mut out = Vec::with_capacity(self.descriptors.len() * CHUNK_DESCRIPTOR_SIZE);
415        for d in &self.descriptors {
416            out.extend_from_slice(&d.to_bytes());
417        }
418        out
419    }
420
421    /// The sum of every non-null chunk's `size_uncompressed`.
422    pub fn total_uncompressed_size(&self) -> u64 {
423        self.entries().map(|d| d.size_uncompressed).sum()
424    }
425}
426
427impl Default for Directory {
428    fn default() -> Self {
429        Directory::new()
430    }
431}
432
433/// The chunk type of the first chunk, or `None` if the directory is all null.
434pub fn first_chunk_type(directory: &Directory) -> Option<ChunkType> {
435    directory
436        .first_entry_index()
437        .map(|i| directory.descriptors[i].chunk_type)
438}
lumen/container.rs

lumen/
container.rs
