iris_format/container.rs
1//! Reading a container that is all in memory.
2//!
3//! This is the untrusted path. Everything behind it is written on the assumption that the bytes
4//! were produced by somebody who wants them to be trouble. The rules it follows are short enough to
5//! check by reading:
6//!
7//! - no indexing that is not bounds checked first, and no `unsafe` at all
8//! - no arithmetic on a length from the file that is not `checked_` or `saturating_`
9//! - nothing is allocated in proportion to a number read out of the file, only in proportion to
10//! bytes that are actually there
11//!
12//! The third rule is the one that gets skipped in hand written parsers, and it is the one that
13//! turns a sixty byte file into an out of memory kill. There is no count field anywhere in the
14//! footer for exactly this reason: the number of sections is however many section records the
15//! footer actually contains, so a liar has to pay for every one.
16//!
17//! The parsing itself is in [`Directory`], because a host that cannot hold the file needs
18//! exactly the same checks on exactly the same bytes. A [`Container`] is a [`Directory`] plus the
19//! payload it describes, and the only thing it adds is the ability to hand a section's bytes over.
20
21use crate::digest::Digest;
22use crate::directory::{Directory, Placement};
23use crate::error::{Error, Result};
24use crate::layout::{HEADER_SIZE, MAGIC, MIN_SIZE};
25use crate::meta::{Dataset, DecoderRef, Schema, Section};
26
27/// The fixed fields at the front of a container.
28#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
29pub struct FileHeader {
30 /// The format major version.
31 pub major: u16,
32 /// The format minor version.
33 pub minor: u16,
34}
35
36/// A parsed container, borrowing the bytes it was parsed from.
37///
38/// Parsing does not copy the payload and does not read it. A container of a hundred gigabytes
39/// parses in the time it takes to hash a footer, and the sections are read when somebody asks for
40/// them.
41#[derive(Clone, Debug)]
42pub struct Container<'a> {
43 bytes: &'a [u8],
44 directory: Directory<'a>,
45}
46
47impl<'a> Container<'a> {
48 /// Parses a container and checks that the footer is the footer that was written.
49 ///
50 /// The root digest covers the header and the footer, so a container that parses has metadata
51 /// nobody has edited since it was written. Section contents are not read here. Call
52 /// [`Container::verify`] for that, which is the expensive one and is a separate decision.
53 ///
54 /// # Errors
55 ///
56 /// Returns [`Error`] describing the first thing that was wrong. It never panics, whatever the
57 /// input is, and there is a fuzz target that exists to keep that true.
58 pub fn parse(bytes: &'a [u8]) -> Result<Self> {
59 Self::open(bytes, Directory::parse)
60 }
61
62 /// Parses a container without checking the root digest.
63 ///
64 /// This exists for the fuzzer. Checking the digest first would mean essentially every generated
65 /// input is rejected in the trailer, and the parser behind it would never be reached, which is
66 /// the part that needs the fuzzing. It is public because the fuzz target lives outside this
67 /// crate, and it is named at length so that nobody reaches for it by accident.
68 ///
69 /// # Errors
70 ///
71 /// The same as [`Container::parse`], minus the digest mismatch.
72 pub fn parse_without_root_digest(bytes: &'a [u8]) -> Result<Self> {
73 Self::open(bytes, Directory::parse_without_root_digest)
74 }
75
76 /// Finds the three ranges the metadata lives in and hands them to one of the two parsers.
77 ///
78 /// The size and magic checks are here rather than in [`Directory`] because they are the
79 /// questions a host asks about a file, and a host reading through a window answers them from
80 /// the file length and the header range instead.
81 fn open(
82 bytes: &'a [u8],
83 parse: fn(&[u8], &'a [u8], Placement) -> Result<Directory<'a>>,
84 ) -> Result<Self> {
85 if bytes.len() < MIN_SIZE {
86 let head = &bytes[..bytes.len().min(MAGIC.len())];
87 if !MAGIC.starts_with(head) {
88 return Err(Error::NotAContainer {
89 found: head.to_vec(),
90 expected: MAGIC.to_vec(),
91 });
92 }
93 return Err(Error::Truncated {
94 what: "the container",
95 needed: MIN_SIZE as u64,
96 available: bytes.len() as u64,
97 });
98 }
99
100 let file_len = bytes.len() as u64;
101 let trailer_at =
102 usize::try_from(Placement::trailer_at(file_len)?).map_err(|_| Error::TooLarge {
103 what: "the trailer offset",
104 needed: file_len,
105 })?;
106 let placement = Placement::read(&bytes[trailer_at..], file_len)?;
107
108 // Both ends of the footer were checked against the file length by the placement, so these
109 // conversions cannot fail on a target where the whole file is already addressable.
110 let start = usize::try_from(placement.footer_at()).map_err(|_| Error::TooLarge {
111 what: "the footer offset",
112 needed: placement.footer_at(),
113 })?;
114 let end = start
115 .checked_add(placement.footer_len())
116 .ok_or(Error::TooLarge {
117 what: "the footer",
118 needed: u64::MAX,
119 })?;
120 let footer = bytes.get(start..end).ok_or(Error::Truncated {
121 what: "the footer",
122 needed: end as u64,
123 available: file_len,
124 })?;
125
126 Ok(Self {
127 bytes,
128 directory: parse(&bytes[..HEADER_SIZE], footer, placement)?,
129 })
130 }
131
132 /// The metadata, without the payload.
133 ///
134 /// This is what a host hands on to anything that does not need the bytes, which is most of what
135 /// sits above this crate.
136 #[must_use]
137 pub const fn directory(&self) -> &Directory<'a> {
138 &self.directory
139 }
140
141 /// The format version this container was written at.
142 #[must_use]
143 pub const fn header(&self) -> FileHeader {
144 self.directory.header()
145 }
146
147 /// The digest that covers the header and the footer.
148 #[must_use]
149 pub const fn root_digest(&self) -> Digest {
150 self.directory.root_digest()
151 }
152
153 /// What the dataset is.
154 #[must_use]
155 pub const fn dataset(&self) -> &Dataset {
156 self.directory.dataset()
157 }
158
159 /// The schema, if there is one.
160 #[must_use]
161 pub const fn schema(&self) -> Option<&Schema<'a>> {
162 self.directory.schema()
163 }
164
165 /// The decoder reference, if there is one.
166 #[must_use]
167 pub const fn decoder(&self) -> Option<&DecoderRef<'a>> {
168 self.directory.decoder()
169 }
170
171 /// Every section, in the order the footer listed them.
172 #[must_use]
173 pub fn sections(&self) -> &[Section] {
174 self.directory.sections()
175 }
176
177 /// The section with this id.
178 #[must_use]
179 pub fn section(&self, id: u32) -> Option<&Section> {
180 self.directory.section(id)
181 }
182
183 /// The bytes of a section.
184 ///
185 /// The bounds were checked during parsing, so this cannot be out of range for a section that
186 /// came from this container. It takes a `&Section` rather than an id so that the only way to
187 /// call it is with one that did.
188 #[must_use]
189 pub fn section_bytes(&self, section: &Section) -> &'a [u8] {
190 let start = usize::try_from(section.offset).unwrap_or(usize::MAX);
191 let end = usize::try_from(section.end().unwrap_or(u64::MAX)).unwrap_or(usize::MAX);
192 self.bytes.get(start..end).unwrap_or(&[])
193 }
194
195 /// The bytes of the embedded decoder module, if the decoder is embedded and the section it
196 /// names exists.
197 #[must_use]
198 pub fn decoder_bytes(&self) -> Option<&'a [u8]> {
199 self.directory
200 .decoder_section()
201 .map(|section| self.section_bytes(section))
202 }
203
204 /// Hashes every section and checks it against the footer.
205 ///
206 /// This reads the whole file, so it is a decision rather than something that happens on every
207 /// open. The honest place for it is once when a dataset arrives and then never again.
208 ///
209 /// # Errors
210 ///
211 /// Returns [`Error::DigestMismatch`] naming the first section whose bytes do not hash to what
212 /// the footer says they should.
213 pub fn verify(&self) -> Result<()> {
214 for section in self.sections() {
215 let actual = Digest::of(self.section_bytes(section));
216 if actual != section.digest {
217 return Err(Error::DigestMismatch {
218 what: format!("section {}", section.id),
219 expected: section.digest.to_string(),
220 actual: actual.to_string(),
221 });
222 }
223 }
224 Ok(())
225 }
226}