zstd_chunked/lib.rs
1//! A library to help read zstd:chunked files
2mod format;
3
4use core::ops::Range;
5use std::{collections::HashMap, io::Write};
6
7use anyhow::{Context, Result, ensure};
8
9use self::format::{Footer, FooterReference, Manifest, TarSplitEntry};
10
11/// A reference to a compressed range in a zstd:chunked file, along with size and checksum
12/// information about the uncompressed data at that range.
13#[derive(Debug, Clone)]
14pub struct ContentReference {
15 /// The range itself, in bytes, in the compressed file.
16 pub range: Range<u64>,
17
18 /// The digest of the data at the range, after decompression.
19 pub digest: String,
20
21 /// The size of the compressed data at the range, after decompression.
22 pub size: u64,
23}
24
25/// A chunk of data in a zstd:chunked stream. Either contains inline data or a reference to a
26/// compressed range (and checksum and size information about the data at that range).
27#[derive(Debug, Clone)]
28pub enum Chunk {
29 /// The literal data appears directly.
30 Inline(Box<[u8]>),
31 /// The data appears at the referenced range, which may need to be fetched and decompressed.
32 External(ContentReference),
33}
34
35/// Represents the layout of a zstd:chunked file. You can reconstruct the original file contents
36/// by iterating over the chunks.
37#[derive(Debug)]
38pub struct Stream {
39 /// The chunks in the file.
40 pub chunks: Vec<Chunk>,
41}
42
43impl Stream {
44 /// Create the metadata structure from the compressed frames referred to by the ranges in the
45 /// OCI layer descriptor annotations or the file footer.
46 ///
47 /// # Errors
48 ///
49 /// This function can fail if any of the metadata isn't in the expected format (zstd-compressed
50 /// JSON) or if there are missing mandatory fields or internal inconsistencies. In all cases,
51 /// it indicates a corrupt zstd:chunked file (or a bug in the library).
52 pub fn new_from_frames(manifest: &[u8], tarsplit: &[u8]) -> Result<Self> {
53 let manifest = zstd::decode_all(manifest)?;
54 let manifest: Manifest = serde_json::from_slice(&manifest)?;
55
56 ensure!(
57 manifest.version == 1,
58 "Incorrect zstd:chunked CRFS manifest version"
59 );
60
61 // Read the manifest entries into a table by filename, taking only the ones that have the
62 // digest, size, offset and end_offset information filled in (ie: regular files). Don't
63 // handle chunks.
64 let manifest_entries: HashMap<String, ContentReference> = manifest
65 .entries
66 .into_iter()
67 .filter_map(|entry| {
68 Some((
69 entry.name,
70 ContentReference {
71 digest: entry.digest?,
72 size: entry.size?,
73 range: entry.offset?..entry.end_offset?,
74 },
75 ))
76 })
77 .collect();
78
79 // Iterate over the chunks in the tarsplit. For inline chunks, store the inline data. For
80 // external chunks, look them up in the manifest_entries and store what we find.
81 let tarsplit = String::from_utf8(zstd::decode_all(tarsplit)?)?;
82 let mut chunks = vec![];
83
84 for line in tarsplit.lines() {
85 let entry: TarSplitEntry = serde_json::from_str(line)?;
86
87 match entry {
88 TarSplitEntry {
89 name: Some(name),
90 size: Some(size),
91 .. // ignored: crc64
92 } => {
93 let reference = manifest_entries.get(&name)
94 .with_context(|| format!("Filename {name} in zstd:chunked tarsplit missing from manifest"))?;
95 ensure!(size == reference.size, "size mismatch");
96 chunks.push(Chunk::External(reference.clone()));
97 }
98 TarSplitEntry {
99 payload: Some(payload),
100 ..
101 } => chunks.push(Chunk::Inline(payload)),
102 _ => {}
103 }
104 }
105
106 Ok(Self { chunks })
107 }
108
109 /// Iterates over all of the references that need to be satisfied for this stream to be
110 /// reconstructed. This might be useful to help prefetch the required items.
111 pub fn references(&self) -> impl Iterator<Item = &ContentReference> {
112 self.chunks.iter().filter_map(|chunk| {
113 if let Chunk::External(reference) = chunk {
114 Some(reference)
115 } else {
116 None
117 }
118 })
119 }
120
121 /// Writes the content of the stream to the given writer. The `resolve_reference()` function
122 /// should return the *decompressed* data corresponding to the reference.
123 ///
124 /// # Errors
125 ///
126 /// This function can fail only in response to external errors: a failure of the
127 /// `resolve_reference()` function or a failure to write to the writer.
128 pub fn write_to(
129 &self,
130 write: &mut impl Write,
131 resolve_reference: impl Fn(&ContentReference) -> Result<Vec<u8>>,
132 ) -> Result<()> {
133 for chunk in &self.chunks {
134 match chunk {
135 Chunk::Inline(data) => {
136 write.write_all(data)?;
137 }
138 Chunk::External(r#ref) => {
139 write.write_all(&resolve_reference(r#ref)?)?;
140 }
141 }
142 }
143 Ok(())
144 }
145}
146
147/// A reference to file metadata, either the manifest or the tarsplit
148#[derive(Debug)]
149pub struct MetadataReference {
150 /// The range itself, in bytes, in the compressed file.
151 pub range: Range<u64>,
152
153 /// The digest of the data at the range, *before* decompression. This will be missing if we
154 /// read from the file footer or if the OCI annotations didn't provide it.
155 pub digest: Option<String>,
156
157 /// The size of the compressed data at the range, after decompression.
158 pub uncompressed_size: u64,
159}
160
161impl MetadataReference {
162 const fn from_footer(value: &FooterReference) -> Self {
163 let start = value.offset.get();
164 let end = start + value.length_compressed.get();
165
166 Self {
167 range: start..end,
168 digest: None,
169 uncompressed_size: value.length_uncompressed.get(),
170 }
171 }
172}
173
174/// References to the manifest and tarsplit metadata. You can read these from the file footer or
175/// from the annotations on the OCI layer descriptor.
176#[derive(Debug)]
177pub struct MetadataReferences {
178 /// The location of the manifest data
179 pub manifest: MetadataReference,
180 /// The location of the tarsplit data
181 pub tarsplit: MetadataReference,
182}
183
184fn to_vec_u64(value: &str) -> Option<Vec<u64>> {
185 value.split(':').map(|s| s.parse().ok()).collect()
186}
187
188impl MetadataReferences {
189 /// Read the metadata references from the file footer. The provided data can be any suffix of
190 /// the file, but it must be at least 72 bytes in length (to contain the footer). Returns None
191 /// if this doesn't appear to be a zstd:chunked file.
192 #[must_use]
193 pub fn from_footer(suffix: &[u8]) -> Option<Self> {
194 let footer = Footer::from_suffix(suffix)?;
195 Some(Self {
196 manifest: MetadataReference::from_footer(&footer.manifest),
197 tarsplit: MetadataReference::from_footer(&footer.tarsplit),
198 })
199 }
200
201 /// Parses the metadata references from OCI layer descriptor annotations. You should provide a
202 /// 'get' closure that returns the requested annotation, or None if it doesn't exist. Returns
203 /// None if this doesn't appear to be a zstd:chunked layer descriptor.
204 pub fn from_oci<'a, S: AsRef<str> + 'a>(get: impl Fn(&str) -> Option<&'a S>) -> Option<Self> {
205 let manifest_digest = get("io.github.containers.zstd-chunked.manifest-checksum");
206 let manifest_position = get("io.github.containers.zstd-chunked.manifest-position")?;
207 let tarsplit_digest = get("io.github.containers.zstd-chunked.tarsplit-checksum");
208 let tarsplit_position = get("io.github.containers.zstd-chunked.tarsplit-position")?;
209
210 Some(Self {
211 manifest: match to_vec_u64(manifest_position.as_ref())?.as_slice() {
212 &[start, length, uncompressed_size, 1] => MetadataReference {
213 range: start..(start + length),
214 digest: manifest_digest.map(|s| s.as_ref().to_owned()),
215 uncompressed_size,
216 },
217 _ => None?,
218 },
219 tarsplit: match to_vec_u64(tarsplit_position.as_ref())?.as_slice() {
220 &[start, length, uncompressed_size] => MetadataReference {
221 range: start..(start + length),
222 digest: tarsplit_digest.map(|s| s.as_ref().to_owned()),
223 uncompressed_size,
224 },
225 _ => None?,
226 },
227 })
228 }
229}