1use std::io::{Read, Seek};
2use std::path::Path;
3
4use crate::Result;
5
6#[allow(dead_code)] const MAX_DECOMPRESSED_BYTES: u64 = 1 << 30;
10
11#[allow(dead_code)] struct LimitReader<R> {
15 inner: R,
16 remaining: u64,
17 limit: u64,
18}
19
20#[allow(dead_code)] impl<R: Read> LimitReader<R> {
22 fn new(inner: R, limit: u64) -> Self {
23 Self {
24 inner,
25 remaining: limit,
26 limit,
27 }
28 }
29}
30
31impl<R: Read> Read for LimitReader<R> {
32 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
33 if buf.is_empty() {
34 return Ok(0);
35 }
36 let allowed = usize::try_from(self.remaining)
37 .unwrap_or(usize::MAX)
38 .saturating_add(1)
39 .min(buf.len());
40 let n = self.inner.read(&mut buf[..allowed])?;
41 if n as u64 > self.remaining {
42 self.remaining = 0;
43 return Err(std::io::Error::other(format!(
44 "decompressed output exceeds {} byte limit (possible decompression bomb)",
45 self.limit
46 )));
47 }
48 self.remaining -= n as u64;
49 Ok(n)
50 }
51}
52
53#[derive(Clone, Copy, Debug, PartialEq, Eq)]
57#[allow(dead_code)] enum Compression {
59 Gzip,
60 Zstd,
61 Lz4,
62}
63
64#[cfg(feature = "mmap")]
66pub struct MmapHandle {
67 pub(crate) mmap: memmap2::Mmap,
68 pub(crate) file: std::fs::File,
69}
70
71#[cfg(feature = "mmap")]
72impl MmapHandle {
73 fn new(file: std::fs::File, prefault: bool) -> Result<Self> {
74 let mmap = unsafe { memmap2::Mmap::map(&file)? };
78 #[cfg(unix)]
79 {
80 if prefault {
81 let _ = mmap.advise(memmap2::Advice::WillNeed);
83 } else {
84 let _ = mmap.advise(memmap2::Advice::Sequential);
86 }
87 }
88 #[cfg(not(unix))]
89 let _ = prefault;
90 Ok(MmapHandle { mmap, file })
91 }
92
93 fn as_slice(&self) -> &[u8] {
94 &self.mmap[..]
95 }
96}
97
98pub enum InputBuffer {
100 #[cfg(feature = "mmap")]
101 Mmap(MmapHandle),
102 Owned(Vec<u8>),
103}
104
105fn detect_compression(file: &mut std::fs::File) -> Result<Option<Compression>> {
106 let mut magic = [0u8; 4];
107 let n = file.read(&mut magic)?;
108 file.rewind()?;
109 #[cfg(feature = "gzip")]
110 if n >= 2 && magic[0] == 0x1f && magic[1] == 0x8b {
111 return Ok(Some(Compression::Gzip));
112 }
113 #[cfg(feature = "zstd")]
114 if n >= 4 && magic == [0x28, 0xb5, 0x2f, 0xfd] {
115 return Ok(Some(Compression::Zstd));
116 }
117 #[cfg(feature = "lz4")]
118 if n >= 4 && magic == [0x04, 0x22, 0x4d, 0x18] {
119 return Ok(Some(Compression::Lz4));
120 }
121 let _ = n;
122 Ok(None)
123}
124
125#[cfg_attr(
128 not(any(feature = "gzip", feature = "zstd", feature = "lz4")),
129 allow(unused_variables, unused_mut)
130)]
131fn decompress(mut file: std::fs::File, codec: Compression) -> Result<Vec<u8>> {
132 match codec {
133 #[cfg(feature = "gzip")]
134 Compression::Gzip => {
135 let mut out = Vec::new();
136 let decoder = flate2::read::GzDecoder::new(&mut file);
137 LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
138 Ok(out)
139 }
140 #[cfg(feature = "zstd")]
141 Compression::Zstd => {
142 let mut out = Vec::new();
143 let decoder = zstd::stream::read::Decoder::new(&mut file)?;
144 LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
145 Ok(out)
146 }
147 #[cfg(feature = "lz4")]
148 Compression::Lz4 => {
149 let mut out = Vec::new();
150 let decoder = lz4_flex::frame::FrameDecoder::new(&mut file);
151 LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
152 Ok(out)
153 }
154 #[allow(unreachable_patterns)]
157 _ => Err(crate::Error::Io(std::io::Error::other(format!(
158 "{codec:?} input detected but its cargo feature is not enabled"
159 )))),
160 }
161}
162
163impl InputBuffer {
164 pub fn open(path: &Path, use_mmap: bool, prefault: bool) -> Result<Self> {
172 let mut file = std::fs::File::open(path)?;
173 if let Some(codec) = detect_compression(&mut file)? {
174 return Ok(InputBuffer::Owned(decompress(file, codec)?));
175 }
176
177 #[cfg(feature = "mmap")]
178 {
179 if use_mmap {
180 return Ok(InputBuffer::Mmap(MmapHandle::new(file, prefault)?));
181 }
182 }
183 let _ = use_mmap;
184 let _ = prefault;
185 let mut bytes = Vec::new();
186 file.read_to_end(&mut bytes)?;
187 Ok(InputBuffer::Owned(bytes))
188 }
189
190 pub fn as_slice(&self) -> &[u8] {
191 match self {
192 #[cfg(feature = "mmap")]
193 InputBuffer::Mmap(handle) => handle.as_slice(),
194 InputBuffer::Owned(bytes) => bytes,
195 }
196 }
197
198 pub fn len(&self) -> usize {
199 self.as_slice().len()
200 }
201
202 pub fn is_empty(&self) -> bool {
203 self.len() == 0
204 }
205}
206
207#[cfg(test)]
208mod tests {
209 use super::*;
210 use std::io::{self, Write};
211
212 #[test]
213 fn decompressed_limit_accepts_exactly_one_gib_without_retaining_output() {
214 let mut reader = LimitReader::new(
215 io::repeat(0).take(MAX_DECOMPRESSED_BYTES),
216 MAX_DECOMPRESSED_BYTES,
217 );
218 assert_eq!(
219 io::copy(&mut reader, &mut io::sink()).unwrap(),
220 MAX_DECOMPRESSED_BYTES
221 );
222 }
223
224 #[test]
225 fn decompressed_limit_rejects_one_extra_byte() {
226 let mut reader = LimitReader::new(
227 io::repeat(0).take(MAX_DECOMPRESSED_BYTES + 1),
228 MAX_DECOMPRESSED_BYTES,
229 );
230 let error = io::copy(&mut reader, &mut io::sink()).unwrap_err();
231 assert!(error
232 .to_string()
233 .contains(&MAX_DECOMPRESSED_BYTES.to_string()));
234 assert_eq!(reader.inner.limit(), 0);
235 }
236
237 #[test]
238 fn limit_reader_does_not_overread_the_limit_or_touch_empty_buffers() {
239 let mut reader = LimitReader::new(&b"123456789"[..], 3);
240 assert_eq!(reader.read(&mut []).unwrap(), 0);
241 let error = reader.read(&mut [0; 1024]).unwrap_err();
242 assert!(error.to_string().contains("3 byte limit"));
243 assert_eq!(reader.inner, b"56789");
244 let mut empty = LimitReader::new(&b""[..], 0);
245 assert_eq!(empty.read(&mut [0; 1]).unwrap(), 0);
246 }
247
248 #[test]
249 fn compressed_expansion_hits_the_same_reader_limit() {
250 let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best());
251 encoder.write_all(&[0; 128 * 1024]).unwrap();
252 let compressed = encoder.finish().unwrap();
253 assert!(compressed.len() < 1024);
254 let decoder = flate2::read::GzDecoder::new(compressed.as_slice());
255 let mut reader = LimitReader::new(decoder, 4096);
256 let error = io::copy(&mut reader, &mut io::sink()).unwrap_err();
257 assert!(error.to_string().contains("4096 byte limit"));
258 }
259}