1use std::io::{Read, Seek};
2use std::path::Path;
3
4use crate::Result;
5
6#[allow(dead_code)] const MAX_DECOMPRESSED_BYTES: u64 = 1 << 30;
10
11#[allow(dead_code)] struct LimitReader<R> {
15 inner: R,
16 remaining: u64,
17 limit: u64,
18}
19
20#[allow(dead_code)] impl<R: Read> LimitReader<R> {
22 fn new(inner: R, limit: u64) -> Self {
23 Self {
24 inner,
25 remaining: limit,
26 limit,
27 }
28 }
29}
30
31impl<R: Read> Read for LimitReader<R> {
32 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
33 if buf.is_empty() {
34 return Ok(0);
35 }
36 let allowed = usize::try_from(self.remaining)
37 .unwrap_or(usize::MAX)
38 .saturating_add(1)
39 .min(buf.len());
40 let n = self.inner.read(&mut buf[..allowed])?;
41 if n as u64 > self.remaining {
42 self.remaining = 0;
43 return Err(std::io::Error::other(format!(
44 "decompressed output exceeds {} byte limit (possible decompression bomb)",
45 self.limit
46 )));
47 }
48 self.remaining -= n as u64;
49 Ok(n)
50 }
51}
52
53#[derive(Clone, Copy, Debug, PartialEq, Eq)]
57#[allow(dead_code)] enum Compression {
59 Gzip,
60 Zstd,
61 Lz4,
62}
63
64#[cfg(feature = "mmap")]
66pub struct MmapHandle {
67 pub(crate) mmap: memmap2::Mmap,
68 pub(crate) file: std::fs::File,
69}
70
71#[cfg(feature = "mmap")]
72impl MmapHandle {
73 fn new(file: std::fs::File, prefault: bool) -> Result<Self> {
74 let mmap = unsafe { memmap2::Mmap::map(&file)? };
76 #[cfg(unix)]
77 {
78 if prefault {
79 let _ = mmap.advise(memmap2::Advice::WillNeed);
81 } else {
82 let _ = mmap.advise(memmap2::Advice::Sequential);
84 }
85 }
86 #[cfg(not(unix))]
87 let _ = prefault;
88 Ok(MmapHandle { mmap, file })
89 }
90
91 fn as_slice(&self) -> &[u8] {
92 &self.mmap[..]
93 }
94}
95
96pub enum InputBuffer {
98 #[cfg(feature = "mmap")]
99 Mmap(MmapHandle),
100 Owned(Vec<u8>),
101}
102
103fn detect_compression(file: &mut std::fs::File) -> Result<Option<Compression>> {
104 let mut magic = [0u8; 4];
105 let n = file.read(&mut magic)?;
106 file.rewind()?;
107 #[cfg(feature = "gzip")]
108 if n >= 2 && magic[0] == 0x1f && magic[1] == 0x8b {
109 return Ok(Some(Compression::Gzip));
110 }
111 #[cfg(feature = "zstd")]
112 if n >= 4 && magic == [0x28, 0xb5, 0x2f, 0xfd] {
113 return Ok(Some(Compression::Zstd));
114 }
115 #[cfg(feature = "lz4")]
116 if n >= 4 && magic == [0x04, 0x22, 0x4d, 0x18] {
117 return Ok(Some(Compression::Lz4));
118 }
119 let _ = n;
120 Ok(None)
121}
122
123#[cfg_attr(
126 not(any(feature = "gzip", feature = "zstd", feature = "lz4")),
127 allow(unused_variables, unused_mut)
128)]
129fn decompress(mut file: std::fs::File, codec: Compression) -> Result<Vec<u8>> {
130 match codec {
131 #[cfg(feature = "gzip")]
132 Compression::Gzip => {
133 let mut out = Vec::new();
134 let decoder = flate2::read::GzDecoder::new(&mut file);
135 LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
136 Ok(out)
137 }
138 #[cfg(feature = "zstd")]
139 Compression::Zstd => {
140 let mut out = Vec::new();
141 let decoder = zstd::stream::read::Decoder::new(&mut file)?;
142 LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
143 Ok(out)
144 }
145 #[cfg(feature = "lz4")]
146 Compression::Lz4 => {
147 let mut out = Vec::new();
148 let decoder = lz4_flex::frame::FrameDecoder::new(&mut file);
149 LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
150 Ok(out)
151 }
152 #[allow(unreachable_patterns)]
155 _ => Err(crate::Error::Io(std::io::Error::other(format!(
156 "{codec:?} input detected but its cargo feature is not enabled"
157 )))),
158 }
159}
160
161impl InputBuffer {
162 pub fn open(path: &Path, use_mmap: bool, prefault: bool) -> Result<Self> {
170 let mut file = std::fs::File::open(path)?;
171 if let Some(codec) = detect_compression(&mut file)? {
172 return Ok(InputBuffer::Owned(decompress(file, codec)?));
173 }
174
175 #[cfg(feature = "mmap")]
176 {
177 if use_mmap {
178 return Ok(InputBuffer::Mmap(MmapHandle::new(file, prefault)?));
179 }
180 }
181 let _ = use_mmap;
182 let _ = prefault;
183 let mut bytes = Vec::new();
184 file.read_to_end(&mut bytes)?;
185 Ok(InputBuffer::Owned(bytes))
186 }
187
188 pub fn as_slice(&self) -> &[u8] {
189 match self {
190 #[cfg(feature = "mmap")]
191 InputBuffer::Mmap(handle) => handle.as_slice(),
192 InputBuffer::Owned(bytes) => bytes,
193 }
194 }
195
196 pub fn len(&self) -> usize {
197 self.as_slice().len()
198 }
199
200 pub fn is_empty(&self) -> bool {
201 self.len() == 0
202 }
203}
204
205#[cfg(test)]
206mod tests {
207 use super::*;
208 use std::io::{self, Write};
209
210 #[test]
211 fn decompressed_limit_accepts_exactly_one_gib_without_retaining_output() {
212 let mut reader = LimitReader::new(
213 io::repeat(0).take(MAX_DECOMPRESSED_BYTES),
214 MAX_DECOMPRESSED_BYTES,
215 );
216 assert_eq!(
217 io::copy(&mut reader, &mut io::sink()).unwrap(),
218 MAX_DECOMPRESSED_BYTES
219 );
220 }
221
222 #[test]
223 fn decompressed_limit_rejects_one_extra_byte() {
224 let mut reader = LimitReader::new(
225 io::repeat(0).take(MAX_DECOMPRESSED_BYTES + 1),
226 MAX_DECOMPRESSED_BYTES,
227 );
228 let error = io::copy(&mut reader, &mut io::sink()).unwrap_err();
229 assert!(error
230 .to_string()
231 .contains(&MAX_DECOMPRESSED_BYTES.to_string()));
232 assert_eq!(reader.inner.limit(), 0);
233 }
234
235 #[test]
236 fn limit_reader_does_not_overread_the_limit_or_touch_empty_buffers() {
237 let mut reader = LimitReader::new(&b"123456789"[..], 3);
238 assert_eq!(reader.read(&mut []).unwrap(), 0);
239 let error = reader.read(&mut [0; 1024]).unwrap_err();
240 assert!(error.to_string().contains("3 byte limit"));
241 assert_eq!(reader.inner, b"56789");
242 let mut empty = LimitReader::new(&b""[..], 0);
243 assert_eq!(empty.read(&mut [0; 1]).unwrap(), 0);
244 }
245
246 #[test]
247 fn compressed_expansion_hits_the_same_reader_limit() {
248 let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best());
249 encoder.write_all(&[0; 128 * 1024]).unwrap();
250 let compressed = encoder.finish().unwrap();
251 assert!(compressed.len() < 1024);
252 let decoder = flate2::read::GzDecoder::new(compressed.as_slice());
253 let mut reader = LimitReader::new(decoder, 4096);
254 let error = io::copy(&mut reader, &mut io::sink()).unwrap_err();
255 assert!(error.to_string().contains("4096 byte limit"));
256 }
257}