Skip to main content

rypipe_core/
input.rs

1use std::io::{Read, Seek};
2use std::path::Path;
3
4use crate::Result;
5
6/// Maximum decompressed size in bytes (1 GiB). Compressed inputs that would
7/// exceed this limit are rejected to prevent decompression-bomb OOM kills.
8#[allow(dead_code)] // used only when a decompression feature is enabled
9const MAX_DECOMPRESSED_BYTES: u64 = 1 << 30;
10
11/// A `Read` wrapper that aborts once the total bytes read exceeds a limit.
12/// Used to cap decompressed output and prevent decompression-bomb OOM kills.
13#[allow(dead_code)] // used only when a decompression feature is enabled
14struct LimitReader<R> {
15    inner: R,
16    remaining: u64,
17    limit: u64,
18}
19
20#[allow(dead_code)] // used only when a decompression feature is enabled
21impl<R: Read> LimitReader<R> {
22    fn new(inner: R, limit: u64) -> Self {
23        Self {
24            inner,
25            remaining: limit,
26            limit,
27        }
28    }
29}
30
31impl<R: Read> Read for LimitReader<R> {
32    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
33        if buf.is_empty() {
34            return Ok(0);
35        }
36        let allowed = usize::try_from(self.remaining)
37            .unwrap_or(usize::MAX)
38            .saturating_add(1)
39            .min(buf.len());
40        let n = self.inner.read(&mut buf[..allowed])?;
41        if n as u64 > self.remaining {
42            self.remaining = 0;
43            return Err(std::io::Error::other(format!(
44                "decompressed output exceeds {} byte limit (possible decompression bomb)",
45                self.limit
46            )));
47        }
48        self.remaining -= n as u64;
49        Ok(n)
50    }
51}
52
53/// Compression codecs recognized by leading magic bytes. Each codec is
54/// compiled in only when its Cargo feature (`gzip`, `zstd`, `lz4`) is
55/// enabled; inputs are detected independently of file extension.
56#[derive(Clone, Copy, Debug, PartialEq, Eq)]
57#[allow(dead_code)] // variants exist only under their cargo features
58enum Compression {
59    Gzip,
60    Zstd,
61    Lz4,
62}
63
64/// Owned handle for a memory-mapped file.
65#[cfg(feature = "mmap")]
66pub struct MmapHandle {
67    pub(crate) mmap: memmap2::Mmap,
68    pub(crate) file: std::fs::File,
69}
70
71#[cfg(feature = "mmap")]
72impl MmapHandle {
73    fn new(file: std::fs::File, prefault: bool) -> Result<Self> {
74        // The input must not be modified or truncated while mapped.
75        let mmap = unsafe { memmap2::Mmap::map(&file)? };
76        #[cfg(unix)]
77        {
78            if prefault {
79                // Pre-fault the entire file into RSS.
80                let _ = mmap.advise(memmap2::Advice::WillNeed);
81            } else {
82                // Let the kernel drop pages behind the sequential reader.
83                let _ = mmap.advise(memmap2::Advice::Sequential);
84            }
85        }
86        #[cfg(not(unix))]
87        let _ = prefault;
88        Ok(MmapHandle { mmap, file })
89    }
90
91    fn as_slice(&self) -> &[u8] {
92        &self.mmap[..]
93    }
94}
95
96/// Input abstraction: either a memory-mapped file or an owned in-memory buffer.
97pub enum InputBuffer {
98    #[cfg(feature = "mmap")]
99    Mmap(MmapHandle),
100    Owned(Vec<u8>),
101}
102
103fn detect_compression(file: &mut std::fs::File) -> Result<Option<Compression>> {
104    let mut magic = [0u8; 4];
105    let n = file.read(&mut magic)?;
106    file.rewind()?;
107    #[cfg(feature = "gzip")]
108    if n >= 2 && magic[0] == 0x1f && magic[1] == 0x8b {
109        return Ok(Some(Compression::Gzip));
110    }
111    #[cfg(feature = "zstd")]
112    if n >= 4 && magic == [0x28, 0xb5, 0x2f, 0xfd] {
113        return Ok(Some(Compression::Zstd));
114    }
115    #[cfg(feature = "lz4")]
116    if n >= 4 && magic == [0x04, 0x22, 0x4d, 0x18] {
117        return Ok(Some(Compression::Lz4));
118    }
119    let _ = n;
120    Ok(None)
121}
122
123/// Decompress `file` with `codec` into an owned buffer.
124/// Aborts early if decompressed output exceeds [`MAX_DECOMPRESSED_BYTES`].
125#[cfg_attr(
126    not(any(feature = "gzip", feature = "zstd", feature = "lz4")),
127    allow(unused_variables, unused_mut)
128)]
129fn decompress(mut file: std::fs::File, codec: Compression) -> Result<Vec<u8>> {
130    match codec {
131        #[cfg(feature = "gzip")]
132        Compression::Gzip => {
133            let mut out = Vec::new();
134            let decoder = flate2::read::GzDecoder::new(&mut file);
135            LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
136            Ok(out)
137        }
138        #[cfg(feature = "zstd")]
139        Compression::Zstd => {
140            let mut out = Vec::new();
141            let decoder = zstd::stream::read::Decoder::new(&mut file)?;
142            LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
143            Ok(out)
144        }
145        #[cfg(feature = "lz4")]
146        Compression::Lz4 => {
147            let mut out = Vec::new();
148            let decoder = lz4_flex::frame::FrameDecoder::new(&mut file);
149            LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
150            Ok(out)
151        }
152        // Only reachable when a codec's cargo feature is disabled; detection
153        // never selects disabled codecs, so this is defensive.
154        #[allow(unreachable_patterns)]
155        _ => Err(crate::Error::Io(std::io::Error::other(format!(
156            "{codec:?} input detected but its cargo feature is not enabled"
157        )))),
158    }
159}
160
161impl InputBuffer {
162    /// Open a path as an input buffer.
163    ///
164    /// When the leading magic bytes identify a compression codec whose Cargo
165    /// feature is enabled, the file is transparently decompressed into an
166    /// owned buffer; all execution modes then operate on the decompressed
167    /// bytes. Otherwise, when `use_mmap` is true and the `"mmap"` feature is
168    /// enabled, the file is mapped; otherwise it is read into memory.
169    pub fn open(path: &Path, use_mmap: bool, prefault: bool) -> Result<Self> {
170        let mut file = std::fs::File::open(path)?;
171        if let Some(codec) = detect_compression(&mut file)? {
172            return Ok(InputBuffer::Owned(decompress(file, codec)?));
173        }
174
175        #[cfg(feature = "mmap")]
176        {
177            if use_mmap {
178                return Ok(InputBuffer::Mmap(MmapHandle::new(file, prefault)?));
179            }
180        }
181        let _ = use_mmap;
182        let _ = prefault;
183        let mut bytes = Vec::new();
184        file.read_to_end(&mut bytes)?;
185        Ok(InputBuffer::Owned(bytes))
186    }
187
188    pub fn as_slice(&self) -> &[u8] {
189        match self {
190            #[cfg(feature = "mmap")]
191            InputBuffer::Mmap(handle) => handle.as_slice(),
192            InputBuffer::Owned(bytes) => bytes,
193        }
194    }
195
196    pub fn len(&self) -> usize {
197        self.as_slice().len()
198    }
199
200    pub fn is_empty(&self) -> bool {
201        self.len() == 0
202    }
203}
204
205#[cfg(test)]
206mod tests {
207    use super::*;
208    use std::io::{self, Write};
209
210    #[test]
211    fn decompressed_limit_accepts_exactly_one_gib_without_retaining_output() {
212        let mut reader = LimitReader::new(
213            io::repeat(0).take(MAX_DECOMPRESSED_BYTES),
214            MAX_DECOMPRESSED_BYTES,
215        );
216        assert_eq!(
217            io::copy(&mut reader, &mut io::sink()).unwrap(),
218            MAX_DECOMPRESSED_BYTES
219        );
220    }
221
222    #[test]
223    fn decompressed_limit_rejects_one_extra_byte() {
224        let mut reader = LimitReader::new(
225            io::repeat(0).take(MAX_DECOMPRESSED_BYTES + 1),
226            MAX_DECOMPRESSED_BYTES,
227        );
228        let error = io::copy(&mut reader, &mut io::sink()).unwrap_err();
229        assert!(error
230            .to_string()
231            .contains(&MAX_DECOMPRESSED_BYTES.to_string()));
232        assert_eq!(reader.inner.limit(), 0);
233    }
234
235    #[test]
236    fn limit_reader_does_not_overread_the_limit_or_touch_empty_buffers() {
237        let mut reader = LimitReader::new(&b"123456789"[..], 3);
238        assert_eq!(reader.read(&mut []).unwrap(), 0);
239        let error = reader.read(&mut [0; 1024]).unwrap_err();
240        assert!(error.to_string().contains("3 byte limit"));
241        assert_eq!(reader.inner, b"56789");
242        let mut empty = LimitReader::new(&b""[..], 0);
243        assert_eq!(empty.read(&mut [0; 1]).unwrap(), 0);
244    }
245
246    #[test]
247    fn compressed_expansion_hits_the_same_reader_limit() {
248        let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best());
249        encoder.write_all(&[0; 128 * 1024]).unwrap();
250        let compressed = encoder.finish().unwrap();
251        assert!(compressed.len() < 1024);
252        let decoder = flate2::read::GzDecoder::new(compressed.as_slice());
253        let mut reader = LimitReader::new(decoder, 4096);
254        let error = io::copy(&mut reader, &mut io::sink()).unwrap_err();
255        assert!(error.to_string().contains("4096 byte limit"));
256    }
257}