Skip to main content

rypipe_core/
input.rs

1use std::io::{Read, Seek};
2use std::path::Path;
3
4use crate::Result;
5
6/// Maximum decompressed size in bytes (1 GiB). Compressed inputs that would
7/// exceed this limit are rejected to prevent decompression-bomb OOM kills.
8#[allow(dead_code)] // used only when a decompression feature is enabled
9const MAX_DECOMPRESSED_BYTES: u64 = 1 << 30;
10
11/// A `Read` wrapper that aborts once the total bytes read exceeds a limit.
12/// Used to cap decompressed output and prevent decompression-bomb OOM kills.
13#[allow(dead_code)] // used only when a decompression feature is enabled
14struct LimitReader<R> {
15    inner: R,
16    remaining: u64,
17    limit: u64,
18}
19
20#[allow(dead_code)] // used only when a decompression feature is enabled
21impl<R: Read> LimitReader<R> {
22    fn new(inner: R, limit: u64) -> Self {
23        Self {
24            inner,
25            remaining: limit,
26            limit,
27        }
28    }
29}
30
31impl<R: Read> Read for LimitReader<R> {
32    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
33        if buf.is_empty() {
34            return Ok(0);
35        }
36        let allowed = usize::try_from(self.remaining)
37            .unwrap_or(usize::MAX)
38            .saturating_add(1)
39            .min(buf.len());
40        let n = self.inner.read(&mut buf[..allowed])?;
41        if n as u64 > self.remaining {
42            self.remaining = 0;
43            return Err(std::io::Error::other(format!(
44                "decompressed output exceeds {} byte limit (possible decompression bomb)",
45                self.limit
46            )));
47        }
48        self.remaining -= n as u64;
49        Ok(n)
50    }
51}
52
53/// Compression codecs recognized by leading magic bytes. Each codec is
54/// compiled in only when its Cargo feature (`gzip`, `zstd`, `lz4`) is
55/// enabled; inputs are detected independently of file extension.
56#[derive(Clone, Copy, Debug, PartialEq, Eq)]
57#[allow(dead_code)] // variants exist only under their cargo features
58enum Compression {
59    Gzip,
60    Zstd,
61    Lz4,
62}
63
64/// Owned handle for a memory-mapped file.
65#[cfg(feature = "mmap")]
66pub struct MmapHandle {
67    pub(crate) mmap: memmap2::Mmap,
68    pub(crate) file: std::fs::File,
69}
70
71#[cfg(feature = "mmap")]
72impl MmapHandle {
73    fn new(file: std::fs::File, prefault: bool) -> Result<Self> {
74        // SAFETY: memmap2 requires the file not be mutated while mapped; the
75        // input file is opened read-only and the caller contract forbids
76        // truncating or modifying it for the life of the mapping.
77        let mmap = unsafe { memmap2::Mmap::map(&file)? };
78        #[cfg(unix)]
79        {
80            if prefault {
81                // Pre-fault the entire file into RSS.
82                let _ = mmap.advise(memmap2::Advice::WillNeed);
83            } else {
84                // Let the kernel drop pages behind the sequential reader.
85                let _ = mmap.advise(memmap2::Advice::Sequential);
86            }
87        }
88        #[cfg(not(unix))]
89        let _ = prefault;
90        Ok(MmapHandle { mmap, file })
91    }
92
93    fn as_slice(&self) -> &[u8] {
94        &self.mmap[..]
95    }
96}
97
98/// Input abstraction: either a memory-mapped file or an owned in-memory buffer.
99pub enum InputBuffer {
100    #[cfg(feature = "mmap")]
101    Mmap(MmapHandle),
102    Owned(Vec<u8>),
103}
104
105fn detect_compression(file: &mut std::fs::File) -> Result<Option<Compression>> {
106    let mut magic = [0u8; 4];
107    let n = file.read(&mut magic)?;
108    file.rewind()?;
109    #[cfg(feature = "gzip")]
110    if n >= 2 && magic[0] == 0x1f && magic[1] == 0x8b {
111        return Ok(Some(Compression::Gzip));
112    }
113    #[cfg(feature = "zstd")]
114    if n >= 4 && magic == [0x28, 0xb5, 0x2f, 0xfd] {
115        return Ok(Some(Compression::Zstd));
116    }
117    #[cfg(feature = "lz4")]
118    if n >= 4 && magic == [0x04, 0x22, 0x4d, 0x18] {
119        return Ok(Some(Compression::Lz4));
120    }
121    let _ = n;
122    Ok(None)
123}
124
125/// Decompress `file` with `codec` into an owned buffer.
126/// Aborts early if decompressed output exceeds [`MAX_DECOMPRESSED_BYTES`].
127#[cfg_attr(
128    not(any(feature = "gzip", feature = "zstd", feature = "lz4")),
129    allow(unused_variables, unused_mut)
130)]
131fn decompress(mut file: std::fs::File, codec: Compression) -> Result<Vec<u8>> {
132    match codec {
133        #[cfg(feature = "gzip")]
134        Compression::Gzip => {
135            let mut out = Vec::new();
136            let decoder = flate2::read::GzDecoder::new(&mut file);
137            LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
138            Ok(out)
139        }
140        #[cfg(feature = "zstd")]
141        Compression::Zstd => {
142            let mut out = Vec::new();
143            let decoder = zstd::stream::read::Decoder::new(&mut file)?;
144            LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
145            Ok(out)
146        }
147        #[cfg(feature = "lz4")]
148        Compression::Lz4 => {
149            let mut out = Vec::new();
150            let decoder = lz4_flex::frame::FrameDecoder::new(&mut file);
151            LimitReader::new(decoder, MAX_DECOMPRESSED_BYTES).read_to_end(&mut out)?;
152            Ok(out)
153        }
154        // Only reachable when a codec's cargo feature is disabled; detection
155        // never selects disabled codecs, so this is defensive.
156        #[allow(unreachable_patterns)]
157        _ => Err(crate::Error::Io(std::io::Error::other(format!(
158            "{codec:?} input detected but its cargo feature is not enabled"
159        )))),
160    }
161}
162
163impl InputBuffer {
164    /// Open a path as an input buffer.
165    ///
166    /// When the leading magic bytes identify a compression codec whose Cargo
167    /// feature is enabled, the file is transparently decompressed into an
168    /// owned buffer; all execution modes then operate on the decompressed
169    /// bytes. Otherwise, when `use_mmap` is true and the `"mmap"` feature is
170    /// enabled, the file is mapped; otherwise it is read into memory.
171    pub fn open(path: &Path, use_mmap: bool, prefault: bool) -> Result<Self> {
172        let mut file = std::fs::File::open(path)?;
173        if let Some(codec) = detect_compression(&mut file)? {
174            return Ok(InputBuffer::Owned(decompress(file, codec)?));
175        }
176
177        #[cfg(feature = "mmap")]
178        {
179            if use_mmap {
180                return Ok(InputBuffer::Mmap(MmapHandle::new(file, prefault)?));
181            }
182        }
183        let _ = use_mmap;
184        let _ = prefault;
185        let mut bytes = Vec::new();
186        file.read_to_end(&mut bytes)?;
187        Ok(InputBuffer::Owned(bytes))
188    }
189
190    pub fn as_slice(&self) -> &[u8] {
191        match self {
192            #[cfg(feature = "mmap")]
193            InputBuffer::Mmap(handle) => handle.as_slice(),
194            InputBuffer::Owned(bytes) => bytes,
195        }
196    }
197
198    pub fn len(&self) -> usize {
199        self.as_slice().len()
200    }
201
202    pub fn is_empty(&self) -> bool {
203        self.len() == 0
204    }
205}
206
207#[cfg(test)]
208mod tests {
209    use super::*;
210    use std::io::{self, Write};
211
212    #[test]
213    fn decompressed_limit_accepts_exactly_one_gib_without_retaining_output() {
214        let mut reader = LimitReader::new(
215            io::repeat(0).take(MAX_DECOMPRESSED_BYTES),
216            MAX_DECOMPRESSED_BYTES,
217        );
218        assert_eq!(
219            io::copy(&mut reader, &mut io::sink()).unwrap(),
220            MAX_DECOMPRESSED_BYTES
221        );
222    }
223
224    #[test]
225    fn decompressed_limit_rejects_one_extra_byte() {
226        let mut reader = LimitReader::new(
227            io::repeat(0).take(MAX_DECOMPRESSED_BYTES + 1),
228            MAX_DECOMPRESSED_BYTES,
229        );
230        let error = io::copy(&mut reader, &mut io::sink()).unwrap_err();
231        assert!(error
232            .to_string()
233            .contains(&MAX_DECOMPRESSED_BYTES.to_string()));
234        assert_eq!(reader.inner.limit(), 0);
235    }
236
237    #[test]
238    fn limit_reader_does_not_overread_the_limit_or_touch_empty_buffers() {
239        let mut reader = LimitReader::new(&b"123456789"[..], 3);
240        assert_eq!(reader.read(&mut []).unwrap(), 0);
241        let error = reader.read(&mut [0; 1024]).unwrap_err();
242        assert!(error.to_string().contains("3 byte limit"));
243        assert_eq!(reader.inner, b"56789");
244        let mut empty = LimitReader::new(&b""[..], 0);
245        assert_eq!(empty.read(&mut [0; 1]).unwrap(), 0);
246    }
247
248    #[test]
249    fn compressed_expansion_hits_the_same_reader_limit() {
250        let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best());
251        encoder.write_all(&[0; 128 * 1024]).unwrap();
252        let compressed = encoder.finish().unwrap();
253        assert!(compressed.len() < 1024);
254        let decoder = flate2::read::GzDecoder::new(compressed.as_slice());
255        let mut reader = LimitReader::new(decoder, 4096);
256        let error = io::copy(&mut reader, &mut io::sink()).unwrap_err();
257        assert!(error.to_string().contains("4096 byte limit"));
258    }
259}