Skip to main content

rypipe_core/
input.rs

1#![allow(unsafe_code)]
2
3use std::io::Read;
4use std::path::Path;
5
6use crate::Result;
7
8/// Compression codecs recognized by leading magic bytes. Each codec is
9/// compiled in only when its Cargo feature (`gzip`, `zstd`, `lz4`) is
10/// enabled; inputs are detected independently of file extension.
11#[derive(Clone, Copy, Debug, PartialEq, Eq)]
12#[allow(dead_code)] // variants exist only under their cargo features
13enum Compression {
14    Gzip,
15    Zstd,
16    Lz4,
17}
18
19/// Owned handle for a memory-mapped file.
20#[cfg(feature = "mmap")]
21pub struct MmapHandle {
22    pub(crate) mmap: memmap2::Mmap,
23}
24
25#[cfg(feature = "mmap")]
26impl MmapHandle {
27    fn new(file: std::fs::File, prefault: bool) -> Result<Self> {
28        let mmap = unsafe { memmap2::Mmap::map(&file)? };
29        #[cfg(unix)]
30        {
31            if prefault {
32                // Pre-fault the entire file into RSS.
33                let _ = mmap.advise(memmap2::Advice::WillNeed);
34            } else {
35                // Let the kernel drop pages behind the sequential reader.
36                let _ = mmap.advise(memmap2::Advice::Sequential);
37            }
38        }
39        Ok(MmapHandle { mmap })
40    }
41
42    fn as_slice(&self) -> &[u8] {
43        &self.mmap[..]
44    }
45}
46
47/// Input abstraction: either a memory-mapped file or an owned in-memory buffer.
48pub enum InputBuffer {
49    #[cfg(feature = "mmap")]
50    Mmap(MmapHandle),
51    Owned(Vec<u8>),
52}
53
54/// Read the first four bytes of `path` and match them against the known
55/// codec magics (gzip `1f 8b`, zstd `28 b5 2f fd`, lz4 frame `04 22 4d 18`).
56/// Unreadable files yield `None` and surface their real error later via the
57/// normal open path.
58fn detect_compression(path: &Path) -> Option<Compression> {
59    let mut magic = [0u8; 4];
60    let n = std::fs::File::open(path).ok()?.read(&mut magic).ok()?;
61    #[cfg(feature = "gzip")]
62    if n >= 2 && magic[0] == 0x1f && magic[1] == 0x8b {
63        return Some(Compression::Gzip);
64    }
65    #[cfg(feature = "zstd")]
66    if n >= 4 && magic == [0x28, 0xb5, 0x2f, 0xfd] {
67        return Some(Compression::Zstd);
68    }
69    #[cfg(feature = "lz4")]
70    if n >= 4 && magic == [0x04, 0x22, 0x4d, 0x18] {
71        return Some(Compression::Lz4);
72    }
73    let _ = n;
74    None
75}
76
77/// Decompress `path` with `codec` into an owned buffer.
78#[cfg_attr(
79    not(any(feature = "gzip", feature = "zstd", feature = "lz4")),
80    allow(unused_variables)
81)]
82fn decompress(path: &Path, codec: Compression) -> Result<Vec<u8>> {
83    match codec {
84        #[cfg(feature = "gzip")]
85        Compression::Gzip => {
86            let mut out = Vec::new();
87            flate2::read::GzDecoder::new(std::fs::File::open(path)?).read_to_end(&mut out)?;
88            Ok(out)
89        }
90        #[cfg(feature = "zstd")]
91        Compression::Zstd => {
92            let mut out = Vec::new();
93            zstd::stream::read::Decoder::new(std::fs::File::open(path)?)?.read_to_end(&mut out)?;
94            Ok(out)
95        }
96        #[cfg(feature = "lz4")]
97        Compression::Lz4 => {
98            let mut out = Vec::new();
99            lz4_flex::frame::FrameDecoder::new(std::fs::File::open(path)?).read_to_end(&mut out)?;
100            Ok(out)
101        }
102        // Only reachable when a codec's cargo feature is disabled; detection
103        // never selects disabled codecs, so this is defensive.
104        #[allow(unreachable_patterns)]
105        _ => Err(crate::Error::Io(std::io::Error::other(format!(
106            "{codec:?} input detected but its cargo feature is not enabled"
107        )))),
108    }
109}
110
111impl InputBuffer {
112    /// Open a path as an input buffer.
113    ///
114    /// When the leading magic bytes identify a compression codec whose Cargo
115    /// feature is enabled, the file is transparently decompressed into an
116    /// owned buffer; all execution modes then operate on the decompressed
117    /// bytes. Otherwise, when `use_mmap` is true and the `"mmap"` feature is
118    /// enabled, the file is mapped; otherwise it is read into memory.
119    pub fn open(path: &Path, use_mmap: bool, prefault: bool) -> Result<Self> {
120        if let Some(codec) = detect_compression(path) {
121            return Ok(InputBuffer::Owned(decompress(path, codec)?));
122        }
123
124        #[cfg(feature = "mmap")]
125        {
126            if use_mmap {
127                let file = std::fs::File::open(path)?;
128                return Ok(InputBuffer::Mmap(MmapHandle::new(file, prefault)?));
129            }
130        }
131        let _ = use_mmap;
132        let _ = prefault;
133        let bytes = std::fs::read(path)?;
134        Ok(InputBuffer::Owned(bytes))
135    }
136
137    pub fn as_slice(&self) -> &[u8] {
138        match self {
139            #[cfg(feature = "mmap")]
140            InputBuffer::Mmap(handle) => handle.as_slice(),
141            InputBuffer::Owned(bytes) => bytes,
142        }
143    }
144
145    pub fn len(&self) -> usize {
146        self.as_slice().len()
147    }
148
149    pub fn is_empty(&self) -> bool {
150        self.len() == 0
151    }
152}