readcon_core/
compression.rs

1//=============================================================================
2// Transparent compression support
3//=============================================================================
4
5use std::io::{self, Read};
6use std::path::Path;
7
8/// Detected compression format based on magic bytes.
9#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10pub enum Compression {
11    None,
12    Gzip,
13}
14
15/// Detect compression format from the first bytes of a file.
16///
17/// - `1f 8b` = gzip
18/// - Otherwise = uncompressed
19pub fn detect_compression(bytes: &[u8]) -> Compression {
20    if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b {
21        return Compression::Gzip;
22    }
23    Compression::None
24}
25
26/// Detect compression format from a file extension.
27///
28/// Returns `Compression::Gzip` for `.gz` extension, `Compression::None` otherwise.
29pub fn detect_compression_from_extension(path: &Path) -> Compression {
30    match path.extension().and_then(|e| e.to_str()) {
31        Some("gz") => Compression::Gzip,
32        _ => Compression::None,
33    }
34}
35
36/// Size threshold below which we use `read_to_string` instead of mmap.
37const MMAP_THRESHOLD: u64 = 64 * 1024;
38
39/// Reads file contents, decompressing if needed.
40///
41/// Detection strategy:
42/// 1. Read first 2 bytes to check magic bytes.
43/// 2. If gzip: decompress entire file to a String.
44/// 3. If uncompressed and < 64 KiB: `read_to_string`.
45/// 4. If uncompressed and >= 64 KiB: memory-mapped I/O.
46pub fn read_file_contents(path: &Path) -> Result<FileContents, Box<dyn std::error::Error>> {
47    let file = std::fs::File::open(path)?;
48    let metadata = file.metadata()?;
49
50    // Read first 2 bytes for magic detection
51    let mut magic = [0u8; 2];
52    let bytes_read = {
53        let mut f = &file;
54        f.read(&mut magic)?
55    };
56
57    let compression = if bytes_read >= 2 {
58        detect_compression(&magic)
59    } else {
60        Compression::None
61    };
62
63    match compression {
64        Compression::Gzip => {
65            // Re-open and decompress the entire file
66            let file = std::fs::File::open(path)?;
67            let mut decoder = flate2::read::GzDecoder::new(file);
68            let mut contents = String::new();
69            decoder.read_to_string(&mut contents)?;
70            Ok(FileContents::Owned(contents))
71        }
72        Compression::None => {
73            if metadata.len() < MMAP_THRESHOLD {
74                let contents = std::fs::read_to_string(path)?;
75                Ok(FileContents::Owned(contents))
76            } else {
77                let file = std::fs::File::open(path)?;
78                let mmap = unsafe { memmap2::Mmap::map(&file)? };
79                Ok(FileContents::Mapped(mmap))
80            }
81        }
82    }
83}
84
85/// Holds file contents either as an owned String or a memory-mapped region.
86pub enum FileContents {
87    Owned(String),
88    Mapped(memmap2::Mmap),
89}
90
91impl FileContents {
92    pub fn as_str(&self) -> Result<&str, std::str::Utf8Error> {
93        match self {
94            FileContents::Owned(s) => Ok(s.as_str()),
95            FileContents::Mapped(m) => std::str::from_utf8(m),
96        }
97    }
98}
99
100/// Creates a gzip-compressed writer wrapping a file at the given path.
101pub fn gzip_writer(path: &Path) -> io::Result<flate2::write::GzEncoder<std::fs::File>> {
102    let file = std::fs::File::create(path)?;
103    Ok(flate2::write::GzEncoder::new(
104        file,
105        flate2::Compression::default(),
106    ))
107}
readcon_core/compression.rs

readcon_core/
compression.rs
