walker_common/compression/
mod.rs

1//! Helpers for using compression/decompression.
2
3mod detecting;
4mod limit;
5
6pub use detecting::*;
7pub use limit::*;
8
9use anyhow::anyhow;
10use bytes::Bytes;
11use std::io::Write;
12
13/// Decompress a stream, or fail if no encoder was configured.
14///
15/// This function will not consume the data, but return `None`, if no decompression was required.
16/// This allows one to hold on to the original, compressed, data if necessary.
17pub fn decompress_opt(data: &[u8], name: &str) -> Option<Result<Bytes, anyhow::Error>> {
18    let detector = Detector {
19        file_name: Some(name),
20        ..Default::default()
21    };
22    let detected = detector.detect(data).map_err(|err| anyhow!("{err}"));
23
24    detected
25        .and_then(|detected| detected.decompress_opt(data).map_err(|err| err.into()))
26        .transpose()
27}
28
29/// Decompress a stream, or fail if no encoder was configured.
30pub fn decompress(data: Bytes, name: &str) -> Result<Bytes, anyhow::Error> {
31    decompress_opt(&data, name).unwrap_or_else(|| Ok(data))
32}
33
34/// Decompress bz2 using `bzip2-rs` (pure Rust version)
35#[cfg(all(feature = "bzip2-rs", not(feature = "bzip2")))]
36#[allow(unused)]
37fn decompress_bzip2(data: &[u8]) -> Result<Bytes, std::io::Error> {
38    #[allow(deprecated)]
39    decompress_bzip2_with(data, &DecompressionOptions::default())
40}
41
42/// Decompress bz2 using `bzip2-rs` (pure Rust version)
43#[cfg(all(feature = "bzip2-rs", not(feature = "bzip2")))]
44fn decompress_bzip2_with(
45    data: &[u8],
46    opts: &DecompressionOptions,
47) -> Result<Bytes, std::io::Error> {
48    let decoder = bzip2_rs::DecoderReader::new(data);
49    decompress_limit(decoder, opts.limit)
50}
51
52/// Decompress bz2 using `bzip2` (based on `libbz2`).
53#[cfg(feature = "bzip2")]
54fn decompress_bzip2_with(
55    data: &[u8],
56    opts: &DecompressionOptions,
57) -> Result<Bytes, std::io::Error> {
58    let decoder = bzip2::read::BzDecoder::new(data);
59    decompress_limit(decoder, opts.limit)
60}
61
62/// Decompress xz using `lzma-rust2`.
63#[cfg(feature = "lzma")]
64fn decompress_xz_with(data: &[u8], opts: &DecompressionOptions) -> Result<Bytes, std::io::Error> {
65    let decoder = lzma_rust2::XzReader::new(data, false);
66    decompress_limit(decoder, opts.limit)
67}
68
69/// Decompress gzip using `zlib-rs` and `flate2`.
70#[cfg(feature = "flate2")]
71fn decompress_gzip_with(data: &[u8], opts: &DecompressionOptions) -> Result<Bytes, std::io::Error> {
72    let decoder = flate2::read::GzDecoder::new(data);
73    decompress_limit(decoder, opts.limit)
74}
75
76/// Decompress with an uncompressed payload limit.
77#[allow(unused)]
78fn decompress_limit(mut reader: impl std::io::Read, limit: usize) -> Result<Bytes, std::io::Error> {
79    let mut data = vec![];
80
81    let data = if limit > 0 {
82        let mut writer = LimitWriter::new(data, limit);
83        std::io::copy(&mut reader, &mut writer)?;
84        writer.flush()?;
85        writer.close()
86    } else {
87        reader.read_to_end(&mut data)?;
88        data
89    };
90
91    Ok(Bytes::from(data))
92}
walker_common/compression/mod.rs

walker_common/compression/
mod.rs
