liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

//! Gzip decompression helpers.

use std::io::Read;

use flate2::read::GzDecoder;

use crate::error::OxenError;

/// Inflate gzipped `data`, erroring if the inflated output would exceed `max_decompressed` bytes.
/// `max_decompressed` protects against gzip bomb attacks by limiting the decompressed size.
pub fn decompress_gzip_capped(data: &[u8], max_decompressed: u64) -> Result<Vec<u8>, OxenError> {
    let mut decoder = GzDecoder::new(data).take(max_decompressed + 1);
    let mut decompressed = Vec::with_capacity(data.len());
    decoder.read_to_end(&mut decompressed).map_err(|e| {
        OxenError::internal_error(format!("Failed to decompress gzipped data: {e}"))
    })?;
    let decompressed_size = decompressed.len() as u64;
    if decompressed_size > max_decompressed {
        return Err(OxenError::internal_error(format!(
            "Decompressed size {decompressed_size} exceeds the {max_decompressed} byte limit"
        )));
    }
    Ok(decompressed)
}