liboxen 0.65.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

//! Gzip decompression helpers.

use std::io::{Read, Write};

use flate2::read::GzDecoder;

use crate::error::OxenError;

/// Inflate gzipped `data`, erroring if the inflated output would exceed `max_decompressed` bytes.
/// `max_decompressed` protects against gzip bomb attacks by limiting the decompressed size.
pub fn decompress_gzip_capped(data: &[u8], max_decompressed: u64) -> Result<Vec<u8>, OxenError> {
    let mut decompressed = Vec::with_capacity(data.len());
    decompress_gzip_capped_into(data, &mut decompressed, max_decompressed)?;
    Ok(decompressed)
}

/// Inflate the gzipped bytes `reader` yields into `writer`, erroring once the inflated output
/// passes `max_decompressed` bytes, and return how many bytes it wrote. On an error, `writer` may
/// hold part of the output.
pub fn decompress_gzip_capped_into(
    reader: impl Read,
    writer: &mut impl Write,
    max_decompressed: u64,
) -> Result<u64, OxenError> {
    let mut decoder = GzDecoder::new(reader).take(max_decompressed + 1);
    let decompressed_size = std::io::copy(&mut decoder, writer).map_err(|e| {
        OxenError::internal_error(format!("Failed to decompress gzipped data: {e}"))
    })?;
    if decompressed_size > max_decompressed {
        return Err(OxenError::internal_error(format!(
            "Decompressed size {decompressed_size} exceeds the {max_decompressed} byte limit"
        )));
    }
    Ok(decompressed_size)
}