heddle-format 0.15.0

An AI-native version control system
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139

// SPDX-License-Identifier: Apache-2.0
//! Bounded zstd encoding and decoding primitives.

#[cfg(feature = "zstd")]
use std::io::{BufReader, Read};

use super::CompressionError;

const MAX_DECOMPRESSED_SIZE: u64 = 256 * 1024 * 1024;

#[cfg(feature = "zstd")]
pub(super) fn compress(data: &[u8], level: i32) -> Result<Vec<u8>, CompressionError> {
    zstd::encode_all(data, level)
        .map_err(|error| CompressionError::CompressionFailed(error.to_string()))
}

#[cfg(feature = "zstd")]
pub(super) fn compress_with_dictionary(
    data: &[u8],
    level: i32,
    dictionary: &[u8],
) -> Result<Vec<u8>, CompressionError> {
    zstd::bulk::Compressor::with_dictionary(level, dictionary)
        .and_then(|mut compressor| compressor.compress(data))
        .map_err(|error| CompressionError::CompressionFailed(error.to_string()))
}

#[cfg(not(feature = "zstd"))]
pub(super) fn compress(_data: &[u8], _level: i32) -> Result<Vec<u8>, CompressionError> {
    Err(CompressionError::InvalidOperation(
        "zstd compression support not compiled into this build".to_string(),
    ))
}

#[cfg(not(feature = "zstd"))]
pub(super) fn compress_with_dictionary(
    _data: &[u8],
    _level: i32,
    _dictionary: &[u8],
) -> Result<Vec<u8>, CompressionError> {
    Err(CompressionError::InvalidOperation(
        "zstd compression support not compiled into this build".to_string(),
    ))
}

#[cfg(feature = "zstd")]
pub(super) fn decompress(data: &[u8], expected_size: u64) -> Result<Vec<u8>, CompressionError> {
    let decoder = zstd::stream::read::Decoder::new(BufReader::new(data))
        .map_err(|error| CompressionError::DecompressionFailed(error.to_string()))?;
    read_bounded(decoder, expected_size)
}

#[cfg(feature = "zstd")]
pub(super) fn decompress_with_dictionary(
    data: &[u8],
    expected_size: u64,
    dictionary: &[u8],
) -> Result<Vec<u8>, CompressionError> {
    let decoder = zstd::stream::read::Decoder::with_dictionary(BufReader::new(data), dictionary)
        .map_err(|error| CompressionError::DecompressionFailed(error.to_string()))?;
    read_bounded(decoder, expected_size)
}

#[cfg(feature = "zstd")]
fn read_bounded<R: Read>(mut decoder: R, expected_size: u64) -> Result<Vec<u8>, CompressionError> {
    validate_size(expected_size)?;
    let expected_capacity = usize::try_from(expected_size).map_err(|_| {
        CompressionError::CorruptedData("zstd expected size exceeds platform limits".to_string())
    })?;
    let mut decompressed = Vec::with_capacity(expected_capacity);
    let mut buffer = [0_u8; 8192];

    loop {
        let bytes_read = decoder
            .read(&mut buffer)
            .map_err(|error| CompressionError::DecompressionFailed(error.to_string()))?;
        if bytes_read == 0 {
            break;
        }
        let next_size = decompressed.len().checked_add(bytes_read).ok_or_else(|| {
            CompressionError::CorruptedData("decompressed size overflows".to_string())
        })?;
        let next_size = u64::try_from(next_size).map_err(|_| {
            CompressionError::CorruptedData("decompressed size exceeds platform limits".to_string())
        })?;
        if next_size > expected_size {
            return Err(CompressionError::CorruptedData(format!(
                "decompressed size exceeds recorded header size: expected {expected_size}, got at least {next_size}",
            )));
        }
        decompressed.extend_from_slice(&buffer[..bytes_read]);
    }

    validate_decompressed_len(expected_size, decompressed.len())?;
    Ok(decompressed)
}

#[cfg(not(feature = "zstd"))]
pub(super) fn decompress(_data: &[u8], expected_size: u64) -> Result<Vec<u8>, CompressionError> {
    validate_size(expected_size)?;
    Err(CompressionError::InvalidOperation(
        "zstd-compressed data is unsupported in this build".to_string(),
    ))
}

#[cfg(not(feature = "zstd"))]
pub(super) fn decompress_with_dictionary(
    _data: &[u8],
    expected_size: u64,
    _dictionary: &[u8],
) -> Result<Vec<u8>, CompressionError> {
    validate_size(expected_size)?;
    Err(CompressionError::InvalidOperation(
        "zstd-compressed data is unsupported in this build".to_string(),
    ))
}

pub(super) fn validate_size(size: u64) -> Result<(), CompressionError> {
    if size > MAX_DECOMPRESSED_SIZE {
        return Err(CompressionError::SizeLimitExceeded {
            size,
            max: MAX_DECOMPRESSED_SIZE,
        });
    }
    Ok(())
}

#[cfg(feature = "zstd")]
fn validate_decompressed_len(expected: u64, actual: usize) -> Result<(), CompressionError> {
    let actual = u64::try_from(actual).map_err(|_| {
        CompressionError::CorruptedData("decompressed size exceeds platform limits".to_string())
    })?;
    if actual != expected {
        return Err(CompressionError::CorruptedData(format!(
            "decompressed size mismatch: expected {expected}, got {actual}",
        )));
    }
    Ok(())
}