Skip to main content

unityrs_core/codec/
encode.rs

1// Ported from UnityPy 1.25.2 (MIT): UnityPy/helpers/CompressionHelper.py. See THIRD_PARTY_LICENSES.
2// Ported from UnityPy git 417d595 (MIT): UnityPy/helpers/CompressionHelper.py. See THIRD_PARTY_LICENSES.
3//! Bundle packers following `helpers/CompressionHelper.py:158-231`.
4//! LZ4 and LZ4HC both use pristine liblz4 1.9.4 HC at level 9.
5
6use std::sync::Arc;
7
8use bytes::Bytes;
9
10use super::Codec;
11use crate::budget::DecodeBuf;
12use crate::error::LimitKind;
13use crate::files::save::{SaveError, Writer, narrow};
14use crate::limits::{Meter, MeteredVec};
15use crate::stream::Endian;
16
17/// One UnityFS block-table row, before it is serialized.
18#[derive(Clone, Copy, Debug, PartialEq, Eq)]
19pub struct BlockInfo {
20    /// Input length of this block.
21    pub uncompressed: u32,
22    /// Stored length of this block.
23    pub compressed: u32,
24    /// The caller's flags, with the codec cleared for a stored chunk.
25    pub flags: u16,
26    /// HEAD format 9's offset in the padded compressed payload.
27    pub offset: Option<u64>,
28}
29
30/// Packed bundle payload and its metered block table.
31#[derive(Debug)]
32pub struct PackedBlocks {
33    /// Concatenated stored blocks, padded to 16 bytes for HEAD format 9.
34    pub data: Bytes,
35    /// Block rows, in payload order.
36    pub blocks: MeteredVec<BlockInfo>,
37}
38
39const _: () = crate::send_sync::<BlockInfo>();
40const _: () = crate::send_sync::<PackedBlocks>();
41
42/// Compress one block, including Unity's five-byte LZMA properties prefix.
43/// Both LZ4 codec IDs select `compress_lz4`
44/// (`helpers/CompressionHelper.py:97-105`).
45pub fn compress(data: &[u8], codec: Codec, meter: &Arc<Meter>) -> Result<Bytes, SaveError> {
46    match codec {
47        Codec::None => {
48            let mut out = Writer::new(meter, Endian::Big)?;
49            out.bytes(data)?;
50            out.finish()
51        }
52        Codec::Lzma => compress_lzma(data, meter),
53        Codec::Lz4 | Codec::Lz4Hc => compress_lz4(data, meter),
54        _ => Err(SaveError::UnsupportedCodec(codec)),
55    }
56}
57
58/// UnityPy 1.25.2's `chunk_based_compress`
59/// (`helpers/CompressionHelper.py:158-231`): 128 KiB for LZ4, at most `u32::MAX`
60/// for LZMA. A chunk is stored only when compression strictly increases its
61/// size; all other flag bits survive. Stored empty data has one empty row;
62/// a compressing packer has no row for empty data.
63pub fn chunk_based_compress(
64    data: &[u8],
65    flags: u16,
66    meter: &Arc<Meter>,
67) -> Result<PackedBlocks, SaveError> {
68    chunks(data, flags, None, meter)
69}
70
71/// HEAD 417d595's `chunk_based_compress`: the bytes already written, with
72/// padding, determine whether the next block is stored. From format 9 each
73/// compressed-packer block has an offset and trailing 16-byte alignment.
74pub fn head_chunk_based_compress(
75    data: &[u8],
76    flags: u16,
77    version: u32,
78    meter: &Arc<Meter>,
79) -> Result<PackedBlocks, SaveError> {
80    chunks(data, flags, Some(version), meter)
81}
82
83fn chunks(
84    data: &[u8],
85    flags: u16,
86    head: Option<u32>,
87    meter: &Arc<Meter>,
88) -> Result<PackedBlocks, SaveError> {
89    chunks_with_registry(data, flags, head, meter, None)
90}
91
92/// Compress through the context's registered block codecs and chunk policy.
93/// `head` selects the later bundle layout when explicitly requested.
94pub fn registered_chunks(
95    data: &[u8],
96    flags: u16,
97    codecs: &super::Codecs,
98    head: Option<u32>,
99    meter: &Arc<Meter>,
100) -> Result<PackedBlocks, SaveError> {
101    chunks_with_registry(data, flags, head, meter, Some(codecs))
102}
103
104/// Compress a single registered block, such as a bundle's directory table.
105pub fn registered_compress(
106    data: &[u8],
107    codec: Codec,
108    codecs: &super::Codecs,
109    meter: &Arc<Meter>,
110) -> Result<Bytes, SaveError> {
111    codecs
112        .get(codec)
113        .ok_or(SaveError::UnsupportedCodec(codec))?
114        .compress(data, codec, meter)
115}
116
117fn chunks_with_registry(
118    data: &[u8],
119    flags: u16,
120    head: Option<u32>,
121    meter: &Arc<Meter>,
122    codecs: Option<&super::Codecs>,
123) -> Result<PackedBlocks, SaveError> {
124    let codec = Codec::from_flags(u32::from(flags));
125    let registered = if codec == Codec::None {
126        None
127    } else {
128        codecs
129            .map(|codecs| {
130                codecs
131                    .get(codec)
132                    .cloned()
133                    .ok_or(SaveError::UnsupportedCodec(codec))
134            })
135            .transpose()?
136    };
137    let captured = registered
138        .as_ref()
139        .map(|encoder| encoder.compressor(codec, meter))
140        .transpose()?
141        .flatten();
142    let encoder = captured.as_ref().or(registered.as_ref());
143    let chunk = if let Some(encoder) = encoder {
144        encoder.chunk_size(codec, meter)?
145    } else {
146        match codec {
147            Codec::None => data.len().max(1),
148            Codec::Lzma => usize::try_from(u32::MAX).unwrap_or(usize::MAX),
149            Codec::Lz4 | Codec::Lz4Hc => 0x20000,
150            _ => return Err(SaveError::UnsupportedCodec(codec)),
151        }
152    };
153    if chunk == 0 {
154        return Err(SaveError::Range("zero codec chunk size"));
155    }
156    let mut blocks = MeteredVec::new(meter);
157    if codec == Codec::None {
158        let len = narrow(data.len(), "stored block length")?;
159        blocks.push(BlockInfo {
160            uncompressed: len,
161            compressed: len,
162            flags,
163            offset: head.filter(|version| *version >= 9).map(|_| 0),
164        })?;
165        return Ok(PackedBlocks {
166            data: compress(data, codec, meter)?,
167            blocks,
168        });
169    }
170    let mut out = Writer::new(meter, Endian::Big)?;
171    for part in data.chunks(chunk) {
172        let packed = if let Some(encoder) = encoder {
173            encoder.compress(part, codec, meter)?
174        } else {
175            compress(part, codec, meter)?
176        };
177        let store = if head.is_some() {
178            out.len() >= chunk
179        } else {
180            packed.len() > part.len()
181        };
182        let (stored, block_flags) = if store {
183            (part, flags ^ u16::from(codec.id()))
184        } else {
185            (packed.as_ref(), flags)
186        };
187        blocks.push(BlockInfo {
188            uncompressed: narrow(part.len(), "uncompressed block length")?,
189            compressed: narrow(stored.len(), "compressed block length")?,
190            flags: block_flags,
191            offset: if head.is_some_and(|version| version >= 9) {
192                Some(narrow(out.len(), "block offset")?)
193            } else {
194                None
195            },
196        })?;
197        out.bytes(stored)?;
198        if head.is_some_and(|version| version >= 9) {
199            out.align(16)?;
200        }
201    }
202    Ok(PackedBlocks {
203        data: out.finish()?,
204        blocks,
205    })
206}
207
208/// `compress_lz4` (`helpers/CompressionHelper.py:97-105`). liblz4's external
209/// state entry point runs exactly the same compressor as its allocating
210/// convenience function, with a caller-provided eight-byte-aligned state.
211fn compress_lz4(data: &[u8], meter: &Arc<Meter>) -> Result<Bytes, SaveError> {
212    let capacity = unityrs_compress_sys::lz4_bound(data.len())
213        .map_err(|e| native_error(e, Codec::Lz4Hc, meter))?;
214    let words = unityrs_compress_sys::lz4_state_words()
215        .map_err(|e| native_error(e, Codec::Lz4Hc, meter))?;
216    // Count the Rust-side state zeroing as well as the input scan before
217    // allocating or touching either compressor buffer.
218    meter.count_decode(
219        u64::try_from(data.len())
220            .unwrap_or(u64::MAX)
221            .saturating_add(u64::try_from(words).unwrap_or(u64::MAX).saturating_mul(8)),
222    )?;
223    let mut state = MeteredVec::with_capacity(meter, words)?;
224    for _ in 0..words {
225        state.push(0u64)?;
226    }
227    let mut out = DecodeBuf::counted(meter, u64::MAX, LimitKind::InputMemory)?;
228    let output = out.grow_zeroed(capacity)?;
229    let written = unityrs_compress_sys::lz4_compress(data, output, state.as_mut_slice())
230        .map_err(|e| native_error(e, Codec::Lz4Hc, meter))?;
231    out.truncate(written);
232    Ok(out.into_bytes()?)
233}
234
235fn native_error(error: unityrs_compress_sys::Error, codec: Codec, meter: &Meter) -> SaveError {
236    use unityrs_compress_sys::Error;
237    match error {
238        Error::Range => SaveError::Range("compressor length"),
239        Error::Allocator(bytes) => {
240            crate::limits::ChargeError::from(meter.allocator_refused(bytes)).into()
241        }
242        Error::Allowance => SaveError::Compress {
243            codec,
244            detail: "native encoder exceeded its charged allowance",
245        },
246        Error::Compress => SaveError::Compress {
247            codec,
248            detail: "native encoder failed",
249        },
250    }
251}
252
253/// `compress_lzma` (`helpers/CompressionHelper.py:47-79`). Filters and
254/// liblzma allocations are owned by the safe native bridge; its complete
255/// memory allowance is charged before the encoder starts.
256fn compress_lzma(data: &[u8], meter: &Arc<Meter>) -> Result<Bytes, SaveError> {
257    meter.count_decode(u64::try_from(data.len()).unwrap_or(u64::MAX))?;
258    let allowance = unityrs_compress_sys::lzma_encoder_memory()
259        .map_err(|e| native_error(e, Codec::Lzma, meter))?;
260    let _state = meter.charge(allowance)?;
261    let mut out = DecodeBuf::counted(meter, u64::MAX, LimitKind::InputMemory)?;
262    out.extend_from_slice(&[0x5d, 0, 0, 0x80, 0])?;
263    unityrs_compress_sys::lzma_compress(data, allowance, |part| out.extend_from_slice(part))
264        .map_err(|error| match error {
265            unityrs_compress_sys::LzmaError::Native(e) => native_error(e, Codec::Lzma, meter),
266            unityrs_compress_sys::LzmaError::Output(e) => SaveError::Charge(e),
267        })?;
268    Ok(out.into_bytes()?)
269}
270
271#[cfg(test)]
272#[path = "encode_tests.rs"]
273mod tests;