Skip to main content

unityrs_core/codec/
mod.rs

1// Ported from UnityPy 1.25.2 (MIT): UnityPy/helpers/CompressionHelper.py. See THIRD_PARTY_LICENSES.
2//! Block codecs with UnityPy's semantics (`helpers/CompressionHelper.py:238-244`).
3//!
4//! - **None** returns the stored bytes as they are, whatever blocks-info says.
5//! - **LZMA** (`:17-44`) is a raw LZMA1 decoder over the 5 property bytes and
6//!   the rest, run to the end of the input: the declared size is ignored, a
7//!   truncated stream yields what decoded, an over-long one yields all of it.
8//!   Output is capped at [`crate::limits::Limits::max_block_output`] (a `PARITY.md` safety
9//!   divergence). Fewer than 5 input bytes raise (`struct.unpack("<BI", ...)`).
10//!   liblzma static, as `docs/spikes/lzma.md` decided.
11//! - **LZ4/LZ4HC** (`:83-94`) is `lz4.block.decompress(data, uncompressed_size)`:
12//!   the declared size is the *maximum* output. A shorter output is accepted;
13//!   a longer one raises (python-lz4: `LZ4BlockError`), and so does an empty
14//!   input, whatever the declared size (`LZ4_decompress_safe` returns -1).
15//! - **LZHAM** raises (`:234-235`), and so does an unknown type
16//!   (`files/BundleFile.py:515-521`; `CompressionFlags` is an `IntFlag`, so the
17//!   map lookup is what raises).
18//!
19//! A block's decoded length can differ from blocks-info; the bundle always
20//! lays entries out by the actual lengths, as UnityPy's join does (plan
21//! section 3.6), so a codec returns what it decoded and never pads or cuts.
22//!
23//! **LZ4 acceptance is liblz4's.** UnityPy's python-lz4 4.4.4 runs liblz4
24//! 1.9.4's `LZ4_decompress_safe`, which refuses some blocks `lz4_flex` decodes:
25//! its end-of-block conditions (a literal run that ends within the last 12
26//! bytes of the declared size or the last 8 bytes of input must be the last
27//! sequence and end exactly at the end of input; a match must end at least 5
28//! bytes before the declared size; the length-extension guards). [`Lz4`]
29//! is a port of liblz4's decoder (`lz4.rs`), so it refuses what liblz4
30//! refuses and decodes what it decodes, and the oracle fixture `tests/fixtures/scaffold/lz4_blocks.json` (python-lz4's verdict and
31//! output for a grid of blocks around each boundary) is the acceptance test.
32//!
33//! **Output.** A codec writes into a [`DecodeBuf`], which checks the block's
34//! cap and charges the input's meter before it grows (plan section 3.6), or
35//! names a range of its input (the stored codec: [`Decoded::Input`]), which
36//! the caller slices. It cannot hand back bytes of its own: a buffer a
37//! registered codec allocated (a Python decompressor's `bytes`) would be
38//! input-sized memory no meter charged (round 5), so its output goes through
39//! `out` like any other. Every block is decoded through [`decode_block`],
40//! which starts the block's segment and checks what the codec returned. A
41//! codec is caller code when a user registers it: the core calls it with no
42//! lock held.
43//!
44//! Owner: package C. The registry type and [`decode_block`] are implemented
45//! by the scaffold.
46
47use std::fmt;
48use std::ops::Range;
49use std::sync::Arc;
50
51use bytes::Bytes;
52
53use crate::budget::DecodeBuf;
54use crate::error::{Error, LimitError, LimitKind};
55
56pub mod encode;
57mod lz4;
58mod lzma;
59mod stored;
60pub mod unitycn;
61
62pub use lz4::Lz4;
63pub use lzma::Lzma;
64pub use stored::Stored;
65
66/// A block's compression, `flags & 0x3F`.
67#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
68pub enum Codec {
69    /// 0: stored.
70    None,
71    /// 1.
72    Lzma,
73    /// 2.
74    Lz4,
75    /// 3.
76    Lz4Hc,
77    /// 4: unsupported, as in UnityPy.
78    Lzham,
79    /// 5-63: a custom codec, if one is registered. Only these ids are
80    /// canonical: `Other(1)` names no codec (it is not `Lzma`), and the
81    /// registry refuses and never answers for it ([`Codec::is_canonical`]).
82    Other(u8),
83}
84
85impl Codec {
86    /// The codec of a block or blocks-info `flags` value.
87    #[must_use]
88    pub fn from_flags(flags: u32) -> Self {
89        match flags & 0x3F {
90            0 => Self::None,
91            1 => Self::Lzma,
92            2 => Self::Lz4,
93            3 => Self::Lz4Hc,
94            4 => Self::Lzham,
95            // `& 0x3F` keeps the value below 64, so the conversion never fails.
96            n => Self::Other(u8::try_from(n).unwrap_or(u8::MAX)),
97        }
98    }
99
100    /// Whether this is the value [`Codec::from_flags`] gives for its id: every
101    /// named variant, and `Other(5..=63)`.
102    #[must_use]
103    pub fn is_canonical(self) -> bool {
104        match self {
105            Self::Other(n) => (5..64).contains(&n),
106            _ => true,
107        }
108    }
109
110    /// The compression type number.
111    #[must_use]
112    pub fn id(self) -> u8 {
113        match self {
114            Self::None => 0,
115            Self::Lzma => 1,
116            Self::Lz4 => 2,
117            Self::Lz4Hc => 3,
118            Self::Lzham => 4,
119            Self::Other(n) => n,
120        }
121    }
122}
123
124/// Where a codec put a block's output ([`BlockCodec::decode`]).
125#[derive(Clone, Debug, PartialEq, Eq)]
126pub enum Decoded {
127    /// In `out`: what the codec wrote into the block's segment.
128    Written,
129    /// In the input: the output is `input[range]`, a zero-copy slice of the
130    /// stored bytes (the stored codec), with nothing written into `out`. A
131    /// range, not `Bytes`, so a codec cannot hand back a buffer of its own.
132    Input(Range<usize>),
133}
134
135/// Decodes one block.
136pub trait BlockCodec: Send + Sync {
137    /// Capture a compressor and chunk policy for one payload, when needed.
138    /// The default uses this registered codec directly.
139    fn compressor(
140        &self,
141        _which: Codec,
142        _meter: &Arc<crate::limits::Meter>,
143    ) -> Result<Option<Arc<dyn BlockCodec>>, crate::files::SaveError> {
144        Ok(None)
145    }
146    /// Decode with the original bundle flags available to extension callbacks.
147    /// Native codecs use only their registered compression type.
148    fn decode_with_flags(
149        &self,
150        input: &Bytes,
151        uncompressed: u32,
152        _flags: u32,
153        out: &mut DecodeBuf,
154    ) -> Result<Decoded, Error> {
155        self.decode(input, uncompressed, out)
156    }
157
158    /// Compress a bundle block. The default preserves the native packer.
159    fn compress(
160        &self,
161        data: &[u8],
162        which: Codec,
163        meter: &Arc<crate::limits::Meter>,
164    ) -> Result<Bytes, crate::files::SaveError> {
165        encode::compress(data, which, meter)
166    }
167
168    /// The writer's chunk bound, looked up once before writing the payload.
169    fn chunk_size(
170        &self,
171        which: Codec,
172        _meter: &Arc<crate::limits::Meter>,
173    ) -> Result<usize, crate::files::SaveError> {
174        match which {
175            Codec::None | Codec::Lzma => Ok(usize::try_from(u32::MAX).unwrap_or(usize::MAX)),
176            Codec::Lz4 | Codec::Lz4Hc => Ok(0x20000),
177            _ => Err(crate::files::SaveError::UnsupportedCodec(which)),
178        }
179    }
180    /// Decodes `input`, a block's stored bytes (a slice of the bundle).
181    /// `uncompressed` is blocks-info's declared size; each codec treats it as
182    /// UnityPy does (module docs). The output goes into `out`, which pays and
183    /// checks the caps before it grows, and the codec returns
184    /// [`Decoded::Written`]; or, with `out` left empty, the codec returns
185    /// [`Decoded::Input`], a range of `input` (the stored codec), which
186    /// [`decode_block`] checks and slices.
187    ///
188    /// ```compile_fail,E0053
189    /// // A codec cannot return bytes of its own (round 5: a buffer it
190    /// // allocated was kept in the bundle's runs, charged to nothing).
191    /// use unityrs_core::budget::DecodeBuf;
192    /// use unityrs_core::codec::BlockCodec;
193    /// use unityrs_core::{Bytes, Error};
194    /// struct Own;
195    /// impl BlockCodec for Own {
196    ///     fn decode(&self, _: &Bytes, _: u32, _: &mut DecodeBuf) -> Result<Option<Bytes>, Error> {
197    ///         Ok(Some(Bytes::from(vec![0u8; 1 << 20])))
198    ///     }
199    /// }
200    /// ```
201    fn decode(
202        &self,
203        input: &Bytes,
204        uncompressed: u32,
205        out: &mut DecodeBuf,
206    ) -> Result<Decoded, Error>;
207}
208
209/// Decodes one block of a bundle with `codec` (the codec of `which`), as
210/// every caller does (package C's bundle; implemented by the scaffold).
211///
212/// Starts the block's segment in `out`, capped at `cap` bytes
213/// ([`DecodeBuf::begin_block`] with [`LimitKind::BlockOutput`], and
214/// `uncompressed` as the size its room grows to while what it writes fits
215/// it), runs the codec, and checks what it returned: `Ok(None)` when the
216/// output is in the segment; `Ok(Some(bytes))`, the slice of `input`, when
217/// the codec named a range of its input, which must lie within the input,
218/// leave the segment empty, and fit `cap` as if it had been written. A range outside the
219/// input, or one returned with bytes written, is the codec's error
220/// ([`Error::Decompress`]); a range longer than `cap` is
221/// [`LimitKind::BlockOutput`]. A codec that left its segment (began one of
222/// its own, which would lift its cap and its work count, or cut the buffer
223/// below its block, erasing earlier blocks of the run) is the codec's error
224/// too, whatever it returned (round 6: `Written` was trusted).
225pub fn decode_block(
226    which: Codec,
227    codec: &dyn BlockCodec,
228    input: &Bytes,
229    uncompressed: u32,
230    out: &mut DecodeBuf,
231    cap: u64,
232) -> Result<Option<Bytes>, Error> {
233    decode_flagged(
234        which,
235        codec,
236        input,
237        uncompressed,
238        out,
239        cap,
240        u32::from(which.id()),
241    )
242}
243
244pub(crate) fn decode_flagged(
245    which: Codec,
246    codec: &dyn BlockCodec,
247    input: &Bytes,
248    uncompressed: u32,
249    out: &mut DecodeBuf,
250    cap: u64,
251    flags: u32,
252) -> Result<Option<Bytes>, Error> {
253    out.begin_block(cap, LimitKind::BlockOutput, u64::from(uncompressed));
254    let epoch = out.segment_epoch();
255    // A pool refusal the codec returns is counted for the Environment's
256    // retry (`env::registry::from_callers`).
257    let decoded = crate::env::registry::from_callers(codec.decode_with_flags(
258        input,
259        uncompressed,
260        flags,
261        out,
262    ))?;
263    let bad = |detail: String| Error::Decompress {
264        codec: which,
265        detail: detail.into(),
266    };
267    if out.segment_epoch() != epoch {
268        return Err(bad(
269            "the codec began a segment of its own or cut the output below its block".to_owned(),
270        ));
271    }
272    let range = match decoded {
273        Decoded::Written => return Ok(None),
274        Decoded::Input(range) => range,
275    };
276    if out.segment_len() != 0 {
277        return Err(bad(format!(
278            "the codec wrote {} bytes and returned a range of its input",
279            out.segment_len()
280        )));
281    }
282    if range.start > range.end || range.end > input.len() {
283        return Err(bad(format!(
284            "the codec returned the range {range:?} of a {}-byte input",
285            input.len()
286        )));
287    }
288    let n = u64::try_from(range.len()).unwrap_or(u64::MAX);
289    if n > cap {
290        return Err(Error::Limit(LimitError::refused(
291            LimitKind::BlockOutput,
292            n,
293            cap,
294        )));
295    }
296    Ok(Some(input.slice(range)))
297}
298
299/// Decrypts one block in place before it is decoded (UnityCN: P3). UnityPy
300/// decrypts a block whose flags carry 0x100 and whose codec is not `None`,
301/// of a bundle whose header's encryption flag built a decryptor
302/// (`files/BundleFile.py:123-124`, `files/BundleFile.py:512-513`); a bundle whose flag is clear
303/// is never decrypted, whatever the context holds.
304pub trait BlockDecryptor: Send + Sync {
305    /// Decrypts block `index` in place.
306    fn decrypt(&self, block: &mut [u8], index: usize) -> Result<(), Error>;
307}
308
309/// Codecs by compression type number (0-63).
310#[derive(Clone)]
311pub struct Codecs {
312    slots: [Option<Arc<dyn BlockCodec>>; 64],
313    /// Slot 0 holds the built-in [`Stored`] that [`Self::unitypy`] puts
314    /// there ([`Self::stores_as_is`]).
315    stored_builtin: bool,
316}
317
318impl Codecs {
319    /// No codec at all, type 0 included.
320    #[must_use]
321    pub fn empty() -> Self {
322        Self {
323            slots: std::array::from_fn(|_| None),
324            stored_builtin: false,
325        }
326    }
327
328    /// UnityPy's set: none, LZMA, LZ4 and LZ4HC.
329    #[must_use]
330    pub fn unitypy() -> Self {
331        let mut c = Self::empty();
332        let lz4: Arc<dyn BlockCodec> = Arc::new(Lz4);
333        for (which, codec) in [
334            (Codec::None, Arc::new(Stored) as Arc<dyn BlockCodec>),
335            (Codec::Lzma, Arc::new(Lzma)),
336            (Codec::Lz4, lz4.clone()),
337            (Codec::Lz4Hc, lz4),
338        ] {
339            if let Some(slot) = c.slots.get_mut(usize::from(which.id())) {
340                *slot = Some(codec);
341            }
342        }
343        c.stored_builtin = true;
344        c
345    }
346
347    /// Registers `codec` for `which` (a custom codec is `Codec::Other(5..=63)`;
348    /// a codec for `Codec::None` replaces how stored blocks decode, as a
349    /// caller may replace UnityPy's `DECOMPRESSION_MAP[NONE]`). A
350    /// non-canonical value (`Other(n)` with `n` below 5 or above 63) names no
351    /// slot and is handed back.
352    pub fn set(&mut self, which: Codec, codec: Arc<dyn BlockCodec>) -> Result<(), Codec> {
353        if !which.is_canonical() {
354            return Err(which);
355        }
356        match self.slots.get_mut(usize::from(which.id())) {
357            Some(slot) => {
358                *slot = Some(codec);
359                if which == Codec::None {
360                    self.stored_builtin = false;
361                }
362                Ok(())
363            }
364            None => Err(which),
365        }
366    }
367
368    /// Whether a stored block (type 0) is its own bytes: slot 0 holds the
369    /// built-in [`Stored`] of [`Self::unitypy`], so a bundle may take such a
370    /// block as a slice of its input, and its length as what was read,
371    /// without calling the codec. Otherwise (a codec registered over it, or
372    /// none) every stored block goes through slot 0 as any block goes
373    /// through its codec, lazily as eagerly, as UnityPy's eager load sends
374    /// every block through `DECOMPRESSION_MAP`.
375    #[must_use]
376    pub fn stores_as_is(&self) -> bool {
377        self.stored_builtin
378    }
379
380    /// The codec for `which`, if registered (never for a non-canonical value).
381    #[must_use]
382    pub fn get(&self, which: Codec) -> Option<&Arc<dyn BlockCodec>> {
383        if !which.is_canonical() {
384            return None;
385        }
386        self.slots.get(usize::from(which.id()))?.as_ref()
387    }
388}
389
390impl Default for Codecs {
391    fn default() -> Self {
392        Self::unitypy()
393    }
394}
395
396impl fmt::Debug for Codecs {
397    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
398        let ids: Vec<usize> = (0..self.slots.len())
399            .filter(|i| self.slots.get(*i).is_some_and(Option::is_some))
400            .collect();
401        f.debug_struct("Codecs")
402            .field("registered", &ids)
403            .field("stored_builtin", &self.stored_builtin)
404            .finish()
405    }
406}
407
408#[cfg(test)]
409mod tests;