unityrs_core/codec/mod.rs
1// Ported from UnityPy 1.25.2 (MIT): UnityPy/helpers/CompressionHelper.py. See THIRD_PARTY_LICENSES.
2//! Block codecs with UnityPy's semantics (`helpers/CompressionHelper.py:238-244`).
3//!
4//! - **None** returns the stored bytes as they are, whatever blocks-info says.
5//! - **LZMA** (`:17-44`) is a raw LZMA1 decoder over the 5 property bytes and
6//! the rest, run to the end of the input: the declared size is ignored, a
7//! truncated stream yields what decoded, an over-long one yields all of it.
8//! Output is capped at [`crate::limits::Limits::max_block_output`] (a `PARITY.md` safety
9//! divergence). Fewer than 5 input bytes raise (`struct.unpack("<BI", ...)`).
10//! liblzma static, as `docs/spikes/lzma.md` decided.
11//! - **LZ4/LZ4HC** (`:83-94`) is `lz4.block.decompress(data, uncompressed_size)`:
12//! the declared size is the *maximum* output. A shorter output is accepted;
13//! a longer one raises (python-lz4: `LZ4BlockError`), and so does an empty
14//! input, whatever the declared size (`LZ4_decompress_safe` returns -1).
15//! - **LZHAM** raises (`:234-235`), and so does an unknown type
16//! (`files/BundleFile.py:515-521`; `CompressionFlags` is an `IntFlag`, so the
17//! map lookup is what raises).
18//!
19//! A block's decoded length can differ from blocks-info; the bundle always
20//! lays entries out by the actual lengths, as UnityPy's join does (plan
21//! section 3.6), so a codec returns what it decoded and never pads or cuts.
22//!
23//! **LZ4 acceptance is liblz4's.** UnityPy's python-lz4 4.4.4 runs liblz4
24//! 1.9.4's `LZ4_decompress_safe`, which refuses some blocks `lz4_flex` decodes:
25//! its end-of-block conditions (a literal run that ends within the last 12
26//! bytes of the declared size or the last 8 bytes of input must be the last
27//! sequence and end exactly at the end of input; a match must end at least 5
28//! bytes before the declared size; the length-extension guards). [`Lz4`]
29//! is a port of liblz4's decoder (`lz4.rs`), so it refuses what liblz4
30//! refuses and decodes what it decodes, and the oracle fixture `tests/fixtures/scaffold/lz4_blocks.json` (python-lz4's verdict and
31//! output for a grid of blocks around each boundary) is the acceptance test.
32//!
33//! **Output.** A codec writes into a [`DecodeBuf`], which checks the block's
34//! cap and charges the input's meter before it grows (plan section 3.6), or
35//! names a range of its input (the stored codec: [`Decoded::Input`]), which
36//! the caller slices. It cannot hand back bytes of its own: a buffer a
37//! registered codec allocated (a Python decompressor's `bytes`) would be
38//! input-sized memory no meter charged (round 5), so its output goes through
39//! `out` like any other. Every block is decoded through [`decode_block`],
40//! which starts the block's segment and checks what the codec returned. A
41//! codec is caller code when a user registers it: the core calls it with no
42//! lock held.
43//!
44//! Owner: package C. The registry type and [`decode_block`] are implemented
45//! by the scaffold.
46
47use std::fmt;
48use std::ops::Range;
49use std::sync::Arc;
50
51use bytes::Bytes;
52
53use crate::budget::DecodeBuf;
54use crate::error::{Error, LimitError, LimitKind};
55
56pub mod encode;
57mod lz4;
58mod lzma;
59mod stored;
60pub mod unitycn;
61
62pub use lz4::Lz4;
63pub use lzma::Lzma;
64pub use stored::Stored;
65
66/// A block's compression, `flags & 0x3F`.
67#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
68pub enum Codec {
69 /// 0: stored.
70 None,
71 /// 1.
72 Lzma,
73 /// 2.
74 Lz4,
75 /// 3.
76 Lz4Hc,
77 /// 4: unsupported, as in UnityPy.
78 Lzham,
79 /// 5-63: a custom codec, if one is registered. Only these ids are
80 /// canonical: `Other(1)` names no codec (it is not `Lzma`), and the
81 /// registry refuses and never answers for it ([`Codec::is_canonical`]).
82 Other(u8),
83}
84
85impl Codec {
86 /// The codec of a block or blocks-info `flags` value.
87 #[must_use]
88 pub fn from_flags(flags: u32) -> Self {
89 match flags & 0x3F {
90 0 => Self::None,
91 1 => Self::Lzma,
92 2 => Self::Lz4,
93 3 => Self::Lz4Hc,
94 4 => Self::Lzham,
95 // `& 0x3F` keeps the value below 64, so the conversion never fails.
96 n => Self::Other(u8::try_from(n).unwrap_or(u8::MAX)),
97 }
98 }
99
100 /// Whether this is the value [`Codec::from_flags`] gives for its id: every
101 /// named variant, and `Other(5..=63)`.
102 #[must_use]
103 pub fn is_canonical(self) -> bool {
104 match self {
105 Self::Other(n) => (5..64).contains(&n),
106 _ => true,
107 }
108 }
109
110 /// The compression type number.
111 #[must_use]
112 pub fn id(self) -> u8 {
113 match self {
114 Self::None => 0,
115 Self::Lzma => 1,
116 Self::Lz4 => 2,
117 Self::Lz4Hc => 3,
118 Self::Lzham => 4,
119 Self::Other(n) => n,
120 }
121 }
122}
123
124/// Where a codec put a block's output ([`BlockCodec::decode`]).
125#[derive(Clone, Debug, PartialEq, Eq)]
126pub enum Decoded {
127 /// In `out`: what the codec wrote into the block's segment.
128 Written,
129 /// In the input: the output is `input[range]`, a zero-copy slice of the
130 /// stored bytes (the stored codec), with nothing written into `out`. A
131 /// range, not `Bytes`, so a codec cannot hand back a buffer of its own.
132 Input(Range<usize>),
133}
134
135/// Decodes one block.
136pub trait BlockCodec: Send + Sync {
137 /// Capture a compressor and chunk policy for one payload, when needed.
138 /// The default uses this registered codec directly.
139 fn compressor(
140 &self,
141 _which: Codec,
142 _meter: &Arc<crate::limits::Meter>,
143 ) -> Result<Option<Arc<dyn BlockCodec>>, crate::files::SaveError> {
144 Ok(None)
145 }
146 /// Decode with the original bundle flags available to extension callbacks.
147 /// Native codecs use only their registered compression type.
148 fn decode_with_flags(
149 &self,
150 input: &Bytes,
151 uncompressed: u32,
152 _flags: u32,
153 out: &mut DecodeBuf,
154 ) -> Result<Decoded, Error> {
155 self.decode(input, uncompressed, out)
156 }
157
158 /// Compress a bundle block. The default preserves the native packer.
159 fn compress(
160 &self,
161 data: &[u8],
162 which: Codec,
163 meter: &Arc<crate::limits::Meter>,
164 ) -> Result<Bytes, crate::files::SaveError> {
165 encode::compress(data, which, meter)
166 }
167
168 /// The writer's chunk bound, looked up once before writing the payload.
169 fn chunk_size(
170 &self,
171 which: Codec,
172 _meter: &Arc<crate::limits::Meter>,
173 ) -> Result<usize, crate::files::SaveError> {
174 match which {
175 Codec::None | Codec::Lzma => Ok(usize::try_from(u32::MAX).unwrap_or(usize::MAX)),
176 Codec::Lz4 | Codec::Lz4Hc => Ok(0x20000),
177 _ => Err(crate::files::SaveError::UnsupportedCodec(which)),
178 }
179 }
180 /// Decodes `input`, a block's stored bytes (a slice of the bundle).
181 /// `uncompressed` is blocks-info's declared size; each codec treats it as
182 /// UnityPy does (module docs). The output goes into `out`, which pays and
183 /// checks the caps before it grows, and the codec returns
184 /// [`Decoded::Written`]; or, with `out` left empty, the codec returns
185 /// [`Decoded::Input`], a range of `input` (the stored codec), which
186 /// [`decode_block`] checks and slices.
187 ///
188 /// ```compile_fail,E0053
189 /// // A codec cannot return bytes of its own (round 5: a buffer it
190 /// // allocated was kept in the bundle's runs, charged to nothing).
191 /// use unityrs_core::budget::DecodeBuf;
192 /// use unityrs_core::codec::BlockCodec;
193 /// use unityrs_core::{Bytes, Error};
194 /// struct Own;
195 /// impl BlockCodec for Own {
196 /// fn decode(&self, _: &Bytes, _: u32, _: &mut DecodeBuf) -> Result<Option<Bytes>, Error> {
197 /// Ok(Some(Bytes::from(vec![0u8; 1 << 20])))
198 /// }
199 /// }
200 /// ```
201 fn decode(
202 &self,
203 input: &Bytes,
204 uncompressed: u32,
205 out: &mut DecodeBuf,
206 ) -> Result<Decoded, Error>;
207}
208
209/// Decodes one block of a bundle with `codec` (the codec of `which`), as
210/// every caller does (package C's bundle; implemented by the scaffold).
211///
212/// Starts the block's segment in `out`, capped at `cap` bytes
213/// ([`DecodeBuf::begin_block`] with [`LimitKind::BlockOutput`], and
214/// `uncompressed` as the size its room grows to while what it writes fits
215/// it), runs the codec, and checks what it returned: `Ok(None)` when the
216/// output is in the segment; `Ok(Some(bytes))`, the slice of `input`, when
217/// the codec named a range of its input, which must lie within the input,
218/// leave the segment empty, and fit `cap` as if it had been written. A range outside the
219/// input, or one returned with bytes written, is the codec's error
220/// ([`Error::Decompress`]); a range longer than `cap` is
221/// [`LimitKind::BlockOutput`]. A codec that left its segment (began one of
222/// its own, which would lift its cap and its work count, or cut the buffer
223/// below its block, erasing earlier blocks of the run) is the codec's error
224/// too, whatever it returned (round 6: `Written` was trusted).
225pub fn decode_block(
226 which: Codec,
227 codec: &dyn BlockCodec,
228 input: &Bytes,
229 uncompressed: u32,
230 out: &mut DecodeBuf,
231 cap: u64,
232) -> Result<Option<Bytes>, Error> {
233 decode_flagged(
234 which,
235 codec,
236 input,
237 uncompressed,
238 out,
239 cap,
240 u32::from(which.id()),
241 )
242}
243
244pub(crate) fn decode_flagged(
245 which: Codec,
246 codec: &dyn BlockCodec,
247 input: &Bytes,
248 uncompressed: u32,
249 out: &mut DecodeBuf,
250 cap: u64,
251 flags: u32,
252) -> Result<Option<Bytes>, Error> {
253 out.begin_block(cap, LimitKind::BlockOutput, u64::from(uncompressed));
254 let epoch = out.segment_epoch();
255 // A pool refusal the codec returns is counted for the Environment's
256 // retry (`env::registry::from_callers`).
257 let decoded = crate::env::registry::from_callers(codec.decode_with_flags(
258 input,
259 uncompressed,
260 flags,
261 out,
262 ))?;
263 let bad = |detail: String| Error::Decompress {
264 codec: which,
265 detail: detail.into(),
266 };
267 if out.segment_epoch() != epoch {
268 return Err(bad(
269 "the codec began a segment of its own or cut the output below its block".to_owned(),
270 ));
271 }
272 let range = match decoded {
273 Decoded::Written => return Ok(None),
274 Decoded::Input(range) => range,
275 };
276 if out.segment_len() != 0 {
277 return Err(bad(format!(
278 "the codec wrote {} bytes and returned a range of its input",
279 out.segment_len()
280 )));
281 }
282 if range.start > range.end || range.end > input.len() {
283 return Err(bad(format!(
284 "the codec returned the range {range:?} of a {}-byte input",
285 input.len()
286 )));
287 }
288 let n = u64::try_from(range.len()).unwrap_or(u64::MAX);
289 if n > cap {
290 return Err(Error::Limit(LimitError::refused(
291 LimitKind::BlockOutput,
292 n,
293 cap,
294 )));
295 }
296 Ok(Some(input.slice(range)))
297}
298
299/// Decrypts one block in place before it is decoded (UnityCN: P3). UnityPy
300/// decrypts a block whose flags carry 0x100 and whose codec is not `None`,
301/// of a bundle whose header's encryption flag built a decryptor
302/// (`files/BundleFile.py:123-124`, `files/BundleFile.py:512-513`); a bundle whose flag is clear
303/// is never decrypted, whatever the context holds.
304pub trait BlockDecryptor: Send + Sync {
305 /// Decrypts block `index` in place.
306 fn decrypt(&self, block: &mut [u8], index: usize) -> Result<(), Error>;
307}
308
309/// Codecs by compression type number (0-63).
310#[derive(Clone)]
311pub struct Codecs {
312 slots: [Option<Arc<dyn BlockCodec>>; 64],
313 /// Slot 0 holds the built-in [`Stored`] that [`Self::unitypy`] puts
314 /// there ([`Self::stores_as_is`]).
315 stored_builtin: bool,
316}
317
318impl Codecs {
319 /// No codec at all, type 0 included.
320 #[must_use]
321 pub fn empty() -> Self {
322 Self {
323 slots: std::array::from_fn(|_| None),
324 stored_builtin: false,
325 }
326 }
327
328 /// UnityPy's set: none, LZMA, LZ4 and LZ4HC.
329 #[must_use]
330 pub fn unitypy() -> Self {
331 let mut c = Self::empty();
332 let lz4: Arc<dyn BlockCodec> = Arc::new(Lz4);
333 for (which, codec) in [
334 (Codec::None, Arc::new(Stored) as Arc<dyn BlockCodec>),
335 (Codec::Lzma, Arc::new(Lzma)),
336 (Codec::Lz4, lz4.clone()),
337 (Codec::Lz4Hc, lz4),
338 ] {
339 if let Some(slot) = c.slots.get_mut(usize::from(which.id())) {
340 *slot = Some(codec);
341 }
342 }
343 c.stored_builtin = true;
344 c
345 }
346
347 /// Registers `codec` for `which` (a custom codec is `Codec::Other(5..=63)`;
348 /// a codec for `Codec::None` replaces how stored blocks decode, as a
349 /// caller may replace UnityPy's `DECOMPRESSION_MAP[NONE]`). A
350 /// non-canonical value (`Other(n)` with `n` below 5 or above 63) names no
351 /// slot and is handed back.
352 pub fn set(&mut self, which: Codec, codec: Arc<dyn BlockCodec>) -> Result<(), Codec> {
353 if !which.is_canonical() {
354 return Err(which);
355 }
356 match self.slots.get_mut(usize::from(which.id())) {
357 Some(slot) => {
358 *slot = Some(codec);
359 if which == Codec::None {
360 self.stored_builtin = false;
361 }
362 Ok(())
363 }
364 None => Err(which),
365 }
366 }
367
368 /// Whether a stored block (type 0) is its own bytes: slot 0 holds the
369 /// built-in [`Stored`] of [`Self::unitypy`], so a bundle may take such a
370 /// block as a slice of its input, and its length as what was read,
371 /// without calling the codec. Otherwise (a codec registered over it, or
372 /// none) every stored block goes through slot 0 as any block goes
373 /// through its codec, lazily as eagerly, as UnityPy's eager load sends
374 /// every block through `DECOMPRESSION_MAP`.
375 #[must_use]
376 pub fn stores_as_is(&self) -> bool {
377 self.stored_builtin
378 }
379
380 /// The codec for `which`, if registered (never for a non-canonical value).
381 #[must_use]
382 pub fn get(&self, which: Codec) -> Option<&Arc<dyn BlockCodec>> {
383 if !which.is_canonical() {
384 return None;
385 }
386 self.slots.get(usize::from(which.id()))?.as_ref()
387 }
388}
389
390impl Default for Codecs {
391 fn default() -> Self {
392 Self::unitypy()
393 }
394}
395
396impl fmt::Debug for Codecs {
397 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
398 let ids: Vec<usize> = (0..self.slots.len())
399 .filter(|i| self.slots.get(*i).is_some_and(Option::is_some))
400 .collect();
401 f.debug_struct("Codecs")
402 .field("registered", &ids)
403 .field("stored_builtin", &self.stored_builtin)
404 .finish()
405 }
406}
407
408#[cfg(test)]
409mod tests;