Skip to main content

unityrs_core/
pystr.rs

1// Ported from UnityPy 1.25.2 (MIT): UnityPy/environment.py, UnityPy/classes/PPtr.py, UnityPy/helpers/ResourceReader.py. See THIRD_PARTY_LICENSES.
2// Ported from CPython 3.11 (PSF-2.0): Lib/ntpath.py, splitdrive and split; Lib/genericpath.py, _splitext. See THIRD_PARTY_LICENSES.
3//! Python string semantics on raw bytes, as UnityPy applies them to names.
4//!
5//! UnityPy compares CAB and resource names after `ntpath.basename` and
6//! `str.lower()` (`environment.py:357-363`, `classes/PPtr.py:66-77`,
7//! `helpers/ResourceReader.py:11-18`). Python lower-cases a surrogate-escaped
8//! string: valid UTF-8 runs with Unicode full case mapping (Python 3.11 has
9//! Unicode 14.0), invalid bytes passed through. Rust's tables are newer, so a
10//! generated exception table ([`crate::tables::lower_exception`]) pins the code
11//! points whose mapping changed after 14.0 (`tools/gen/`, package A; plan risk R7).
12//! The `ntpath` functions behave as Python 3.11's do, which
13//! `tools/fixtures/env.py` records (`tests/fixtures/d/pystr.json`).
14//!
15//! **Which bytes are characters.** A surrogate-escaped string is UTF-8
16//! decoded with `surrogateescape`: every valid UTF-8 sequence is one
17//! character, and every byte of an invalid sequence is a lone surrogate
18//! (U+DC80-U+DCFF), which `lower()` keeps and the encoder turns back into
19//! the byte. Python's decoder and [`std::str::Utf8Chunks`] split a byte string
20//! into the same valid runs (both reject surrogates, overlong forms and code
21//! points past U+10FFFF), so a run of [`std::str::Utf8Chunks`] is lowered as a
22//! `str` and its invalid bytes are copied.
23//!
24//! **Final sigma.** `str.lower()` maps U+03A3 to U+03C2 in the
25//! `Final_Sigma` context (`handle_capital_sigma` in Python's
26//! `unicodeobject.c`): a cased character before it and none after it,
27//! skipping case-ignorable ones. The two classes come from Python 3.11's
28//! own `str.lower()` ([`crate::tables::is_sigma_cased`],
29//! [`crate::tables::is_case_ignorable`], Unicode 14.0), not from Rust's newer
30//! tables, which differ for characters Unicode assigned after 14.0. A lone
31//! surrogate (an invalid byte) is neither cased nor case-ignorable to
32//! Python, so the context never crosses a run's edge and a run lowered on
33//! its own gets Python's answer.
34//!
35//! **Memory.** Each function that returns a string computes it from an
36//! input's name (a directory path, an external path, a resource path, up to
37//! the input's size) and charges it to that input's meter before it is
38//! allocated ([`RawStr::charged`]'s cost): a refusal allocates nothing. A
39//! result equal to its input (a name already lower-case and without a
40//! directory) is returned as the input's own bytes, uncopied, and a result
41//! that is a part of its input (a basename already lower-case) as a slice of
42//! them. Comparisons that only need to know whether two names are equal
43//! ([`lower_eq`]) compare without building the lowered string at all, and
44//! nothing else is allocated: a capital sigma's context is read in place.
45//!
46//! Owner: package D.
47
48use std::sync::Arc;
49
50use crate::limits::{ChargeError, Meter, MeteredVec};
51use crate::raw::RawStr;
52use crate::tables::{is_case_ignorable, is_sigma_cased, lower_exception, nd_digit};
53
54/// U+03A3 GREEK CAPITAL LETTER SIGMA, the one character `lower()` maps by
55/// context.
56const CAPITAL_SIGMA: char = '\u{3A3}';
57
58/// Emits `s.lower()` piece by piece, in order, to `emit`, which returns
59/// `false` to stop early. A piece is one lowered character or one run of
60/// invalid bytes. Returns whether every piece was emitted.
61fn lower_pieces(s: &[u8], mut emit: impl FnMut(&[u8]) -> bool) -> bool {
62    for chunk in s.utf8_chunks() {
63        let run = chunk.valid();
64        if !run.is_empty() && !lower_run(run, &mut emit) {
65            return false;
66        }
67        let bad = chunk.invalid();
68        // Lone surrogates: `lower()` keeps them, the encoder restores the bytes.
69        if !bad.is_empty() && !emit(bad) {
70            return false;
71        }
72    }
73    true
74}
75
76/// One valid run's lowering, character by character.
77fn lower_run(run: &str, emit: &mut impl FnMut(&[u8]) -> bool) -> bool {
78    let mut buf = [0u8; 12];
79    for (i, c) in run.char_indices() {
80        let piece: &[u8] = if c == CAPITAL_SIGMA {
81            sigma_form(run, i).encode_utf8(&mut buf).as_bytes()
82        } else if let Some(py) = lower_exception(c) {
83            py.as_bytes()
84        } else {
85            let mut n = 0;
86            for l in c.to_lowercase() {
87                n += l.encode_utf8(buf.get_mut(n..).unwrap_or_default()).len();
88            }
89            buf.get(..n).unwrap_or_default()
90        };
91        if !emit(piece) {
92            return false;
93        }
94    }
95    true
96}
97
98/// The lower-case form of the capital sigma at byte `at` of `run`: final
99/// (U+03C2) when the nearest character before it that is not
100/// case-ignorable is cased, and the nearest such one after it is not cased
101/// or absent (`handle_capital_sigma` in `CPython`'s `Objects/unicodeobject.c`).
102fn sigma_form(run: &str, at: usize) -> char {
103    let before = run.get(..at).unwrap_or_default();
104    let after = run.get(at + CAPITAL_SIGMA.len_utf8()..).unwrap_or_default();
105    let mut back = before.chars().rev().filter(|&c| !is_case_ignorable(c));
106    let mut ahead = after.chars().filter(|&c| !is_case_ignorable(c));
107    let fin = back.next().is_some_and(is_sigma_cased) && !ahead.next().is_some_and(is_sigma_cased);
108    if fin { '\u{3C2}' } else { '\u{3C3}' }
109}
110
111/// `sub.lower()`, where `sub` lies inside `whole`: `sub`'s own bytes (a
112/// zero-copy slice of `whole`) when lowering changes nothing, else a new
113/// string charged to `meter` before it is allocated.
114fn lower_within(whole: &RawStr, sub: &[u8], meter: &Arc<Meter>) -> Result<RawStr, ChargeError> {
115    let mut len = 0usize;
116    let mut same = true;
117    lower_pieces(sub, |piece| {
118        let end = len.saturating_add(piece.len());
119        same = same && sub.get(len..end) == Some(piece);
120        len = end;
121        true
122    });
123    if same && len == sub.len() {
124        return Ok(slice_of(whole, sub));
125    }
126    let mut out = MeteredVec::with_capacity(meter, len)?;
127    let mut failed = None;
128    lower_pieces(sub, |piece| match out.extend_from_slice(piece) {
129        Ok(()) => true,
130        Err(e) => {
131            failed = Some(e);
132            false
133        }
134    });
135    if let Some(e) = failed {
136        return Err(e);
137    }
138    RawStr::from_metered(out, meter)
139}
140
141/// `sub` as a zero-copy slice of `whole` (`sub` lies inside it).
142fn slice_of(whole: &RawStr, sub: &[u8]) -> RawStr {
143    if sub.is_empty() {
144        return RawStr::default();
145    }
146    whole.slice_ref(sub)
147}
148
149/// `str.lower()` on surrogate-escaped bytes, charged to `meter`.
150pub fn py_lower(s: &RawStr, meter: &Arc<Meter>) -> Result<RawStr, ChargeError> {
151    lower_within(s, s.as_bytes(), meter)
152}
153
154/// Whether `a.lower() == lowered`, compared piece by piece without building
155/// `a.lower()` (`classes/PPtr.py:75-76` compares each `files` key so); it
156/// allocates nothing.
157#[must_use]
158pub fn lower_eq(a: &[u8], lowered: &[u8]) -> bool {
159    let mut at = 0usize;
160    let all = lower_pieces(a, |piece| {
161        let end = at.saturating_add(piece.len());
162        let same = lowered.get(at..end) == Some(piece);
163        at = end;
164        same
165    });
166    all && at == lowered.len()
167}
168
169/// The byte length of the first Python character of `p`: a valid UTF-8
170/// sequence, or one byte of an invalid one (a lone surrogate).
171fn first_char_len(p: &[u8]) -> usize {
172    p.utf8_chunks()
173        .next()
174        .map_or(0, |c| c.valid().chars().next().map_or(1, char::len_utf8))
175}
176
177/// `ntpath.splitdrive` as Python 3.11 splits: the drive or UNC share, and
178/// the rest. Every rule but the drive letter's compares ASCII bytes at
179/// ASCII positions, so it holds on bytes; the drive letter is any one
180/// character followed by `:` (`normp[1:2] == colon` indexes characters).
181fn splitdrive(p: &[u8]) -> (&[u8], &[u8]) {
182    let is_sep = |b: &u8| *b == b'\\' || *b == b'/';
183    // `normp[0:2] == sep * 2`: UNC drives and device drives.
184    if p.first().is_some_and(is_sep) && p.get(1).is_some_and(is_sep) {
185        // `normp[:8].upper() == '\\\\?\\UNC\\'`: only ASCII `u`, `n` and `c`
186        // upper-case to the prefix's letters (probed on Python 3.11 over
187        // every code point), so the first eight bytes decide it.
188        let unc = p.get(..8).is_some_and(|head| {
189            head.iter().zip(b"\\\\?\\UNC\\").all(|(&b, &want)| {
190                let b = if b == b'/' { b'\\' } else { b };
191                b.to_ascii_uppercase() == want
192            })
193        });
194        let start = if unc { 8 } else { 2 };
195        let find = |from: usize| {
196            p.get(from..)
197                .and_then(|t| t.iter().position(is_sep))
198                .map(|i| i + from)
199        };
200        let Some(index) = find(start) else {
201            return (p, &[]);
202        };
203        let Some(index2) = find(index + 1) else {
204            return (p, &[]);
205        };
206        return p.split_at(index2);
207    }
208    // `normp[1:2] == colon`: the second character.
209    let first = first_char_len(p);
210    if first > 0 && p.get(first) == Some(&b':') {
211        return p.split_at(first + 1);
212    }
213    (&[], p)
214}
215
216/// `ntpath.basename`: after `splitdrive`, the part after the last `/` or `\`.
217#[must_use]
218pub fn ntpath_basename(p: &[u8]) -> &[u8] {
219    let (_, rest) = splitdrive(p);
220    let start = rest
221        .iter()
222        .rposition(|&b| b == b'\\' || b == b'/')
223        .map_or(0, |i| i + 1);
224    rest.get(start..).unwrap_or_default()
225}
226
227/// `ntpath.splitext`: `(root, ext)`, leading dots not counting as an extension.
228#[must_use]
229pub fn ntpath_splitext(p: &[u8]) -> (&[u8], &[u8]) {
230    // `genericpath._splitext(p, '\\', '/', '.')`
231    let sep = p.iter().rposition(|&b| b == b'\\' || b == b'/');
232    let Some(dot) = p.iter().rposition(|&b| b == b'.') else {
233        return (p, &[]);
234    };
235    if sep.is_none_or(|s| dot > s) {
236        let name = sep.map_or(0, |s| s + 1);
237        // Leading dots are part of the name, not an extension.
238        if p.get(name..dot)
239            .is_some_and(|lead| lead.iter().any(|&b| b != b'.'))
240        {
241            return p.split_at(dot);
242        }
243    }
244    (p, &[])
245}
246
247/// A transient CAB lookup key. Keeps computed bytes in a fallibly allocated,
248/// charged vector instead of constructing a `Bytes` shared owner (CR10).
249pub fn simplify_name_vec(
250    name: &[u8],
251    meter: &Arc<Meter>,
252) -> Result<crate::limits::MeteredVec<u8>, ChargeError> {
253    let name = ntpath_basename(name);
254    let mut len = 0usize;
255    lower_pieces(name, |piece| {
256        len = len.saturating_add(piece.len());
257        true
258    });
259    let mut out = crate::limits::MeteredVec::with_capacity(meter, len)?;
260    let mut error = None;
261    lower_pieces(name, |piece| match out.extend_from_slice(piece) {
262        Ok(()) => true,
263        Err(e) => {
264            error = Some(e);
265            false
266        }
267    });
268    match error {
269        Some(e) => Err(e),
270        None => Ok(out),
271    }
272}
273
274/// `simplify_name`: `ntpath.basename(name).lower()`, the CAB registry's key,
275/// charged to `meter`.
276pub fn simplify_name(name: &RawStr, meter: &Arc<Meter>) -> Result<RawStr, ChargeError> {
277    // UnityPy environment.py:357-363
278    lower_within(name, ntpath_basename(name.as_bytes()), meter)
279}
280
281/// `PPtr.deref`'s external path cleaning: strip a leading `archive:/`, then a
282/// leading `assets/`, keep what follows the last `/`, lower-case it; charged
283/// to `meter`.
284pub fn clean_external_path(path: &RawStr, meter: &Arc<Meter>) -> Result<RawStr, ChargeError> {
285    // UnityPy classes/PPtr.py:68-73
286    let mut p = path.as_bytes();
287    if let Some(rest) = p.strip_prefix(b"archive:/") {
288        p = rest;
289    }
290    if let Some(rest) = p.strip_prefix(b"assets/") {
291        p = rest;
292    }
293    // `rsplit("/")[-1]`
294    let start = p.iter().rposition(|&b| b == b'/').map_or(0, |i| i + 1);
295    lower_within(path, p.get(start..).unwrap_or_default(), meter)
296}
297
298/// Whether UnityPy's `load_file` reads a path as the parts of a split file:
299/// `reSplit.match(path)` (`environment.py:25`, `:111-116`), with `reSplit`
300/// `(.*?([^\/\\]+?))\.split\d+`, a match at the start that need not reach
301/// the end. So `.split` and a digit must follow at least one character that
302/// is no `/` or `\`, and no `\n` (which `.` does not match) may come before
303/// the last separator ahead of it. Python's `\d` on a `str` pattern is
304/// `Py_UNICODE_ISDECIMAL` (`sre`'s `SRE_UNI_IS_DIGIT`): a character with a
305/// decimal digit value, which Unicode gives exactly the category `Nd`, read
306/// from Python 3.11's own table ([`crate::tables::nd_digit`], Unicode 14.0),
307/// so a name such as `x.split²` (`No`) or `x.splitⅠ` (`Nl`) is one file, as
308/// UnityPy reads it (`tests/fixtures/d/pystr.json` records `\d` over every
309/// code point). Linear in the path.
310#[must_use]
311pub fn is_split_path(path: &[u8]) -> bool {
312    const SPLIT: &[u8] = b".split";
313    let mut first_newline = None;
314    let mut last_sep = None;
315    for (i, &b) in path.iter().enumerate() {
316        if path.get(i..).is_some_and(|rest| rest.starts_with(SPLIT))
317            && i > 0
318            && path.get(i - 1).is_some_and(|&p| p != b'/' && p != b'\\')
319            && first_newline.is_none_or(|n| last_sep.is_none_or(|s| n > s))
320            && path
321                .get(i + SPLIT.len()..)
322                .and_then(|rest| rest.utf8_chunks().next())
323                .and_then(|c| c.valid().chars().next())
324                .is_some_and(|c| nd_digit(c).is_some())
325        {
326            return true;
327        }
328        match b {
329            b'\n' if first_newline.is_none() => first_newline = Some(i),
330            b'/' | b'\\' => last_sep = Some(i),
331            _ => {}
332        }
333    }
334    false
335}
336
337#[cfg(test)]
338mod tests {
339    use std::sync::Arc;
340
341    use super::{
342        clean_external_path, lower_eq, ntpath_basename, ntpath_splitext, py_lower, simplify_name,
343        splitdrive,
344    };
345    use crate::budget::{OWNER_COST, Reservation, Unbounded};
346    use crate::limits::{Meter, heap_cost};
347    use crate::raw::RawStr;
348
349    fn meter() -> Arc<Meter> {
350        Meter::new(1 << 30, Arc::new(Unbounded), Reservation::Input)
351    }
352
353    fn lower(s: &[u8]) -> Vec<u8> {
354        py_lower(&RawStr::from(s), &meter())
355            .unwrap()
356            .as_bytes()
357            .to_vec()
358    }
359
360    #[test]
361    fn an_unchanged_name_is_its_own_bytes_and_charges_nothing() {
362        let m = meter();
363        let name = RawStr::from_shared(bytes::Bytes::from(b"dir/cab-0123".to_vec()));
364        let key = simplify_name(&name, &m).unwrap();
365        assert_eq!(key.as_bytes(), b"cab-0123");
366        assert_eq!(
367            key.as_bytes().as_ptr(),
368            name.as_bytes()[4..].as_ptr(),
369            "a slice of the name"
370        );
371        let same = py_lower(&name, &m).unwrap();
372        assert_eq!(same.as_bytes().as_ptr(), name.as_bytes().as_ptr());
373        assert_eq!(m.used(), 0);
374    }
375
376    #[test]
377    fn a_lowered_name_is_charged_and_returns_its_charge() {
378        let m = meter();
379        let key = simplify_name(&RawStr::from("a\\CAB-ABC"), &m).unwrap();
380        assert_eq!(key.as_bytes(), b"cab-abc");
381        assert_eq!(m.used(), heap_cost(7) + OWNER_COST);
382        drop(key);
383        assert_eq!(m.used(), 0);
384        let full = Meter::new(OWNER_COST, Arc::new(Unbounded), Reservation::Input);
385        assert!(py_lower(&RawStr::from("ABC"), &full).is_err());
386        assert_eq!(full.used(), 0);
387    }
388
389    #[test]
390    fn invalid_bytes_pass_through_and_valid_runs_lower() {
391        assert_eq!(
392            lower(b"A\xffB\xc3\x84\xed\xa0\x80Z"),
393            b"a\xffb\xc3\xa4\xed\xa0\x80z"
394        );
395        // U+0130 lowers to two characters, as Python's full mapping does.
396        assert_eq!(lower("\u{130}".as_bytes()), "i\u{307}".as_bytes());
397    }
398
399    #[test]
400    fn a_final_sigma_lowers_by_context() {
401        // Python 3.11 (tests/fixtures/d/pystr.json records more): 'ΑΣ' ->
402        // 'ας', 'ΑΣΑ' -> 'ασα', 'Σ' -> 'σ', 'ΑΣ.txt' -> 'ασ.txt' ('.' is
403        // case-ignorable, 't' cased), and a lone surrogate ends the word.
404        assert_eq!(lower("ΑΣ".as_bytes()), "ας".as_bytes());
405        assert_eq!(lower("ΑΣΑ".as_bytes()), "ασα".as_bytes());
406        assert_eq!(lower("Σ".as_bytes()), "σ".as_bytes());
407        assert_eq!(lower("ΑΣ.txt".as_bytes()), "ασ.txt".as_bytes());
408        assert_eq!(lower("ΑΣ.".as_bytes()), "ας.".as_bytes());
409        let mut b = "ΑΣ".as_bytes().to_vec();
410        b.push(0xff);
411        b.extend_from_slice("Α".as_bytes());
412        let mut want = "ας".as_bytes().to_vec();
413        want.push(0xff);
414        want.extend_from_slice("α".as_bytes());
415        assert_eq!(lower(&b), want);
416        // The context is Python 3.11's (Unicode 14.0), not Rust's newer one:
417        // U+A7CB (Unicode 16) is not cased there, and U+1E030 (Unicode 15)
418        // is not case-ignorable, so neither makes the sigma final.
419        assert_eq!(lower("\u{A7CB}Σ".as_bytes()), "\u{A7CB}σ".as_bytes());
420        assert_eq!(lower("Α\u{1E030}Σ".as_bytes()), "α\u{1E030}σ".as_bytes());
421        assert_eq!(lower("ΑΣ\u{1E030}".as_bytes()), "ας\u{1E030}".as_bytes());
422        // The result alone is charged, and returned.
423        let m = meter();
424        let s = py_lower(&RawStr::from("ΑΣ"), &m).unwrap();
425        assert_eq!(m.used(), heap_cost(4) + OWNER_COST);
426        drop(s);
427        assert_eq!(m.used(), 0);
428    }
429
430    #[test]
431    fn lower_eq_compares_without_building() {
432        assert!(lower_eq(b"CAB-Abc", b"cab-abc"));
433        assert!(!lower_eq(b"CAB-Abc", b"cab-ab"));
434        assert!(!lower_eq(b"CAB-Ab", b"cab-abc"));
435        assert!(lower_eq("ΑΣ".as_bytes(), "ας".as_bytes()));
436        assert!(!lower_eq("ΑΣ".as_bytes(), "ασ".as_bytes()));
437        assert!(lower_eq(b"", b""));
438    }
439
440    #[test]
441    fn splitdrive_and_basename_follow_ntpath() {
442        let sd = |p: &str| {
443            let (a, b) = splitdrive(p.as_bytes());
444            (
445                String::from_utf8(a.to_vec()).unwrap(),
446                String::from_utf8(b.to_vec()).unwrap(),
447            )
448        };
449        let s = |a: &str, b: &str| (a.to_owned(), b.to_owned());
450        // Probed on Python 3.11.17.
451        assert_eq!(sd("c:x/y"), s("c:", "x/y"));
452        assert_eq!(sd("//a/b/c"), s("//a/b", "/c"));
453        assert_eq!(sd("//a/b"), s("//a/b", ""));
454        assert_eq!(sd("///a/b"), s("///a", "/b"));
455        assert_eq!(sd("//"), s("//", ""));
456        assert_eq!(sd("é:foo"), s("é:", "foo"));
457        assert_eq!(sd(":"), s("", ":"));
458        assert_eq!(sd("\\\\?\\UNC\\s\\sh\\x"), s("\\\\?\\UNC\\s\\sh", "\\x"));
459        assert_eq!(sd("//?/unc/s/sh/x"), s("//?/unc/s/sh", "/x"));
460        assert_eq!(ntpath_basename(b"a/b\\c"), b"c");
461        assert_eq!(ntpath_basename(b"c:x"), b"x");
462        assert_eq!(ntpath_basename(b"//srv/share"), b"");
463        assert_eq!(ntpath_basename(b"dir/"), b"");
464    }
465
466    #[test]
467    fn splitext_keeps_leading_dots() {
468        assert_eq!(ntpath_splitext(b"a.b.c"), (&b"a.b"[..], &b".c"[..]));
469        assert_eq!(ntpath_splitext(b".resS"), (&b".resS"[..], &b""[..]));
470        assert_eq!(ntpath_splitext(b"..x"), (&b"..x"[..], &b""[..]));
471        assert_eq!(ntpath_splitext(b"..x.y"), (&b"..x"[..], &b".y"[..]));
472        assert_eq!(ntpath_splitext(b"a.d/b"), (&b"a.d/b"[..], &b""[..]));
473        assert_eq!(ntpath_splitext(b"x."), (&b"x"[..], &b"."[..]));
474    }
475
476    #[test]
477    fn an_external_path_is_cleaned_as_pptr_does() {
478        let m = meter();
479        let c = |p: &str| clean_external_path(&RawStr::from(p), &m).unwrap();
480        assert_eq!(c("archive:/CAB-X/CAB-Y").as_bytes(), b"cab-y");
481        assert_eq!(c("assets/Foo/Bar.asset").as_bytes(), b"bar.asset");
482        assert_eq!(c("archive:/assets/x").as_bytes(), b"x");
483        assert_eq!(c("Library/a\\B").as_bytes(), b"a\\b");
484        assert_eq!(c("").as_bytes(), b"");
485    }
486}