Skip to main content

unityrs_core/
raw.rs

1// Ported from UnityPy 1.25.2 (MIT): UnityPy/streams/EndianBinaryReader.py. See THIRD_PARTY_LICENSES.
2//! Raw strings: the bytes UnityPy holds as a `str` decoded with `surrogateescape`.
3//!
4//! Every string unityrs reads from a file is kept as bytes, never as `String`:
5//! UnityPy decodes with `surrogateescape` (`streams/EndianBinaryReader.py:144`,
6//! `:282`), so its `str` can hold any bytes, and the canonical dump writes a
7//! name that is not valid UTF-8 as hex (`docs/CANONICAL_DUMP.md` section 4.2).
8//!
9//! **Zero-copy.** An input `RawStr` borrows [`Bytes`]: a string read from a file is a
10//! slice of the file's buffer ([`RawStr::from_shared`]), never a copy, so a
11//! typetree's node names (each row naming an offset in one string buffer),
12//! metadata strings and `Value::Str` cost a handle each, whatever their
13//! length and however often a hostile file repeats them (plan section 3.15).
14//! Common strings are static. Only strings the core computes (a decimal
15//! common-string offset, `py_lower`, a registry key) allocate, and each is
16//! made with [`RawStr::charged`], which charges the input's meter for its
17//! buffer and its owner record and keeps the charge inside the buffer, so a
18//! clone of it that outlives its row still counts against the input.
19//!
20//! Owner: the scaffold (implemented). Python string semantics on top of
21//! these bytes (`str.lower`, `ntpath`) are in [`crate::pystr`].
22
23use std::borrow::Borrow;
24use std::fmt;
25use std::sync::Arc;
26
27use bytes::Bytes;
28
29use crate::budget::OWNER_COST;
30use crate::limits::{Charge, ChargeError, Meter, MeteredVec, heap_cost, reserve_exact};
31
32/// Bytes UnityPy holds as a `str`. Valid UTF-8 or not; computed strings use
33/// fallible shared storage, while input strings borrow their `Bytes`.
34#[derive(Clone)]
35pub struct RawStr(Storage);
36
37#[derive(Clone)]
38enum Storage {
39    Bytes(Bytes),
40    Computed {
41        owner: triomphe::Arc<Computed>,
42        start: usize,
43        end: usize,
44    },
45}
46struct Computed {
47    bytes: Vec<u8>,
48    _charge: Charge,
49}
50
51impl RawStr {
52    /// Wraps bytes (a `Vec`, a boxed slice, a static slice or a shared buffer).
53    #[must_use]
54    pub fn new(bytes: impl Into<Bytes>) -> Self {
55        Self(Storage::Bytes(bytes.into()))
56    }
57
58    /// A slice of a file's shared buffer, kept without a copy.
59    #[must_use]
60    pub fn from_shared(bytes: Bytes) -> Self {
61        Self(Storage::Bytes(bytes))
62    }
63
64    /// A string the core computes from an input (a decimal common-string
65    /// offset, a lower-cased name), charged to `meter` before it is
66    /// allocated: its buffer at [`heap_cost`] and the owner record
67    /// ([`crate::budget::OWNER_COST`]) that keeps the charge inside it, so
68    /// every clone counts until the last one drops. An empty string is
69    /// static and charges nothing.
70    pub fn charged(bytes: &[u8], meter: &Arc<Meter>) -> Result<Self, ChargeError> {
71        Self::charged_concat(&[bytes], meter)
72    }
73
74    /// The concatenation of `parts` (a resource candidate's stem and its
75    /// suffix), computed and charged as [`Self::charged`] is: one buffer,
76    /// charged before it is allocated.
77    pub fn charged_concat(parts: &[&[u8]], meter: &Arc<Meter>) -> Result<Self, ChargeError> {
78        let len = parts.iter().fold(0_usize, |n, p| n.saturating_add(p.len()));
79        if len == 0 {
80            return Ok(Self::default());
81        }
82        let n = u64::try_from(len).unwrap_or(u64::MAX);
83        let charge = meter.charge(heap_cost(n))?;
84        let mut buf = Vec::new();
85        reserve_exact(&mut buf, len, n, meter).map_err(ChargeError::Budget)?;
86        for p in parts {
87            buf.extend_from_slice(p);
88        }
89        Self::from_metered(MeteredVec::from_charged_parts(buf, charge)?, meter)
90    }
91
92    /// Keep computed bytes and their charge in a fallibly allocated shared
93    /// owner. Cloning or slicing the resulting name never allocates.
94    pub(crate) fn from_metered(
95        bytes: MeteredVec<u8>,
96        meter: &Arc<Meter>,
97    ) -> Result<Self, ChargeError> {
98        let (bytes, mut charge) = bytes.into_parts();
99        let end = bytes.len();
100        charge.grow(OWNER_COST)?;
101        let owner = triomphe::Arc::try_new(Computed {
102            bytes,
103            _charge: charge,
104        })
105        .map_err(|_| ChargeError::Allocator(meter.allocator_refused(OWNER_COST)))?;
106        Ok(Self(Storage::Computed {
107            owner,
108            start: 0,
109            end,
110        }))
111    }
112
113    /// A borrowed subrange of this name, preserving either shared owner.
114    pub(crate) fn slice_ref(&self, sub: &[u8]) -> Self {
115        let start = (sub.as_ptr() as usize).saturating_sub(self.as_bytes().as_ptr() as usize);
116        let end = start.saturating_add(sub.len());
117        match &self.0 {
118            Storage::Bytes(bytes) => Self::from_shared(bytes.slice(start..end)),
119            Storage::Computed {
120                owner, start: base, ..
121            } => Self(Storage::Computed {
122                owner: owner.clone(),
123                start: base + start,
124                end: base + end,
125            }),
126        }
127    }
128
129    /// A static string (a common string, a constant name).
130    #[must_use]
131    pub const fn from_static(bytes: &'static [u8]) -> Self {
132        Self(Storage::Bytes(Bytes::from_static(bytes)))
133    }
134
135    /// The bytes, as read.
136    #[must_use]
137    pub fn as_bytes(&self) -> &[u8] {
138        match &self.0 {
139            Storage::Bytes(bytes) => bytes,
140            Storage::Computed { owner, start, end } => {
141                owner.bytes.get(*start..*end).unwrap_or_default()
142            }
143        }
144    }
145
146    /// The string when the bytes are strict UTF-8 (overlong forms and encoded
147    /// surrogates are invalid, as `CANONICAL_DUMP.md` section 3 defines it).
148    #[must_use]
149    pub fn to_str(&self) -> Option<&str> {
150        std::str::from_utf8(self.as_bytes()).ok()
151    }
152
153    /// Length in bytes.
154    #[must_use]
155    pub fn len(&self) -> usize {
156        self.as_bytes().len()
157    }
158
159    /// Whether the string is empty.
160    #[must_use]
161    pub fn is_empty(&self) -> bool {
162        self.as_bytes().is_empty()
163    }
164
165    /// The shared bytes, consuming the string. Computed strings need a
166    /// `Bytes` owner adapter here; name lookups and slices avoid that adapter.
167    #[must_use]
168    pub fn into_bytes(self) -> Bytes {
169        match self.0 {
170            Storage::Bytes(bytes) => bytes,
171            computed @ Storage::Computed { .. } => Bytes::from_owner(Self(computed)),
172        }
173    }
174}
175
176impl Default for RawStr {
177    fn default() -> Self {
178        Self::from_static(b"")
179    }
180}
181impl PartialEq for RawStr {
182    fn eq(&self, other: &Self) -> bool {
183        self.as_bytes() == other.as_bytes()
184    }
185}
186impl Eq for RawStr {}
187impl std::hash::Hash for RawStr {
188    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
189        self.as_bytes().hash(state);
190    }
191}
192impl PartialOrd for RawStr {
193    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
194        Some(self.cmp(other))
195    }
196}
197impl Ord for RawStr {
198    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
199        self.as_bytes().cmp(other.as_bytes())
200    }
201}
202
203impl fmt::Debug for RawStr {
204    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
205        write!(f, "\"{}\"", self.as_bytes().escape_ascii())
206    }
207}
208
209impl fmt::Display for RawStr {
210    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
211        write!(f, "{}", self.as_bytes().escape_ascii())
212    }
213}
214
215impl AsRef<[u8]> for RawStr {
216    fn as_ref(&self) -> &[u8] {
217        self.as_bytes()
218    }
219}
220
221impl Borrow<[u8]> for RawStr {
222    fn borrow(&self) -> &[u8] {
223        self.as_bytes()
224    }
225}
226
227impl From<&[u8]> for RawStr {
228    /// Copies: a borrowed slice has no shared buffer to keep.
229    fn from(b: &[u8]) -> Self {
230        Self::from_shared(Bytes::copy_from_slice(b))
231    }
232}
233
234impl From<Vec<u8>> for RawStr {
235    fn from(b: Vec<u8>) -> Self {
236        Self::from_shared(Bytes::from(b))
237    }
238}
239
240impl From<&str> for RawStr {
241    /// Copies (use [`RawStr::from_static`] for a constant).
242    fn from(s: &str) -> Self {
243        Self::from_shared(Bytes::copy_from_slice(s.as_bytes()))
244    }
245}
246
247impl From<Bytes> for RawStr {
248    fn from(b: Bytes) -> Self {
249        Self::from_shared(b)
250    }
251}
252
253#[cfg(test)]
254mod tests {
255    use super::RawStr;
256
257    #[test]
258    fn a_computed_string_carries_its_charge() {
259        use std::sync::Arc;
260
261        use crate::budget::{OWNER_COST, Reservation, Unbounded};
262        use crate::limits::{Meter, heap_cost};
263        // Round 3: an unknown common-string offset's decimal string (and any
264        // string the core computes) was allocated outside the meter.
265        let m = Meter::new(1 << 20, Arc::new(Unbounded), Reservation::Input);
266        let s = RawStr::charged(b"2147483661", &m).unwrap();
267        assert_eq!(s.as_bytes(), b"2147483661");
268        assert_eq!(m.used(), heap_cost(10) + OWNER_COST);
269        let kept = s.clone();
270        drop(s);
271        assert_eq!(m.used(), heap_cost(10) + OWNER_COST, "a clone keeps it");
272        drop(kept);
273        assert_eq!(m.used(), 0);
274        let full = Meter::new(OWNER_COST, Arc::new(Unbounded), Reservation::Input);
275        assert!(RawStr::charged(b"0123456789", &full).is_err());
276        assert_eq!(full.used(), 0);
277        assert_eq!(RawStr::charged(b"", &full).unwrap(), RawStr::default());
278    }
279
280    #[test]
281    fn keeps_invalid_utf8() {
282        let s = RawStr::from(&b"a\xffb"[..]);
283        assert_eq!(s.as_bytes(), b"a\xffb");
284        assert_eq!(s.to_str(), None);
285        assert_eq!(format!("{s:?}"), "\"a\\xffb\"");
286    }
287
288    #[test]
289    fn a_string_read_from_a_buffer_shares_it() {
290        // A thousand rows naming one 32 KiB string cost a handle each, not a copy.
291        let buf = bytes::Bytes::from(vec![b'x'; 32 * 1024]);
292        let names: Vec<RawStr> = (0..1000)
293            .map(|_| RawStr::from_shared(buf.slice(..)))
294            .collect();
295        assert!(names.iter().all(|n| n.as_bytes().as_ptr() == buf.as_ptr()));
296        assert_eq!(names[999].len(), 32 * 1024);
297    }
298
299    #[test]
300    fn surrogate_encodings_are_invalid() {
301        // ED A0 80 encodes U+D800: Python's strict decoder and Rust's agree it is invalid.
302        assert_eq!(RawStr::from(&b"\xed\xa0\x80"[..]).to_str(), None);
303        assert_eq!(RawStr::from("ok").to_str(), Some("ok"));
304    }
305}