cadmpeg_core/decode/
utf16.rs

1// SPDX-License-Identifier: Apache-2.0
2//! UTF-16LE text decoding with charged scans and owned storage.
3
4use crate::CodecError;
5
6use super::{u64_from_index, DecodeContext, ScopedReservation, View};
7
8#[derive(Clone, Copy)]
9enum Surrogates {
10    Reject,
11    Replace,
12}
13
14impl DecodeContext<'_> {
15    /// Validates a UTF-16LE window and retains its exact UTF-8 text size.
16    pub fn utf16le_text(
17        &self,
18        bytes: &[u8],
19        units: usize,
20        trim_nul: bool,
21        operation: &'static str,
22    ) -> Result<String, CodecError> {
23        let (bytes, length) =
24            admit_text(self, bytes, units, trim_nul, Surrogates::Reject, operation)?;
25        let mut text = self.retained_string(length, operation)?;
26        write_text(&mut text, bytes, trim_nul, Surrogates::Reject)?;
27        Ok(text)
28    }
29
30    /// Decodes strict UTF-16LE into storage held by a scoped reservation.
31    pub fn utf16le_scoped_text<'ctx>(
32        &'ctx self,
33        bytes: &[u8],
34        units: usize,
35        trim_nul: bool,
36        operation: &'static str,
37    ) -> Result<(String, ScopedReservation<'ctx>), CodecError> {
38        let (bytes, length) =
39            admit_text(self, bytes, units, trim_nul, Surrogates::Reject, operation)?;
40        let (mut text, reservation) = self.scoped_string(length, operation)?;
41        write_text(&mut text, bytes, trim_nul, Surrogates::Reject)?;
42        Ok((text, reservation))
43    }
44
45    /// Retains exact UTF-8 text while replacing unpaired surrogates with U+FFFD.
46    pub fn utf16le_lossy_text(
47        &self,
48        bytes: &[u8],
49        units: usize,
50        trim_nul: bool,
51        operation: &'static str,
52    ) -> Result<String, CodecError> {
53        let (bytes, length) =
54            admit_text(self, bytes, units, trim_nul, Surrogates::Replace, operation)?;
55        let mut text = self.retained_string(length, operation)?;
56        write_text(&mut text, bytes, trim_nul, Surrogates::Replace)?;
57        Ok(text)
58    }
59
60    /// Holds replacement UTF-8 text in an exact-byte scoped reservation.
61    pub fn utf16le_lossy_scoped_text<'ctx>(
62        &'ctx self,
63        bytes: &[u8],
64        units: usize,
65        trim_nul: bool,
66        operation: &'static str,
67    ) -> Result<(String, ScopedReservation<'ctx>), CodecError> {
68        let (bytes, length) =
69            admit_text(self, bytes, units, trim_nul, Surrogates::Replace, operation)?;
70        let (mut text, reservation) = self.scoped_string(length, operation)?;
71        write_text(&mut text, bytes, trim_nul, Surrogates::Replace)?;
72        Ok((text, reservation))
73    }
74}
75
76fn admit_text<'a>(
77    ctx: &DecodeContext<'_>,
78    bytes: &'a [u8],
79    units: usize,
80    trim_nul: bool,
81    surrogates: Surrogates,
82    operation: &'static str,
83) -> Result<(&'a [u8], usize), CodecError> {
84    let byte_len = units
85        .checked_mul(2)
86        .ok_or_else(|| CodecError::malformed("UTF-16LE byte length overflow"))?;
87    let bytes = bytes
88        .get(..byte_len)
89        .ok_or_else(|| CodecError::malformed("truncated UTF-16LE text"))?;
90    ctx.charge_work(u64_from_index(units), operation)?;
91    let mut length = 0_usize;
92    for character in characters(bytes, trim_nul, surrogates) {
93        let character =
94            character.map_err(|_| CodecError::malformed("invalid UTF-16LE surrogate sequence"))?;
95        length = length
96            .checked_add(character.len_utf8())
97            .ok_or_else(|| CodecError::malformed("UTF-16LE text length overflow"))?;
98    }
99    ctx.charge_work(u64_from_index(units), operation)?;
100    ctx.charge_work(u64_from_index(length), operation)?;
101    Ok((bytes, length))
102}
103
104fn write_text(
105    text: &mut String,
106    bytes: &[u8],
107    trim_nul: bool,
108    surrogates: Surrogates,
109) -> Result<(), CodecError> {
110    for character in characters(bytes, trim_nul, surrogates) {
111        text.push(
112            character.map_err(|_| CodecError::malformed("invalid UTF-16LE surrogate sequence"))?,
113        );
114    }
115    Ok(())
116}
117
118fn characters(
119    bytes: &[u8],
120    trim_nul: bool,
121    surrogates: Surrogates,
122) -> impl Iterator<Item = Result<char, std::char::DecodeUtf16Error>> + '_ {
123    let mut view = View::over_retained(bytes);
124    char::decode_utf16(
125        std::iter::from_fn(move || view.u16_le()).take_while(move |unit| !trim_nul || *unit != 0),
126    )
127    .map(move |character| match (surrogates, character) {
128        (Surrogates::Replace, Err(_)) => Ok(char::REPLACEMENT_CHARACTER),
129        (_, character) => character,
130    })
131}
132
133#[cfg(test)]
134mod tests {
135    use crate::decode::{DecodeArena, DecodeContext, DecodePolicy, ResourceDimension};
136    use crate::CodecError;
137
138    #[test]
139    fn utf16le_lossy_text_charges_exact_replacement_bytes() {
140        let bytes = [0, 0xd8, 0x41, 0, 0, 0xdc, 0x3d, 0xd8, 0, 0xde];
141        let arena = DecodeArena::new();
142        let mut policy = DecodePolicy::default();
143        policy.limits.max_retained_bytes = 11;
144        policy.limits.max_work_units = 21;
145        let (ctx, _) = DecodeContext::from_root_bytes(&[], &arena, &policy).expect("empty root");
146        let text = ctx
147            .utf16le_lossy_text(&bytes, 5, false, "replacement test")
148            .expect("exact byte and work admission");
149        assert_eq!(text, "�A�😀");
150        assert!(
151            matches!(ctx.charge_retained(1, "after replacement"), Err(CodecError::ResourceLimit(limit))
152            if limit.dimension == ResourceDimension::RetainedBytes && limit.used == 11)
153        );
154    }
155
156    #[test]
157    fn utf16le_lossy_text_refuses_retained_before_allocation() {
158        let arena = DecodeArena::new();
159        let mut policy = DecodePolicy::default();
160        policy.limits.max_retained_bytes = 2;
161        let (ctx, _) = DecodeContext::from_root_bytes(&[], &arena, &policy).expect("empty root");
162        assert!(
163            matches!(ctx.utf16le_lossy_text(&[0, 0xd8], 1, false, "replacement test"), Err(CodecError::ResourceLimit(limit))
164            if limit.dimension == ResourceDimension::RetainedBytes && limit.used == 0 && limit.additional == 3)
165        );
166    }
167
168    #[test]
169    fn utf16le_lossy_text_trims_first_nul() {
170        let arena = DecodeArena::new();
171        let (ctx, _) = DecodeContext::from_root_bytes(&[], &arena, &DecodePolicy::default())
172            .expect("empty root");
173        assert_eq!(
174            ctx.utf16le_lossy_text(&[0, 0xd8, 0, 0, 0, 0xdc], 3, true, "replacement test")
175                .expect("terminated replacement text"),
176            "�"
177        );
178    }
179
180    #[test]
181    fn utf16le_lossy_text_refuses_work_before_scanning() {
182        let arena = DecodeArena::new();
183        let mut policy = DecodePolicy::default();
184        policy.limits.max_work_units = 0;
185        let (ctx, _) = DecodeContext::from_root_bytes(&[], &arena, &policy).expect("empty root");
186        assert!(
187            matches!(ctx.utf16le_lossy_text(&[0, 0xd8], 1, false, "replacement test"), Err(CodecError::ResourceLimit(limit))
188            if limit.dimension == ResourceDimension::WorkUnits)
189        );
190    }
191
192    #[test]
193    fn utf16le_lossy_scoped_text_charges_replacements_and_holds_storage() {
194        for temporary in [2, 3] {
195            let arena = DecodeArena::new();
196            let mut policy = DecodePolicy::default();
197            policy.limits.max_retained_bytes = 0;
198            policy.limits.max_materialized_bytes = temporary;
199            let (ctx, _) =
200                DecodeContext::from_root_bytes(&[], &arena, &policy).expect("empty root");
201            let result = ctx.utf16le_lossy_scoped_text(&[0, 0xd8], 1, false, "replacement test");
202            if temporary == 2 {
203                assert!(matches!(result, Err(CodecError::ResourceLimit(limit))
204                    if limit.dimension == ResourceDimension::MaterializedBytes && limit.additional == 3));
205            } else {
206                let (text, reservation) = result.expect("exact temporary budget");
207                assert_eq!(text, "�");
208                drop(text);
209                drop(reservation);
210                let (text, _reservation) = ctx
211                    .utf16le_lossy_scoped_text(&[0, 0xdc], 1, false, "replacement test")
212                    .expect("prior temporary storage released");
213                assert_eq!(text, "�");
214            }
215        }
216    }
217
218    #[test]
219    fn utf16le_scoped_text_holds_exact_temporary_bytes() {
220        let arena = DecodeArena::new();
221        let mut policy = DecodePolicy::default();
222        policy.limits.max_retained_bytes = 0;
223        policy.limits.max_materialized_bytes = 3;
224        let (ctx, _) =
225            DecodeContext::from_root_bytes(b"", &arena, &policy).expect("empty root context");
226        let (text, reservation) = ctx
227            .utf16le_scoped_text(&[0, 8], 1, false, "scoped UTF-16 test")
228            .expect("three-byte scoped text");
229        assert_eq!(text, "ࠀ");
230        drop(text);
231        drop(reservation);
232        let (text, _reservation) = ctx
233            .utf16le_scoped_text(&[0, 8], 1, false, "scoped UTF-16 test")
234            .expect("released temporary storage");
235        assert_eq!(text, "ࠀ");
236    }
237
238    #[test]
239    fn utf16le_scoped_text_refuses_temporary_limit() {
240        let arena = DecodeArena::new();
241        let mut policy = DecodePolicy::default();
242        policy.limits.max_materialized_bytes = 2;
243        let (ctx, _) =
244            DecodeContext::from_root_bytes(b"", &arena, &policy).expect("empty root context");
245        assert!(
246            matches!(ctx.utf16le_scoped_text(&[0, 8], 1, false, "scoped UTF-16 test"),
247            Err(CodecError::ResourceLimit(limit)) if limit.dimension == ResourceDimension::MaterializedBytes && limit.additional == 3)
248        );
249    }
250
251    #[test]
252    fn utf16le_text_charges_exact_retained_utf8_bytes() {
253        let arena = DecodeArena::new();
254        let mut policy = DecodePolicy::default();
255        policy.limits.max_retained_bytes = 10;
256        let (ctx, _) =
257            DecodeContext::from_root_bytes(b"", &arena, &policy).expect("empty root context");
258        let bytes = [0x41, 0, 0xe9, 0, 0x00, 0x08, 0x3d, 0xd8, 0x00, 0xde];
259        assert_eq!(
260            ctx.utf16le_text(&bytes, 5, false, "UTF-16 test")
261                .expect("valid UTF-16 fits exact budget"),
262            "Aéࠀ😀"
263        );
264        assert!(matches!(ctx.charge_retained(1, "after UTF-16"),
265            Err(CodecError::ResourceLimit(limit))
266                if limit.dimension == ResourceDimension::RetainedBytes && limit.used == 10));
267    }
268
269    #[test]
270    fn utf16le_text_refuses_retained_limit_before_allocation() {
271        let arena = DecodeArena::new();
272        let mut policy = DecodePolicy::default();
273        policy.limits.max_retained_bytes = 2;
274        let (ctx, _) =
275            DecodeContext::from_root_bytes(b"", &arena, &policy).expect("empty root context");
276        assert!(
277            matches!(ctx.utf16le_text(&[0x00, 0x08], 1, false, "UTF-16 test"),
278            Err(CodecError::ResourceLimit(limit))
279                if limit.dimension == ResourceDimension::RetainedBytes
280                    && limit.used == 0 && limit.additional == 3 && limit.operation == "UTF-16 test")
281        );
282    }
283
284    #[test]
285    fn utf16le_text_rejects_odd_byte_window() {
286        let arena = DecodeArena::new();
287        let (ctx, _) = DecodeContext::from_root_bytes(b"", &arena, &DecodePolicy::default())
288            .expect("empty root context");
289        assert!(matches!(
290            ctx.utf16le_text(&[0x41], 1, false, "UTF-16 test"),
291            Err(CodecError::Malformed(_))
292        ));
293        assert!(matches!(
294            ctx.utf16le_text(&[], usize::MAX, false, "UTF-16 test"),
295            Err(CodecError::Malformed(_))
296        ));
297    }
298
299    #[test]
300    fn utf16le_text_rejects_invalid_surrogates_before_retaining() {
301        let arena = DecodeArena::new();
302        let mut policy = DecodePolicy::default();
303        policy.limits.max_retained_bytes = 0;
304        let (ctx, _) =
305            DecodeContext::from_root_bytes(b"", &arena, &policy).expect("empty root context");
306        for bytes in [&[0, 0xd8][..], &[0, 0xdc][..], &[0, 0xd8, 0x41, 0][..]] {
307            assert!(matches!(
308                ctx.utf16le_text(bytes, bytes.len() / 2, false, "UTF-16 test"),
309                Err(CodecError::Malformed(_))
310            ));
311        }
312    }
313
314    #[test]
315    fn utf16le_text_trims_at_first_nul() {
316        let arena = DecodeArena::new();
317        let mut policy = DecodePolicy::default();
318        policy.limits.max_retained_bytes = 1;
319        let (ctx, _) =
320            DecodeContext::from_root_bytes(b"", &arena, &policy).expect("empty root context");
321        assert_eq!(
322            ctx.utf16le_text(&[0x41, 0, 0, 0, 0, 0xd8], 3, true, "UTF-16 test")
323                .expect("first NUL ends valid text"),
324            "A"
325        );
326        assert_eq!(
327            ctx.utf16le_text(&[0, 0], 1, true, "UTF-16 test")
328                .expect("empty text needs no retained bytes"),
329            ""
330        );
331        assert!(matches!(
332            ctx.charge_retained(1, "after UTF-16"),
333            Err(CodecError::ResourceLimit(_))
334        ));
335    }
336
337    #[test]
338    fn utf16le_text_refuses_work_before_scanning() {
339        let arena = DecodeArena::new();
340        let mut policy = DecodePolicy::default();
341        policy.limits.max_work_units = 0;
342        let (ctx, _) =
343            DecodeContext::from_root_bytes(b"", &arena, &policy).expect("empty root context");
344        assert!(
345            matches!(ctx.utf16le_text(&[0x41, 0], 1, false, "UTF-16 test"),
346            Err(CodecError::ResourceLimit(limit)) if limit.dimension == ResourceDimension::WorkUnits)
347        );
348    }
349}
cadmpeg_core/decode/utf16.rs

cadmpeg_core/decode/
utf16.rs
