hermes_parser/
utf8.rs

1//! UTF-8 decode helpers, ported from include/hermes/Support/UTF8.h (decode side).
2//!
3//! These mirror the inline classifiers and `_decodeUTF8SlowPath`/`decodeUTF8`
4//! from `Support/UTF8.h`. The C++ uses an advancing `const char *&from`; here we
5//! use a slice plus a `&mut usize` index, so the raw-pointer parity lives only
6//! in `cursor.rs`. The lexer always passes the NUL-terminated buffer, so an
7//! out-of-range continuation read sees `0x00` (a non-continuation byte) and is
8//! correctly rejected; we also guard indexes against `bytes.len()` defensively.
9
10use hermes_unicode::{
11    is_high_surrogate, is_low_surrogate, utf16_surrogate_pair_to_code_point,
12    UNICODE_MAX_VALUE, UNICODE_REPLACEMENT_CHARACTER, UNICODE_SURROGATE_FIRST,
13    UNICODE_SURROGATE_LAST, UTF16_HIGH_SURROGATE, UTF16_LOW_SURROGATE,
14};
15
16/// First byte of the UTF-8 encoding of U+2028/U+2029 (e2 80 a8/a9).
17pub const UTF8_LINE_TERMINATOR_CHAR0: u8 = 0xe2;
18
19/// Check whether a byte is a regular ASCII or a UTF8 starting byte.
20/// \return true if it is UTF8 starting byte.
21#[inline]
22pub fn is_utf8_start(ch: u8) -> bool {
23    (ch & 0x80) != 0
24}
25
26/// \return true if this is a UTF-8 leading byte.
27#[inline]
28pub fn is_utf8_leading_byte(ch: u8) -> bool {
29    (ch & 0xC0) == 0xC0
30}
31
32/// \return true if this is a UTF-8 continuation byte, or in other words, this
33/// is a byte in the "middle" of a UTF-8 codepoint.
34#[inline]
35pub fn is_utf8_continuation_byte(ch: u8) -> bool {
36    (ch & 0xC0) == 0x80
37}
38
39/// \return true if `bytes` starts with the UTF-8 encoding of U+2028 or U+2029.
40/// `bytes[0]` is assumed to be UTF8_LINE_TERMINATOR_CHAR0 (the caller checked).
41///
42/// Line separator   UTF8 encoded is      : e2 80 a8
43/// Paragraph separator   UTF8 encoded is : e2 80 a9
44#[inline]
45pub fn match_unicode_line_terminator_offset1(bytes: &[u8]) -> bool {
46    bytes.len() >= 3
47        && bytes[0] == 0xe2
48        && bytes[1] == 0x80
49        && (bytes[2] == 0xa8 || bytes[2] == 0xa9)
50}
51
52/// Read `bytes[i]`, or `0` if `i` is out of range. The lexer's buffer is
53/// NUL-terminated, so the only way to read past the end here is in unit tests
54/// of malformed/truncated sequences, where `0` (a non-continuation byte)
55/// reproduces the buffer's NUL terminator and the same rejection behavior.
56#[inline]
57fn at(bytes: &[u8], i: usize) -> u32 {
58    bytes.get(i).copied().unwrap_or(0) as u32
59}
60
61/// Decode a sequence of UTF8 encoded bytes when it is known that the first byte
62/// is a start of a UTF8 sequence. Port of `_decodeUTF8SlowPath` (UTF8.h:77-162),
63/// reading `bytes` from `*i` and advancing `*i` past the consumed bytes.
64/// On malformed input it invokes `error` and returns the replacement character.
65///
66/// \tparam ALLOW_SURROGATES when false, values in the surrogate range are
67///     reported as errors.
68pub fn decode_utf8_slow_path<const ALLOW_SURROGATES: bool>(
69    bytes: &[u8],
70    i: &mut usize,
71    mut error: impl FnMut(&str),
72) -> u32 {
73    let ch = at(bytes, *i);
74    let result: u32;
75
76    debug_assert!(is_utf8_start(ch as u8));
77
78    if (ch & 0xE0) == 0xC0 {
79        let ch1 = at(bytes, *i + 1);
80        if (ch1 & 0xC0) != 0x80 {
81            *i += 1;
82            error("Invalid UTF-8 continuation byte");
83            return UNICODE_REPLACEMENT_CHARACTER;
84        }
85
86        *i += 2;
87        result = ((ch & 0x1F) << 6) | (ch1 & 0x3F);
88        if result <= 0x7F {
89            error("Non-canonical UTF-8 encoding");
90            return UNICODE_REPLACEMENT_CHARACTER;
91        }
92    } else if (ch & 0xF0) == 0xE0 {
93        let ch1 = at(bytes, *i + 1);
94        if (ch1 & 0x40) != 0 || (ch1 & 0x80) == 0 {
95            *i += 1;
96            error("Invalid UTF-8 continuation byte");
97            return UNICODE_REPLACEMENT_CHARACTER;
98        }
99        let ch2 = at(bytes, *i + 2);
100        if (ch2 & 0x40) != 0 || (ch2 & 0x80) == 0 {
101            *i += 2;
102            error("Invalid UTF-8 continuation byte");
103            return UNICODE_REPLACEMENT_CHARACTER;
104        }
105        *i += 3;
106        result = ((ch & 0x0F) << 12) | ((ch1 & 0x3F) << 6) | (ch2 & 0x3F);
107        if result <= 0x7FF {
108            error("Non-canonical UTF-8 encoding");
109            return UNICODE_REPLACEMENT_CHARACTER;
110        }
111        if result >= UNICODE_SURROGATE_FIRST && result <= UNICODE_SURROGATE_LAST && !ALLOW_SURROGATES
112        {
113            error(&format!("Invalid UTF-8 code point 0x{:X}", result));
114            return UNICODE_REPLACEMENT_CHARACTER;
115        }
116    } else if (ch & 0xF8) == 0xF0 {
117        let ch1 = at(bytes, *i + 1);
118        if (ch1 & 0x40) != 0 || (ch1 & 0x80) == 0 {
119            *i += 1;
120            error("Invalid UTF-8 continuation byte");
121            return UNICODE_REPLACEMENT_CHARACTER;
122        }
123        let ch2 = at(bytes, *i + 2);
124        if (ch2 & 0x40) != 0 || (ch2 & 0x80) == 0 {
125            *i += 2;
126            error("Invalid UTF-8 continuation byte");
127            return UNICODE_REPLACEMENT_CHARACTER;
128        }
129        let ch3 = at(bytes, *i + 3);
130        if (ch3 & 0x40) != 0 || (ch3 & 0x80) == 0 {
131            *i += 3;
132            error("Invalid UTF-8 continuation byte");
133            return UNICODE_REPLACEMENT_CHARACTER;
134        }
135        *i += 4;
136        result =
137            ((ch & 0x07) << 18) | ((ch1 & 0x3F) << 12) | ((ch2 & 0x3F) << 6) | (ch3 & 0x3F);
138        if result <= 0xFFFF {
139            error("Non-canonical UTF-8 encoding");
140            return UNICODE_REPLACEMENT_CHARACTER;
141        }
142        if result > UNICODE_MAX_VALUE {
143            error(&format!("Invalid UTF-8 code point 0x{:X}", result));
144            return UNICODE_REPLACEMENT_CHARACTER;
145        }
146    } else {
147        *i += 1;
148        error(&format!("Invalid UTF-8 lead byte 0x{:X}", ch & 0xFF));
149        return UNICODE_REPLACEMENT_CHARACTER;
150    }
151
152    result
153}
154
155/// Decode a sequence of UTF8 encoded bytes into a Unicode codepoint, ASCII fast
156/// path. Port of `decodeUTF8` (UTF8.h:187-193). In case of decoding errors, the
157/// provided callback is invoked with an appropriate message and
158/// UNICODE_REPLACEMENT_CHARACTER is returned.
159///
160/// \tparam ALLOW_SURROGATES when false, values in the surrogate range are
161///     reported as errors.
162#[inline]
163pub fn decode_utf8<const ALLOW_SURROGATES: bool>(
164    bytes: &[u8],
165    i: &mut usize,
166    error: impl FnMut(&str),
167) -> u32 {
168    if *i < bytes.len() && (bytes[*i] & 0x80) == 0 {
169        // Ordinary ASCII?
170        let c = bytes[*i] as u32;
171        *i += 1;
172        return c;
173    }
174    decode_utf8_slow_path::<ALLOW_SURROGATES>(bytes, i, error)
175}
176
177/// Encode a Unicode code point as UTF-8 (up to the legacy 6-byte form, matching
178/// `encodeUTF8`), appending the bytes to `out`. Port of `UTF8.cpp:encodeUTF8`.
179#[inline]
180pub fn encode_utf8(out: &mut Vec<u8>, cp: u32) {
181    if cp <= 0x7F {
182        out.push(cp as u8);
183    } else if cp <= 0x7FF {
184        out.push(((cp >> 6) & 0x1F) as u8 | 0xC0);
185        out.push((cp & 0x3F) as u8 | 0x80);
186    } else if cp <= 0xFFFF {
187        out.push(((cp >> 12) & 0x0F) as u8 | 0xE0);
188        out.push(((cp >> 6) & 0x3F) as u8 | 0x80);
189        out.push((cp & 0x3F) as u8 | 0x80);
190    } else if cp <= 0x1FFFFF {
191        out.push(((cp >> 18) & 0x07) as u8 | 0xF0);
192        out.push(((cp >> 12) & 0x3F) as u8 | 0x80);
193        out.push(((cp >> 6) & 0x3F) as u8 | 0x80);
194        out.push((cp & 0x3F) as u8 | 0x80);
195    } else if cp <= 0x3FFFFFF {
196        out.push(((cp >> 24) & 0x03) as u8 | 0xF8);
197        out.push(((cp >> 18) & 0x3F) as u8 | 0x80);
198        out.push(((cp >> 12) & 0x3F) as u8 | 0x80);
199        out.push(((cp >> 6) & 0x3F) as u8 | 0x80);
200        out.push((cp & 0x3F) as u8 | 0x80);
201    } else {
202        out.push(((cp >> 30) & 0x01) as u8 | 0xFC);
203        out.push(((cp >> 24) & 0x3F) as u8 | 0x80);
204        out.push(((cp >> 18) & 0x3F) as u8 | 0x80);
205        out.push(((cp >> 12) & 0x3F) as u8 | 0x80);
206        out.push(((cp >> 6) & 0x3F) as u8 | 0x80);
207        out.push((cp & 0x3F) as u8 | 0x80);
208    }
209}
210
211/// Encode `cp` into `storage` like the lexer's `appendUnicodeToStorage`
212/// (JSLexer.h:1125-1143): code points above 0xFFFF are first split into a
213/// UTF-16 surrogate pair, and each surrogate is encoded individually into UTF-8
214/// (technically invalid UTF-8 / WTF-8, which JS string & identifier storage
215/// allows).
216#[inline]
217pub fn append_unicode_to_storage(storage: &mut Vec<u8>, cp: u32) {
218    // We need to normalize code points which would be encoded with a surrogate
219    // pair. Note that this produces technically invalid UTF-8.
220    if cp < 0x10000 {
221        encode_utf8(storage, cp);
222    } else {
223        debug_assert!(cp <= UNICODE_MAX_VALUE, "invalid Unicode value");
224        let cp = cp - 0x10000;
225        encode_utf8(storage, UTF16_HIGH_SURROGATE + ((cp >> 10) & 0x3FF));
226        encode_utf8(storage, UTF16_LOW_SURROGATE + (cp & 0x3FF));
227    }
228}
229
230/// Encode a 32-bit value into UTF-16, appending to `out`. If the value is a
231/// part of a surrogate pair, it is encoded without any conversion. Port of
232/// `encodeUTF16` (UTF8.h:197-210).
233#[inline]
234pub fn encode_utf16(out: &mut Vec<u16>, cp: u32) {
235    if cp < 0x10000 {
236        out.push(cp as u16);
237    } else {
238        debug_assert!(cp <= UNICODE_MAX_VALUE, "invalid Unicode value");
239        let cp = cp - 0x10000;
240        out.push((UTF16_HIGH_SURROGATE + ((cp >> 10) & 0x3FF)) as u16);
241        out.push((UTF16_LOW_SURROGATE + (cp & 0x3FF)) as u16);
242    }
243}
244
245/// Decode a UTF-8 sequence, which is assumed to be valid, but may possibly
246/// contain explicitly encoded surrogate pairs, into a UTF-16 sequence. Port of
247/// `convertUTF8WithSurrogatesToUTF16` (UTF8.h:216-225).
248pub fn convert_utf8_with_surrogates_to_utf16(bytes: &[u8]) -> Vec<u16> {
249    let mut out = Vec::with_capacity(bytes.len());
250    let mut i = 0usize;
251    while i < bytes.len() {
252        // Surrogates are ALLOWED; the input is assumed valid, so the error
253        // callback is unreachable (a no-op here).
254        let cp = decode_utf8::<true>(bytes, &mut i, |_| {});
255        encode_utf16(&mut out, cp);
256    }
257    out
258}
259
260/// Inspect the code unit at `u16s[i]`. If it is a high surrogate followed by a
261/// low surrogate, decode the surrogate pair into a single code point. If it is
262/// an unpaired surrogate, replace the value with `UNICODE_REPLACEMENT_CHARACTER`
263/// (U+FFFD). Port of `convertToCodePointAt` (UTF8.cpp:77-96).
264///
265/// \return a pair with the first element being the Unicode code point, and the
266///         second being how many code units were consumed.
267#[inline]
268fn convert_to_code_point_at(u16s: &[u16], i: usize) -> (u32, usize) {
269    let c = u16s[i] as u32;
270    if is_low_surrogate(c) {
271        // Unpaired low surrogate.
272        (UNICODE_REPLACEMENT_CHARACTER, 1)
273    } else if is_high_surrogate(c) {
274        // Leading high surrogate. See if the next character is a low surrogate.
275        if i + 1 >= u16s.len() || !is_low_surrogate(u16s[i + 1] as u32) {
276            // Trailing or unpaired high surrogate.
277            (UNICODE_REPLACEMENT_CHARACTER, 1)
278        } else {
279            // Decode surrogate pair and consume two chars.
280            (utf16_surrogate_pair_to_code_point(c, u16s[i + 1] as u32), 2)
281        }
282    } else {
283        // Not a surrogate.
284        (c, 1)
285    }
286}
287
288/// Convert a UTF-16 encoded string `u16s` to valid UTF-8, combining surrogate
289/// pairs into supplementary-plane characters and replacing unpaired surrogates
290/// with U+FFFD. Port of `convertUTF16ToUTF8WithReplacements` (UTF8.cpp:99-133),
291/// dropping the `maxCharacters` parameter (the lexer always passes 0/unbounded).
292pub fn convert_utf16_to_utf8_with_replacements(u16s: &[u16]) -> Vec<u8> {
293    let mut out = Vec::with_capacity(u16s.len());
294    let mut cur = 0usize;
295    while cur < u16s.len() {
296        let c = u16s[cur] as u32;
297        // ASCII fast-path.
298        if c <= 0x7F {
299            out.push(c as u8);
300            cur += 1;
301            continue;
302        }
303
304        let (c32, input_consumed) = convert_to_code_point_at(u16s, cur);
305        cur += input_consumed;
306
307        // The code point to be encoded here is guaranteed to be a valid unicode
308        // code point and not a surrogate. Because of the
309        // convert_to_code_point_at() process.
310        encode_utf8(&mut out, c32);
311    }
312    out
313}
314
315#[cfg(test)]
316mod tests {
317    use super::*;
318    #[test]
319    fn classifiers() {
320        assert!(!is_utf8_start(b'a'));
321        assert!(is_utf8_start(0xc3));
322        assert!(is_utf8_leading_byte(0xc3));
323        assert!(!is_utf8_leading_byte(0x80));
324        assert!(is_utf8_continuation_byte(0x80));
325    }
326    #[test]
327    fn decode_ascii_and_multibyte() {
328        // ASCII fast path advances by 1.
329        let buf = b"a";
330        let mut i = 0usize;
331        assert_eq!(decode_utf8::<false>(buf, &mut i, |_| {}), 'a' as u32);
332        assert_eq!(i, 1);
333        // é = U+00E9 = c3 a9
334        let buf = b"\xc3\xa9";
335        let mut i = 0usize;
336        assert_eq!(decode_utf8::<false>(buf, &mut i, |_| {}), 0x00E9);
337        assert_eq!(i, 2);
338        // U+1F600 = f0 9f 98 80
339        let buf = b"\xf0\x9f\x98\x80";
340        let mut i = 0usize;
341        assert_eq!(decode_utf8::<false>(buf, &mut i, |_| {}), 0x1F600);
342        assert_eq!(i, 4);
343    }
344    #[test]
345    fn line_terminator_match() {
346        assert!(match_unicode_line_terminator_offset1(b"\xe2\x80\xa8")); // U+2028
347        assert!(match_unicode_line_terminator_offset1(b"\xe2\x80\xa9")); // U+2029
348        assert!(!match_unicode_line_terminator_offset1(b"\xe2\x80\xaa"));
349    }
350    #[test]
351    fn encode_basic() {
352        let mut v = vec![];
353        encode_utf8(&mut v, 'a' as u32);
354        assert_eq!(v, b"a");
355        let mut v = vec![];
356        encode_utf8(&mut v, 0x00E9);
357        assert_eq!(v, b"\xc3\xa9"); // é
358        let mut v = vec![];
359        encode_utf8(&mut v, 0x4E2D);
360        assert_eq!(v, b"\xe4\xb8\xad"); // 中
361    }
362
363    #[test]
364    fn append_storage_surrogate_pair() {
365        // BMP: plain UTF-8.
366        let mut v = vec![];
367        append_unicode_to_storage(&mut v, 0x00E9);
368        assert_eq!(v, b"\xc3\xa9");
369        // Astral U+1F600: split into surrogate pair, each encoded as 3-byte
370        // WTF-8. high = 0xD83D, low = 0xDE00 -> ed a0 bd  ed b8 80
371        let mut v = vec![];
372        append_unicode_to_storage(&mut v, 0x1F600);
373        assert_eq!(v, b"\xed\xa0\xbd\xed\xb8\x80");
374    }
375
376    #[test]
377    fn utf16_roundtrip_and_replacement() {
378        // encode_utf16: BMP -> 1 u16, astral -> surrogate pair.
379        let mut v = vec![];
380        encode_utf16(&mut v, 0x41);
381        assert_eq!(v, [0x41]);
382        let mut v = vec![];
383        encode_utf16(&mut v, 0x1F600);
384        assert_eq!(v, [0xD83D, 0xDE00]);
385
386        // convert_utf8_with_surrogates_to_utf16: WTF-8 astral (surrogate pair,
387        // 3 bytes each) -> 2 u16.
388        let wtf8: &[u8] = b"\xed\xa0\xbd\xed\xb8\x80"; // U+1F600 as a surrogate pair (WTF-8)
389        let u16s = convert_utf8_with_surrogates_to_utf16(wtf8);
390        assert_eq!(u16s, [0xD83D, 0xDE00]);
391
392        // convert_utf16_to_utf8_with_replacements: surrogate pair -> 4-byte
393        // UTF-8; lone surrogate -> U+FFFD.
394        assert_eq!(
395            convert_utf16_to_utf8_with_replacements(&[0xD83D, 0xDE00]),
396            b"\xf0\x9f\x98\x80".to_vec()
397        );
398        assert_eq!(
399            convert_utf16_to_utf8_with_replacements(&[0xD800]),
400            "\u{FFFD}".as_bytes().to_vec()
401        ); // lone high
402        assert_eq!(
403            convert_utf16_to_utf8_with_replacements(&[0xDC00]),
404            "\u{FFFD}".as_bytes().to_vec()
405        ); // lone low
406        assert_eq!(
407            convert_utf16_to_utf8_with_replacements(&[0x41, 0x42]),
408            b"AB".to_vec()
409        );
410    }
411
412    #[test]
413    fn invalid_reports_error_and_replacement() {
414        let buf = b"\xc3\x20"; // 0x20 is not a continuation byte
415        let mut i = 0usize;
416        let mut errs = 0;
417        let cp = decode_utf8::<false>(buf, &mut i, |_| errs += 1);
418        assert_eq!(cp, 0xFFFD);
419        assert_eq!(errs, 1);
420    }
421}
hermes_parser/utf8.rs

hermes_parser/
utf8.rs
