mago_syntax_core/
utils.rs

1use mago_allocator::prelude::*;
2
3use crate::input::Input;
4use crate::number_separator;
5
6/// Parses a PHP literal string, handling all escape sequences, and allocates the result in an arena.
7///
8/// # Returns
9///
10/// An `Option` containing the parsed `&'arena str` or `None` if the input is invalid.
11///
12/// # Panics
13///
14/// Panics if internal assumptions about character parsing are violated (e.g., invalid hex or octal digits
15/// after validation). This should not occur with valid PHP strings.
16pub fn parse_literal_string_in<'arena, A>(
17    arena: &'arena A,
18    s: &'arena [u8],
19    quote_char: Option<u8>,
20    has_quote: bool,
21) -> Option<&'arena [u8]>
22where
23    A: Arena,
24{
25    if s.is_empty() {
26        return Some(b"");
27    }
28
29    let s = if has_quote
30        && (s.starts_with(b"b\"") || s.starts_with(b"b'") || s.starts_with(b"B\"") || s.starts_with(b"B'"))
31    {
32        &s[1..]
33    } else {
34        s
35    };
36
37    let (quote_char, content) = if let Some(quote_char) = quote_char {
38        (Some(quote_char), s)
39    } else if !has_quote {
40        (None, s)
41    } else if s.starts_with(b"\"") && s.ends_with(b"\"") && s.len() >= 2 {
42        (Some(b'"'), &s[1..s.len() - 1])
43    } else if s.starts_with(b"'") && s.ends_with(b"'") && s.len() >= 2 {
44        (Some(b'\''), &s[1..s.len() - 1])
45    } else {
46        return None;
47    };
48
49    let needs_processing = content.contains(&b'\\') || quote_char.is_some_and(|q| content.contains(&q));
50    if !needs_processing {
51        return Some(content);
52    }
53
54    let mut result = Vec::with_capacity_in(content.len(), arena);
55    let mut i = 0;
56
57    while i < content.len() {
58        let b = content[i];
59        if b != b'\\' {
60            result.push(b);
61            i += 1;
62            continue;
63        }
64
65        let next_index = i + 1;
66        let Some(&next) = content.get(next_index) else {
67            result.push(b'\\');
68            i += 1;
69            continue;
70        };
71
72        // Most escapes consume two bytes (`\` + the next byte). The hex and octal
73        // forms scan additional digit bytes and update `i` themselves.
74        let mut consumed = 2;
75
76        match next {
77            b'\\' => result.push(b'\\'),
78            b'\'' if quote_char == Some(b'\'') => result.push(b'\''),
79            b'"' if quote_char == Some(b'"') => result.push(b'"'),
80            b'$' if quote_char == Some(b'"') => result.push(b'$'),
81            b'n' if quote_char == Some(b'"') => result.push(b'\n'),
82            b't' if quote_char == Some(b'"') => result.push(b'\t'),
83            b'r' if quote_char == Some(b'"') => result.push(b'\r'),
84            b'v' if quote_char == Some(b'"') => result.push(0x0B),
85            b'e' if quote_char == Some(b'"') => result.push(0x1B),
86            b'f' if quote_char == Some(b'"') => result.push(0x0C),
87            b'x' if quote_char == Some(b'"') => {
88                let mut hex_val = 0u8;
89                let mut hex_len = 0;
90                let mut j = i + 2;
91                while hex_len < 2 && j < content.len() {
92                    let c = content[j];
93                    let digit = if c.is_ascii_digit() {
94                        c - b'0'
95                    } else if (b'a'..=b'f').contains(&c) {
96                        c - b'a' + 10
97                    } else if (b'A'..=b'F').contains(&c) {
98                        c - b'A' + 10
99                    } else {
100                        break;
101                    };
102                    hex_val = hex_val * 16 + digit;
103                    hex_len += 1;
104                    j += 1;
105                }
106                if hex_len > 0 {
107                    result.push(hex_val);
108                    consumed = 2 + hex_len;
109                } else {
110                    // Invalid `\x` sequence, treat as literal `\x`
111                    result.push(b'\\');
112                    result.push(b'x');
113                }
114            }
115            b'u' if quote_char == Some(b'"') && content.get(i + 2) == Some(&b'{') => {
116                let mut code_point = 0u32;
117                let mut hex_len = 0;
118                let mut overflowed = false;
119                let mut j = i + 3;
120                while j < content.len() {
121                    let c = content[j];
122                    let digit = if c.is_ascii_digit() {
123                        c - b'0'
124                    } else if (b'a'..=b'f').contains(&c) {
125                        c - b'a' + 10
126                    } else if (b'A'..=b'F').contains(&c) {
127                        c - b'A' + 10
128                    } else {
129                        break;
130                    };
131
132                    match code_point.checked_mul(16).and_then(|value| value.checked_add(u32::from(digit))) {
133                        Some(value) => code_point = value,
134                        None => {
135                            overflowed = true;
136                            break;
137                        }
138                    }
139
140                    hex_len += 1;
141                    j += 1;
142                }
143
144                if hex_len > 0 && !overflowed && content.get(j) == Some(&b'}') && code_point <= 0x10_FFFF {
145                    match code_point {
146                        0x0000..=0x007F => result.push(code_point as u8),
147                        0x0080..=0x07FF => {
148                            result.push(0xC0 | (code_point >> 6) as u8);
149                            result.push(0x80 | (code_point & 0x3F) as u8);
150                        }
151                        0x0800..=0xFFFF => {
152                            result.push(0xE0 | (code_point >> 12) as u8);
153                            result.push(0x80 | ((code_point >> 6) & 0x3F) as u8);
154                            result.push(0x80 | (code_point & 0x3F) as u8);
155                        }
156                        _ => {
157                            result.push(0xF0 | (code_point >> 18) as u8);
158                            result.push(0x80 | ((code_point >> 12) & 0x3F) as u8);
159                            result.push(0x80 | ((code_point >> 6) & 0x3F) as u8);
160                            result.push(0x80 | (code_point & 0x3F) as u8);
161                        }
162                    }
163
164                    consumed = (j + 1) - i;
165                } else {
166                    // Invalid `\u{...}` sequence, return None to indicate failure to parse.
167                    return None;
168                }
169            }
170            c if quote_char == Some(b'"') && c.is_ascii_digit() => {
171                let mut octal_val = 0u16;
172                let mut octal_len = 0;
173                let mut j = i + 1;
174                while octal_len < 3 && j < content.len() {
175                    let d = content[j];
176                    if d.is_ascii_digit() && d <= b'7' {
177                        octal_val = octal_val * 8 + u16::from(d - b'0');
178                        octal_len += 1;
179                        j += 1;
180                    } else {
181                        break;
182                    }
183                }
184                if octal_len > 0 {
185                    // Truncate to u8 (matches PHP behavior for octal sequences > 255)
186                    result.push(octal_val as u8);
187                    consumed = 1 + octal_len;
188                } else {
189                    result.push(b'\\');
190                    result.push(next);
191                }
192            }
193            _ => {
194                // Unrecognized escape sequence
195                result.push(b'\\');
196                result.push(next);
197            }
198        }
199
200        i += consumed;
201    }
202
203    Some(result.leak())
204}
205
206/// Parses a PHP literal float, handling underscore separators.
207#[inline]
208#[must_use]
209pub fn parse_literal_float(value: &[u8]) -> Option<f64> {
210    if memchr::memchr(b'_', value).is_none() {
211        return std::str::from_utf8(value).ok()?.parse::<f64>().ok();
212    }
213
214    let mut buf = [0u8; 64];
215    let mut len = 0;
216
217    for &b in value {
218        if b != b'_' {
219            if len < 64 {
220                buf[len] = b;
221                len += 1;
222            } else {
223                let source: std::vec::Vec<u8> = value.iter().copied().filter(|&b| b != b'_').collect();
224                return std::str::from_utf8(&source).ok()?.parse::<f64>().ok();
225            }
226        }
227    }
228
229    std::str::from_utf8(&buf[..len]).ok()?.parse::<f64>().ok()
230}
231
232/// Parses a PHP literal integer with support for binary, octal, decimal, and hex.
233///
234/// Optimized to use byte-level iteration instead of Unicode chars.
235#[inline]
236#[must_use]
237pub fn parse_literal_integer(bytes: &[u8]) -> Option<u64> {
238    if bytes.is_empty() {
239        return None;
240    }
241
242    let (radix, start) = match bytes {
243        [b'0', b'x' | b'X', ..] => (16u128, 2),
244        [b'0', b'o' | b'O', ..] => (8u128, 2),
245        [b'0', b'b' | b'B', ..] => (2u128, 2),
246        [b'0', _, ..] if bytes[1..].iter().all(|&b| b == b'_' || (b'0'..=b'7').contains(&b)) => (8u128, 1), // Legacy octal
247        [b'0', _, ..] => (10u128, 0), // Invalid octal (contains 8/9), treat as decimal
248        _ => (10u128, 0),
249    };
250
251    let mut result: u128 = 0;
252    let mut has_digits = false;
253
254    for &b in &bytes[start..] {
255        if b == b'_' {
256            continue;
257        }
258
259        let digit = if b.is_ascii_digit() {
260            (b - b'0') as u128
261        } else if (b'a'..=b'f').contains(&b) {
262            (b - b'a' + 10) as u128
263        } else if (b'A'..=b'F').contains(&b) {
264            (b - b'A' + 10) as u128
265        } else {
266            return None;
267        };
268
269        if digit >= radix {
270            return None;
271        }
272
273        has_digits = true;
274
275        result = match result.checked_mul(radix) {
276            Some(r) => r,
277            None => return Some(u64::MAX),
278        };
279
280        result = match result.checked_add(digit) {
281            Some(r) => r,
282            None => return Some(u64::MAX),
283        };
284    }
285
286    if !has_digits {
287        return None;
288    }
289
290    Some(result.min(u64::MAX as u128) as u64)
291}
292
293/// Parses a PHP literal integer's magnitude as an `f64`.
294///
295/// This is how PHP widens an integer literal that overflows the platform `int` into a float: it
296/// reflects the real magnitude rather than clamping at `u64::MAX` like [`parse_literal_integer`].
297/// Supports binary (`0b`), octal (`0o`/legacy `0`), decimal, and hexadecimal (`0x`), matching the
298/// base detection of [`parse_literal_integer`].
299#[inline]
300#[must_use]
301pub fn parse_literal_integer_as_float(bytes: &[u8]) -> Option<f64> {
302    if bytes.is_empty() {
303        return None;
304    }
305
306    let (radix, start) = match bytes {
307        [b'0', b'x' | b'X', ..] => (16u128, 2),
308        [b'0', b'o' | b'O', ..] => (8u128, 2),
309        [b'0', b'b' | b'B', ..] => (2u128, 2),
310        [b'0', _, ..] if bytes[1..].iter().all(|&b| b == b'_' || (b'0'..=b'7').contains(&b)) => (8u128, 1),
311        _ => (10u128, 0),
312    };
313
314    if radix == 10 {
315        return parse_literal_float(bytes);
316    }
317
318    let mut result: u128 = 0;
319    let mut has_digits = false;
320
321    for &b in &bytes[start..] {
322        if b == b'_' {
323            continue;
324        }
325
326        let digit = if b.is_ascii_digit() {
327            (b - b'0') as u128
328        } else if (b'a'..=b'f').contains(&b) {
329            (b - b'a' + 10) as u128
330        } else if (b'A'..=b'F').contains(&b) {
331            (b - b'A' + 10) as u128
332        } else {
333            return None;
334        };
335
336        if digit >= radix {
337            return None;
338        }
339
340        has_digits = true;
341        result = match result.checked_mul(radix).and_then(|value| value.checked_add(digit)) {
342            Some(value) => value,
343            None => return Some(f64::INFINITY),
344        };
345    }
346
347    if !has_digits {
348        return None;
349    }
350
351    Some(result as f64)
352}
353
354/// Lookup table for identifier start characters (a-z, A-Z, _)
355/// Index by byte value, true if valid start of identifier
356static IS_IDENT_START: [bool; 256] = {
357    let mut table = [false; 256];
358    let mut i = 0u8;
359    loop {
360        table[i as usize] = matches!(i, b'a'..=b'z' | b'A'..=b'Z' | b'_');
361        if i == 255 {
362            break;
363        }
364        i += 1;
365    }
366
367    table
368};
369
370/// Lookup table for identifier continuation characters (a-z, A-Z, 0-9, _, or >= 0x80)
371/// Index by byte value, true if valid part of identifier
372static IS_IDENT_PART: [bool; 256] = {
373    let mut table = [false; 256];
374    let mut i = 0u8;
375    loop {
376        table[i as usize] = matches!(i, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | 0x80..=0xFF);
377        if i == 255 {
378            break;
379        }
380        i += 1;
381    }
382    table
383};
384
385/// Check if a byte can start an identifier (a-z, A-Z, _)
386#[inline(always)]
387#[must_use]
388pub const fn is_start_of_identifier(byte: &u8) -> bool {
389    IS_IDENT_START[*byte as usize]
390}
391
392/// Check if a byte can be part of an identifier (a-z, A-Z, 0-9, _, or >= 0x80)
393#[inline(always)]
394#[must_use]
395pub const fn is_part_of_identifier(byte: &u8) -> bool {
396    IS_IDENT_PART[*byte as usize]
397}
398
399/// Scans an identifier starting at `offset` in the byte slice and returns the length.
400///
401/// Assumes the first byte is already validated as a start of identifier.
402/// Returns the total length of the identifier (including the first byte).
403/// Stops at the first byte that is not a valid identifier character.
404#[inline(always)]
405#[must_use]
406pub fn scan_identifier_length(bytes: &[u8], offset: usize) -> usize {
407    let mut len = 1;
408    let remaining = &bytes[offset + 1..];
409
410    for &b in remaining {
411        if IS_IDENT_PART[b as usize] {
412            len += 1;
413        } else {
414            break;
415        }
416    }
417
418    len
419}
420
421/// Reads a sequence of bytes representing digits in a specific numerical base.
422///
423/// This utility function iterates through the input byte slice, consuming bytes
424/// as long as they represent valid digits for the given `base`. It handles
425/// decimal digits ('0'-'9') and hexadecimal digits ('a'-'f', 'A'-'F').
426///
427/// It stops consuming at the first byte that is not a valid digit character,
428/// or is a digit character whose value is greater than or equal to the specified `base`
429/// (e.g., '8' in base 8, or 'A' in base 10).
430///
431/// This function is primarily intended as a helper for lexer implementations
432/// when tokenizing the digit part of number literals (binary, octal, decimal, hexadecimal).
433///
434/// # Arguments
435///
436/// * `input` - A byte slice starting at the potential first digit of the number.
437/// * `base` - The numerical base (e.g., 2, 8, 10, 16) to use for validating digits.
438///   Must be between 2 and 36 (inclusive) for hex characters to be potentially valid.
439///
440/// # Returns
441///
442/// The number of bytes (`usize`) consumed from the beginning of the `input` slice
443/// that constitute a valid sequence of digits for the specified `base`. Returns 0 if
444/// the first byte is not a valid digit for the base.
445#[inline]
446pub fn read_digits_of_base(input: &Input, offset: usize, base: u8) -> usize {
447    if base == 16 {
448        read_digits_with(input, offset, u8::is_ascii_hexdigit)
449    } else {
450        let max = b'0' + base;
451
452        read_digits_with(input, offset, |b| b >= &b'0' && b < &max)
453    }
454}
455
456#[inline]
457fn read_digits_with<F>(input: &Input, offset: usize, is_digit: F) -> usize
458where
459    F: Fn(&u8) -> bool,
460{
461    let bytes = input.bytes;
462    let total = input.length;
463    let start = input.offset;
464    let mut pos = start + offset; // Compute the absolute position.
465
466    while pos < total {
467        let current = bytes[pos];
468        if is_digit(&current) {
469            pos += 1;
470        } else if pos + 1 < total && bytes[pos] == number_separator!() && is_digit(&bytes[pos + 1]) {
471            pos += 2; // Skip the separator and the digit.
472        } else {
473            break;
474        }
475    }
476
477    // Return the relative length from the start of the current position.
478    pos - start
479}
480
481#[cfg(test)]
482mod tests {
483    use mago_allocator::LocalArena;
484
485    use super::*;
486
487    macro_rules! parse_int {
488        ($input:expr, $expected:expr) => {
489            assert_eq!(parse_literal_integer($input), $expected);
490        };
491    }
492
493    #[test]
494    fn test_unicode_escape_in_double_quoted() {
495        let arena = LocalArena::new();
496
497        assert_eq!(parse_literal_string_in(&arena, b"A\\u{1F600}", Some(b'"'), false), Some(&b"A\xF0\x9F\x98\x80"[..]),);
498        assert_eq!(parse_literal_string_in(&arena, b"\\u{41}", Some(b'"'), false), Some(&b"A"[..]));
499        assert_eq!(parse_literal_string_in(&arena, b"\\u{E9}", Some(b'"'), false), Some(&b"\xC3\xA9"[..]));
500        assert_eq!(parse_literal_string_in(&arena, b"\\u{D800}", Some(b'"'), false), Some(&b"\xED\xA0\x80"[..]));
501        assert_eq!(parse_literal_string_in(&arena, b"\\u{}", Some(b'"'), false), None);
502        assert_eq!(parse_literal_string_in(&arena, b"\\u{12", Some(b'"'), false), None);
503        assert_eq!(parse_literal_string_in(&arena, b"\\u{ZZ}", Some(b'"'), false), None);
504        assert_eq!(parse_literal_string_in(&arena, b"\\u{110000}", Some(b'"'), false), None);
505        assert_eq!(parse_literal_string_in(&arena, b"\\uABC", Some(b'"'), false), Some(&b"\\uABC"[..]));
506        assert_eq!(parse_literal_string_in(&arena, b"\\u{41}", Some(b'\''), false), Some(&b"\\u{41}"[..]));
507        assert_eq!(parse_literal_string_in(&arena, b"\\x", Some(b'"'), false), Some(&b"\\x"[..]));
508        assert_eq!(parse_literal_string_in(&arena, b"\\q", Some(b'"'), false), Some(&b"\\q"[..]));
509    }
510
511    #[test]
512    fn test_parse_literal_integer() {
513        parse_int!(b"123", Some(123));
514        parse_int!(b"0", Some(0));
515        parse_int!(b"0b1010", Some(10));
516        parse_int!(b"0o17", Some(15));
517        parse_int!(b"0x1A3F", Some(6719));
518        parse_int!(b"0XFF", Some(255));
519        parse_int!(b"0_1_2_3", Some(83));
520        parse_int!(b"0b1_0_1_0", Some(10));
521        parse_int!(b"0o1_7", Some(15));
522        parse_int!(b"0x1_A_3_F", Some(6719));
523        parse_int!(b"", None);
524        parse_int!(b"0xGHI", None);
525        parse_int!(b"0b102", None);
526        parse_int!(b"0o89", None);
527    }
528
529    #[test]
530    fn test_parse_literal_integer_as_float() {
531        assert_eq!(parse_literal_integer_as_float(b"0"), Some(0.0));
532        assert_eq!(parse_literal_integer_as_float(b"255"), Some(255.0));
533        assert_eq!(parse_literal_integer_as_float(b"0xff"), Some(255.0));
534        assert_eq!(parse_literal_integer_as_float(b"0o17"), Some(15.0));
535        assert_eq!(parse_literal_integer_as_float(b"017"), Some(15.0));
536        assert_eq!(parse_literal_integer_as_float(b"0b101"), Some(5.0));
537        assert_eq!(parse_literal_integer_as_float(b"1_000"), Some(1000.0));
538        assert_eq!(parse_literal_integer_as_float(b"111111111111111111111"), Some(1.111_111_111_111_111_1e20));
539        assert_eq!(parse_literal_integer_as_float(b"0xFFFFFFFFFFFFFFFF0"), Some(2.951_479_051_793_528_3e20));
540        assert_eq!(parse_literal_integer_as_float(&[b'9'; 400]), Some(f64::INFINITY));
541        assert_eq!(parse_literal_integer_as_float(b""), None);
542        assert_eq!(parse_literal_integer_as_float(b"0xGHI"), None);
543        assert_eq!(parse_literal_integer_as_float(b"0b102"), None);
544    }
545}
mago_syntax_core/utils.rs

mago_syntax_core/
utils.rs
