boa_parser/lexer/
regex.rs

1//! Boa's lexing for ECMAScript regex literals.
2
3use crate::lexer::{Cursor, Error, Token, TokenKind, Tokenizer};
4use crate::source::ReadChar;
5use bitflags::bitflags;
6use boa_ast::PositionGroup;
7use boa_interner::Interner;
8use regress::Flags;
9use std::fmt::{Display, Write};
10use std::str::{self, FromStr};
11
12const MAXIMUM_REGEX_FLAGS: usize = 8;
13
14/// Regex literal lexing.
15///
16/// Lexes Division, Assigndiv or Regex literal.
17///
18/// Expects: Initial '/' to already be consumed by cursor.
19///
20/// More information:
21///  - [ECMAScript reference][spec]
22///  - [MDN documentation][mdn]
23///
24/// [spec]: https://tc39.es/ecma262/#sec-literals-regular-expression-literals
25/// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions
26#[derive(Debug, Clone, Copy)]
27pub(super) struct RegexLiteral {
28    // If there is more cases than only `/=`
29    // then use `Option<u8>` or (more correct) `Option<enum>`
30    init_with_eq: bool,
31}
32
33impl RegexLiteral {
34    /// `init_with_eq` is '=' after `/` already consumed?
35    pub(super) fn new(init_with_eq: bool) -> Self {
36        Self { init_with_eq }
37    }
38}
39
40impl<R> Tokenizer<R> for RegexLiteral {
41    fn lex(
42        &mut self,
43        cursor: &mut Cursor<R>,
44        start_pos: PositionGroup,
45        interner: &mut Interner,
46    ) -> Result<Token, Error>
47    where
48        R: ReadChar,
49    {
50        let mut body = Vec::new();
51        if self.init_with_eq {
52            body.push(u32::from(b'='));
53        }
54
55        let mut is_class_char = false;
56
57        // Lex RegularExpressionBody.
58        loop {
59            match cursor.next_char()? {
60                None => {
61                    // Abrupt end.
62                    return Err(Error::syntax(
63                        "abrupt end on regular expression",
64                        cursor.pos(),
65                    ));
66                }
67                Some(b) => {
68                    match b {
69                        // /
70                        0x2F if !is_class_char => break, // RegularExpressionBody finished.
71                        // [
72                        0x5B => {
73                            is_class_char = true;
74                            body.push(b);
75                        }
76                        // ]
77                        0x5D if is_class_char => {
78                            is_class_char = false;
79                            body.push(b);
80                        }
81                        // \n | \r | \u{2028} | \u{2029}
82                        0xA | 0xD | 0x2028 | 0x2029 => {
83                            // Not allowed in Regex literal.
84                            return Err(Error::syntax(
85                                "new lines are not allowed in regular expressions",
86                                cursor.pos(),
87                            ));
88                        }
89                        // \
90                        0x5C => {
91                            // Escape sequence
92                            body.push(b);
93                            if let Some(sc) = cursor.next_char()? {
94                                match sc {
95                                    // \n | \r | \u{2028} | \u{2029}
96                                    0xA | 0xD | 0x2028 | 0x2029 => {
97                                        // Not allowed in Regex literal.
98                                        return Err(Error::syntax(
99                                            "new lines are not allowed in regular expressions",
100                                            cursor.pos(),
101                                        ));
102                                    }
103                                    b => body.push(b),
104                                }
105                            } else {
106                                // Abrupt end of regex.
107                                return Err(Error::syntax(
108                                    "abrupt end on regular expression",
109                                    cursor.pos(),
110                                ));
111                            }
112                        }
113                        _ => body.push(b),
114                    }
115                }
116            }
117        }
118
119        let mut flags: [u32; MAXIMUM_REGEX_FLAGS] = [0; MAXIMUM_REGEX_FLAGS];
120        let n = cursor.take_array_alphabetic(&mut flags)?;
121        if n > MAXIMUM_REGEX_FLAGS {
122            // There can only be a maximum of 8 flags.
123            return Err(Error::syntax(
124                "Invalid regular expression: too many flags",
125                start_pos,
126            ));
127        }
128        let flags: RegExpFlags =
129            RegExpFlags::try_from(&flags[..n]).map_err(|e| Error::syntax(e, start_pos))?;
130
131        // We have a vague hint of the size of this vector in the best case scenario.
132        let mut body_utf16 = Vec::with_capacity(body.len());
133
134        // We convert the body to UTF-16 since it may contain code points that are not valid UTF-8.
135        // We already know that the body is valid UTF-16. Casting is fine.
136        #[allow(clippy::cast_possible_truncation)]
137        for cp in &body {
138            let cp = *cp;
139            if cp <= 0xFFFF {
140                body_utf16.push(cp as u16);
141            } else {
142                let cp = cp - 0x1_0000;
143                let high = 0xD800 | ((cp >> 10) as u16);
144                let low = 0xDC00 | ((cp as u16) & 0x3FF);
145                body_utf16.push(high);
146                body_utf16.push(low);
147            }
148        }
149
150        // Only try to parse and validate, do not optimize/compile.
151        drop(
152            regress::backends::try_parse(body.into_iter(), flags.into()).map_err(|error| {
153                Error::syntax(
154                    format!("Invalid regular expression literal: {error}"),
155                    start_pos,
156                )
157            })?,
158        );
159
160        let mut flags_buf = [0u8; MAXIMUM_REGEX_FLAGS];
161        let flags_str = flags.write_to_buf(&mut flags_buf);
162
163        Ok(Token::new_by_position_group(
164            TokenKind::regular_expression_literal(
165                interner.get_or_intern(body_utf16.as_slice()),
166                interner.get_or_intern(flags_str),
167            ),
168            start_pos,
169            cursor.pos_group(),
170        ))
171    }
172}
173
174bitflags! {
175    /// Flags of a regular expression.
176    #[derive(Debug, Default, Copy, Clone)]
177    pub struct RegExpFlags: u8 {
178        /// Whether to test the regular expression against all possible matches in a string,
179        /// or only against the first.
180        const GLOBAL = 0b0000_0001;
181
182        /// Whether to ignore case while attempting a match in a string.
183        const IGNORE_CASE = 0b0000_0010;
184
185        /// Whether or not to search in strings across multiple lines.
186        const MULTILINE = 0b0000_0100;
187
188        /// Whether `.` matches newlines or not.
189        const DOT_ALL = 0b0000_1000;
190
191        /// Whether or not Unicode features are enabled.
192        const UNICODE = 0b0001_0000;
193
194        /// Whether or not the search is sticky.
195        const STICKY = 0b0010_0000;
196
197        /// Whether the regular expression result exposes the start and end indices of
198        /// captured substrings.
199        const HAS_INDICES = 0b0100_0000;
200
201        /// Whether or not UnicodeSets features are enabled.
202        const UNICODE_SETS = 0b1000_0000;
203    }
204}
205
206impl TryFrom<&[u32]> for RegExpFlags {
207    type Error = String;
208
209    fn try_from(value: &[u32]) -> Result<Self, Self::Error> {
210        let mut flags = Self::default();
211        for c in value {
212            let c = char::from_u32(*c)
213                .ok_or_else(|| format!("Invalid regular expression flag: {c}"))?;
214
215            let new_flag = match c {
216                'g' => Self::GLOBAL,
217                'i' => Self::IGNORE_CASE,
218                'm' => Self::MULTILINE,
219                's' => Self::DOT_ALL,
220                'u' => Self::UNICODE,
221                'y' => Self::STICKY,
222                'd' => Self::HAS_INDICES,
223                'v' => Self::UNICODE_SETS,
224                _ => return Err(format!("invalid regular expression flag {c}")),
225            };
226
227            if flags.contains(new_flag) {
228                return Err(format!("repeated regular expression flag {c}"));
229            }
230            flags.insert(new_flag);
231        }
232
233        if flags.contains(Self::UNICODE) && flags.contains(Self::UNICODE_SETS) {
234            return Err("cannot use both 'u' and 'v' flags".into());
235        }
236
237        Ok(flags)
238    }
239}
240
241impl FromStr for RegExpFlags {
242    type Err = String;
243
244    fn from_str(s: &str) -> Result<Self, Self::Err> {
245        let mut flags = Self::default();
246        for c in s.bytes() {
247            let new_flag = match c {
248                b'g' => Self::GLOBAL,
249                b'i' => Self::IGNORE_CASE,
250                b'm' => Self::MULTILINE,
251                b's' => Self::DOT_ALL,
252                b'u' => Self::UNICODE,
253                b'y' => Self::STICKY,
254                b'd' => Self::HAS_INDICES,
255                b'v' => Self::UNICODE_SETS,
256                _ => return Err(format!("invalid regular expression flag {}", char::from(c))),
257            };
258
259            if flags.contains(new_flag) {
260                return Err(format!(
261                    "repeated regular expression flag {}",
262                    char::from(c)
263                ));
264            }
265            flags.insert(new_flag);
266        }
267
268        if flags.contains(Self::UNICODE) && flags.contains(Self::UNICODE_SETS) {
269            return Err("cannot use both 'u' and 'v' flags".into());
270        }
271
272        Ok(flags)
273    }
274}
275
276impl RegExpFlags {
277    /// Writes the flags string to a buffer and returns it as `&str`.
278    /// Avoids heap allocation when interning regex flags during lexing.
279    /// The buffer must be at least 8 bytes (maximum number of flags).
280    #[inline]
281    pub fn write_to_buf<'a>(&self, buf: &'a mut [u8; MAXIMUM_REGEX_FLAGS]) -> &'a str {
282        let mut len = 0;
283        if self.contains(Self::HAS_INDICES) {
284            buf[len] = b'd';
285            len += 1;
286        }
287        if self.contains(Self::GLOBAL) {
288            buf[len] = b'g';
289            len += 1;
290        }
291        if self.contains(Self::IGNORE_CASE) {
292            buf[len] = b'i';
293            len += 1;
294        }
295        if self.contains(Self::MULTILINE) {
296            buf[len] = b'm';
297            len += 1;
298        }
299        if self.contains(Self::DOT_ALL) {
300            buf[len] = b's';
301            len += 1;
302        }
303        if self.contains(Self::UNICODE) {
304            buf[len] = b'u';
305            len += 1;
306        }
307        if self.contains(Self::STICKY) {
308            buf[len] = b'y';
309            len += 1;
310        }
311        if self.contains(Self::UNICODE_SETS) {
312            buf[len] = b'v';
313            len += 1;
314        }
315        // SAFETY: We only wrote ASCII bytes (d, g, i, m, s, u, y, v).
316        unsafe { str::from_utf8_unchecked(&buf[..len]) }
317    }
318}
319
320impl Display for RegExpFlags {
321    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
322        if self.contains(Self::HAS_INDICES) {
323            f.write_char('d')?;
324        }
325        if self.contains(Self::GLOBAL) {
326            f.write_char('g')?;
327        }
328        if self.contains(Self::IGNORE_CASE) {
329            f.write_char('i')?;
330        }
331        if self.contains(Self::MULTILINE) {
332            f.write_char('m')?;
333        }
334        if self.contains(Self::DOT_ALL) {
335            f.write_char('s')?;
336        }
337        if self.contains(Self::UNICODE) {
338            f.write_char('u')?;
339        }
340        if self.contains(Self::STICKY) {
341            f.write_char('y')?;
342        }
343        if self.contains(Self::UNICODE_SETS) {
344            f.write_char('v')?;
345        }
346        Ok(())
347    }
348}
349
350impl From<RegExpFlags> for Flags {
351    fn from(value: RegExpFlags) -> Self {
352        Self {
353            icase: value.contains(RegExpFlags::IGNORE_CASE),
354            multiline: value.contains(RegExpFlags::MULTILINE),
355            dot_all: value.contains(RegExpFlags::DOT_ALL),
356            unicode: value.contains(RegExpFlags::UNICODE),
357            unicode_sets: value.contains(RegExpFlags::UNICODE_SETS),
358            ..Self::default()
359        }
360    }
361}
boa_parser/lexer/regex.rs

boa_parser/lexer/
regex.rs
