boa_parser/lexer/
regex.rs

1//! Boa's lexing for ECMAScript regex literals.
2
3use crate::lexer::{Cursor, Error, Token, TokenKind, Tokenizer};
4use crate::source::ReadChar;
5use bitflags::bitflags;
6use boa_ast::PositionGroup;
7use boa_interner::Interner;
8use regress::Flags;
9use std::fmt::{Display, Write};
10use std::str::{self, FromStr};
11
12const MAXIMUM_REGEX_FLAGS: usize = 8;
13
14/// Regex literal lexing.
15///
16/// Lexes Division, Assigndiv or Regex literal.
17///
18/// Expects: Initial '/' to already be consumed by cursor.
19///
20/// More information:
21///  - [ECMAScript reference][spec]
22///  - [MDN documentation][mdn]
23///
24/// [spec]: https://tc39.es/ecma262/#sec-literals-regular-expression-literals
25/// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions
26#[derive(Debug, Clone, Copy)]
27pub(super) struct RegexLiteral {
28    // If there is more cases than only `/=`
29    // then use `Option<u8>` or (more correct) `Option<enum>`
30    init_with_eq: bool,
31}
32
33impl RegexLiteral {
34    /// `init_with_eq` is '=' after `/` already consumed?
35    pub(super) fn new(init_with_eq: bool) -> Self {
36        Self { init_with_eq }
37    }
38}
39
40impl<R> Tokenizer<R> for RegexLiteral {
41    fn lex(
42        &mut self,
43        cursor: &mut Cursor<R>,
44        start_pos: PositionGroup,
45        interner: &mut Interner,
46    ) -> Result<Token, Error>
47    where
48        R: ReadChar,
49    {
50        let mut body = Vec::new();
51        if self.init_with_eq {
52            body.push(u32::from(b'='));
53        }
54
55        let mut is_class_char = false;
56
57        // Lex RegularExpressionBody.
58        loop {
59            match cursor.next_char()? {
60                None => {
61                    // Abrupt end.
62                    return Err(Error::syntax(
63                        "abrupt end on regular expression",
64                        cursor.pos(),
65                    ));
66                }
67                Some(b) => {
68                    match b {
69                        // /
70                        0x2F if !is_class_char => break, // RegularExpressionBody finished.
71                        // [
72                        0x5B => {
73                            is_class_char = true;
74                            body.push(b);
75                        }
76                        // ]
77                        0x5D if is_class_char => {
78                            is_class_char = false;
79                            body.push(b);
80                        }
81                        // \n | \r | \u{2028} | \u{2029}
82                        0xA | 0xD | 0x2028 | 0x2029 => {
83                            // Not allowed in Regex literal.
84                            return Err(Error::syntax(
85                                "new lines are not allowed in regular expressions",
86                                cursor.pos(),
87                            ));
88                        }
89                        // \
90                        0x5C => {
91                            // Escape sequence
92                            body.push(b);
93                            if let Some(sc) = cursor.next_char()? {
94                                match sc {
95                                    // \n | \r | \u{2028} | \u{2029}
96                                    0xA | 0xD | 0x2028 | 0x2029 => {
97                                        // Not allowed in Regex literal.
98                                        return Err(Error::syntax(
99                                            "new lines are not allowed in regular expressions",
100                                            cursor.pos(),
101                                        ));
102                                    }
103                                    b => body.push(b),
104                                }
105                            } else {
106                                // Abrupt end of regex.
107                                return Err(Error::syntax(
108                                    "abrupt end on regular expression",
109                                    cursor.pos(),
110                                ));
111                            }
112                        }
113                        _ => body.push(b),
114                    }
115                }
116            }
117        }
118
119        let mut flags: [u32; MAXIMUM_REGEX_FLAGS] = [0; MAXIMUM_REGEX_FLAGS];
120        let n = cursor.take_array_alphabetic(&mut flags)?;
121        if n > MAXIMUM_REGEX_FLAGS {
122            // There can only be a maximum of 8 flags.
123            return Err(Error::syntax(
124                "Invalid regular expression: too many flags",
125                start_pos,
126            ));
127        }
128        let flags: RegExpFlags =
129            RegExpFlags::try_from(&flags[..n]).map_err(|e| Error::syntax(e, start_pos))?;
130
131        // We have a vague hint of the size of this vector in the best case scenario.
132        let mut body_utf16 = Vec::with_capacity(body.len());
133
134        // We convert the body to UTF-16 since it may contain code points that are not valid UTF-8.
135        // We already know that the body is valid UTF-16. Casting is fine.
136        #[allow(clippy::cast_possible_truncation)]
137        for cp in &body {
138            let cp = *cp;
139            if cp <= 0xFFFF {
140                body_utf16.push(cp as u16);
141            } else {
142                let cp = cp - 0x1_0000;
143                let high = 0xD800 | ((cp >> 10) as u16);
144                let low = 0xDC00 | ((cp as u16) & 0x3FF);
145                body_utf16.push(high);
146                body_utf16.push(low);
147            }
148        }
149
150        // Only try to parse and validate, do not optimize/compile.
151        drop(
152            regress::backends::try_parse(body.into_iter(), flags.into()).map_err(|error| {
153                Error::syntax(
154                    format!("Invalid regular expression literal: {error}"),
155                    start_pos,
156                )
157            })?,
158        );
159
160        Ok(Token::new_by_position_group(
161            TokenKind::regular_expression_literal(
162                interner.get_or_intern(body_utf16.as_slice()),
163                interner.get_or_intern(flags.to_string().as_str()),
164            ),
165            start_pos,
166            cursor.pos_group(),
167        ))
168    }
169}
170
171bitflags! {
172    /// Flags of a regular expression.
173    #[derive(Debug, Default, Copy, Clone)]
174    pub struct RegExpFlags: u8 {
175        /// Whether to test the regular expression against all possible matches in a string,
176        /// or only against the first.
177        const GLOBAL = 0b0000_0001;
178
179        /// Whether to ignore case while attempting a match in a string.
180        const IGNORE_CASE = 0b0000_0010;
181
182        /// Whether or not to search in strings across multiple lines.
183        const MULTILINE = 0b0000_0100;
184
185        /// Whether `.` matches newlines or not.
186        const DOT_ALL = 0b0000_1000;
187
188        /// Whether or not Unicode features are enabled.
189        const UNICODE = 0b0001_0000;
190
191        /// Whether or not the search is sticky.
192        const STICKY = 0b0010_0000;
193
194        /// Whether the regular expression result exposes the start and end indices of
195        /// captured substrings.
196        const HAS_INDICES = 0b0100_0000;
197
198        /// Whether or not UnicodeSets features are enabled.
199        const UNICODE_SETS = 0b1000_0000;
200    }
201}
202
203impl TryFrom<&[u32]> for RegExpFlags {
204    type Error = String;
205
206    fn try_from(value: &[u32]) -> Result<Self, Self::Error> {
207        let mut flags = Self::default();
208        for c in value {
209            let c = char::from_u32(*c)
210                .ok_or_else(|| format!("Invalid regular expression flag: {c}"))?;
211
212            let new_flag = match c {
213                'g' => Self::GLOBAL,
214                'i' => Self::IGNORE_CASE,
215                'm' => Self::MULTILINE,
216                's' => Self::DOT_ALL,
217                'u' => Self::UNICODE,
218                'y' => Self::STICKY,
219                'd' => Self::HAS_INDICES,
220                'v' => Self::UNICODE_SETS,
221                _ => return Err(format!("invalid regular expression flag {c}")),
222            };
223
224            if flags.contains(new_flag) {
225                return Err(format!("repeated regular expression flag {c}"));
226            }
227            flags.insert(new_flag);
228        }
229
230        if flags.contains(Self::UNICODE) && flags.contains(Self::UNICODE_SETS) {
231            return Err("cannot use both 'u' and 'v' flags".into());
232        }
233
234        Ok(flags)
235    }
236}
237
238impl FromStr for RegExpFlags {
239    type Err = String;
240
241    fn from_str(s: &str) -> Result<Self, Self::Err> {
242        let mut flags = Self::default();
243        for c in s.bytes() {
244            let new_flag = match c {
245                b'g' => Self::GLOBAL,
246                b'i' => Self::IGNORE_CASE,
247                b'm' => Self::MULTILINE,
248                b's' => Self::DOT_ALL,
249                b'u' => Self::UNICODE,
250                b'y' => Self::STICKY,
251                b'd' => Self::HAS_INDICES,
252                b'v' => Self::UNICODE_SETS,
253                _ => return Err(format!("invalid regular expression flag {}", char::from(c))),
254            };
255
256            if flags.contains(new_flag) {
257                return Err(format!(
258                    "repeated regular expression flag {}",
259                    char::from(c)
260                ));
261            }
262            flags.insert(new_flag);
263        }
264
265        if flags.contains(Self::UNICODE) && flags.contains(Self::UNICODE_SETS) {
266            return Err("cannot use both 'u' and 'v' flags".into());
267        }
268
269        Ok(flags)
270    }
271}
272
273impl Display for RegExpFlags {
274    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
275        if self.contains(Self::HAS_INDICES) {
276            f.write_char('d')?;
277        }
278        if self.contains(Self::GLOBAL) {
279            f.write_char('g')?;
280        }
281        if self.contains(Self::IGNORE_CASE) {
282            f.write_char('i')?;
283        }
284        if self.contains(Self::MULTILINE) {
285            f.write_char('m')?;
286        }
287        if self.contains(Self::DOT_ALL) {
288            f.write_char('s')?;
289        }
290        if self.contains(Self::UNICODE) {
291            f.write_char('u')?;
292        }
293        if self.contains(Self::STICKY) {
294            f.write_char('y')?;
295        }
296        if self.contains(Self::UNICODE_SETS) {
297            f.write_char('v')?;
298        }
299        Ok(())
300    }
301}
302
303impl From<RegExpFlags> for Flags {
304    fn from(value: RegExpFlags) -> Self {
305        Self {
306            icase: value.contains(RegExpFlags::IGNORE_CASE),
307            multiline: value.contains(RegExpFlags::MULTILINE),
308            dot_all: value.contains(RegExpFlags::DOT_ALL),
309            unicode: value.contains(RegExpFlags::UNICODE),
310            unicode_sets: value.contains(RegExpFlags::UNICODE_SETS),
311            ..Self::default()
312        }
313    }
314}
boa_parser/lexer/regex.rs

boa_parser/lexer/
regex.rs
