nodejs/
lexer.rs

1//! JavaScript tokenizer.
2//!
3//! Produces a flat token stream ending in `Eof`. Unlike Python, JS is not
4//! indentation-sensitive: blocks are brace-delimited and statements are
5//! semicolon-terminated, with Automatic Semicolon Insertion (ASI) filling in
6//! for newline-terminated statements. Each token records whether a line break
7//! preceded it (`newline_before`) so the parser can apply ASI. `//` and `/* */`
8//! comments are stripped here. Template literals are emitted as a single
9//! `Template` token carrying the cooked quasis plus the raw source of each
10//! `${...}` field; the parser recursively parses those fields.
11
12/// A lexical token.
13#[derive(Debug, Clone, PartialEq)]
14pub enum Tok {
15    Num(f64),
16    Str(String),
17    /// A template literal: `quasis.len() == exprs.len() + 1`. Each `exprs`
18    /// entry is the raw source text between `${` and its matching `}`.
19    Template {
20        quasis: Vec<String>,
21        exprs: Vec<String>,
22    },
23    Ident(String),
24    /// An operator or delimiter, e.g. `+`, `===`, `=>`, `(`, `{`, `.`, `?.`.
25    Punct(String),
26    Eof,
27}
28
29/// A token plus its 1-based source line and whether a newline preceded it.
30#[derive(Debug, Clone, PartialEq)]
31pub struct Token {
32    pub tok: Tok,
33    pub line: u32,
34    pub newline_before: bool,
35}
36
37struct Lexer {
38    src: Vec<char>,
39    pos: usize,
40    line: u32,
41    out: Vec<Token>,
42    pending_newline: bool,
43}
44
45/// Multi-char operators, longest first so the scanner is greedy.
46const OPS4: &[&str] = &[">>>="];
47const OPS3: &[&str] = &[
48    "===", "!==", "**=", "...", ">>>", "<<=", ">>=", "&&=", "||=", "??=",
49];
50const OPS2: &[&str] = &[
51    "==", "!=", "<=", ">=", "&&", "||", "??", "?.", "=>", "++", "--", "+=", "-=", "*=", "/=", "%=",
52    "&=", "|=", "^=", "<<", ">>", "**",
53];
54
55/// Tokenize `src` into a token stream ending in `Eof`.
56pub fn lex(src: &str) -> Result<Vec<Token>, String> {
57    let mut lx = Lexer {
58        src: src.chars().collect(),
59        pos: 0,
60        line: 1,
61        out: Vec::new(),
62        pending_newline: false,
63    };
64    lx.run()?;
65    Ok(lx.out)
66}
67
68impl Lexer {
69    fn peek(&self) -> Option<char> {
70        self.src.get(self.pos).copied()
71    }
72    fn peek_at(&self, n: usize) -> Option<char> {
73        self.src.get(self.pos + n).copied()
74    }
75    fn bump(&mut self) -> Option<char> {
76        let c = self.src.get(self.pos).copied();
77        if let Some(ch) = c {
78            self.pos += 1;
79            if ch == '\n' {
80                self.line += 1;
81            }
82        }
83        c
84    }
85    fn push(&mut self, tok: Tok) {
86        self.out.push(Token {
87            tok,
88            line: self.line,
89            newline_before: self.pending_newline,
90        });
91        self.pending_newline = false;
92    }
93
94    fn run(&mut self) -> Result<(), String> {
95        loop {
96            match self.peek() {
97                None => break,
98                Some('\n') => {
99                    self.bump();
100                    self.pending_newline = true;
101                }
102                Some(c) if c == ' ' || c == '\t' || c == '\r' => {
103                    self.bump();
104                }
105                Some('/') if self.peek_at(1) == Some('/') => {
106                    while let Some(c) = self.peek() {
107                        if c == '\n' {
108                            break;
109                        }
110                        self.bump();
111                    }
112                }
113                Some('/') if self.peek_at(1) == Some('*') => {
114                    self.bump();
115                    self.bump();
116                    while let Some(c) = self.peek() {
117                        if c == '*' && self.peek_at(1) == Some('/') {
118                            self.bump();
119                            self.bump();
120                            break;
121                        }
122                        if c == '\n' {
123                            self.pending_newline = true;
124                        }
125                        self.bump();
126                    }
127                }
128                Some(_) => self.scan_token()?,
129            }
130        }
131        self.push(Tok::Eof);
132        Ok(())
133    }
134
135    fn scan_token(&mut self) -> Result<(), String> {
136        let c = self.peek().unwrap();
137        if c == '"' || c == '\'' {
138            return self.scan_string(c);
139        }
140        if c == '`' {
141            return self.scan_template();
142        }
143        if c.is_ascii_alphabetic() || c == '_' || c == '$' {
144            return self.scan_name();
145        }
146        if c.is_ascii_digit() || (c == '.' && self.peek_at(1).map(|d| d.is_ascii_digit()).unwrap_or(false))
147        {
148            return self.scan_number();
149        }
150        self.scan_op()
151    }
152
153    fn scan_name(&mut self) -> Result<(), String> {
154        let mut s = String::new();
155        while let Some(c) = self.peek() {
156            if c.is_alphanumeric() || c == '_' || c == '$' {
157                s.push(c);
158                self.pos += 1;
159            } else {
160                break;
161            }
162        }
163        self.push(Tok::Ident(s));
164        Ok(())
165    }
166
167    fn scan_string(&mut self, quote: char) -> Result<(), String> {
168        self.bump(); // opening quote
169        let mut raw = String::new();
170        loop {
171            match self.peek() {
172                None => return Err(format!("SyntaxError: unterminated string (line {})", self.line)),
173                Some(c) if c == quote => {
174                    self.bump();
175                    break;
176                }
177                Some('\\') => {
178                    self.bump();
179                    if let Some(e) = self.bump() {
180                        push_escape(&mut raw, e, self);
181                    }
182                }
183                Some('\n') => {
184                    return Err(format!(
185                        "SyntaxError: unterminated string literal (line {})",
186                        self.line
187                    ))
188                }
189                Some(c) => {
190                    raw.push(c);
191                    self.bump();
192                }
193            }
194        }
195        self.push(Tok::Str(raw));
196        Ok(())
197    }
198
199    /// Scan a `` `...${expr}...` `` template. Cooked quasis are decoded; each
200    /// `${...}` field's raw source (with balanced braces) is captured for the
201    /// parser to re-parse.
202    fn scan_template(&mut self) -> Result<(), String> {
203        self.bump(); // opening backtick
204        let mut quasis = Vec::new();
205        let mut exprs = Vec::new();
206        let mut cur = String::new();
207        loop {
208            match self.peek() {
209                None => return Err(format!("SyntaxError: unterminated template (line {})", self.line)),
210                Some('`') => {
211                    self.bump();
212                    break;
213                }
214                Some('\\') => {
215                    self.bump();
216                    if let Some(e) = self.bump() {
217                        push_escape(&mut cur, e, self);
218                    }
219                }
220                Some('$') if self.peek_at(1) == Some('{') => {
221                    self.bump();
222                    self.bump();
223                    quasis.push(std::mem::take(&mut cur));
224                    // Capture raw source until the matching `}` (brace-balanced,
225                    // skipping strings).
226                    let mut depth = 1;
227                    let mut src = String::new();
228                    loop {
229                        match self.peek() {
230                            None => {
231                                return Err(format!(
232                                    "SyntaxError: unterminated template expression (line {})",
233                                    self.line
234                                ))
235                            }
236                            Some('{') => {
237                                depth += 1;
238                                src.push('{');
239                                self.bump();
240                            }
241                            Some('}') => {
242                                depth -= 1;
243                                self.bump();
244                                if depth == 0 {
245                                    break;
246                                }
247                                src.push('}');
248                            }
249                            Some(q) if q == '"' || q == '\'' || q == '`' => {
250                                src.push(q);
251                                self.bump();
252                                while let Some(cc) = self.peek() {
253                                    src.push(cc);
254                                    self.bump();
255                                    if cc == '\\' {
256                                        if let Some(n) = self.peek() {
257                                            src.push(n);
258                                            self.bump();
259                                        }
260                                    } else if cc == q {
261                                        break;
262                                    }
263                                }
264                            }
265                            Some(cc) => {
266                                src.push(cc);
267                                self.bump();
268                            }
269                        }
270                    }
271                    exprs.push(src);
272                }
273                Some(c) => {
274                    cur.push(c);
275                    self.bump();
276                }
277            }
278        }
279        quasis.push(cur);
280        self.push(Tok::Template { quasis, exprs });
281        Ok(())
282    }
283
284    fn scan_number(&mut self) -> Result<(), String> {
285        // Radix prefixes: 0x / 0o / 0b.
286        if self.peek() == Some('0') {
287            if let Some(r) = self.peek_at(1) {
288                if matches!(r, 'x' | 'X' | 'o' | 'O' | 'b' | 'B') {
289                    self.bump();
290                    self.bump();
291                    let radix = match r.to_ascii_lowercase() {
292                        'x' => 16,
293                        'o' => 8,
294                        _ => 2,
295                    };
296                    let mut digits = String::new();
297                    while let Some(c) = self.peek() {
298                        if c == '_' {
299                            self.pos += 1;
300                        } else if c.is_digit(radix) {
301                            digits.push(c);
302                            self.pos += 1;
303                        } else {
304                            break;
305                        }
306                    }
307                    let n = i64::from_str_radix(&digits, radix)
308                        .map_err(|_| format!("SyntaxError: bad number (line {})", self.line))?;
309                    self.push(Tok::Num(n as f64));
310                    return Ok(());
311                }
312            }
313        }
314        let mut s = String::new();
315        while let Some(c) = self.peek() {
316            match c {
317                '0'..='9' => {
318                    s.push(c);
319                    self.pos += 1;
320                }
321                '_' => {
322                    self.pos += 1;
323                }
324                '.' => {
325                    s.push(c);
326                    self.pos += 1;
327                }
328                'e' | 'E' => {
329                    s.push('e');
330                    self.pos += 1;
331                    if matches!(self.peek(), Some('+') | Some('-')) {
332                        s.push(self.peek().unwrap());
333                        self.pos += 1;
334                    }
335                }
336                _ => break,
337            }
338        }
339        let v: f64 = s
340            .parse()
341            .map_err(|_| format!("SyntaxError: bad number '{s}' (line {})", self.line))?;
342        self.push(Tok::Num(v));
343        Ok(())
344    }
345
346    fn scan_op(&mut self) -> Result<(), String> {
347        let slice: String = self.src[self.pos..(self.pos + 4).min(self.src.len())]
348            .iter()
349            .collect();
350        for op in OPS4 {
351            if slice.starts_with(op) {
352                self.pos += 4;
353                self.push(Tok::Punct((*op).to_string()));
354                return Ok(());
355            }
356        }
357        for op in OPS3 {
358            if slice.starts_with(op) {
359                self.pos += 3;
360                self.push(Tok::Punct((*op).to_string()));
361                return Ok(());
362            }
363        }
364        for op in OPS2 {
365            if slice.starts_with(op) {
366                self.pos += 2;
367                self.push(Tok::Punct((*op).to_string()));
368                return Ok(());
369            }
370        }
371        let c = self.bump().unwrap();
372        if "+-*/%<>=!&|^~?:;,.(){}[]".contains(c) {
373            self.push(Tok::Punct(c.to_string()));
374            Ok(())
375        } else {
376            Err(format!(
377                "SyntaxError: unexpected character {c:?} (line {})",
378                self.line
379            ))
380        }
381    }
382}
383
384/// Append one escape sequence's decoded character(s) to `out`. `\xNN` and
385/// `\uNNNN` / `\u{...}` are decoded; unknown escapes keep the literal char.
386fn push_escape(out: &mut String, e: char, lx: &mut Lexer) {
387    match e {
388        'n' => out.push('\n'),
389        't' => out.push('\t'),
390        'r' => out.push('\r'),
391        'b' => out.push('\u{08}'),
392        'f' => out.push('\u{0C}'),
393        'v' => out.push('\u{0B}'),
394        '0' => out.push('\0'),
395        '\\' => out.push('\\'),
396        '\'' => out.push('\''),
397        '"' => out.push('"'),
398        '`' => out.push('`'),
399        '\n' => {} // line continuation
400        'x' => {
401            let mut h = String::new();
402            for _ in 0..2 {
403                if let Some(c) = lx.peek() {
404                    if c.is_ascii_hexdigit() {
405                        h.push(c);
406                        lx.bump();
407                    }
408                }
409            }
410            if let Ok(n) = u32::from_str_radix(&h, 16) {
411                if let Some(ch) = char::from_u32(n) {
412                    out.push(ch);
413                }
414            }
415        }
416        'u' => {
417            if lx.peek() == Some('{') {
418                lx.bump();
419                let mut h = String::new();
420                while let Some(c) = lx.peek() {
421                    if c == '}' {
422                        lx.bump();
423                        break;
424                    }
425                    h.push(c);
426                    lx.bump();
427                }
428                if let Ok(n) = u32::from_str_radix(&h, 16) {
429                    if let Some(ch) = char::from_u32(n) {
430                        out.push(ch);
431                    }
432                }
433            } else {
434                let mut h = String::new();
435                for _ in 0..4 {
436                    if let Some(c) = lx.peek() {
437                        if c.is_ascii_hexdigit() {
438                            h.push(c);
439                            lx.bump();
440                        }
441                    }
442                }
443                if let Ok(n) = u32::from_str_radix(&h, 16) {
444                    if let Some(ch) = char::from_u32(n) {
445                        out.push(ch);
446                    }
447                }
448            }
449        }
450        other => out.push(other),
451    }
452}
nodejs/lexer.rs

nodejs/
lexer.rs
