Skip to main content

fatou_parser/parser/
lexer.rs

1//! Hand-written, lossless tokenizer for the Julia subset.
2//!
3//! Every byte of the input ends up in exactly one [`Token`] (including
4//! whitespace, newlines, and comments), so the token stream can be reassembled
5//! into the original text. Unrecognized bytes become [`TokKind::Unknown`]
6//! single-byte tokens rather than being dropped, which keeps losslessness a
7//! property of the lexer alone.
8//!
9use super::unicode_ops::unicode_op_kind;
10use crate::keywords::keyword_table;
11use crate::tokens::token_table;
12
13mod operators;
14
15/// Generate [`TokKind`] from the shared token table. Only the tokens that do
16/// not materialize 1:1 as a `SyntaxKind` are written out here.
17macro_rules! define_tok_kind {
18    ($($(#[$meta:meta])* $tok:ident $syn:ident,)*) => {
19/// A lexed token kind. Maps to a [`crate::syntax::SyntaxKind`] in
20/// [`crate::parser::tree_builder::syntax_kind_for`].
21#[derive(Debug, Clone, Copy, PartialEq, Eq)]
22pub(crate) enum TokKind {
23    $($(#[$meta])* $tok,)*
24
25    // Single-codepoint Unicode operators, grouped by precedence tier. The exact
26    // operator text is carried by the token; the parser only needs the tier (for
27    // binding power) and the projector reads the text. The lexer classifies each
28    // operator char via the generated `unicode_op_kind` table. All six tiers
29    // materialize as one `SyntaxKind::UNICODE_OP`, which is why they are not
30    // rows of the token table (the assignment tier and the radicals, which do
31    // map 1:1, are).
32    UniArrow,
33    UniComparison,
34    UniColon,
35    UniPlus,
36    UniTimes,
37    UniPower,
38}
39    };
40}
41
42token_table!(define_tok_kind);
43
44impl TokKind {
45    /// Whether this token is trivia (whitespace, newline, or a comment) — never
46    /// part of the grammar, always carried through as it is.
47    pub(crate) fn is_trivia(self) -> bool {
48        matches!(
49            self,
50            TokKind::Whitespace | TokKind::Newline | TokKind::Comment | TokKind::BlockComment
51        )
52    }
53}
54
55#[derive(Debug, Clone, PartialEq, Eq)]
56pub(crate) struct Token<'src> {
57    pub(crate) kind: TokKind,
58    pub(crate) text: &'src str,
59    pub(crate) start: usize,
60    pub(crate) end: usize,
61}
62
63/// Tokenize `input` into a lossless token stream. Tokens borrow their text
64/// from `input`: the text is always a verbatim slice, so no token needs an
65/// allocation of its own.
66pub(crate) fn lex(input: &str) -> Vec<Token<'_>> {
67    Lexer::new(input).run()
68}
69
70/// The lexer's context stack. The base context is normal Julia code; opening a
71/// string/command delimiter pushes a `Str`/`Cmd` frame, and a `$(` interpolation
72/// inside one pushes an `Interp` frame (back to normal lexing) until its matching
73/// `)` pops it. A nested string inside `$(...)` simply pushes another `Str` frame.
74#[derive(Debug, Clone, Copy, PartialEq, Eq)]
75enum Mode {
76    /// Inside a `"..."` / `"""..."""` string body.
77    Str {
78        triple: bool,
79        /// Non-standard (prefixed) literal: body is taken verbatim, no `$`/escape
80        /// splitting, and a trailing flag run is lexed as a suffix.
81        raw: bool,
82        prefixed: bool,
83    },
84    /// Inside a `` `...` `` / ` ```...``` ` command body.
85    Cmd {
86        triple: bool,
87        raw: bool,
88        prefixed: bool,
89    },
90    /// Inside a `$( ... )` interpolation; `depth` counts unbalanced `(`.
91    Interp { depth: usize },
92}
93
94pub(super) struct Lexer<'a> {
95    input: &'a str,
96    pub(super) bytes: &'a [u8],
97    pub(super) pos: usize,
98    tokens: Vec<Token<'a>>,
99    mode_stack: Vec<Mode>,
100}
101
102impl<'a> Lexer<'a> {
103    fn new(input: &'a str) -> Self {
104        Self {
105            input,
106            bytes: input.as_bytes(),
107            pos: 0,
108            tokens: Vec::new(),
109            mode_stack: Vec::new(),
110        }
111    }
112
113    fn run(mut self) -> Vec<Token<'a>> {
114        while self.pos < self.bytes.len() {
115            self.next_token();
116        }
117        self.tokens
118    }
119
120    fn peek(&self, ahead: usize) -> Option<u8> {
121        self.bytes.get(self.pos + ahead).copied()
122    }
123
124    fn push(&mut self, kind: TokKind, start: usize, end: usize) {
125        self.tokens.push(Token {
126            kind,
127            text: &self.input[start..end],
128            start,
129            end,
130        });
131    }
132
133    /// Push an operator token, first absorbing any trailing sub/superscript or
134    /// prime suffix into its text when the operator kind takes one (`+₁`,
135    /// `-->₁`, `f'ᵀ`). The kind is unchanged — only the text grows — so the
136    /// parser still keys on the precedence tier and the projector reads the full
137    /// operator text.
138    fn push_op(&mut self, kind: TokKind, start: usize) {
139        if op_takes_suffix(kind) {
140            while self.pos < self.bytes.len() {
141                let c = self.char_at(self.pos);
142                if is_op_suffix_char(c) {
143                    self.pos += c.len_utf8();
144                } else {
145                    break;
146                }
147            }
148        }
149        self.push(kind, start, self.pos);
150    }
151
152    fn next_token(&mut self) {
153        // Inside a string/command body, the body lexer owns the bytes until the
154        // closing delimiter (or an interpolation, which pushes its own frame).
155        if matches!(
156            self.mode_stack.last(),
157            Some(Mode::Str { .. } | Mode::Cmd { .. })
158        ) {
159            self.lex_in_string_mode();
160            return;
161        }
162
163        let start = self.pos;
164        let b = self.bytes[self.pos];
165
166        // Inside a `$( ... )` interpolation, track paren nesting so the matching
167        // `)` returns us to the enclosing string/command body.
168        if matches!(self.mode_stack.last(), Some(Mode::Interp { .. })) {
169            if b == b'(' {
170                self.pos += 1;
171                self.push(TokKind::LParen, start, self.pos);
172                if let Some(Mode::Interp { depth }) = self.mode_stack.last_mut() {
173                    *depth += 1;
174                }
175                return;
176            }
177            if b == b')' {
178                self.pos += 1;
179                self.push(TokKind::RParen, start, self.pos);
180                if matches!(self.mode_stack.last(), Some(Mode::Interp { depth }) if *depth == 1) {
181                    self.mode_stack.pop();
182                } else if let Some(Mode::Interp { depth }) = self.mode_stack.last_mut() {
183                    *depth -= 1;
184                }
185                return;
186            }
187        }
188
189        match b {
190            b' ' | b'\t' => self.lex_whitespace(start),
191            b'\r' | b'\n' => self.lex_newline(start),
192            b'#' => self.lex_comment(start),
193            b'"' => self.lex_open_string(start, false),
194            b'`' => self.lex_open_cmd(start, false),
195            b'\'' if self.prev_ends_value() || self.prev_is_dot() => {
196                self.pos += 1;
197                self.push_op(TokKind::Transpose, start);
198            }
199            b'\'' => self.lex_char_literal(start),
200            b'0'..=b'9' => self.lex_number(start),
201            b'.' if self.peek(1).is_some_and(|c| c.is_ascii_digit()) => self.lex_number(start),
202            _ => {
203                if is_ident_start(self.char_at(self.pos)) {
204                    self.lex_ident_or_keyword(start);
205                } else {
206                    self.lex_operator_or_unknown(start);
207                }
208            }
209        }
210    }
211
212    /// The `char` beginning at byte offset `at` (for unicode identifier checks).
213    pub(super) fn char_at(&self, at: usize) -> char {
214        self.input[at..].chars().next().unwrap_or('\0')
215    }
216
217    fn lex_whitespace(&mut self, start: usize) {
218        while matches!(self.peek(0), Some(b' ' | b'\t')) {
219            self.pos += 1;
220        }
221        self.push(TokKind::Whitespace, start, self.pos);
222    }
223
224    fn lex_newline(&mut self, start: usize) {
225        // A single newline token per line break: `\r\n`, `\r`, or `\n`.
226        match self.peek(0) {
227            Some(b'\r') if self.peek(1) == Some(b'\n') => self.pos += 2,
228            _ => self.pos += 1,
229        }
230        self.push(TokKind::Newline, start, self.pos);
231    }
232
233    fn lex_comment(&mut self, start: usize) {
234        if self.peek(1) == Some(b'=') {
235            self.lex_block_comment(start);
236            return;
237        }
238        // Line comment: `#` to end of line (newline excluded).
239        self.pos += 1;
240        while !matches!(self.peek(0), Some(b'\n' | b'\r') | None) {
241            self.pos += 1;
242        }
243        self.push(TokKind::Comment, start, self.pos);
244    }
245
246    /// Nested block comment `#= ... =#` (Julia allows nesting). Unterminated
247    /// comments run to end of input — still lossless.
248    fn lex_block_comment(&mut self, start: usize) {
249        self.pos += 2; // consume `#=`
250        let mut depth = 1usize;
251        while depth > 0 && self.pos < self.bytes.len() {
252            match (self.peek(0), self.peek(1)) {
253                (Some(b'#'), Some(b'=')) => {
254                    self.pos += 2;
255                    depth += 1;
256                }
257                (Some(b'='), Some(b'#')) => {
258                    self.pos += 2;
259                    depth -= 1;
260                }
261                _ => self.pos += 1,
262            }
263        }
264        self.push(TokKind::BlockComment, start, self.pos);
265    }
266
267    /// Open a `"..."` / `"""..."""` string: emit the opening delimiter token and
268    /// push a `Str` body frame. `prefixed` is set when a non-standard literal
269    /// prefix (`r`, `raw`, …) directly precedes the quote, which makes the body
270    /// raw (no `$`/escape processing) and enables a trailing suffix scan.
271    fn lex_open_string(&mut self, start: usize, prefixed: bool) {
272        let triple = self.peek(1) == Some(b'"') && self.peek(2) == Some(b'"');
273        self.pos += if triple { 3 } else { 1 };
274        self.push(TokKind::StringDelimOpen, start, self.pos);
275        self.mode_stack.push(Mode::Str {
276            triple,
277            raw: prefixed,
278            prefixed,
279        });
280    }
281
282    /// Open a `` `...` `` / ` ```...``` ` command literal, analogous to a string.
283    fn lex_open_cmd(&mut self, start: usize, prefixed: bool) {
284        let triple = self.peek(1) == Some(b'`') && self.peek(2) == Some(b'`');
285        self.pos += if triple { 3 } else { 1 };
286        self.push(TokKind::CmdDelimOpen, start, self.pos);
287        self.mode_stack.push(Mode::Cmd {
288            triple,
289            raw: prefixed,
290            prefixed,
291        });
292    }
293
294    /// Lex one token inside a string/command body: a literal-content chunk, a
295    /// closing delimiter (plus optional suffix), or an interpolation sigil.
296    fn lex_in_string_mode(&mut self) {
297        let frame = *self.mode_stack.last().expect("string mode frame");
298        let (quote, triple, raw, prefixed) = match frame {
299            Mode::Str {
300                triple,
301                raw,
302                prefixed,
303            } => (b'"', triple, raw, prefixed),
304            Mode::Cmd {
305                triple,
306                raw,
307                prefixed,
308            } => (b'`', triple, raw, prefixed),
309            Mode::Interp { .. } => unreachable!("lex_in_string_mode called in interp mode"),
310        };
311        let close_kind = if quote == b'"' {
312            TokKind::StringDelimClose
313        } else {
314            TokKind::CmdDelimClose
315        };
316
317        let start = self.pos;
318
319        // Closing delimiter at the very start of this call: empty trailing chunk.
320        if self.at_close_delim(quote, triple) {
321            self.pos += if triple { 3 } else { 1 };
322            self.push(close_kind, start, self.pos);
323            self.mode_stack.pop();
324            if prefixed {
325                self.lex_suffix();
326            }
327            return;
328        }
329
330        // Accumulate a content chunk until the close delimiter, an interpolation,
331        // or EOF. A newline does *not* stop it, single-quoted strings included
332        // (`"a\nb"` is one content run), which is why an unterminated literal
333        // always runs to the end of the file.
334        while self.pos < self.bytes.len() {
335            // In a raw (prefixed) string, a backslash run immediately before the
336            // closing quote escapes it when the run length is odd (Julia's
337            // raw-string rule: `\"` ⇒ literal quote, `\\\"` ⇒ `\` then literal
338            // quote). Consume the run plus the escaped quote so it stays content.
339            if raw && self.peek(0) == Some(b'\\') {
340                let mut run = 0;
341                while self.peek(run) == Some(b'\\') {
342                    run += 1;
343                }
344                if self.peek(run) == Some(quote) && run % 2 == 1 {
345                    self.pos += run + 1;
346                } else {
347                    self.pos += run;
348                }
349                continue;
350            }
351            if self.at_close_delim(quote, triple) {
352                break;
353            }
354            if !raw && self.peek(0) == Some(b'$') && self.is_interp_start(1) {
355                break;
356            }
357            self.consume_body_byte(raw);
358        }
359
360        if self.pos > start {
361            self.push(TokKind::StringContent, start, self.pos);
362        }
363
364        // Decide what stopped the chunk.
365        if self.at_close_delim(quote, triple) {
366            let delim_start = self.pos;
367            self.pos += if triple { 3 } else { 1 };
368            self.push(close_kind, delim_start, self.pos);
369            self.mode_stack.pop();
370            if prefixed {
371                self.lex_suffix();
372            }
373        } else if !raw && self.peek(0) == Some(b'$') && self.is_interp_start(1) {
374            self.lex_interp_sigil();
375        } else {
376            // Unterminated (newline or EOF): leave the body frame; the parser
377            // assembles whatever was emitted. Losslessness still holds.
378            self.mode_stack.pop();
379        }
380    }
381
382    /// Whether a closing delimiter (`triple` → three of `quote`) begins at `pos`.
383    fn at_close_delim(&self, quote: u8, triple: bool) -> bool {
384        if triple {
385            self.peek(0) == Some(quote)
386                && self.peek(1) == Some(quote)
387                && self.peek(2) == Some(quote)
388        } else {
389            self.peek(0) == Some(quote)
390        }
391    }
392
393    /// Whether the byte at `self.pos + ahead` begins an interpolation operand
394    /// (an identifier-start char or an opening paren).
395    fn is_interp_start(&self, ahead: usize) -> bool {
396        match self.peek(ahead) {
397            Some(b'(') => true,
398            Some(_) => is_ident_start(self.char_at(self.pos + ahead)),
399            None => false,
400        }
401    }
402
403    /// Emit the `$` sigil and set up the interpolation operand: either a bare
404    /// identifier (lexed inline) or a `(` that opens an `Interp` frame.
405    fn lex_interp_sigil(&mut self) {
406        let dollar = self.pos;
407        self.pos += 1;
408        self.push(TokKind::Dollar, dollar, self.pos);
409        if self.peek(0) == Some(b'(') {
410            let paren = self.pos;
411            self.pos += 1;
412            self.push(TokKind::LParen, paren, self.pos);
413            self.mode_stack.push(Mode::Interp { depth: 1 });
414        } else {
415            // `$ident`: lex the longest identifier (so `$foo.bar` interpolates
416            // `foo` and `.bar` stays content).
417            self.lex_interp_ident();
418        }
419    }
420
421    /// Consume one body character. In non-raw mode a backslash escapes the
422    /// next character (so `\"`, `\$`, `\n` stay inside the content chunk) —
423    /// the whole character, since an invalid escape may name a multi-byte one
424    /// (`"\α"`). A `\`-newline line continuation may span a CRLF, so the
425    /// whole `\r\n` is consumed with the backslash — otherwise the trailing
426    /// `\n` would leak out and terminate a single-line string.
427    fn consume_body_byte(&mut self, raw: bool) {
428        if !raw && self.peek(0) == Some(b'\\') && self.pos + 1 < self.bytes.len() {
429            if self.peek(1) == Some(b'\r') && self.peek(2) == Some(b'\n') {
430                self.pos += 3;
431            } else {
432                self.pos += 1 + self.char_at(self.pos + 1).len_utf8();
433            }
434        } else {
435            self.pos += self.char_at(self.pos).len_utf8();
436        }
437    }
438
439    /// After a prefixed literal closes, lex an identifier-shaped flag suffix as a
440    /// single suffix token (e.g. the `ims` in `r"pat"ims`, the `i2` in `x"s"i2`).
441    /// The suffix must start with a letter; a digit-led suffix is a numeric macro
442    /// argument instead (handled in `parse_string_literal`), not a flag string.
443    fn lex_suffix(&mut self) {
444        let start = self.pos;
445        if matches!(self.peek(0), Some(c) if c.is_ascii_alphabetic()) {
446            self.pos += 1;
447            while matches!(self.peek(0), Some(c) if c.is_ascii_alphanumeric()) {
448                self.pos += 1;
449            }
450        }
451        if self.pos > start {
452            self.push(TokKind::StringSuffix, start, self.pos);
453        }
454    }
455
456    /// Whether the immediately preceding token ends a value, making a following
457    /// `'` a postfix transpose/adjoint rather than the start of a char literal.
458    /// The check is on the *immediately* preceding token (not skipping trivia),
459    /// which mirrors Julia's whitespace sensitivity: `A'` is transpose but `A '`
460    /// is `A` followed by a (here unterminated) char literal.
461    fn prev_ends_value(&self) -> bool {
462        matches!(
463            self.tokens.last().map(|t| t.kind),
464            Some(
465                TokKind::Ident
466                    | TokKind::Integer
467                    | TokKind::Float
468                    | TokKind::Char
469                    | TokKind::TrueKw
470                    | TokKind::FalseKw
471                    | TokKind::RParen
472                    | TokKind::RBracket
473                    | TokKind::RBrace
474                    | TokKind::StringDelimClose
475                    | TokKind::CmdDelimClose
476                    | TokKind::StringSuffix
477                    | TokKind::Transpose
478            )
479        )
480    }
481
482    /// Whether the preceding significant token is a field-access `.`. A `'`
483    /// after a `.` on the same line is the removed `.'` transpose operator,
484    /// not the start of a char literal. Horizontal whitespace and block comments
485    /// after the dot are insignificant, but a newline ends the form.
486    fn prev_is_dot(&self) -> bool {
487        self.tokens
488            .iter()
489            .rev()
490            .find(|t| !matches!(t.kind, TokKind::Whitespace | TokKind::BlockComment))
491            .map(|t| t.kind)
492            == Some(TokKind::Dot)
493    }
494
495    /// `'` begins a char literal when it is *not* a postfix adjoint/transpose
496    /// (see [`Self::prev_ends_value`]). It is always lexed as a [`TokKind::Char`]:
497    /// a closing `'` (within one char or a backslash escape) terminates it; a
498    /// newline or end of input without one leaves an *unterminated* char token
499    /// spanning the opening quote and any content. JuliaSyntax also reads the
500    /// unterminated form as a char and recovers with a missing-close marker
501    /// (`'` ⇒ `(char (error))`, `'a` ⇒ `(char 'a' (error-t))`); the parser flags
502    /// it with `UnterminatedLiteral` and the projector replays that shape.
503    fn lex_char_literal(&mut self, start: usize) {
504        // Scan to the closing `'`, skipping a backslash escape's following byte
505        // so an escaped quote (`'\''`) does not terminate the literal. The
506        // content may be several escapes (`'\xce\xb1'`) or over-long (`'ab'`);
507        // validity (single codepoint, well-formed escapes) is decided later. A
508        // newline is *content*, not a terminator — Julia scans to the next `'`
509        // or end of input (`'\n'` is the newline char; `'a\n` is unterminated
510        // over-long content), so the only stop short of `'` is EOF.
511        let mut idx = self.pos + 1;
512        let mut found = false;
513        while idx < self.bytes.len() {
514            match self.bytes[idx] {
515                b'\'' => {
516                    found = true;
517                    break;
518                }
519                b'\\' => {
520                    idx += 1;
521                    if idx < self.bytes.len() {
522                        idx += self.char_at(idx).len_utf8();
523                    }
524                }
525                _ => idx += self.char_at(idx).len_utf8(),
526            }
527        }
528
529        // Include the closing quote when present; otherwise span only the opening
530        // quote and content (no close) so the parser can detect the unterminated
531        // form from the token text.
532        self.pos = if found { idx + 1 } else { idx };
533        self.push(TokKind::Char, start, self.pos);
534    }
535
536    fn lex_number(&mut self, start: usize) {
537        // Base-prefixed integers (`0x`, `0o`, `0b`). Julia's prefixes are
538        // lowercase only, so `0X1` is *not* a hex literal — it falls through to
539        // the decimal path and lexes as `0` followed by the identifier `X1`.
540        if self.peek(0) == Some(b'0') {
541            match self.peek(1) {
542                Some(b'x') => {
543                    self.pos += 2;
544                    // Hex literal classification mirrors Julia's tokenizer. A
545                    // `.`-fraction or `p`/`P` binary exponent makes it a (always
546                    // Float64) hex float, but Julia constrains the shape: a hex
547                    // literal needs at least one mantissa digit (integer or
548                    // fractional), a `.` fraction requires a `p`/`P` exponent, and
549                    // a `p`/`P` requires at least one exponent digit. Each
550                    // violation is one of JuliaSyntax's two numeric error tokens,
551                    // not a valid literal (e.g. `0x1p` is `(ErrorInvalidNumeric-
552                    // Constant)`, `0x1.8` is `(ErrorHexFloatMustContainP)`).
553                    let int_digits = self.consume_digits(|c| c.is_ascii_hexdigit());
554                    // A `.` followed by another `.` is the `..` range operator
555                    // (`0x1..n`), not a decimal point.
556                    let mut had_dot = false;
557                    let mut frac_digits = false;
558                    if self.peek(0) == Some(b'.') && self.peek(1) != Some(b'.') {
559                        had_dot = true;
560                        self.pos += 1;
561                        frac_digits = self.consume_digits(|c| c.is_ascii_hexdigit());
562                    }
563                    let mut had_exponent = false;
564                    let mut exp_digits = false;
565                    if matches!(self.peek(0), Some(b'p' | b'P')) {
566                        had_exponent = true;
567                        self.pos += 1;
568                        if matches!(self.peek(0), Some(b'+' | b'-')) {
569                            self.pos += 1;
570                        }
571                        exp_digits = self.consume_digits(|c| c.is_ascii_digit());
572                    }
573                    let has_mantissa = int_digits || frac_digits;
574                    let kind = if had_exponent {
575                        if has_mantissa && exp_digits {
576                            TokKind::Float
577                        } else {
578                            TokKind::ErrorInvalidNumber
579                        }
580                    } else if had_dot {
581                        TokKind::ErrorHexFloatNoP
582                    } else if int_digits {
583                        TokKind::HexInt
584                    } else {
585                        TokKind::ErrorInvalidNumber
586                    };
587                    self.push(kind, start, self.pos);
588                    return;
589                }
590                Some(b'o') => {
591                    self.pos += 2;
592                    self.consume_digits(|c| (b'0'..=b'7').contains(&c));
593                    self.push(TokKind::OctInt, start, self.pos);
594                    return;
595                }
596                Some(b'b') => {
597                    self.pos += 2;
598                    self.consume_digits(|c| matches!(c, b'0' | b'1'));
599                    self.push(TokKind::BinInt, start, self.pos);
600                    return;
601                }
602                _ => {}
603            }
604        }
605
606        let mut is_float = false;
607        let mut is_f32 = false;
608        self.consume_digits(|c| c.is_ascii_digit());
609        // Fractional part: a `.` followed by a digit, or a trailing `.`. A `.`
610        // that is itself followed by another `.` belongs to the `..` range
611        // operator (`1..n` is `1 .. n`), so it is not consumed as a decimal point.
612        if self.peek(0) == Some(b'.') && self.peek(1) != Some(b'.') {
613            is_float = true;
614            self.pos += 1;
615            self.consume_digits(|c| c.is_ascii_digit());
616        }
617        // Exponent: `e`/`E` mark a `Float`, `f` marks a `Float32`; both take an
618        // optional sign. The marker only belongs to the number when a digit or
619        // a sign follows it; otherwise it starts an identifier and the number
620        // ends here (`3E₁` is `3 * E₁`, not a malformed float, and `3f` is
621        // `3 * f`).
622        if matches!(self.peek(0), Some(b'e' | b'E' | b'f'))
623            && matches!(self.peek(1), Some(c) if c.is_ascii_digit() || c == b'+' || c == b'-')
624        {
625            if self.peek(0) == Some(b'f') {
626                is_f32 = true;
627            } else {
628                is_float = true;
629            }
630            self.pos += 1;
631            if matches!(self.peek(0), Some(b'+' | b'-')) {
632                self.pos += 1;
633            }
634            while matches!(self.peek(0), Some(c) if c.is_ascii_digit()) {
635                self.pos += 1;
636            }
637        }
638
639        let kind = if is_f32 {
640            TokKind::Float32
641        } else if is_float {
642            TokKind::Float
643        } else {
644            TokKind::Integer
645        };
646        self.push(kind, start, self.pos);
647    }
648
649    /// Advance past a run of digits accepted by `is_digit`, with `_` allowed as
650    /// a digit separator anywhere within the run. Returns whether at least one
651    /// real (non-`_`) digit was consumed, which the hex path uses to tell a
652    /// digit-bearing mantissa/exponent from an empty one (`0x1p` vs `0x1p3`).
653    fn consume_digits(&mut self, is_digit: impl Fn(u8) -> bool) -> bool {
654        let mut seen = false;
655        while matches!(self.peek(0), Some(c) if is_digit(c) || c == b'_') {
656            seen |= self.peek(0) != Some(b'_');
657            self.pos += 1;
658        }
659        seen
660    }
661
662    fn lex_ident_or_keyword(&mut self, start: usize) {
663        self.scan_ident();
664        let text = &self.input[start..self.pos];
665        let keyword = keyword_kind(text);
666        // Non-standard literal: an identifier immediately before `"`/`` ` `` with
667        // no intervening whitespace is a prefix (`r"..."`, `raw"..."`, `` v`...` ``).
668        // A quote after contextual `typegroup` starts an ordinary string in
669        // the block body, even when the parser reports an edition error.
670        if keyword.is_none()
671            && match self.peek(0) {
672                Some(b'"') => text != "typegroup",
673                Some(b'`') => true,
674                _ => false,
675            }
676        {
677            self.push(TokKind::StringPrefix, start, self.pos);
678            let open = self.pos;
679            if self.peek(0) == Some(b'"') {
680                self.lex_open_string(open, true);
681            } else {
682                self.lex_open_cmd(open, true);
683            }
684            return;
685        }
686        self.push(keyword.unwrap_or(TokKind::Ident), start, self.pos);
687    }
688
689    /// Lex a bare `$ident` interpolation operand. Unlike [`Self::lex_ident_or_keyword`]
690    /// this never treats a following quote as a prefix, so the closing quote of
691    /// `"$x"` is not mistaken for the start of a non-standard literal.
692    fn lex_interp_ident(&mut self) {
693        let start = self.pos;
694        self.scan_ident();
695        self.push(TokKind::Ident, start, self.pos);
696    }
697
698    /// Advance `pos` over a full identifier (the first char is already known to
699    /// be an identifier start).
700    fn scan_ident(&mut self) {
701        self.pos += self.char_at(self.pos).len_utf8();
702        loop {
703            let c = self.char_at(self.pos);
704            // A `!` immediately followed by `=` is the start of the `!=`/`!==`
705            // operator, not an identifier suffix — stop here so the operator
706            // lexer can claim it (`a!=b` ⇒ `a` `!=` `b`, `a!!=b` ⇒ `a!` `!=`
707            // `b`), while a `!` followed by anything else stays in the
708            // identifier (`a!b`, `push!`).
709            if c == '!' && self.peek(1) == Some(b'=') {
710                break;
711            }
712            if self.pos < self.bytes.len() && is_ident_continue(c) {
713                self.pos += c.len_utf8();
714            } else {
715                break;
716            }
717        }
718    }
719}
720
721/// Every operator, delimiter, and punctuator with a fixed ASCII spelling,
722/// paired with the kind it lexes as.
723///
724/// Entries are grouped by first byte and ordered longest-first within each
725/// group. [`build_op_index`] enforces both invariants at compile time.
726///
727/// Spellings that are not fixed bytes are not here: the Unicode operators come
728/// from a generated code-point table (see [`Lexer::unicode_op_at`]), and the
729/// contextual `'` (transpose vs. char literal) is decided in [`Lexer::next_token`].
730#[rustfmt::skip]
731const OPS: &[(&[u8], TokKind)] = &[
732    // `.` — the broadcast forms, plus `..`/`...` and the lone dot. Every
733    // multi-byte entry fuses `.` to an operator; `.` before an identifier,
734    // a `(`, or a digit is not here, so those stay `Dot` (field access,
735    // `f.(x)`, and the `.5` the number lexer takes first).
736    (b".>>>=", TokKind::DotUShrEq),
737    (b".<-->", TokKind::DotLeftRightArrow),
738    (b".>>>",  TokKind::DotUShr),
739    (b".//=",  TokKind::DotSlashSlashEq),
740    (b".<<=",  TokKind::DotShlEq),
741    (b".>>=",  TokKind::DotShrEq),
742    (b".-->",  TokKind::DotLongArrow),
743    (b".<--",  TokKind::DotLeftLongArrow),
744    (b".===",  TokKind::DotEqEqEq),
745    (b".!==",  TokKind::DotNotEqEq),
746    (b".+%=",  TokKind::DotPlusPercentEq),
747    (b".-%=",  TokKind::DotMinusPercentEq),
748    (b".*%=",  TokKind::DotStarPercentEq),
749    (b"...",   TokKind::DotDotDot),
750    (b".==",   TokKind::DotEqEq),
751    (b".!=",   TokKind::DotNotEq),
752    (b".<=",   TokKind::DotLe),
753    (b".>=",   TokKind::DotGe),
754    (b".<<",   TokKind::DotShl),
755    (b".>>",   TokKind::DotShr),
756    (b".<:",   TokKind::DotSubtype),
757    (b".>:",   TokKind::DotSupertype),
758    (b".//",   TokKind::DotSlashSlash),
759    // The broadcast invalid doubled operators.
760    (b".**",   TokKind::DotStarStar),
761    (b".--",   TokKind::DotMinusMinus),
762    (b".=>",   TokKind::DotFatArrow),
763    (b".|>",   TokKind::DotPipeGt),
764    (b".&&",   TokKind::DotAndAnd),
765    (b".||",   TokKind::DotOrOr),
766    (b".+%",   TokKind::DotPlusPercent),
767    (b".-%",   TokKind::DotMinusPercent),
768    (b".*%",   TokKind::DotStarPercent),
769    // Broadcast augmented assignment `.op=`.
770    (b".+=",   TokKind::DotPlusEq),
771    (b".-=",   TokKind::DotMinusEq),
772    (b".*=",   TokKind::DotStarEq),
773    (b"./=",   TokKind::DotSlashEq),
774    (b".\\=",  TokKind::DotBackslashEq),
775    (b".^=",   TokKind::DotCaretEq),
776    (b".%=",   TokKind::DotPercentEq),
777    (b".&=",   TokKind::DotAmpEq),
778    (b".|=",   TokKind::DotPipeEq),
779    (b"..",    TokKind::DotDot),
780    (b".+",    TokKind::DotPlus),
781    (b".-",    TokKind::DotMinus),
782    (b".*",    TokKind::DotStar),
783    (b"./",    TokKind::DotSlash),
784    (b".\\",   TokKind::DotBackslash),
785    (b".^",    TokKind::DotCaret),
786    (b".%",    TokKind::DotPercent),
787    (b".=",    TokKind::DotEq),
788    (b".<",    TokKind::DotLt),
789    (b".>",    TokKind::DotGt),
790    (b".~",    TokKind::DotTilde),
791    (b".&",    TokKind::DotAmp),
792    (b".|",    TokKind::DotPipe),
793    // The prefix broadcast-not. `.!=`/`.!==` are longer, so they win above.
794    (b".!",    TokKind::DotBang),
795    (b".",     TokKind::Dot),
796    // `<` — note a lone `<-` is not an operator: it stays `Lt` + unary minus.
797    (b"<-->",  TokKind::LeftRightArrow),
798    (b"<--",   TokKind::LeftLongArrow),
799    (b"<<=",   TokKind::ShlEq),
800    (b"<=",    TokKind::Le),
801    (b"<:",    TokKind::Subtype),
802    (b"<|",    TokKind::PipeLt),
803    (b"<<",    TokKind::Shl),
804    (b"<",     TokKind::Lt),
805    // `>`
806    (b">>>=",  TokKind::UShrEq),
807    (b">>>",   TokKind::UShr),
808    (b">>=",   TokKind::ShrEq),
809    (b">=",    TokKind::Ge),
810    (b">:",    TokKind::Supertype),
811    (b">>",    TokKind::Shr),
812    (b">",     TokKind::Gt),
813    // `=`
814    (b"===",   TokKind::EqEqEq),
815    (b"==",    TokKind::EqEq),
816    (b"=>",    TokKind::FatArrow),
817    (b"=",     TokKind::Eq),
818    // `!`. The lexer only reaches these where `scan_ident` stopped at a `!`
819    // followed by `=` (`a!=b` is `a` `!=` `b`, `a!!=b` is `a!` `!=` `b`).
820    (b"!==",   TokKind::NotEqEq),
821    (b"!=",    TokKind::NotEq),
822    (b"!",     TokKind::Bang),
823    // `+`. `+++` falls out as `++` then `+`, and `++=` as `++` then `=`
824    // (there is no augmented `++=` form), both matching JuliaSyntax.
825    (b"+%=",   TokKind::PlusPercentEq),
826    (b"+%",    TokKind::PlusPercent),
827    (b"++",    TokKind::PlusPlus),
828    (b"+=",    TokKind::PlusEq),
829    (b"+",     TokKind::Plus),
830    // `-`
831    (b"-->",   TokKind::LongArrow),
832    (b"-%=",   TokKind::MinusPercentEq),
833    (b"--",    TokKind::MinusMinus),
834    (b"->",    TokKind::Arrow),
835    (b"-%",    TokKind::MinusPercent),
836    (b"-=",    TokKind::MinusEq),
837    (b"-",     TokKind::Minus),
838    // `*`
839    (b"*%=",   TokKind::StarPercentEq),
840    (b"**",    TokKind::StarStar),
841    (b"*%",    TokKind::StarPercent),
842    (b"*=",    TokKind::StarEq),
843    (b"*",     TokKind::Star),
844    // `/`
845    (b"//=",   TokKind::SlashSlashEq),
846    (b"//",    TokKind::SlashSlash),
847    (b"/=",    TokKind::SlashEq),
848    (b"/",     TokKind::Slash),
849    // `\`
850    (b"\\=",   TokKind::BackslashEq),
851    (b"\\",    TokKind::Backslash),
852    // `^`
853    (b"^=",    TokKind::CaretEq),
854    (b"^",     TokKind::Caret),
855    // `%`
856    (b"%=",    TokKind::PercentEq),
857    (b"%",     TokKind::Percent),
858    // `&`
859    (b"&&",    TokKind::AndAnd),
860    (b"&=",    TokKind::AmpEq),
861    (b"&",     TokKind::Amp),
862    // `|`
863    (b"||",    TokKind::OrOr),
864    (b"|>",    TokKind::PipeGt),
865    (b"|=",    TokKind::PipeEq),
866    (b"|",     TokKind::Pipe),
867    // `:`
868    (b"::",    TokKind::ColonColon),
869    (b":=",    TokKind::ColonEq),
870    (b":",     TokKind::Colon),
871    // `$`. A `$` elsewhere is the interpolation sigil, which never precedes
872    // an `=`.
873    (b"$=",    TokKind::DollarEq),
874    (b"$",     TokKind::Dollar),
875    // The remaining single bytes: the prefix operators and the delimiters.
876    (b"~",     TokKind::Tilde),
877    (b"?",     TokKind::Question),
878    (b"@",     TokKind::At),
879    (b"(",     TokKind::LParen),
880    (b")",     TokKind::RParen),
881    (b"[",     TokKind::LBracket),
882    (b"]",     TokKind::RBracket),
883    (b"{",     TokKind::LBrace),
884    (b"}",     TokKind::RBrace),
885    (b",",     TokKind::Comma),
886    (b";",     TokKind::Semicolon),
887];
888
889/// The [`OPS`] entries sharing one first byte.
890#[derive(Clone, Copy)]
891struct OpGroup {
892    /// The group's `[start, end)` range in [`OPS`]; empty when the byte begins
893    /// no operator at all.
894    start: u8,
895    end: u8,
896    /// The bytes that continue a multi-byte spelling in this group, as a set
897    /// (all spellings are ASCII, so 128 bits hold it). When the byte after the
898    /// first is not in this set, no multi-byte entry can match and the whole
899    /// group can be skipped for its single-byte spelling — which is what keeps
900    /// `a.b` from walking all 50 broadcast operators to reach the lone `Dot`.
901    seconds: u128,
902}
903
904impl OpGroup {
905    /// Whether `next`, the byte after the group's own, could continue one of
906    /// its multi-byte spellings.
907    fn extends(&self, next: Option<&u8>) -> bool {
908        matches!(next, Some(&b) if b < 128 && self.seconds >> b & 1 == 1)
909    }
910}
911
912/// [`OPS`] indexed by first byte.
913const OP_INDEX: [OpGroup; 256] = build_op_index();
914
915/// Build [`OP_INDEX`], asserting [`OPS`]'s invariants as it goes: spellings are
916/// ASCII, entries sharing a first byte are contiguous, within such a group the
917/// spellings get no longer, and every group ends with its single-byte spelling
918/// (which is what lets [`try_ascii_op`] answer from the group's last entry
919/// alone). All of it is checked at compile time, which is what makes longest
920/// match a property of the table rather than of arm order.
921const fn build_op_index() -> [OpGroup; 256] {
922    assert!(OPS.len() < u8::MAX as usize, "OPS outgrew its u8 index");
923    let mut index = [OpGroup {
924        start: 0,
925        end: 0,
926        seconds: 0,
927    }; 256];
928    let mut i = 0;
929    while i < OPS.len() {
930        let first = OPS[i].0[0];
931        assert!(
932            index[first as usize].start == index[first as usize].end,
933            "OPS entries sharing a first byte must be contiguous"
934        );
935        let start = i;
936        let mut len = usize::MAX;
937        let mut seconds = 0u128;
938        while i < OPS.len() && OPS[i].0[0] == first {
939            let text = OPS[i].0;
940            assert!(
941                text.len() <= len,
942                "OPS entries must run longest-first within a first-byte group"
943            );
944            len = text.len();
945            if len > 1 {
946                assert!(
947                    text[1] < 128,
948                    "OPS holds ASCII spellings only; a Unicode operator belongs in `unicode_op_at`"
949                );
950                seconds |= 1u128 << text[1];
951            }
952            i += 1;
953        }
954        assert!(
955            len == 1,
956            "every OPS group must end with its own single-byte spelling"
957        );
958        index[first as usize] = OpGroup {
959            start: start as u8,
960            end: i as u8,
961            seconds,
962        };
963    }
964    index
965}
966
967/// The longest [`OPS`] spelling that starts `rest`, with its byte length.
968///
969/// Only the entries sharing `rest`'s first byte are scanned, and they are
970/// ordered longest-first, so the first hit is the answer.
971fn try_ascii_op(rest: &[u8]) -> Option<(TokKind, usize)> {
972    let group = &OP_INDEX[*rest.first()? as usize];
973    let entries = &OPS[group.start as usize..group.end as usize];
974    // Nothing longer can match, so take the group's single-byte spelling —
975    // its last entry, which `build_op_index` checks every group has.
976    if !group.extends(rest.get(1)) {
977        let (text, kind) = *entries.last()?;
978        return Some((kind, text.len()));
979    }
980    let next = rest[1]; // `extends` said there is one
981    entries
982        .iter()
983        .find(|(text, _)| match text.get(1) {
984            // Reject on the second byte before comparing the whole spelling.
985            // The single-byte entry is last, so falling through to it means
986            // nothing longer matched (`<-` is `Lt` then a unary minus).
987            Some(&second) => second == next && rest.starts_with(text),
988            None => true,
989        })
990        .map(|&(text, kind)| (kind, text.len()))
991}
992
993/// Generate the lexer's three keyword tables from the shared keyword table: the
994/// `KEYWORDS` slice, the text -> [`TokKind`] classifier, and the `TokKind`
995/// predicate.
996macro_rules! define_keyword_tables {
997    ($($text:literal $tok:ident $syn:ident,)*) => {
998        /// Every Julia keyword, as written. Shared by the lexer's
999        /// `keyword_kind` classification and the language server's keyword
1000        /// completion.
1001        pub const KEYWORDS: &[&str] = &[$($text),*];
1002
1003        /// The keyword `text` spells, or `None` when it is an ordinary
1004        /// identifier.
1005        fn keyword_kind(text: &str) -> Option<TokKind> {
1006            Some(match text {
1007                $($text => TokKind::$tok,)*
1008                _ => return None,
1009            })
1010        }
1011
1012        impl TokKind {
1013            /// Whether this token is a reserved keyword. Used to recognize a
1014            /// keyword quoted as a symbol (`:end`, `:function`).
1015            pub(crate) fn is_keyword(self) -> bool {
1016                matches!(self, $(TokKind::$tok)|*)
1017            }
1018        }
1019    };
1020}
1021
1022keyword_table!(define_keyword_tables);
1023
1024/// Whether `c` may begin an identifier, mirroring JuliaSyntax's
1025/// `is_identifier_start_char` (`Base.is_id_start_char`). ASCII is handled inline;
1026/// non-ASCII code points defer to the generated `unicode_ident` tables.
1027///
1028/// `🢲` is JuliaSyntax's one hard-coded divergence from `Base.is_id_start_char`:
1029/// it became an arrow operator, so it must not start an identifier. The
1030/// generated table tracks `Base`, which still admits it, so the carve-out lives
1031/// here rather than in the table.
1032pub fn is_ident_start(c: char) -> bool {
1033    if c.is_ascii() {
1034        c == '_' || c.is_ascii_alphabetic()
1035    } else {
1036        c != '\u{1f8b2}' && super::unicode_ident::is_unicode_ident_start(c)
1037    }
1038}
1039
1040/// Whether `c` may continue an identifier, mirroring JuliaSyntax's
1041/// `is_identifier_char` (`Base.is_id_char`). ASCII is handled inline; non-ASCII
1042/// code points defer to the generated `unicode_ident` tables. The `!=`
1043/// operator split for a trailing `!` is handled by the caller in `scan_ident`.
1044pub fn is_ident_continue(c: char) -> bool {
1045    if c.is_ascii() {
1046        c == '_' || c == '!' || c.is_ascii_alphanumeric()
1047    } else {
1048        super::unicode_ident::is_unicode_ident_continue(c)
1049    }
1050}
1051
1052/// The explicit operator-suffix characters Julia allows after an operator
1053/// (subscripts, superscripts, and primes), mirroring JuliaSyntax's `isopsuffix`
1054/// "additional allowed cases" set. The combining-mark categories (Mn/Mc/Me) that
1055/// `isopsuffix` also accepts are not handled here.
1056const OP_SUFFIX_CHARS: &str = "²³¹ʰʲʳʷʸˡˢˣ˱˲ᴬᴮᴰᴱᴳᴴᴵᴶᴷᴸᴹᴺᴼᴾᴿᵀᵁᵂᵃᵅᵇᵈᵉᵋᵍᵏᵐᵒᵖᵗᵘᵛᵝᵞᵟᵠᵡᵢᵣᵤᵥᵦᵧᵨᵩᵪᶜᶠᶥᶦᶫᶰᶲᶸᶻᶿ′″‴‵‶‷⁗⁰ⁱ⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎ₐₑₒₓₔₕₖₗₘₙₚₛₜ₝₞₟ⱼⱽꜛꜜꜝ𝿐";
1057
1058/// Whether `c` can extend an operator token as a sub/superscript or prime suffix
1059/// (`+₁`, `-->₁`, `f'ᵀ`). See [`OP_SUFFIX_CHARS`].
1060pub(crate) fn is_op_suffix_char(c: char) -> bool {
1061    OP_SUFFIX_CHARS.contains(c)
1062}
1063
1064/// Whether an operator of the given kind may absorb trailing suffix characters.
1065/// Mirrors JuliaSyntax's `optakessuffix`: assignments, the short-circuit/type
1066/// operators, `: :: .. ... ! ~ -> ? $` and the radicals do **not** take a suffix;
1067/// the arithmetic/comparison/bitwise/arrow operators (and their broadcast forms)
1068/// do.
1069fn op_takes_suffix(kind: TokKind) -> bool {
1070    use TokKind::*;
1071    matches!(
1072        kind,
1073        Plus | Minus
1074            | Star
1075            | Slash
1076            | Backslash
1077            | SlashSlash
1078            | Caret
1079            | Percent
1080            | PlusPercent
1081            | MinusPercent
1082            | StarPercent
1083            | EqEq
1084            | NotEq
1085            | EqEqEq
1086            | NotEqEq
1087            | Lt
1088            | Le
1089            | Gt
1090            | Ge
1091            | Amp
1092            | Pipe
1093            | Shl
1094            | Shr
1095            | UShr
1096            | PipeGt
1097            | PipeLt
1098            | FatArrow
1099            | LongArrow
1100            | LeftRightArrow
1101            | LeftLongArrow
1102            | Transpose
1103            | DotPlus
1104            | DotMinus
1105            | DotStar
1106            | DotPlusPercent
1107            | DotMinusPercent
1108            | DotStarPercent
1109            | DotSlash
1110            | DotBackslash
1111            | DotSlashSlash
1112            | DotCaret
1113            | DotPercent
1114            | DotEqEq
1115            | DotNotEq
1116            | DotEqEqEq
1117            | DotNotEqEq
1118            | DotLt
1119            | DotLe
1120            | DotGt
1121            | DotGe
1122            | DotShl
1123            | DotShr
1124            | DotUShr
1125            | DotFatArrow
1126            | DotLongArrow
1127            | DotLeftLongArrow
1128            | DotLeftRightArrow
1129            | DotPipeGt
1130            | DotAmp
1131            | DotPipe
1132            | UniArrow
1133            | UniComparison
1134            | UniColon
1135            | UniPlus
1136            | UniTimes
1137            | UniPower
1138    )
1139}
1140
1141/// U+2212 MINUS SIGN, which Julia treats as the ASCII `-`.
1142const MINUS_SIGN: char = '\u{2212}';
1143
1144/// U+00F7 `÷` and U+22BB `⊻`, the two Unicode operators with an
1145/// augmented-assign form (`÷=`, `⊻=`).
1146const DIVIDE_SIGN: char = '\u{f7}';
1147const XOR_SIGN: char = '\u{22bb}';
1148
1149/// Whether a Unicode operator tier is an infix `call-i` tier (so a broadcast
1150/// `.` may fuse to it as `dotcall-i`). Excludes the prefix-only radicals and the
1151/// assignment tier, whose broadcast forms are not modeled yet.
1152fn is_unicode_infix_tier(kind: TokKind) -> bool {
1153    use TokKind::*;
1154    matches!(
1155        kind,
1156        UniArrow | UniComparison | UniColon | UniPlus | UniTimes | UniPower
1157    )
1158}
1159
1160#[cfg(test)]
1161mod tests {
1162    use super::*;
1163
1164    fn kinds(input: &str) -> Vec<TokKind> {
1165        lex(input).into_iter().map(|t| t.kind).collect()
1166    }
1167
1168    fn roundtrips(input: &str) -> bool {
1169        let joined: String = lex(input).into_iter().map(|t| t.text).collect();
1170        joined == input
1171    }
1172
1173    #[test]
1174    fn every_keyword_lexes_as_one_keyword_token() {
1175        // The tables all come from one list, so what is left to check is the
1176        // path through it: every keyword must reach the lexer's classification
1177        // as a single keyword token, and materialize as a keyword `SyntaxKind`.
1178        // Completion offers `KEYWORDS` verbatim, so a word the lexer treats as
1179        // an identifier would be offered as one.
1180        for kw in KEYWORDS {
1181            assert_eq!(kinds(kw), vec![keyword_kind(kw).unwrap()], "lexing {kw}");
1182            assert!(keyword_kind(kw).unwrap().is_keyword(), "{kw} is a keyword");
1183            assert!(
1184                crate::parser::tree_builder::syntax_kind_for(keyword_kind(kw).unwrap())
1185                    .is_keyword(),
1186                "{kw} materializes as a keyword kind"
1187            );
1188        }
1189    }
1190
1191    /// `\u{1f8b2}` is an arrow operator in JuliaSyntax 1.0 even though
1192    /// `Base.is_id_start_char` still accepts it as an identifier start, so the
1193    /// lexer must prefer the operator reading.
1194    #[test]
1195    fn the_wide_arrow_operator_is_not_an_identifier_start() {
1196        assert_eq!(
1197            kinds("x \u{1f8b2} y"),
1198            vec![
1199                TokKind::Ident,
1200                TokKind::Whitespace,
1201                TokKind::UniArrow,
1202                TokKind::Whitespace,
1203                TokKind::Ident,
1204            ]
1205        );
1206        // It still *continues* an identifier: JuliaSyntax's `is_identifier_char`
1207        // has no matching carve-out, so `a\u{1f8b2}b` stays one name.
1208        assert_eq!(kinds("a\u{1f8b2}b"), vec![TokKind::Ident]);
1209    }
1210
1211    #[test]
1212    fn lossless_over_assorted_input() {
1213        for input in [
1214            "x = 1 + 2\n",
1215            "f(a, b)\n",
1216            "#= a #= nested =# b =#\n",
1217            "function g(x)\n    x ^ 2\nend\n",
1218            "s = \"hello\\n\"\nc = 'a'\n",
1219            "if a >= b\n    a\nelseif c\n    c\nelse\n    b\nend\n",
1220            "for i in 1:10\n    x\nend\n",
1221            "while x > 0\n    x\nend\n",
1222            "map(xs) do x, y\n    x + y\nend\n",
1223            "try\n    f()\ncatch e\n    g()\nfinally\n    h()\nend\n",
1224            "mutable struct Counter\n    n\nend\n",
1225            "module M\nend\n",
1226            "α = β + 1\n",
1227            "0x1f + 0b1010\n",
1228            "x = 0o755\ny = 0x1.8p3\nz = 1f0\n",
1229            "r = 3//4 + 1_000\nq = a .// b\n",
1230            "n = 1.5e-3\nm = .5\nk = 2.\n",
1231            "s = \"a$x b\"\n",
1232            "s = \"a$(f(x))b\"\n",
1233            "s = \"\"\"x$(y)\"\"\"\n",
1234            "c = `echo $x`\n",
1235            "r = raw\"\\d+\"\n",
1236            "m = r\"pat\"ims\n",
1237            "v = v\"1.2.3\"\n",
1238            "b = b\"\\x00\"\n",
1239            "s = \"$foo.bar\"\n",
1240            "s = \"\\$lit\"\n",
1241            "s = \"$$\"\n",
1242            "s = \"unterminated\n",
1243            "s = \"$(g(\"nested\"))\"\n",
1244            // An invalid escape of a multi-byte character must not split it
1245            // (a byte-wise skip panics on the mid-character boundary).
1246            "s = \"a\\αb\"\n",
1247            "s = \"\\α",
1248        ] {
1249            assert!(roundtrips(input), "did not round-trip: {input:?}");
1250        }
1251    }
1252
1253    #[test]
1254    fn interpolation_with_bare_ident() {
1255        assert_eq!(
1256            kinds("\"a$x\""),
1257            vec![
1258                TokKind::StringDelimOpen,
1259                TokKind::StringContent,
1260                TokKind::Dollar,
1261                TokKind::Ident,
1262                TokKind::StringDelimClose,
1263            ]
1264        );
1265    }
1266
1267    #[test]
1268    fn interpolation_with_parenthesized_expr() {
1269        assert_eq!(
1270            kinds("\"$(y)\""),
1271            vec![
1272                TokKind::StringDelimOpen,
1273                TokKind::Dollar,
1274                TokKind::LParen,
1275                TokKind::Ident,
1276                TokKind::RParen,
1277                TokKind::StringDelimClose,
1278            ]
1279        );
1280    }
1281
1282    #[test]
1283    fn nested_parens_in_interpolation() {
1284        // `$(f(x))`: the inner `)` must not close the interpolation early.
1285        assert_eq!(
1286            kinds("\"$(f(x))\""),
1287            vec![
1288                TokKind::StringDelimOpen,
1289                TokKind::Dollar,
1290                TokKind::LParen,
1291                TokKind::Ident,
1292                TokKind::LParen,
1293                TokKind::Ident,
1294                TokKind::RParen,
1295                TokKind::RParen,
1296                TokKind::StringDelimClose,
1297            ]
1298        );
1299    }
1300
1301    #[test]
1302    fn raw_literal_does_not_interpolate() {
1303        // `raw"..."` keeps `$x` and the backslash as literal content.
1304        assert_eq!(
1305            kinds("raw\"$x\\d\""),
1306            vec![
1307                TokKind::StringPrefix,
1308                TokKind::StringDelimOpen,
1309                TokKind::StringContent,
1310                TokKind::StringDelimClose,
1311            ]
1312        );
1313    }
1314
1315    #[test]
1316    fn prefix_and_suffix_flags() {
1317        assert_eq!(
1318            kinds("r\"pat\"ims"),
1319            vec![
1320                TokKind::StringPrefix,
1321                TokKind::StringDelimOpen,
1322                TokKind::StringContent,
1323                TokKind::StringDelimClose,
1324                TokKind::StringSuffix,
1325            ]
1326        );
1327    }
1328
1329    #[test]
1330    fn command_literal_interpolates() {
1331        assert_eq!(
1332            kinds("`cmd $x`"),
1333            vec![
1334                TokKind::CmdDelimOpen,
1335                TokKind::StringContent,
1336                TokKind::Dollar,
1337                TokKind::Ident,
1338                TokKind::CmdDelimClose,
1339            ]
1340        );
1341    }
1342
1343    #[test]
1344    fn escaped_dollar_is_content() {
1345        // `\$` does not introduce an interpolation.
1346        assert_eq!(
1347            kinds("\"\\$x\""),
1348            vec![
1349                TokKind::StringDelimOpen,
1350                TokKind::StringContent,
1351                TokKind::StringDelimClose,
1352            ]
1353        );
1354    }
1355
1356    #[test]
1357    fn plain_string_adjacent_ident_is_not_a_suffix() {
1358        // Only prefixed literals take a suffix; `"a"b` is a string then an ident.
1359        assert_eq!(
1360            kinds("\"a\"b"),
1361            vec![
1362                TokKind::StringDelimOpen,
1363                TokKind::StringContent,
1364                TokKind::StringDelimClose,
1365                TokKind::Ident,
1366            ]
1367        );
1368    }
1369
1370    #[test]
1371    fn prefix_requires_adjacent_quote() {
1372        // A space between the ident and the quote means it is a plain variable.
1373        assert_eq!(
1374            kinds("r \"x\""),
1375            vec![
1376                TokKind::Ident,
1377                TokKind::Whitespace,
1378                TokKind::StringDelimOpen,
1379                TokKind::StringContent,
1380                TokKind::StringDelimClose,
1381            ]
1382        );
1383    }
1384
1385    #[test]
1386    fn keywords_and_operators() {
1387        assert_eq!(
1388            kinds("a == b"),
1389            vec![
1390                TokKind::Ident,
1391                TokKind::Whitespace,
1392                TokKind::EqEq,
1393                TokKind::Whitespace,
1394                TokKind::Ident
1395            ]
1396        );
1397        assert_eq!(keyword_kind("function"), Some(TokKind::FunctionKw));
1398        assert_eq!(keyword_kind("ends"), None);
1399    }
1400
1401    #[test]
1402    fn bang_in_identifier() {
1403        assert_eq!(kinds("push!"), vec![TokKind::Ident]);
1404        // A `!` stays in the identifier unless immediately followed by `=`, where
1405        // it begins the `!=`/`!==` operator instead (matching Julia's munch).
1406        assert_eq!(kinds("a!b"), vec![TokKind::Ident]);
1407        assert_eq!(
1408            kinds("a!=b"),
1409            vec![TokKind::Ident, TokKind::NotEq, TokKind::Ident]
1410        );
1411        assert_eq!(
1412            kinds("a!==b"),
1413            vec![TokKind::Ident, TokKind::NotEqEq, TokKind::Ident]
1414        );
1415        // Only the `!` before the `=` splits off: `a!` stays an identifier.
1416        assert_eq!(
1417            kinds("a!!=b"),
1418            vec![TokKind::Ident, TokKind::NotEq, TokKind::Ident]
1419        );
1420    }
1421
1422    #[test]
1423    fn unicode_identifier_chars() {
1424        // Combining marks (category Mn) continue an identifier.
1425        assert_eq!(kinds("x\u{304}"), vec![TokKind::Ident]);
1426        assert_eq!(kinds("x\u{302}r"), vec![TokKind::Ident]);
1427        // Primes and a math symbol also continue an identifier (`α′`, `ρ∞`).
1428        assert_eq!(kinds("\u{3b1}\u{2032}"), vec![TokKind::Ident]);
1429        assert_eq!(kinds("\u{3c1}\u{221e}"), vec![TokKind::Ident]);
1430        // `∇` (U+2207) is a valid identifier *start* char (`∇batchnorm`).
1431        assert_eq!(kinds("\u{2207}batchnorm"), vec![TokKind::Ident]);
1432        // A leading identifier-continuation character has a more specific error
1433        // token than a wholly unknown character.
1434        assert_eq!(kinds("\u{2081}"), vec![TokKind::ErrorIdentifierStart]);
1435        // Every case lexes losslessly.
1436        assert!(roundtrips("x\u{304} = 1"));
1437        assert!(roundtrips("\u{2207}f(x) = x"));
1438    }
1439
1440    #[test]
1441    fn unicode_17_identifier_chars() {
1442        assert_eq!(kinds("\u{88f}"), vec![TokKind::Ident]);
1443        assert_eq!(kinds("\u{11db0}"), vec![TokKind::Ident]);
1444        assert_eq!(kinds("x\u{11b60}"), vec![TokKind::Ident]);
1445        assert_eq!(kinds("\u{11b60}"), vec![TokKind::ErrorIdentifierStart]);
1446        assert!(roundtrips("\u{88f} = \u{11db0} + x\u{11b60}"));
1447    }
1448
1449    #[test]
1450    fn unicode_18_identifier_chars() {
1451        for c in [
1452            '\u{558}',
1453            '\u{209d}',
1454            '\u{209e}',
1455            '\u{209f}',
1456            '\u{a7dd}',
1457            '\u{107bb}',
1458            '\u{10ed9}',
1459            '\u{12550}',
1460            '\u{18e00}',
1461            '\u{1dfd0}',
1462            '\u{3d000}',
1463            '\u{3fc3f}',
1464        ] {
1465            for input in [c.to_string(), format!("x{c}!")] {
1466                assert_eq!(kinds(&input), vec![TokKind::Ident], "{input:?}");
1467                assert!(roundtrips(&input));
1468            }
1469        }
1470        for c in [
1471            '\u{5c8}',
1472            '\u{b53}',
1473            '\u{1ade}',
1474            '\u{10ec9}',
1475            '\u{11df0}',
1476            '\u{1d127}',
1477            '\u{1d250}',
1478        ] {
1479            assert_eq!(kinds(&format!("x{c}")), vec![TokKind::Ident], "{c:?}");
1480            assert_eq!(
1481                kinds(&c.to_string()),
1482                vec![TokKind::ErrorIdentifierStart],
1483                "{c:?}"
1484            );
1485            assert!(roundtrips(&format!("{c} x{c}")));
1486        }
1487        for c in ['\u{3cfff}', '\u{3fc40}'] {
1488            assert_eq!(kinds(&c.to_string()), vec![TokKind::Unknown], "{c:?}");
1489        }
1490    }
1491
1492    #[test]
1493    fn unicode_18_operator_suffixes() {
1494        for suffix in "ᵅᵋᶲ˱˲ₔ₝₞₟\u{1dfd0}".chars() {
1495            for (op, kind) in [
1496                ("+", TokKind::Plus),
1497                (".+", TokKind::DotPlus),
1498                ("⊗", TokKind::UniTimes),
1499                (".⊗", TokKind::UniTimes),
1500                ("-->", TokKind::LongArrow),
1501                ("'", TokKind::Transpose),
1502            ] {
1503                let operator = format!("{op}{suffix}");
1504                let input = format!("a{operator}b");
1505                let tokens = lex(&input);
1506                assert_eq!(tokens.len(), 3, "{input:?}: {tokens:?}");
1507                assert_eq!(tokens[1].kind, kind, "{input:?}");
1508                assert_eq!(tokens[1].text, operator, "{input:?}");
1509                assert_eq!(tokens[1].start, 1);
1510                assert_eq!(tokens[1].end, 1 + operator.len());
1511                assert_eq!(tokens[2].text, "b");
1512                assert!(roundtrips(&input));
1513            }
1514            for op in ["=", ":", "::", "&&", "||", "->", "<:", "!", "~", "+="] {
1515                let input = format!("{op}{suffix}");
1516                assert_eq!(lex(&input)[0].text, op, "{input:?}");
1517                assert!(roundtrips(&input));
1518            }
1519        }
1520    }
1521
1522    #[test]
1523    fn identity_operators() {
1524        // `===`/`!==` beat `==`/`!=` in longest match.
1525        assert_eq!(
1526            kinds("a===b"),
1527            vec![TokKind::Ident, TokKind::EqEqEq, TokKind::Ident]
1528        );
1529        assert_eq!(
1530            kinds("a==b"),
1531            vec![TokKind::Ident, TokKind::EqEq, TokKind::Ident]
1532        );
1533    }
1534
1535    #[test]
1536    fn numeric_literal_kinds() {
1537        // Decimal int, big int (still a plain integer token), and underscores.
1538        assert_eq!(kinds("123"), vec![TokKind::Integer]);
1539        assert_eq!(kinds("1_000"), vec![TokKind::Integer]);
1540        assert_eq!(
1541            kinds("12345678901234567890123456789"),
1542            vec![TokKind::Integer]
1543        );
1544        // Base-prefixed integers.
1545        assert_eq!(kinds("0x1f"), vec![TokKind::HexInt]);
1546        assert_eq!(kinds("0o755"), vec![TokKind::OctInt]);
1547        assert_eq!(kinds("0b1010"), vec![TokKind::BinInt]);
1548        // Floats: fractional, leading/trailing dot, scientific.
1549        assert_eq!(kinds("3.14"), vec![TokKind::Float]);
1550        assert_eq!(kinds(".5"), vec![TokKind::Float]);
1551        assert_eq!(kinds("2."), vec![TokKind::Float]);
1552        assert_eq!(kinds("1.5e-3"), vec![TokKind::Float]);
1553        // `f` exponent marks Float32; hex floats are Float64.
1554        assert_eq!(kinds("1f0"), vec![TokKind::Float32]);
1555        assert_eq!(kinds("2.5f-3"), vec![TokKind::Float32]);
1556        assert_eq!(kinds("0x1p0"), vec![TokKind::Float]);
1557        assert_eq!(kinds("0x1.8p3"), vec![TokKind::Float]);
1558        // A valid hex float needs a mantissa digit and, once a `p`/`P` appears,
1559        // an exponent digit; the fraction/exponent forms below are all valid.
1560        assert_eq!(kinds("0x1.p3"), vec![TokKind::Float]);
1561        assert_eq!(kinds("0x.8p3"), vec![TokKind::Float]);
1562        assert_eq!(kinds("0x1P+2"), vec![TokKind::Float]);
1563        assert_eq!(kinds("0x1_2p3"), vec![TokKind::Float]);
1564        // `e`/`f` are hex digits, not exponent markers, so no `p` means an int.
1565        assert_eq!(kinds("0x1e3"), vec![TokKind::HexInt]);
1566    }
1567
1568    #[test]
1569    fn malformed_hex_literal_kinds() {
1570        // A `p`/`P` binary exponent with no digits is an invalid constant, as is
1571        // a hex literal with no mantissa digit at all (Julia keeps the whole run
1572        // as one error token rather than splitting it).
1573        for src in [
1574            "0x1p", "0x1p+", "0x1.8p", "0x1.p", "0x.p3", "0xp3", "0x", "0x_",
1575        ] {
1576            assert_eq!(
1577                kinds(src),
1578                vec![TokKind::ErrorInvalidNumber],
1579                "{src} should be an invalid numeric constant"
1580            );
1581        }
1582        // A `.` fraction with no `p`/`P` exponent must contain a `p`.
1583        for src in ["0x1.8", "0x1.", "0x.8", "0x.", "0x1.8e2"] {
1584            assert_eq!(
1585                kinds(src),
1586                vec![TokKind::ErrorHexFloatNoP],
1587                "{src} should require a p exponent"
1588            );
1589        }
1590    }
1591
1592    #[test]
1593    fn rational_operators() {
1594        assert_eq!(
1595            kinds("3//4"),
1596            vec![TokKind::Integer, TokKind::SlashSlash, TokKind::Integer]
1597        );
1598        assert_eq!(
1599            kinds("a .// b"),
1600            vec![
1601                TokKind::Ident,
1602                TokKind::Whitespace,
1603                TokKind::DotSlashSlash,
1604                TokKind::Whitespace,
1605                TokKind::Ident,
1606            ]
1607        );
1608    }
1609
1610    #[test]
1611    fn left_division_operators() {
1612        // `\` (left division) is a single-char infix operator; its augmented and
1613        // broadcast forms follow the slash family. The backslash byte must not be
1614        // confused with a string escape here (it never starts a string).
1615        assert_eq!(
1616            kinds("a\\b"),
1617            vec![TokKind::Ident, TokKind::Backslash, TokKind::Ident]
1618        );
1619        assert_eq!(
1620            kinds("a\\=b"),
1621            vec![TokKind::Ident, TokKind::BackslashEq, TokKind::Ident]
1622        );
1623        assert_eq!(
1624            kinds("a .\\ b"),
1625            vec![
1626                TokKind::Ident,
1627                TokKind::Whitespace,
1628                TokKind::DotBackslash,
1629                TokKind::Whitespace,
1630                TokKind::Ident,
1631            ]
1632        );
1633        assert_eq!(
1634            kinds("a.\\=b"),
1635            vec![TokKind::Ident, TokKind::DotBackslashEq, TokKind::Ident]
1636        );
1637    }
1638
1639    #[test]
1640    fn left_arrow_operators() {
1641        // `<--` lexes as one arrow-tier token (longest match: `<-->` beats it,
1642        // and it beats `<` + `--`); likewise the broadcast `.<--`/`.<-->` beat
1643        // `.<`. A lone `<-` stays `<` + unary minus.
1644        assert_eq!(
1645            kinds("a<--b"),
1646            vec![TokKind::Ident, TokKind::LeftLongArrow, TokKind::Ident]
1647        );
1648        assert_eq!(
1649            kinds("a<-->b"),
1650            vec![TokKind::Ident, TokKind::LeftRightArrow, TokKind::Ident]
1651        );
1652        assert_eq!(
1653            kinds("a.<--b"),
1654            vec![TokKind::Ident, TokKind::DotLeftLongArrow, TokKind::Ident]
1655        );
1656        assert_eq!(
1657            kinds("a.<-->b"),
1658            vec![TokKind::Ident, TokKind::DotLeftRightArrow, TokKind::Ident]
1659        );
1660        assert_eq!(
1661            kinds("a<-b"),
1662            vec![TokKind::Ident, TokKind::Lt, TokKind::Minus, TokKind::Ident]
1663        );
1664    }
1665
1666    #[test]
1667    fn compound_shift_and_unicode_assignments() {
1668        // Bitshift augmented assignments lex as one token (longest match beats
1669        // the bare shift `<<`/`>>`/`>>>` and, for `>>>=`, the unsigned shift).
1670        assert_eq!(
1671            kinds("a<<=b"),
1672            vec![TokKind::Ident, TokKind::ShlEq, TokKind::Ident]
1673        );
1674        assert_eq!(
1675            kinds("a>>=b"),
1676            vec![TokKind::Ident, TokKind::ShrEq, TokKind::Ident]
1677        );
1678        assert_eq!(
1679            kinds("a>>>=b"),
1680            vec![TokKind::Ident, TokKind::UShrEq, TokKind::Ident]
1681        );
1682        // The two Unicode augmented assignments `÷=` and `⊻=` fuse the `=`.
1683        assert_eq!(
1684            kinds("a÷=b"),
1685            vec![TokKind::Ident, TokKind::DivEq, TokKind::Ident]
1686        );
1687        assert_eq!(
1688            kinds("a⊻=b"),
1689            vec![TokKind::Ident, TokKind::XorEq, TokKind::Ident]
1690        );
1691        // Broadcast forms fuse the leading `.` as well.
1692        assert_eq!(
1693            kinds("a.<<=b"),
1694            vec![TokKind::Ident, TokKind::DotShlEq, TokKind::Ident]
1695        );
1696        assert_eq!(
1697            kinds("a.>>>=b"),
1698            vec![TokKind::Ident, TokKind::DotUShrEq, TokKind::Ident]
1699        );
1700        assert_eq!(
1701            kinds("a.÷=b"),
1702            vec![TokKind::Ident, TokKind::DotDivEq, TokKind::Ident]
1703        );
1704        assert_eq!(
1705            kinds("a.⊻=b"),
1706            vec![TokKind::Ident, TokKind::DotXorEq, TokKind::Ident]
1707        );
1708    }
1709
1710    #[test]
1711    fn uppercase_hex_prefix_is_not_a_literal() {
1712        // Julia's base prefixes are lowercase only: `0X1` is `0` then ident `X1`.
1713        assert_eq!(kinds("0X1"), vec![TokKind::Integer, TokKind::Ident]);
1714    }
1715
1716    #[test]
1717    fn inf_and_nan_are_identifiers() {
1718        // `Inf`/`NaN` are ordinary identifiers in Julia, not numeric literals.
1719        assert_eq!(kinds("Inf"), vec![TokKind::Ident]);
1720        assert_eq!(kinds("NaN"), vec![TokKind::Ident]);
1721        assert_eq!(kinds("Inf32"), vec![TokKind::Ident]);
1722    }
1723
1724    #[test]
1725    fn subtype_and_supertype_operators() {
1726        assert_eq!(
1727            kinds("T<:U"),
1728            vec![TokKind::Ident, TokKind::Subtype, TokKind::Ident]
1729        );
1730        assert_eq!(
1731            kinds("T>:U"),
1732            vec![TokKind::Ident, TokKind::Supertype, TokKind::Ident]
1733        );
1734    }
1735
1736    #[test]
1737    fn splat_is_three_dots() {
1738        assert_eq!(kinds("x..."), vec![TokKind::Ident, TokKind::DotDotDot]);
1739        // Longest match: `...` is the splat, `..` is the range operator.
1740        assert_eq!(kinds(".."), vec![TokKind::DotDot]);
1741        assert_eq!(
1742            kinds("a.b"),
1743            vec![TokKind::Ident, TokKind::Dot, TokKind::Ident]
1744        );
1745    }
1746
1747    #[test]
1748    fn broadcasting_operators() {
1749        assert_eq!(
1750            kinds("a .+ b"),
1751            vec![
1752                TokKind::Ident,
1753                TokKind::Whitespace,
1754                TokKind::DotPlus,
1755                TokKind::Whitespace,
1756                TokKind::Ident
1757            ]
1758        );
1759        // Longest match: `.==` is `DotEqEq`, `.=` is `DotEq`.
1760        assert_eq!(kinds("x .== y").get(2), Some(&TokKind::DotEqEq));
1761        assert_eq!(kinds("x .= y").get(2), Some(&TokKind::DotEq));
1762        assert_eq!(kinds("a .<= b").get(2), Some(&TokKind::DotLe));
1763        // Longest match: the 4-char `.===`/`.!==` beat the 3-char `.==`/`.!=`.
1764        assert_eq!(kinds("x .=== y").get(2), Some(&TokKind::DotEqEqEq));
1765        assert_eq!(kinds("x .!== y").get(2), Some(&TokKind::DotNotEqEq));
1766        // Broadcast unary-not `.!` — the `.!=`/`.!==` inequalities still win the
1767        // longest match, so a lone `!` after the dot is `DotBang`.
1768        assert_eq!(kinds(".!y").first(), Some(&TokKind::DotBang));
1769        assert_eq!(kinds("x .!= y").get(2), Some(&TokKind::DotNotEq));
1770        // The broadcast Unicode radicals fuse into one `UniRadical` token
1771        // spanning `.op`, like the infix tiers (`.×`) above.
1772        assert_eq!(kinds(".√[3]").first(), Some(&TokKind::UniRadical));
1773        assert_eq!(kinds(".¬a").first(), Some(&TokKind::UniRadical));
1774        // Broadcast bitwise augmented assignment `.&=`/`.|=` — distinct from the
1775        // `.&`/`.|` bitwise ops and the `.&&`/`.||`/`.|>` triples.
1776        assert_eq!(kinds("a .&= b").get(2), Some(&TokKind::DotAmpEq));
1777        assert_eq!(kinds("a .|= b").get(2), Some(&TokKind::DotPipeEq));
1778        assert_eq!(kinds("a .& b").get(2), Some(&TokKind::DotAmp));
1779        assert_eq!(kinds("a .| b").get(2), Some(&TokKind::DotPipe));
1780        assert_eq!(kinds("a .|> b").get(2), Some(&TokKind::DotPipeGt));
1781        // A `.` fuses to operators but never to an ident (`a.b` field access).
1782        assert_eq!(
1783            kinds("a.b"),
1784            vec![TokKind::Ident, TokKind::Dot, TokKind::Ident]
1785        );
1786        // `..` is its own range operator, not two lone dots or a broadcast `.`.
1787        assert_eq!(kinds(".."), vec![TokKind::DotDot]);
1788        // `f.(` stays `Dot LParen` so the parser can form a broadcast call.
1789        assert_eq!(
1790            kinds("f.(x)"),
1791            vec![
1792                TokKind::Ident,
1793                TokKind::Dot,
1794                TokKind::LParen,
1795                TokKind::Ident,
1796                TokKind::RParen
1797            ]
1798        );
1799    }
1800
1801    #[test]
1802    fn where_is_a_keyword() {
1803        assert_eq!(keyword_kind("where"), Some(TokKind::WhereKw));
1804        assert_eq!(kinds("where"), vec![TokKind::WhereKw]);
1805    }
1806
1807    #[test]
1808    fn every_ops_entry_lexes_as_itself() {
1809        // Each spelling must come back as exactly one token of its own kind.
1810        // A duplicated or shadowed entry — one whose prefix is claimed by an
1811        // earlier, shorter entry — fails here, which is what keeps every row
1812        // of the table reachable.
1813        for &(text, kind) in OPS {
1814            let spelling = std::str::from_utf8(text).expect("OPS spellings are ASCII");
1815            let tokens = lex(spelling);
1816            assert_eq!(
1817                tokens.iter().map(|t| t.kind).collect::<Vec<_>>(),
1818                vec![kind],
1819                "{spelling:?} did not lex as one {kind:?}"
1820            );
1821            assert_eq!(tokens[0].text, spelling);
1822        }
1823    }
1824
1825    #[test]
1826    fn longest_match_beats_every_shared_prefix() {
1827        // The truncation the table exists to prevent: each of these has a
1828        // shorter operator as a prefix, and must still lex as one token.
1829        for spelling in [
1830            ".>>>=", ".<-->", ".<--", ".//=", ".===", ".!==", "...", "<-->", ">>>=", "-->", "//=",
1831            "+%=", ".+%", ".-%", ".*%", ".+%=", ".-%=", ".*%=", "!==", "===",
1832        ] {
1833            assert_eq!(kinds(spelling).len(), 1, "{spelling:?} was split");
1834        }
1835        // And the other side of it: a prefix that is *not* an operator falls
1836        // back to the shorter spelling rather than being consumed.
1837        assert_eq!(kinds("<-"), vec![TokKind::Lt, TokKind::Minus]);
1838        assert_eq!(kinds("+++"), vec![TokKind::PlusPlus, TokKind::Plus]);
1839        assert_eq!(kinds("++="), vec![TokKind::PlusPlus, TokKind::Eq]);
1840        assert_eq!(
1841            kinds("a.b"),
1842            vec![TokKind::Ident, TokKind::Dot, TokKind::Ident]
1843        );
1844    }
1845
1846    #[test]
1847    fn unicode_operators_beat_the_ascii_table() {
1848        // `.` fused to a non-ASCII operator outranks the lone `Dot` the ASCII
1849        // table would otherwise hand back.
1850        assert_eq!(kinds(".×"), vec![TokKind::UniTimes]);
1851        assert_eq!(kinds(".√"), vec![TokKind::UniRadical]);
1852        assert_eq!(kinds(".÷="), vec![TokKind::DotDivEq]);
1853        assert_eq!(kinds(".⊻="), vec![TokKind::DotXorEq]);
1854        // U+2212 MINUS SIGN is the ASCII `-`, augmented form included.
1855        assert_eq!(kinds("−"), vec![TokKind::Minus]);
1856        assert_eq!(kinds("−="), vec![TokKind::MinusEq]);
1857        assert_eq!(kinds(".−="), vec![TokKind::DotMinusEq]);
1858        assert_eq!(kinds("÷="), vec![TokKind::DivEq]);
1859        // The assignment tier does not fuse, so its `.` stays a lone `Dot`.
1860        assert_eq!(kinds(".⩴"), vec![TokKind::Dot, TokKind::UniAssign]);
1861    }
1862}