ruff_python_trivia/
tokenizer.rs

1use unicode_ident::{is_xid_continue, is_xid_start};
2
3use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
4
5use crate::{Cursor, is_python_whitespace};
6
7/// Searches for the first non-trivia character after `offset`.
8///
9/// The search skips over any whitespace and comments.
10///
11/// Returns `Some` if the source code after `offset` contains any non-trivia character.///
12/// Returns `None` if the text after `offset` is empty or only contains trivia (whitespace or comments).
13pub fn first_non_trivia_token(offset: TextSize, code: &str) -> Option<SimpleToken> {
14    SimpleTokenizer::starts_at(offset, code)
15        .skip_trivia()
16        .next()
17}
18
19/// Returns the only non-trivia, non-closing parenthesis token in `range`.
20///
21/// Includes debug assertions that the range only contains that single token.
22pub fn find_only_token_in_range(
23    range: TextRange,
24    token_kind: SimpleTokenKind,
25    code: &str,
26) -> SimpleToken {
27    let mut tokens = SimpleTokenizer::new(code, range)
28        .skip_trivia()
29        .skip_while(|token| token.kind == SimpleTokenKind::RParen);
30    let token = tokens.next().expect("Expected a token");
31    debug_assert_eq!(token.kind(), token_kind);
32    let mut tokens = tokens.skip_while(|token| token.kind == SimpleTokenKind::LParen);
33    #[expect(clippy::debug_assert_with_mut_call)]
34    {
35        debug_assert_eq!(tokens.next(), None);
36    }
37    token
38}
39
40/// Returns the number of newlines between `offset` and the first non whitespace character in the source code.
41pub fn lines_before(offset: TextSize, code: &str) -> u32 {
42    let mut cursor = Cursor::new(&code[TextRange::up_to(offset)]);
43
44    let mut newlines = 0u32;
45    while let Some(c) = cursor.bump_back() {
46        match c {
47            '\n' => {
48                cursor.eat_char_back('\r');
49                newlines += 1;
50            }
51            '\r' => {
52                newlines += 1;
53            }
54            c if is_python_whitespace(c) => {
55                continue;
56            }
57            _ => {
58                break;
59            }
60        }
61    }
62
63    newlines
64}
65
66/// Counts the empty lines between `offset` and the first non-whitespace character.
67pub fn lines_after(offset: TextSize, code: &str) -> u32 {
68    let mut cursor = Cursor::new(&code[offset.to_usize()..]);
69
70    let mut newlines = 0u32;
71    while let Some(c) = cursor.bump() {
72        match c {
73            '\n' => {
74                newlines += 1;
75            }
76            '\r' => {
77                cursor.eat_char('\n');
78                newlines += 1;
79            }
80            c if is_python_whitespace(c) => {
81                continue;
82            }
83            _ => {
84                break;
85            }
86        }
87    }
88
89    newlines
90}
91
92/// Counts the empty lines after `offset`, ignoring any trailing trivia: end-of-line comments,
93/// own-line comments, and any intermediary newlines.
94pub fn lines_after_ignoring_trivia(offset: TextSize, code: &str) -> u32 {
95    let mut newlines = 0u32;
96    for token in SimpleTokenizer::starts_at(offset, code) {
97        match token.kind() {
98            SimpleTokenKind::Newline => {
99                newlines += 1;
100            }
101            SimpleTokenKind::Whitespace => {}
102            // If we see a comment, reset the newlines counter.
103            SimpleTokenKind::Comment => {
104                newlines = 0;
105            }
106            // As soon as we see a non-trivia token, we're done.
107            _ => {
108                break;
109            }
110        }
111    }
112    newlines
113}
114
115/// Counts the empty lines after `offset`, ignoring any trailing trivia on the same line as
116/// `offset`.
117#[expect(clippy::cast_possible_truncation)]
118pub fn lines_after_ignoring_end_of_line_trivia(offset: TextSize, code: &str) -> u32 {
119    // SAFETY: We don't support files greater than 4GB, so casting to u32 is safe.
120    SimpleTokenizer::starts_at(offset, code)
121        .skip_while(|token| token.kind != SimpleTokenKind::Newline && token.kind.is_trivia())
122        .take_while(|token| {
123            token.kind == SimpleTokenKind::Newline || token.kind == SimpleTokenKind::Whitespace
124        })
125        .filter(|token| token.kind == SimpleTokenKind::Newline)
126        .count() as u32
127}
128
129fn is_identifier_start(c: char) -> bool {
130    if c.is_ascii() {
131        c.is_ascii_alphabetic() || c == '_'
132    } else {
133        is_xid_start(c)
134    }
135}
136
137// Checks if the character c is a valid continuation character as described
138// in https://docs.python.org/3/reference/lexical_analysis.html#identifiers
139fn is_identifier_continuation(c: char) -> bool {
140    // Arrange things such that ASCII codepoints never
141    // result in the slower `is_xid_continue` getting called.
142    if c.is_ascii() {
143        matches!(c, 'a'..='z' | 'A'..='Z' | '_' | '0'..='9')
144    } else {
145        is_xid_continue(c)
146    }
147}
148
149fn to_keyword_or_other(source: &str) -> SimpleTokenKind {
150    match source {
151        "and" => SimpleTokenKind::And,
152        "as" => SimpleTokenKind::As,
153        "assert" => SimpleTokenKind::Assert,
154        "async" => SimpleTokenKind::Async,
155        "await" => SimpleTokenKind::Await,
156        "break" => SimpleTokenKind::Break,
157        "class" => SimpleTokenKind::Class,
158        "continue" => SimpleTokenKind::Continue,
159        "def" => SimpleTokenKind::Def,
160        "del" => SimpleTokenKind::Del,
161        "elif" => SimpleTokenKind::Elif,
162        "else" => SimpleTokenKind::Else,
163        "except" => SimpleTokenKind::Except,
164        "finally" => SimpleTokenKind::Finally,
165        "for" => SimpleTokenKind::For,
166        "from" => SimpleTokenKind::From,
167        "global" => SimpleTokenKind::Global,
168        "if" => SimpleTokenKind::If,
169        "import" => SimpleTokenKind::Import,
170        "in" => SimpleTokenKind::In,
171        "is" => SimpleTokenKind::Is,
172        "lazy" => SimpleTokenKind::Lazy, // Lazy is a soft keyword that depends on the context but we can always lex it as a keyword and leave it to the caller (parser) to decide if it should be handled as an identifier or keyword.
173        "lambda" => SimpleTokenKind::Lambda,
174        "nonlocal" => SimpleTokenKind::Nonlocal,
175        "not" => SimpleTokenKind::Not,
176        "or" => SimpleTokenKind::Or,
177        "pass" => SimpleTokenKind::Pass,
178        "raise" => SimpleTokenKind::Raise,
179        "return" => SimpleTokenKind::Return,
180        "try" => SimpleTokenKind::Try,
181        "while" => SimpleTokenKind::While,
182        "match" => SimpleTokenKind::Match, // Match is a soft keyword that depends on the context but we can always lex it as a keyword and leave it to the caller (parser) to decide if it should be handled as an identifier or keyword.
183        "type" => SimpleTokenKind::Type, // Type is a soft keyword that depends on the context but we can always lex it as a keyword and leave it to the caller (parser) to decide if it should be handled as an identifier or keyword.
184        "case" => SimpleTokenKind::Case,
185        "with" => SimpleTokenKind::With,
186        "yield" => SimpleTokenKind::Yield,
187        _ => SimpleTokenKind::Name, // Potentially an identifier, but only if it isn't a string prefix. The caller (SimpleTokenizer) is responsible for enforcing that constraint.
188    }
189}
190
191#[derive(Clone, Debug, Eq, PartialEq, Hash)]
192pub struct SimpleToken {
193    pub kind: SimpleTokenKind,
194    pub range: TextRange,
195}
196
197impl SimpleToken {
198    pub const fn kind(&self) -> SimpleTokenKind {
199        self.kind
200    }
201}
202
203impl Ranged for SimpleToken {
204    fn range(&self) -> TextRange {
205        self.range
206    }
207}
208
209#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
210pub enum SimpleTokenKind {
211    /// A comment, not including the trailing new line.
212    Comment,
213
214    /// Sequence of ' ' or '\t'
215    Whitespace,
216
217    /// Start or end of the file
218    EndOfFile,
219
220    /// `\\`
221    Continuation,
222
223    /// `\n` or `\r` or `\r\n`
224    Newline,
225
226    /// `(`
227    LParen,
228
229    /// `)`
230    RParen,
231
232    /// `{`
233    LBrace,
234
235    /// `}`
236    RBrace,
237
238    /// `[`
239    LBracket,
240
241    /// `]`
242    RBracket,
243
244    /// `,`
245    Comma,
246
247    /// `:`
248    Colon,
249
250    /// `;`
251    Semi,
252
253    /// `/`
254    Slash,
255
256    /// `*`
257    Star,
258
259    /// `.`
260    Dot,
261
262    /// `+`
263    Plus,
264
265    /// `-`
266    Minus,
267
268    /// `=`
269    Equals,
270
271    /// `>`
272    Greater,
273
274    /// `<`
275    Less,
276
277    /// `%`
278    Percent,
279
280    /// `&`
281    Ampersand,
282
283    /// `^`
284    Circumflex,
285
286    /// `|`
287    Vbar,
288
289    /// `@`
290    At,
291
292    /// `~`
293    Tilde,
294
295    /// `==`
296    EqEqual,
297
298    /// `!=`
299    NotEqual,
300
301    /// `<=`
302    LessEqual,
303
304    /// `>=`
305    GreaterEqual,
306
307    /// `<<`
308    LeftShift,
309
310    /// `>>`
311    RightShift,
312
313    /// `**`
314    DoubleStar,
315
316    /// `**=`
317    DoubleStarEqual,
318
319    /// `+=`
320    PlusEqual,
321
322    /// `-=`
323    MinusEqual,
324
325    /// `*=`
326    StarEqual,
327
328    /// `/=`
329    SlashEqual,
330
331    /// `%=`
332    PercentEqual,
333
334    /// `&=`
335    AmperEqual,
336
337    /// `|=`
338    VbarEqual,
339
340    /// `^=`
341    CircumflexEqual,
342
343    /// `<<=`
344    LeftShiftEqual,
345
346    /// `>>=`
347    RightShiftEqual,
348
349    /// `//`
350    DoubleSlash,
351
352    /// `//=`
353    DoubleSlashEqual,
354
355    /// `:=`
356    ColonEqual,
357
358    /// `...`
359    Ellipsis,
360
361    /// `@=`
362    AtEqual,
363
364    /// `->`
365    RArrow,
366
367    /// `and`
368    And,
369
370    /// `as`
371    As,
372
373    /// `assert`
374    Assert,
375
376    /// `async`
377    Async,
378
379    /// `await`
380    Await,
381
382    /// `break`
383    Break,
384
385    /// `class`
386    Class,
387
388    /// `continue`
389    Continue,
390
391    /// `def`
392    Def,
393
394    /// `del`
395    Del,
396
397    /// `elif`
398    Elif,
399
400    /// `else`
401    Else,
402
403    /// `except`
404    Except,
405
406    /// `finally`
407    Finally,
408
409    /// `for`
410    For,
411
412    /// `from`
413    From,
414
415    /// `global`
416    Global,
417
418    /// `if`
419    If,
420
421    /// `import`
422    Import,
423
424    /// `in`
425    In,
426
427    /// `is`
428    Is,
429
430    /// `lambda`
431    Lambda,
432
433    /// `nonlocal`
434    Nonlocal,
435
436    /// `not`
437    Not,
438
439    /// `or`
440    Or,
441
442    /// `pass`
443    Pass,
444
445    /// `raise`
446    Raise,
447
448    /// `return`
449    Return,
450
451    /// `try`
452    Try,
453
454    /// `while`
455    While,
456
457    /// `lazy`
458    Lazy,
459
460    /// `match`
461    Match,
462
463    /// `type`
464    Type,
465
466    /// `case`
467    Case,
468
469    /// `with`
470    With,
471
472    /// `yield`
473    Yield,
474
475    /// An identifier or keyword.
476    Name,
477
478    /// Any other non trivia token.
479    Other,
480
481    /// Returned for each character after [`SimpleTokenKind::Other`] has been returned once.
482    Bogus,
483}
484
485impl SimpleTokenKind {
486    pub const fn is_trivia(self) -> bool {
487        matches!(
488            self,
489            SimpleTokenKind::Whitespace
490                | SimpleTokenKind::Newline
491                | SimpleTokenKind::Comment
492                | SimpleTokenKind::Continuation
493        )
494    }
495
496    pub const fn is_comment(self) -> bool {
497        matches!(self, SimpleTokenKind::Comment)
498    }
499}
500
501/// Simple zero allocation tokenizer handling most tokens.
502///
503/// The tokenizer must start at an offset that is trivia (e.g. not inside of a multiline string).
504///
505/// In case it finds something it can't parse, the tokenizer will return a
506/// [`SimpleTokenKind::Other`] and then only a final [`SimpleTokenKind::Bogus`] afterwards.
507pub struct SimpleTokenizer<'a> {
508    offset: TextSize,
509    /// `true` when it is known that the current `back` line has no comment for sure.
510    bogus: bool,
511    source: &'a str,
512    cursor: Cursor<'a>,
513}
514
515impl<'a> SimpleTokenizer<'a> {
516    pub fn new(source: &'a str, range: TextRange) -> Self {
517        Self {
518            offset: range.start(),
519            bogus: false,
520            source,
521            cursor: Cursor::new(&source[range]),
522        }
523    }
524
525    pub fn starts_at(offset: TextSize, source: &'a str) -> Self {
526        let range = TextRange::new(offset, source.text_len());
527        Self::new(source, range)
528    }
529
530    fn next_token(&mut self) -> SimpleToken {
531        self.cursor.start_token();
532
533        let Some(first) = self.cursor.bump() else {
534            return SimpleToken {
535                kind: SimpleTokenKind::EndOfFile,
536                range: TextRange::empty(self.offset),
537            };
538        };
539
540        if self.bogus {
541            // Emit a single final bogus token
542            let token = SimpleToken {
543                kind: SimpleTokenKind::Bogus,
544                range: TextRange::new(self.offset, self.source.text_len()),
545            };
546
547            // Set the cursor to EOF
548            self.cursor = Cursor::new("");
549            self.offset = self.source.text_len();
550            return token;
551        }
552
553        let kind = self.next_token_inner(first);
554
555        let token_len = self.cursor.token_len();
556
557        let token = SimpleToken {
558            kind,
559            range: TextRange::at(self.offset, token_len),
560        };
561
562        self.offset += token_len;
563
564        token
565    }
566
567    fn next_token_inner(&mut self, first: char) -> SimpleTokenKind {
568        match first {
569            // Keywords and identifiers
570            c if is_identifier_start(c) => {
571                self.cursor.eat_while(is_identifier_continuation);
572                let token_len = self.cursor.token_len();
573
574                let range = TextRange::at(self.offset, token_len);
575                let kind = to_keyword_or_other(&self.source[range]);
576
577                // If the next character is a quote, we may be in a string prefix. For example:
578                // `f"foo`.
579                if kind == SimpleTokenKind::Name
580                    && matches!(self.cursor.first(), '"' | '\'')
581                    && matches!(
582                        &self.source[range],
583                        "B" | "BR"
584                            | "Br"
585                            | "F"
586                            | "FR"
587                            | "Fr"
588                            | "R"
589                            | "RB"
590                            | "RF"
591                            | "Rb"
592                            | "Rf"
593                            | "U"
594                            | "b"
595                            | "bR"
596                            | "br"
597                            | "f"
598                            | "fR"
599                            | "fr"
600                            | "r"
601                            | "rB"
602                            | "rF"
603                            | "rb"
604                            | "rf"
605                            | "u"
606                            | "T"
607                            | "TR"
608                            | "Tr"
609                            | "RT"
610                            | "Rt"
611                            | "t"
612                            | "tR"
613                            | "tr"
614                            | "rT"
615                            | "rt"
616                    )
617                {
618                    self.bogus = true;
619                    SimpleTokenKind::Other
620                } else {
621                    kind
622                }
623            }
624
625            // Space, tab, or form feed. We ignore the true semantics of form feed, and treat it as
626            // whitespace.
627            ' ' | '\t' | '\x0C' => {
628                self.cursor.eat_while(|c| matches!(c, ' ' | '\t' | '\x0C'));
629                SimpleTokenKind::Whitespace
630            }
631
632            '\n' => SimpleTokenKind::Newline,
633
634            '\r' => {
635                self.cursor.eat_char('\n');
636                SimpleTokenKind::Newline
637            }
638
639            '#' => {
640                self.cursor.eat_while(|c| !matches!(c, '\n' | '\r'));
641                SimpleTokenKind::Comment
642            }
643
644            '\\' => SimpleTokenKind::Continuation,
645
646            // Non-trivia, non-keyword tokens
647            '=' => {
648                if self.cursor.eat_char('=') {
649                    SimpleTokenKind::EqEqual
650                } else {
651                    SimpleTokenKind::Equals
652                }
653            }
654            '+' => {
655                if self.cursor.eat_char('=') {
656                    SimpleTokenKind::PlusEqual
657                } else {
658                    SimpleTokenKind::Plus
659                }
660            }
661            '*' => {
662                if self.cursor.eat_char('=') {
663                    SimpleTokenKind::StarEqual
664                } else if self.cursor.eat_char('*') {
665                    if self.cursor.eat_char('=') {
666                        SimpleTokenKind::DoubleStarEqual
667                    } else {
668                        SimpleTokenKind::DoubleStar
669                    }
670                } else {
671                    SimpleTokenKind::Star
672                }
673            }
674            '/' => {
675                if self.cursor.eat_char('=') {
676                    SimpleTokenKind::SlashEqual
677                } else if self.cursor.eat_char('/') {
678                    if self.cursor.eat_char('=') {
679                        SimpleTokenKind::DoubleSlashEqual
680                    } else {
681                        SimpleTokenKind::DoubleSlash
682                    }
683                } else {
684                    SimpleTokenKind::Slash
685                }
686            }
687            '%' => {
688                if self.cursor.eat_char('=') {
689                    SimpleTokenKind::PercentEqual
690                } else {
691                    SimpleTokenKind::Percent
692                }
693            }
694            '|' => {
695                if self.cursor.eat_char('=') {
696                    SimpleTokenKind::VbarEqual
697                } else {
698                    SimpleTokenKind::Vbar
699                }
700            }
701            '^' => {
702                if self.cursor.eat_char('=') {
703                    SimpleTokenKind::CircumflexEqual
704                } else {
705                    SimpleTokenKind::Circumflex
706                }
707            }
708            '&' => {
709                if self.cursor.eat_char('=') {
710                    SimpleTokenKind::AmperEqual
711                } else {
712                    SimpleTokenKind::Ampersand
713                }
714            }
715            '-' => {
716                if self.cursor.eat_char('=') {
717                    SimpleTokenKind::MinusEqual
718                } else if self.cursor.eat_char('>') {
719                    SimpleTokenKind::RArrow
720                } else {
721                    SimpleTokenKind::Minus
722                }
723            }
724            '@' => {
725                if self.cursor.eat_char('=') {
726                    SimpleTokenKind::AtEqual
727                } else {
728                    SimpleTokenKind::At
729                }
730            }
731            '!' if self.cursor.eat_char('=') => SimpleTokenKind::NotEqual,
732            '~' => SimpleTokenKind::Tilde,
733            ':' => {
734                if self.cursor.eat_char('=') {
735                    SimpleTokenKind::ColonEqual
736                } else {
737                    SimpleTokenKind::Colon
738                }
739            }
740            ';' => SimpleTokenKind::Semi,
741            '<' => {
742                if self.cursor.eat_char('<') {
743                    if self.cursor.eat_char('=') {
744                        SimpleTokenKind::LeftShiftEqual
745                    } else {
746                        SimpleTokenKind::LeftShift
747                    }
748                } else if self.cursor.eat_char('=') {
749                    SimpleTokenKind::LessEqual
750                } else {
751                    SimpleTokenKind::Less
752                }
753            }
754            '>' => {
755                if self.cursor.eat_char('>') {
756                    if self.cursor.eat_char('=') {
757                        SimpleTokenKind::RightShiftEqual
758                    } else {
759                        SimpleTokenKind::RightShift
760                    }
761                } else if self.cursor.eat_char('=') {
762                    SimpleTokenKind::GreaterEqual
763                } else {
764                    SimpleTokenKind::Greater
765                }
766            }
767            ',' => SimpleTokenKind::Comma,
768            '.' => {
769                if self.cursor.first() == '.' && self.cursor.second() == '.' {
770                    self.cursor.bump();
771                    self.cursor.bump();
772                    SimpleTokenKind::Ellipsis
773                } else {
774                    SimpleTokenKind::Dot
775                }
776            }
777
778            // Bracket tokens
779            '(' => SimpleTokenKind::LParen,
780            ')' => SimpleTokenKind::RParen,
781            '[' => SimpleTokenKind::LBracket,
782            ']' => SimpleTokenKind::RBracket,
783            '{' => SimpleTokenKind::LBrace,
784            '}' => SimpleTokenKind::RBrace,
785
786            _ => {
787                self.bogus = true;
788                SimpleTokenKind::Other
789            }
790        }
791    }
792
793    pub fn skip_trivia(self) -> impl Iterator<Item = SimpleToken> + 'a {
794        self.filter(|t| !t.kind().is_trivia())
795    }
796}
797
798impl Iterator for SimpleTokenizer<'_> {
799    type Item = SimpleToken;
800
801    fn next(&mut self) -> Option<Self::Item> {
802        let token = self.next_token();
803
804        if token.kind == SimpleTokenKind::EndOfFile {
805            None
806        } else {
807            Some(token)
808        }
809    }
810}
811
812/// Simple zero allocation backwards tokenizer for finding preceding tokens.
813///
814/// The tokenizer must start at an offset that is trivia (e.g. not inside of a multiline string).
815/// It will fail when reaching a string.
816///
817/// In case it finds something it can't parse, the tokenizer will return a
818/// [`SimpleTokenKind::Other`] and then only a final [`SimpleTokenKind::Bogus`] afterwards.
819pub struct BackwardsTokenizer<'a> {
820    offset: TextSize,
821    back_offset: TextSize,
822    /// Not `&CommentRanges` to avoid a circular dependency.
823    comment_ranges: &'a [TextRange],
824    bogus: bool,
825    source: &'a str,
826    cursor: Cursor<'a>,
827}
828
829impl<'a> BackwardsTokenizer<'a> {
830    pub fn new(source: &'a str, range: TextRange, comment_range: &'a [TextRange]) -> Self {
831        Self {
832            offset: range.start(),
833            back_offset: range.end(),
834            // Throw out any comments that follow the range.
835            comment_ranges: &comment_range
836                [..comment_range.partition_point(|comment| comment.start() <= range.end())],
837            bogus: false,
838            source,
839            cursor: Cursor::new(&source[range]),
840        }
841    }
842
843    pub fn up_to(offset: TextSize, source: &'a str, comment_range: &'a [TextRange]) -> Self {
844        Self::new(source, TextRange::up_to(offset), comment_range)
845    }
846
847    pub fn skip_trivia(self) -> impl Iterator<Item = SimpleToken> + 'a {
848        self.filter(|t| !t.kind().is_trivia())
849    }
850
851    fn next_token(&mut self) -> SimpleToken {
852        self.cursor.start_token();
853        self.back_offset = self.cursor.text_len() + self.offset;
854
855        let Some(last) = self.cursor.bump_back() else {
856            return SimpleToken {
857                kind: SimpleTokenKind::EndOfFile,
858                range: TextRange::empty(self.back_offset),
859            };
860        };
861
862        if self.bogus {
863            let token = SimpleToken {
864                kind: SimpleTokenKind::Bogus,
865                range: TextRange::up_to(self.back_offset),
866            };
867
868            // Set the cursor to EOF
869            self.cursor = Cursor::new("");
870            self.back_offset = TextSize::new(0);
871            return token;
872        }
873
874        if let Some(comment) = self
875            .comment_ranges
876            .last()
877            .filter(|comment| comment.contains_inclusive(self.back_offset))
878        {
879            self.comment_ranges = &self.comment_ranges[..self.comment_ranges.len() - 1];
880
881            // Skip the comment without iterating over the chars manually.
882            self.cursor = Cursor::new(&self.source[TextRange::new(self.offset, comment.start())]);
883            debug_assert_eq!(self.cursor.text_len() + self.offset, comment.start());
884            return SimpleToken {
885                kind: SimpleTokenKind::Comment,
886                range: comment.range(),
887            };
888        }
889
890        let kind = match last {
891            // Space, tab, or form feed. We ignore the true semantics of form feed, and treat it as
892            // whitespace. Note that this will lex-out trailing whitespace from a comment as
893            // whitespace rather than as part of the comment token, but this shouldn't matter for
894            // our use case.
895            ' ' | '\t' | '\x0C' => {
896                self.cursor
897                    .eat_back_while(|c| matches!(c, ' ' | '\t' | '\x0C'));
898                SimpleTokenKind::Whitespace
899            }
900
901            '\r' => SimpleTokenKind::Newline,
902            '\n' => {
903                self.cursor.eat_char_back('\r');
904                SimpleTokenKind::Newline
905            }
906            _ => self.next_token_inner(last),
907        };
908
909        let token_len = self.cursor.token_len();
910        let start = self.back_offset - token_len;
911        SimpleToken {
912            kind,
913            range: TextRange::at(start, token_len),
914        }
915    }
916
917    /// Helper to parser the previous token once we skipped all whitespace
918    fn next_token_inner(&mut self, last: char) -> SimpleTokenKind {
919        match last {
920            // Keywords and identifiers
921            c if is_identifier_continuation(c) => {
922                // if we only have identifier continuations but no start (e.g. 555) we
923                // don't want to consume the chars, so in that case, we want to rewind the
924                // cursor to here
925                let savepoint = self.cursor.clone();
926                self.cursor.eat_back_while(is_identifier_continuation);
927
928                let token_len = self.cursor.token_len();
929                let range = TextRange::at(self.back_offset - token_len, token_len);
930
931                if self.source[range]
932                    .chars()
933                    .next()
934                    .is_some_and(is_identifier_start)
935                {
936                    to_keyword_or_other(&self.source[range])
937                } else {
938                    self.cursor = savepoint;
939                    self.bogus = true;
940                    SimpleTokenKind::Other
941                }
942            }
943
944            // Non-trivia tokens that are unambiguous when lexing backwards.
945            // In other words: these are characters that _don't_ appear at the
946            // end of a multi-character token (like `!=`).
947            '\\' => SimpleTokenKind::Continuation,
948            ':' => SimpleTokenKind::Colon,
949            '~' => SimpleTokenKind::Tilde,
950            '%' => SimpleTokenKind::Percent,
951            '|' => SimpleTokenKind::Vbar,
952            ',' => SimpleTokenKind::Comma,
953            ';' => SimpleTokenKind::Semi,
954            '(' => SimpleTokenKind::LParen,
955            ')' => SimpleTokenKind::RParen,
956            '[' => SimpleTokenKind::LBracket,
957            ']' => SimpleTokenKind::RBracket,
958            '{' => SimpleTokenKind::LBrace,
959            '}' => SimpleTokenKind::RBrace,
960            '&' => SimpleTokenKind::Ampersand,
961            '^' => SimpleTokenKind::Circumflex,
962            '+' => SimpleTokenKind::Plus,
963            '-' => SimpleTokenKind::Minus,
964
965            // Non-trivia tokens that _are_ ambiguous when lexing backwards.
966            // In other words: these are characters that _might_ mark the end
967            // of a multi-character token (like `!=` or `->` or `//` or `**`).
968            '=' | '*' | '/' | '@' | '!' | '<' | '>' | '.' => {
969                // This could be a single-token token, like `+` in `x + y`, or a
970                // multi-character token, like `+=` in `x += y`. It could also be a sequence
971                // of multi-character tokens, like `x ==== y`, which is invalid, _but_ it's
972                // important that we produce the same token stream when lexing backwards as
973                // we do when lexing forwards. So, identify the range of the sequence, lex
974                // forwards, and return the last token.
975                let mut cursor = self.cursor.clone();
976                cursor.eat_back_while(|c| {
977                    matches!(
978                        c,
979                        ':' | '~'
980                            | '%'
981                            | '|'
982                            | '&'
983                            | '^'
984                            | '+'
985                            | '-'
986                            | '='
987                            | '*'
988                            | '/'
989                            | '@'
990                            | '!'
991                            | '<'
992                            | '>'
993                            | '.'
994                    )
995                });
996
997                let token_len = cursor.token_len();
998                let range = TextRange::at(self.back_offset - token_len, token_len);
999
1000                let forward_lexer = SimpleTokenizer::new(self.source, range);
1001                if let Some(token) = forward_lexer.last() {
1002                    // If the token spans multiple characters, bump the cursor. Note,
1003                    // though, that we already bumped the cursor to past the last character
1004                    // in the token at the very start of `next_token_back`.y
1005                    for _ in self.source[token.range].chars().rev().skip(1) {
1006                        self.cursor.bump_back().unwrap();
1007                    }
1008                    token.kind()
1009                } else {
1010                    self.bogus = true;
1011                    SimpleTokenKind::Other
1012                }
1013            }
1014            _ => {
1015                self.bogus = true;
1016                SimpleTokenKind::Other
1017            }
1018        }
1019    }
1020}
1021
1022impl Iterator for BackwardsTokenizer<'_> {
1023    type Item = SimpleToken;
1024
1025    fn next(&mut self) -> Option<Self::Item> {
1026        let token = self.next_token();
1027
1028        if token.kind == SimpleTokenKind::EndOfFile {
1029            None
1030        } else {
1031            Some(token)
1032        }
1033    }
1034}
ruff_python_trivia/tokenizer.rs

ruff_python_trivia/
tokenizer.rs
