cairo_lang_parser/
lexer.rs

1#[cfg(test)]
2#[path = "lexer_test.rs"]
3mod test;
4
5use std::sync::Arc;
6
7use cairo_lang_filesystem::ids::SmolStrId;
8use cairo_lang_filesystem::span::{TextOffset, TextSpan, TextWidth};
9use cairo_lang_syntax::node::Token;
10use cairo_lang_syntax::node::ast::{
11    TokenNewline, TokenSingleLineComment, TokenSingleLineDocComment, TokenSingleLineInnerComment,
12    TokenWhitespace, TriviumGreen,
13};
14use cairo_lang_syntax::node::kind::SyntaxKind;
15use salsa::Database;
16
17#[derive(Clone, PartialEq, Eq, Hash)]
18pub struct Lexer {
19    text: Arc<str>,
20    previous_position: TextOffset,
21    current_position: TextOffset,
22}
23
24impl Lexer {
25    /// Creates a new lexer with the given text.
26    pub fn new(text: Arc<str>) -> Self {
27        Self { text, previous_position: TextOffset::START, current_position: TextOffset::START }
28    }
29
30    // Helpers.
31    fn peek(&self) -> Option<char> {
32        self.current_position.take_from(&self.text).chars().next()
33    }
34
35    fn peek_nth(&self, n: usize) -> Option<char> {
36        self.current_position.take_from(&self.text).chars().nth(n)
37    }
38
39    fn take(&mut self) -> Option<char> {
40        let res = self.peek()?;
41        self.current_position = self.current_position.add_width(TextWidth::from_char(res));
42        Some(res)
43    }
44
45    /// Takes a character while the given function returns true.
46    fn take_while<F>(&mut self, f: F)
47    where
48        F: Fn(char) -> bool,
49    {
50        while self.peek().map(&f).unwrap_or(false) {
51            self.take();
52        }
53    }
54
55    fn peek_text_span(&self) -> TextSpan {
56        TextSpan::new(self.previous_position, self.current_position)
57    }
58
59    fn consume_text_span(&mut self) -> TextSpan {
60        let val = self.peek_text_span();
61        self.previous_position = self.current_position;
62        val
63    }
64
65    // Trivia matchers.
66    fn match_trivia<'a>(&mut self, db: &'a dyn Database, leading: bool) -> Vec<TriviumGreen<'a>> {
67        let mut res: Vec<TriviumGreen<'a>> = Vec::new();
68        while let Some(current) = self.peek() {
69            let trivium = match current {
70                ' ' | '\r' | '\t' => self.match_trivium_whitespace(db),
71                '\n' => self.match_trivium_newline(db),
72                '/' if self.peek_nth(1) == Some('/') => self.match_trivium_single_line_comment(db),
73                _ => break,
74            };
75            res.push(trivium);
76            if current == '\n' && !leading {
77                break;
78            }
79        }
80        res
81    }
82
83    /// Assumes the next character is one of [' ', '\r', '\t'].
84    fn match_trivium_whitespace<'a>(&mut self, db: &'a dyn Database) -> TriviumGreen<'a> {
85        self.take_while(|s| matches!(s, ' ' | '\r' | '\t'));
86        let span = self.consume_text_span();
87        let text = span.take(&self.text);
88        TokenWhitespace::new_green(db, SmolStrId::from(db, text)).into()
89    }
90
91    /// Assumes the next character is '\n'.
92    fn match_trivium_newline<'a>(&mut self, db: &'a dyn Database) -> TriviumGreen<'a> {
93        self.take();
94        let span = self.consume_text_span();
95        let text = span.take(&self.text);
96        TokenNewline::new_green(db, SmolStrId::from(db, text)).into()
97    }
98
99    /// Assumes the next 2 characters are "//".
100    fn match_trivium_single_line_comment<'a>(&mut self, db: &'a dyn Database) -> TriviumGreen<'a> {
101        match self.peek_nth(2) {
102            // `///` is a doc comment, but `////` (4+ slashes) is a regular comment. This matches
103            // `cairo-lang-doc`, which discards a doc comment whose content starts with `/`.
104            Some('/') if self.peek_nth(3) != Some('/') => {
105                self.take_while(|c| c != '\n');
106                let span = self.consume_text_span();
107                let text = span.take(&self.text);
108                TokenSingleLineDocComment::new_green(db, SmolStrId::from(db, text)).into()
109            }
110            Some('!') => {
111                self.take_while(|c| c != '\n');
112                let span = self.consume_text_span();
113                let text = span.take(&self.text);
114                TokenSingleLineInnerComment::new_green(db, SmolStrId::from(db, text)).into()
115            }
116            _ => {
117                self.take_while(|c| c != '\n');
118                let span = self.consume_text_span();
119                let text = span.take(&self.text);
120                TokenSingleLineComment::new_green(db, SmolStrId::from(db, text)).into()
121            }
122        }
123    }
124
125    // Token matchers.
126    // =================================================================================
127
128    /// Takes a number. May be decimal, hex, oct or bin.
129    fn take_token_literal_number(&mut self) -> TokenKind {
130        let special = if self.peek() == Some('0') {
131            self.take();
132            match self.peek() {
133                Some('x' | 'o' | 'b') => {
134                    match self.take() {
135                        Some('x') => self.take_while(|c| c.is_ascii_hexdigit()),
136                        Some('o') => self.take_while(|c| matches!(c, '0'..='7')),
137                        Some('b') => self.take_while(|c| matches!(c, '0'..='1')),
138                        _ => unreachable!(),
139                    }
140                    true
141                }
142                _ => false,
143            }
144        } else {
145            false
146        };
147        // Not a special case - so just reading the rest of the digits.
148        if !special {
149            self.take_while(|c| c.is_ascii_digit());
150        }
151
152        // Parse _type suffix.
153        if self.peek() == Some('_') {
154            self.take_while(|c| c.is_ascii_alphanumeric() || c == '_');
155        }
156        TokenKind::LiteralNumber
157    }
158
159    /// Takes a short string.
160    fn take_token_short_string(&mut self) -> TokenKind {
161        self.take_token_string_helper('\'');
162
163        // Parse _type suffix.
164        if self.peek() == Some('_') {
165            self.take_while(|c| c.is_ascii_alphanumeric() || c == '_');
166        }
167        TokenKind::ShortString
168    }
169
170    /// Takes a string.
171    fn take_token_string(&mut self) -> TokenKind {
172        self.take_token_string_helper('"');
173        TokenKind::String
174    }
175
176    fn take_token_string_helper(&mut self, delimiter: char) {
177        self.take();
178        let mut escaped = false;
179        while let Some(token) = self.peek() {
180            self.take();
181            match token {
182                _ if escaped => escaped = false,
183                '\\' => escaped = true,
184                _ if token == delimiter => {
185                    break;
186                }
187                _ => {}
188            };
189        }
190    }
191
192    /// Assumes the next character is [a-zA-Z_].
193    fn take_token_identifier(&mut self) -> TokenKind {
194        // TODO(spapini): Support or explicitly report general unicode characters.
195        self.take_while(|c| c.is_ascii_alphanumeric() || c == '_');
196
197        let span = self.peek_text_span();
198        match span.take(&self.text) {
199            "as" => TokenKind::As,
200            "const" => TokenKind::Const,
201            "false" => TokenKind::False,
202            "true" => TokenKind::True,
203            "extern" => TokenKind::Extern,
204            "type" => TokenKind::Type,
205            "fn" => TokenKind::Function,
206            "trait" => TokenKind::Trait,
207            "impl" => TokenKind::Impl,
208            "of" => TokenKind::Of,
209            "mod" => TokenKind::Module,
210            "struct" => TokenKind::Struct,
211            "enum" => TokenKind::Enum,
212            "let" => TokenKind::Let,
213            "return" => TokenKind::Return,
214            "match" => TokenKind::Match,
215            "macro" => TokenKind::Macro,
216            "if" => TokenKind::If,
217            "loop" => TokenKind::Loop,
218            "continue" => TokenKind::Continue,
219            "break" => TokenKind::Break,
220            "else" => TokenKind::Else,
221            "while" => TokenKind::While,
222            "use" => TokenKind::Use,
223            "implicits" => TokenKind::Implicits,
224            "ref" => TokenKind::Ref,
225            "mut" => TokenKind::Mut,
226            "for" => TokenKind::For,
227            "nopanic" => TokenKind::NoPanic,
228            "pub" => TokenKind::Pub,
229            "_" => TokenKind::Underscore,
230            _ => TokenKind::Identifier,
231        }
232    }
233
234    /// Takes a single character and returns the given kind.
235    fn take_token_of_kind(&mut self, kind: TokenKind) -> TokenKind {
236        self.take();
237        kind
238    }
239
240    /// If the next character is `second_char`, returns `long_kind`, otherwise returns `short_kind`.
241    fn pick_kind(
242        &mut self,
243        second_char: char,
244        long_kind: TokenKind,
245        short_kind: TokenKind,
246    ) -> TokenKind {
247        self.take();
248        if self.peek() == Some(second_char) {
249            self.take();
250            long_kind
251        } else {
252            short_kind
253        }
254    }
255
256    pub fn match_terminal<'a>(&mut self, db: &'a dyn Database) -> LexerTerminal<'a> {
257        let leading_trivia = self.match_trivia(db, true);
258
259        let kind = if let Some(current) = self.peek() {
260            match current {
261                '0'..='9' => self.take_token_literal_number(),
262                '\'' => self.take_token_short_string(),
263                '"' => self.take_token_string(),
264                ',' => self.take_token_of_kind(TokenKind::Comma),
265                ';' => self.take_token_of_kind(TokenKind::Semicolon),
266                '?' => self.take_token_of_kind(TokenKind::QuestionMark),
267                '{' => self.take_token_of_kind(TokenKind::LBrace),
268                '}' => self.take_token_of_kind(TokenKind::RBrace),
269                '[' => self.take_token_of_kind(TokenKind::LBrack),
270                ']' => self.take_token_of_kind(TokenKind::RBrack),
271                '(' => self.take_token_of_kind(TokenKind::LParen),
272                ')' => self.take_token_of_kind(TokenKind::RParen),
273                '.' => {
274                    self.take();
275                    match self.peek() {
276                        Some('.') => self.pick_kind('=', TokenKind::DotDotEq, TokenKind::DotDot),
277                        _ => TokenKind::Dot,
278                    }
279                }
280                '*' => self.pick_kind('=', TokenKind::MulEq, TokenKind::Mul),
281                '/' => self.pick_kind('=', TokenKind::DivEq, TokenKind::Div),
282                '%' => self.pick_kind('=', TokenKind::ModEq, TokenKind::Mod),
283                '+' => self.pick_kind('=', TokenKind::PlusEq, TokenKind::Plus),
284                '#' => self.take_token_of_kind(TokenKind::Hash),
285                '$' => self.take_token_of_kind(TokenKind::Dollar),
286                '-' => {
287                    self.take();
288                    match self.peek() {
289                        Some('>') => self.take_token_of_kind(TokenKind::Arrow),
290                        Some('=') => self.take_token_of_kind(TokenKind::MinusEq),
291                        _ => TokenKind::Minus,
292                    }
293                }
294                '<' => self.pick_kind('=', TokenKind::LE, TokenKind::LT),
295                '>' => self.pick_kind('=', TokenKind::GE, TokenKind::GT),
296                'a'..='z' | 'A'..='Z' | '_' => self.take_token_identifier(),
297                ':' => self.pick_kind(':', TokenKind::ColonColon, TokenKind::Colon),
298                '!' => self.pick_kind('=', TokenKind::Neq, TokenKind::Not),
299                '~' => self.take_token_of_kind(TokenKind::BitNot),
300                '=' => {
301                    self.take();
302                    match self.peek() {
303                        Some('=') => self.take_token_of_kind(TokenKind::EqEq),
304                        Some('>') => self.take_token_of_kind(TokenKind::MatchArrow),
305                        _ => TokenKind::Eq,
306                    }
307                }
308                '&' => self.pick_kind('&', TokenKind::AndAnd, TokenKind::And),
309                '|' => self.pick_kind('|', TokenKind::OrOr, TokenKind::Or),
310                '^' => self.take_token_of_kind(TokenKind::Xor),
311                '@' => self.take_token_of_kind(TokenKind::At),
312                _ => self.take_token_of_kind(TokenKind::BadCharacters),
313            }
314        } else {
315            TokenKind::EndOfFile
316        };
317
318        let span = self.consume_text_span();
319        let text = SmolStrId::from(db, span.take(&self.text));
320        let trailing_trivia = self.match_trivia(db, false);
321        let terminal_kind = token_kind_to_terminal_syntax_kind(kind);
322
323        // TODO(yuval): log(verbose) "consumed text: ..."
324        LexerTerminal { text, kind: terminal_kind, leading_trivia, trailing_trivia }
325    }
326}
327
328/// Output terminal emitted by the lexer.
329#[derive(Clone, PartialEq, Eq, Debug, salsa::Update)]
330pub struct LexerTerminal<'a> {
331    pub text: SmolStrId<'a>,
332    /// The kind of the inner token of this terminal.
333    pub kind: SyntaxKind,
334    pub leading_trivia: Vec<TriviumGreen<'a>>,
335    pub trailing_trivia: Vec<TriviumGreen<'a>>,
336}
337impl<'a> LexerTerminal<'a> {
338    pub fn width(&self, db: &dyn Database) -> TextWidth {
339        self.leading_trivia.iter().map(|t| t.0.width(db)).sum::<TextWidth>()
340            + TextWidth::from_str(self.text.long(db))
341            + self.trailing_trivia.iter().map(|t| t.0.width(db)).sum::<TextWidth>()
342    }
343
344    pub fn text(&self, db: &'a dyn Database) -> &'a str {
345        self.text.long(db)
346    }
347}
348
349#[derive(Clone, Copy, PartialEq, Debug, Eq, Hash)]
350enum TokenKind {
351    Identifier,
352
353    // Literals.
354    LiteralNumber,
355    ShortString,
356    String,
357
358    // Keywords.
359    As,
360    Const,
361    False,
362    True,
363    Extern,
364    Type,
365    Function,
366    Trait,
367    Impl,
368    Of,
369    Module,
370    Struct,
371    Enum,
372    Let,
373    Return,
374    Match,
375    Macro,
376    If,
377    While,
378    For,
379    Loop,
380    Continue,
381    Break,
382    Else,
383    Use,
384    Implicits,
385    NoPanic,
386    Pub,
387
388    // Modifiers.
389    Ref,
390    Mut,
391
392    // Punctuation.
393    And,
394    AndAnd,
395    At,
396    Or,
397    OrOr,
398    Xor,
399    EqEq,
400    Neq,
401    GE,
402    GT,
403    LE,
404    LT,
405    Not,
406    BitNot,
407    Plus,
408    PlusEq,
409    Minus,
410    MinusEq,
411    Mul,
412    MulEq,
413    Div,
414    DivEq,
415    Mod,
416    ModEq,
417
418    Colon,
419    ColonColon,
420    Comma,
421    Dollar,
422    Dot,
423    DotDot,
424    DotDotEq,
425    Eq,
426    Hash,
427    Semicolon,
428    QuestionMark,
429    Underscore,
430    LBrace,
431    RBrace,
432    LBrack,
433    RBrack,
434    LParen,
435    RParen,
436    Arrow,
437    MatchArrow,
438
439    // Meta.
440    EndOfFile,
441    BadCharacters,
442}
443
444fn token_kind_to_terminal_syntax_kind(kind: TokenKind) -> SyntaxKind {
445    match kind {
446        TokenKind::As => SyntaxKind::TerminalAs,
447        TokenKind::Const => SyntaxKind::TerminalConst,
448        TokenKind::Identifier => SyntaxKind::TerminalIdentifier,
449        TokenKind::LiteralNumber => SyntaxKind::TerminalLiteralNumber,
450        TokenKind::ShortString => SyntaxKind::TerminalShortString,
451        TokenKind::String => SyntaxKind::TerminalString,
452        TokenKind::False => SyntaxKind::TerminalFalse,
453        TokenKind::True => SyntaxKind::TerminalTrue,
454        TokenKind::Extern => SyntaxKind::TerminalExtern,
455        TokenKind::Type => SyntaxKind::TerminalType,
456        TokenKind::Function => SyntaxKind::TerminalFunction,
457        TokenKind::Trait => SyntaxKind::TerminalTrait,
458        TokenKind::Impl => SyntaxKind::TerminalImpl,
459        TokenKind::Of => SyntaxKind::TerminalOf,
460        TokenKind::Module => SyntaxKind::TerminalModule,
461        TokenKind::Struct => SyntaxKind::TerminalStruct,
462        TokenKind::Enum => SyntaxKind::TerminalEnum,
463        TokenKind::Let => SyntaxKind::TerminalLet,
464        TokenKind::Return => SyntaxKind::TerminalReturn,
465        TokenKind::Match => SyntaxKind::TerminalMatch,
466        TokenKind::If => SyntaxKind::TerminalIf,
467        TokenKind::While => SyntaxKind::TerminalWhile,
468        TokenKind::For => SyntaxKind::TerminalFor,
469        TokenKind::Loop => SyntaxKind::TerminalLoop,
470        TokenKind::Continue => SyntaxKind::TerminalContinue,
471        TokenKind::Break => SyntaxKind::TerminalBreak,
472        TokenKind::Else => SyntaxKind::TerminalElse,
473        TokenKind::Use => SyntaxKind::TerminalUse,
474        TokenKind::Implicits => SyntaxKind::TerminalImplicits,
475        TokenKind::NoPanic => SyntaxKind::TerminalNoPanic,
476        TokenKind::Pub => SyntaxKind::TerminalPub,
477        TokenKind::Macro => SyntaxKind::TerminalMacro,
478        TokenKind::And => SyntaxKind::TerminalAnd,
479        TokenKind::AndAnd => SyntaxKind::TerminalAndAnd,
480        TokenKind::At => SyntaxKind::TerminalAt,
481        TokenKind::Or => SyntaxKind::TerminalOr,
482        TokenKind::OrOr => SyntaxKind::TerminalOrOr,
483        TokenKind::Xor => SyntaxKind::TerminalXor,
484        TokenKind::EqEq => SyntaxKind::TerminalEqEq,
485        TokenKind::Neq => SyntaxKind::TerminalNeq,
486        TokenKind::GE => SyntaxKind::TerminalGE,
487        TokenKind::GT => SyntaxKind::TerminalGT,
488        TokenKind::LE => SyntaxKind::TerminalLE,
489        TokenKind::LT => SyntaxKind::TerminalLT,
490        TokenKind::Not => SyntaxKind::TerminalNot,
491        TokenKind::BitNot => SyntaxKind::TerminalBitNot,
492        TokenKind::Plus => SyntaxKind::TerminalPlus,
493        TokenKind::PlusEq => SyntaxKind::TerminalPlusEq,
494        TokenKind::Minus => SyntaxKind::TerminalMinus,
495        TokenKind::MinusEq => SyntaxKind::TerminalMinusEq,
496        TokenKind::Mul => SyntaxKind::TerminalMul,
497        TokenKind::MulEq => SyntaxKind::TerminalMulEq,
498        TokenKind::Div => SyntaxKind::TerminalDiv,
499        TokenKind::DivEq => SyntaxKind::TerminalDivEq,
500        TokenKind::Mod => SyntaxKind::TerminalMod,
501        TokenKind::ModEq => SyntaxKind::TerminalModEq,
502        TokenKind::Colon => SyntaxKind::TerminalColon,
503        TokenKind::ColonColon => SyntaxKind::TerminalColonColon,
504        TokenKind::Comma => SyntaxKind::TerminalComma,
505        TokenKind::Dollar => SyntaxKind::TerminalDollar,
506        TokenKind::Dot => SyntaxKind::TerminalDot,
507        TokenKind::DotDot => SyntaxKind::TerminalDotDot,
508        TokenKind::DotDotEq => SyntaxKind::TerminalDotDotEq,
509        TokenKind::Eq => SyntaxKind::TerminalEq,
510        TokenKind::Hash => SyntaxKind::TerminalHash,
511        TokenKind::Semicolon => SyntaxKind::TerminalSemicolon,
512        TokenKind::QuestionMark => SyntaxKind::TerminalQuestionMark,
513        TokenKind::Underscore => SyntaxKind::TerminalUnderscore,
514        TokenKind::LBrace => SyntaxKind::TerminalLBrace,
515        TokenKind::RBrace => SyntaxKind::TerminalRBrace,
516        TokenKind::LBrack => SyntaxKind::TerminalLBrack,
517        TokenKind::RBrack => SyntaxKind::TerminalRBrack,
518        TokenKind::LParen => SyntaxKind::TerminalLParen,
519        TokenKind::RParen => SyntaxKind::TerminalRParen,
520        TokenKind::Ref => SyntaxKind::TerminalRef,
521        TokenKind::Mut => SyntaxKind::TerminalMut,
522        TokenKind::Arrow => SyntaxKind::TerminalArrow,
523        TokenKind::MatchArrow => SyntaxKind::TerminalMatchArrow,
524        TokenKind::BadCharacters => SyntaxKind::TerminalBadCharacters,
525        TokenKind::EndOfFile => SyntaxKind::TerminalEndOfFile,
526    }
527}
cairo_lang_parser/lexer.rs

cairo_lang_parser/
lexer.rs
