oak_racket/lexer/
mod.rs

1use oak_core::{
2    Range,
3    lexer::{LexOutput, Lexer as CoreLexer, LexerCache, LexerState},
4    source::{Source, TextEdit},
5};
6
7use crate::language::RacketLanguage;
8mod token_type;
9pub use token_type::TokenType;
10
11/// Lexer for Racket source code.
12pub struct Lexer;
13
14impl CoreLexer<RacketLanguage> for Lexer {
15    fn lex<'a, S: Source + ?Sized>(&self, text: &S, edits: &[TextEdit], cache: &'a mut impl LexerCache<RacketLanguage>) -> LexOutput<RacketLanguage> {
16        let mut state = LexerState::new_with_cache(text, edits.last().map(|edit| edit.span.start).unwrap_or(0), cache);
17
18        while state.not_at_end() {
19            let safe_point = state.get_position();
20
21            let whitespace_range = state.skip_ascii_whitespace();
22            if whitespace_range.end > whitespace_range.start {
23                state.add_token(TokenType::Whitespace, whitespace_range.start, whitespace_range.end);
24                continue;
25            }
26
27            if state.scan_line_comment(TokenType::Comment, "//") {
28                continue;
29            }
30
31            if let Some(ch) = state.peek() {
32                match ch {
33                    '(' => {
34                        state.advance(1);
35                        state.add_token(TokenType::LParen, safe_point, state.get_position());
36                    }
37                    ')' => {
38                        state.advance(1);
39                        state.add_token(TokenType::RParen, safe_point, state.get_position());
40                    }
41                    '[' => {
42                        state.advance(1);
43                        state.add_token(TokenType::LBracket, safe_point, state.get_position());
44                    }
45                    ']' => {
46                        state.advance(1);
47                        state.add_token(TokenType::RBracket, safe_point, state.get_position());
48                    }
49                    '{' => {
50                        state.advance(1);
51                        state.add_token(TokenType::LBrace, safe_point, state.get_position());
52                    }
53                    '}' => {
54                        state.advance(1);
55                        state.add_token(TokenType::RBrace, safe_point, state.get_position());
56                    }
57                    ',' => {
58                        state.advance(1);
59                        state.add_token(TokenType::Comma, safe_point, state.get_position());
60                    }
61                    '.' => {
62                        state.advance(1);
63                        state.add_token(TokenType::Dot, safe_point, state.get_position());
64                    }
65                    ':' => {
66                        state.advance(1);
67                        state.add_token(TokenType::Colon, safe_point, state.get_position());
68                    }
69                    ';' => {
70                        state.advance(1);
71                        state.add_token(TokenType::Semicolon, safe_point, state.get_position());
72                    }
73                    '+' => {
74                        state.advance(1);
75                        state.add_token(TokenType::Plus, safe_point, state.get_position());
76                    }
77                    '-' => {
78                        state.advance(1);
79                        state.add_token(TokenType::Minus, safe_point, state.get_position());
80                    }
81                    '*' => {
82                        state.advance(1);
83                        state.add_token(TokenType::Multiply, safe_point, state.get_position());
84                    }
85                    '/' => {
86                        state.advance(1);
87                        state.add_token(TokenType::Divide, safe_point, state.get_position());
88                    }
89                    '%' => {
90                        state.advance(1);
91                        state.add_token(TokenType::Modulo, safe_point, state.get_position());
92                    }
93                    '=' => {
94                        state.advance(1);
95                        state.add_token(TokenType::Equals, safe_point, state.get_position());
96                    }
97                    '!' => {
98                        state.advance(1);
99                        if state.starts_with("=") {
100                            state.advance(1);
101                            state.add_token(TokenType::NotEquals, safe_point, state.get_position());
102                        }
103                        else {
104                            state.add_token(TokenType::Not, safe_point, state.get_position());
105                        }
106                    }
107                    '<' => {
108                        state.advance(1);
109                        if state.starts_with("=") {
110                            state.advance(1);
111                            state.add_token(TokenType::LessThanOrEqual, safe_point, state.get_position());
112                        }
113                        else {
114                            state.add_token(TokenType::LessThan, safe_point, state.get_position());
115                        }
116                    }
117                    '>' => {
118                        state.advance(1);
119                        if state.starts_with("=") {
120                            state.advance(1);
121                            state.add_token(TokenType::GreaterThanOrEqual, safe_point, state.get_position());
122                        }
123                        else {
124                            state.add_token(TokenType::GreaterThan, safe_point, state.get_position());
125                        }
126                    }
127                    'a'..='z' | 'A'..='Z' | '_' => {
128                        let start = state.get_position();
129                        state.advance(1);
130                        state.skip_ascii_ident_continue();
131                        let end = state.get_position();
132                        let range = Range { start, end };
133                        let identifier = state.get_text_in(range).to_string();
134
135                        let token_type = match identifier.as_str() {
136                            "for" => TokenType::For,
137                            "in" => TokenType::In,
138                            _ => TokenType::Identifier,
139                        };
140
141                        state.add_token(token_type, start, end);
142                    }
143                    '0'..='9' => {
144                        let start = state.get_position();
145                        state.advance(1);
146                        while state.not_at_end() {
147                            if let Some(ch) = state.peek() {
148                                if ch.is_ascii_digit() || ch == '.' {
149                                    state.advance(1);
150                                }
151                                else {
152                                    break;
153                                }
154                            }
155                            else {
156                                break;
157                            }
158                        }
159                        let end = state.get_position();
160                        state.add_token(TokenType::Number, start, end);
161                    }
162                    '"' => {
163                        let start = state.get_position();
164                        state.advance(1);
165                        while state.not_at_end() {
166                            if let Some(ch) = state.peek() {
167                                if ch != '"' {
168                                    state.advance(1);
169                                }
170                                else {
171                                    state.advance(1);
172                                    break;
173                                }
174                            }
175                            else {
176                                break;
177                            }
178                        }
179                        let end = state.get_position();
180                        state.add_token(TokenType::String, start, end);
181                    }
182                    _ => {
183                        state.advance(1);
184                        state.add_token(TokenType::Identifier, safe_point, state.get_position());
185                    }
186                }
187            }
188
189            state.advance_if_dead_lock(safe_point);
190        }
191
192        state.add_eof();
193        state.finish_with_cache(Ok(()), cache)
194    }
195}
oak_racket/lexer/mod.rs

oak_racket/lexer/
mod.rs
