1use anyhow::{Context, Result};
2
3use super::utils::{compute_indent, is_identifier_start, lex_identifier, lex_number};
4use crate::language::syntax::tokens::{Keyword, Token, TokenKind, keyword_from_ident};
5
6#[derive(Debug, Default)]
7pub struct Lexer {
8 source: String,
9}
10
11impl Lexer {
12 pub fn new(content: impl Into<String>) -> Self {
13 Self {
14 source: content.into(),
15 }
16 }
17
18 pub fn with_source(mut self, content: impl Into<String>) -> Self {
19 self.source = content.into();
20 self
21 }
22
23 pub fn lex(self) -> Result<Vec<Token>> {
24 lex_source(&self.source)
25 }
26}
27
28fn lex_source(source: &str) -> Result<Vec<Token>> {
29 let mut tokens = Vec::new();
30 let mut indent_stack = vec![0usize];
31
32 for (line_idx, raw_line) in source.lines().enumerate() {
33 let (indent_level, cursor_start) = compute_indent(raw_line);
34 let trimmed = raw_line[cursor_start..].trim_end();
35
36 let current_indent = *indent_stack.last().context("indent stack corruption")?;
37 let line_number = line_idx + 1;
38
39 if indent_level > current_indent {
40 indent_stack.push(indent_level);
41 tokens.push(Token::new(
42 TokenKind::Indent,
43 String::new(),
44 indent_level,
45 line_number,
46 1,
47 ));
48 } else {
49 while indent_stack
50 .last()
51 .is_some_and(|last_indent| indent_level < *last_indent)
52 {
53 indent_stack.pop();
54 tokens.push(Token::new(
55 TokenKind::Dedent,
56 String::new(),
57 indent_level,
58 line_number,
59 1,
60 ));
61 }
62 }
63
64 let mut cursor = cursor_start;
65 let bytes = raw_line.as_bytes();
66 let len = raw_line.len();
67
68 while cursor < len {
69 let ch = raw_line.as_bytes()[cursor];
70 let column = cursor + 1;
71
72 match ch {
73 b' ' | b'\t' => {
74 cursor += 1;
75 }
76 b'#' => {
77 if !trimmed.is_empty() {
79 tokens.push(Token::new(
80 TokenKind::Comment,
81 raw_line[cursor..].trim().to_string(),
82 indent_level,
83 line_number,
84 column,
85 ));
86 }
87 break;
88 }
89 b'"' | b'\'' => {
90 let quote = ch as char;
91 let mut end = cursor + 1;
92 let mut escaped = false;
93 while end < len {
94 let c = bytes[end] as char;
95 if c == quote && !escaped {
96 end += 1;
97 break;
98 }
99 escaped = !escaped && c == '\\';
100 end += 1;
101 }
102 let lexeme = &raw_line[cursor..end];
103 tokens.push(Token::new(
104 TokenKind::String,
105 lexeme,
106 indent_level,
107 line_number,
108 column,
109 ));
110 cursor = end;
111 }
112 b'0'..=b'9' => {
113 let (end, kind) = lex_number(raw_line, cursor);
114 let lexeme = &raw_line[cursor..end];
115 tokens.push(Token::new(kind, lexeme, indent_level, line_number, column));
116 cursor = end;
117 }
118 b'@' => {
119 tokens.push(Token::new(
120 TokenKind::Keyword(Keyword::At),
121 "@",
122 indent_level,
123 line_number,
124 column,
125 ));
126 cursor += 1;
127 }
128 b'-' => {
129 if cursor + 1 < len && bytes[cursor + 1] == b'>' {
130 tokens.push(Token::new(
131 TokenKind::Arrow,
132 "->",
133 indent_level,
134 line_number,
135 column,
136 ));
137 cursor += 2;
138 } else {
139 tokens.push(Token::new(
140 TokenKind::Minus,
141 "-",
142 indent_level,
143 line_number,
144 column,
145 ));
146 cursor += 1;
147 }
148 }
149 b'=' => {
150 if cursor + 1 < len && bytes[cursor + 1] == b'=' {
151 tokens.push(Token::new(
152 TokenKind::DoubleEquals,
153 "==",
154 indent_level,
155 line_number,
156 column,
157 ));
158 cursor += 2;
159 } else {
160 tokens.push(Token::new(
161 TokenKind::Equals,
162 "=",
163 indent_level,
164 line_number,
165 column,
166 ));
167 cursor += 1;
168 }
169 }
170 b'!' => {
171 if cursor + 1 < len && bytes[cursor + 1] == b'=' {
172 tokens.push(Token::new(
173 TokenKind::NotEquals,
174 "!=",
175 indent_level,
176 line_number,
177 column,
178 ));
179 cursor += 2;
180 } else {
181 tokens.push(Token::new(
182 TokenKind::Unknown,
183 "!",
184 indent_level,
185 line_number,
186 column,
187 ));
188 cursor += 1;
189 }
190 }
191 b'>' => {
192 if cursor + 1 < len && bytes[cursor + 1] == b'=' {
193 tokens.push(Token::new(
194 TokenKind::GreaterEqual,
195 ">=",
196 indent_level,
197 line_number,
198 column,
199 ));
200 cursor += 2;
201 } else {
202 tokens.push(Token::new(
203 TokenKind::Greater,
204 ">",
205 indent_level,
206 line_number,
207 column,
208 ));
209 cursor += 1;
210 }
211 }
212 b'<' => {
213 if cursor + 1 < len && bytes[cursor + 1] == b'=' {
214 tokens.push(Token::new(
215 TokenKind::LessEqual,
216 "<=",
217 indent_level,
218 line_number,
219 column,
220 ));
221 cursor += 2;
222 } else {
223 tokens.push(Token::new(
224 TokenKind::Less,
225 "<",
226 indent_level,
227 line_number,
228 column,
229 ));
230 cursor += 1;
231 }
232 }
233 b'{' => {
234 tokens.push(Token::new(
235 TokenKind::LBrace,
236 "{",
237 indent_level,
238 line_number,
239 column,
240 ));
241 cursor += 1;
242 }
243 b'}' => {
244 tokens.push(Token::new(
245 TokenKind::RBrace,
246 "}",
247 indent_level,
248 line_number,
249 column,
250 ));
251 cursor += 1;
252 }
253 b'[' => {
254 tokens.push(Token::new(
255 TokenKind::LBracket,
256 "[",
257 indent_level,
258 line_number,
259 column,
260 ));
261 cursor += 1;
262 }
263 b']' => {
264 tokens.push(Token::new(
265 TokenKind::RBracket,
266 "]",
267 indent_level,
268 line_number,
269 column,
270 ));
271 cursor += 1;
272 }
273 b'(' => {
274 tokens.push(Token::new(
275 TokenKind::LParen,
276 "(",
277 indent_level,
278 line_number,
279 column,
280 ));
281 cursor += 1;
282 }
283 b')' => {
284 tokens.push(Token::new(
285 TokenKind::RParen,
286 ")",
287 indent_level,
288 line_number,
289 column,
290 ));
291 cursor += 1;
292 }
293 b',' => {
294 tokens.push(Token::new(
295 TokenKind::Comma,
296 ",",
297 indent_level,
298 line_number,
299 column,
300 ));
301 cursor += 1;
302 }
303 b':' => {
304 tokens.push(Token::new(
305 TokenKind::Colon,
306 ":",
307 indent_level,
308 line_number,
309 column,
310 ));
311 cursor += 1;
312 }
313 b'+' => {
314 tokens.push(Token::new(
315 TokenKind::Plus,
316 "+",
317 indent_level,
318 line_number,
319 column,
320 ));
321 cursor += 1;
322 }
323 b'*' => {
324 tokens.push(Token::new(
325 TokenKind::Asterisk,
326 "*",
327 indent_level,
328 line_number,
329 column,
330 ));
331 cursor += 1;
332 }
333 b'/' => {
334 tokens.push(Token::new(
335 TokenKind::Slash,
336 "/",
337 indent_level,
338 line_number,
339 column,
340 ));
341 cursor += 1;
342 }
343 b'.' => {
344 tokens.push(Token::new(
345 TokenKind::Dot,
346 ".",
347 indent_level,
348 line_number,
349 column,
350 ));
351 cursor += 1;
352 }
353 _ => {
354 if is_identifier_start(ch as char) {
355 let end = lex_identifier(raw_line, cursor);
356 let ident = &raw_line[cursor..end];
357 let lower = ident.to_ascii_lowercase();
358 let kind = if let Some(keyword) = keyword_from_ident(&lower) {
359 TokenKind::Keyword(keyword)
360 } else if lower == "true" || lower == "false" {
361 TokenKind::Boolean
362 } else {
363 TokenKind::Identifier
364 };
365 tokens.push(Token::new(kind, ident, indent_level, line_number, column));
366 cursor = end;
367 } else {
368 tokens.push(Token::new(
369 TokenKind::Unknown,
370 (ch as char).to_string(),
371 indent_level,
372 line_number,
373 column,
374 ));
375 cursor += 1;
376 }
377 }
378 }
379 }
380
381 if !trimmed.is_empty() {
382 tokens.push(Token::new(
383 TokenKind::Newline,
384 "\\n",
385 indent_level,
386 line_number,
387 raw_line.len() + 1,
388 ));
389 }
390 }
391
392 while indent_stack.len() > 1 {
393 indent_stack.pop();
394 tokens.push(Token::new(
395 TokenKind::Dedent,
396 String::new(),
397 0,
398 source.lines().count() + 1,
399 1,
400 ));
401 }
402
403 tokens.push(Token::new(
404 TokenKind::Eof,
405 String::new(),
406 0,
407 source.lines().count() + 1,
408 1,
409 ));
410
411 Ok(tokens)
412}