1use crate::lexer::codes::{is_token_type, token_type};
2use crate::lexer::cursor::{Cursor, CursorItem};
3use crate::lexer::error::LexerResult;
4use crate::lexer::token::{
5 Bracket, ComparisonOperator, Identifier, LogicalOperator, Operator, Token, TokenKind,
6};
7use crate::lexer::{LexerError, QuotationMark, TemplateString};
8use bumpalo::collections::Vec as BumpVec;
9use bumpalo::Bump;
10use std::str::FromStr;
11
12#[derive(Debug, Default)]
13pub struct Lexer {
14 capacity_hint: usize,
15}
16
17impl Lexer {
18 pub fn new() -> Self {
19 Self::default()
20 }
21
22 pub fn tokenize<'arena>(
23 &mut self,
24 bump: &'arena Bump,
25 source: &'arena str,
26 ) -> LexerResult<BumpVec<'arena, Token<'arena>>> {
27 let mut tokens = BumpVec::with_capacity_in(self.capacity_hint, bump);
28
29 Scanner::new(source, &mut tokens).scan()?;
30 self.capacity_hint = self.capacity_hint.max(tokens.len());
31 Ok(tokens)
32 }
33}
34
35struct Scanner<'arena, 'self_ref> {
36 cursor: Cursor<'arena>,
37 tokens: &'self_ref mut BumpVec<'arena, Token<'arena>>,
38 source: &'arena str,
39}
40
41impl<'arena, 'self_ref> Scanner<'arena, 'self_ref> {
42 pub fn new(source: &'arena str, tokens: &'self_ref mut BumpVec<'arena, Token<'arena>>) -> Self {
43 Self {
44 cursor: Cursor::from(source),
45 source,
46 tokens,
47 }
48 }
49
50 pub fn scan(&mut self) -> LexerResult<()> {
51 while let Some(cursor_item) = self.cursor.peek() {
52 self.scan_cursor_item(cursor_item)?;
53 }
54
55 Ok(())
56 }
57
58 pub(crate) fn scan_cursor_item(&mut self, cursor_item: CursorItem) -> LexerResult<()> {
59 let (i, s) = cursor_item;
60
61 match s {
62 token_type!("space") => {
63 self.cursor.next();
64 Ok(())
65 }
66 '\'' => self.string(QuotationMark::SingleQuote),
67 '"' => self.string(QuotationMark::DoubleQuote),
68 token_type!("digit") => self.number(),
69 token_type!("bracket") => self.bracket(),
70 token_type!("cmp_operator") => self.cmp_operator(),
71 token_type!("operator") => self.operator(),
72 token_type!("question_mark") => self.question_mark(),
73 '=' => self.equals(),
74 '`' => self.template_string(),
75 '.' => self.dot(),
76 ';' => self.semi(),
77 token_type!("alpha") => self.identifier(),
78 _ => Err(LexerError::UnmatchedSymbol {
79 symbol: s,
80 position: i as u32,
81 }),
82 }
83 }
84
85 fn next(&self) -> LexerResult<CursorItem> {
86 self.cursor.next().ok_or_else(|| {
87 let (a, b) = self.cursor.peek_back().unwrap_or((0, ' '));
88
89 LexerError::UnexpectedEof {
90 symbol: b,
91 position: a as u32,
92 }
93 })
94 }
95
96 fn push(&mut self, token: Token<'arena>) {
97 self.tokens.push(token);
98 }
99
100 fn template_string(&mut self) -> LexerResult<()> {
101 let (start, _) = self.next()?;
102
103 self.tokens.push(Token {
104 kind: TokenKind::QuotationMark(QuotationMark::Backtick),
105 span: (start as u32, (start + 1) as u32),
106 value: QuotationMark::Backtick.into(),
107 });
108
109 let mut in_expression = false;
110 let mut str_start = start + 1;
111 loop {
112 let Some((e, c)) = self.cursor.next() else {
113 let end = self.source.len();
114 if !in_expression && str_start < end {
115 self.tokens.push(Token {
116 kind: TokenKind::Literal,
117 span: (str_start as u32, end as u32),
118 value: &self.source[str_start..end],
119 });
120 }
121
122 self.tokens.push(Token {
123 kind: TokenKind::QuotationMark(QuotationMark::Backtick),
124 span: (end as u32, end as u32),
125 value: QuotationMark::Backtick.into(),
126 });
127
128 break;
129 };
130
131 match (c, in_expression) {
132 ('`', _) => {
133 if str_start < e {
134 self.tokens.push(Token {
135 kind: TokenKind::Literal,
136 span: (str_start as u32, e as u32),
137 value: &self.source[str_start..e],
138 });
139 }
140
141 self.tokens.push(Token {
142 kind: TokenKind::QuotationMark(QuotationMark::Backtick),
143 span: (e as u32, (e + 1) as u32),
144 value: QuotationMark::Backtick.into(),
145 });
146
147 break;
148 }
149 ('$', false) => {
150 in_expression = self.cursor.next_if_is("{");
151 if in_expression {
152 self.tokens.push(Token {
153 kind: TokenKind::Literal,
154 span: (str_start as u32, e as u32),
155 value: &self.source[str_start..e],
156 });
157
158 self.tokens.push(Token {
159 kind: TokenKind::TemplateString(TemplateString::ExpressionStart),
160 span: (e as u32, (e + 2) as u32),
161 value: TemplateString::ExpressionStart.into(),
162 });
163 }
164 }
165 ('}', true) => {
166 in_expression = false;
167 self.tokens.push(Token {
168 kind: TokenKind::TemplateString(TemplateString::ExpressionEnd),
169 span: (str_start as u32, e as u32),
170 value: TemplateString::ExpressionEnd.into(),
171 });
172
173 str_start = e + 1;
174 }
175 (_, false) => {
176 }
178 (_, true) => {
179 self.cursor.back();
180 self.scan_cursor_item((e, c))?;
181 }
182 }
183 }
184
185 Ok(())
186 }
187
188 fn string(&mut self, quote_kind: QuotationMark) -> LexerResult<()> {
189 let (start, opener) = self.next()?;
190 let end: usize;
191
192 loop {
193 let (e, c) = self.next()?;
194 if c == opener {
195 end = e;
196 break;
197 }
198 }
199
200 self.push(Token {
201 kind: TokenKind::QuotationMark(quote_kind),
202 span: (start as u32, (start + 1) as u32),
203 value: quote_kind.into(),
204 });
205
206 self.push(Token {
207 kind: TokenKind::Literal,
208 span: ((start + 1) as u32, end as u32),
209 value: &self.source[start + 1..end],
210 });
211
212 self.push(Token {
213 kind: TokenKind::QuotationMark(quote_kind),
214 span: (end as u32, (end + 1) as u32),
215 value: quote_kind.into(),
216 });
217
218 Ok(())
219 }
220
221 fn number(&mut self) -> LexerResult<()> {
222 let (start, _) = self.next()?;
223 let mut end = start;
224 let mut fractal = false;
225
226 while let Some((e, c)) = self
227 .cursor
228 .next_if(|c| is_token_type!(c, "digit") || c == '_' || c == '.')
229 {
230 if fractal && c == '.' {
231 self.cursor.back();
232 break;
233 }
234
235 if c == '.' {
236 if let Some((_, p)) = self.cursor.peek() {
237 if p == '.' {
238 self.cursor.back();
239 break;
240 }
241
242 fractal = true
243 }
244 }
245
246 end = e;
247 }
248
249 if let Some((e_pos, _)) = self.cursor.next_if(|c| c == 'e') {
250 end = e_pos;
251
252 if let Some((sign_pos, _)) = self.cursor.next_if(|c| c == '+' || c == '-') {
253 end = sign_pos;
254 }
255
256 let mut has_exponent_digits = false;
257 while let Some((exp_pos, _)) = self.cursor.next_if(|c| is_token_type!(c, "digit")) {
258 end = exp_pos;
259 has_exponent_digits = true;
260 }
261
262 if !has_exponent_digits {
263 while self.cursor.position() > e_pos {
264 self.cursor.back();
265 }
266
267 end = e_pos - 1;
268 }
269 }
270
271 self.push(Token {
272 kind: TokenKind::Number,
273 span: (start as u32, (end + 1) as u32),
274 value: &self.source[start..=end],
275 });
276
277 Ok(())
278 }
279
280 fn bracket(&mut self) -> LexerResult<()> {
281 let (start, _) = self.next()?;
282
283 let value = &self.source[start..=start];
284 let span = (start as u32, (start + 1) as u32);
285 self.push(Token {
286 kind: TokenKind::Bracket(Bracket::from_str(value).map_err(|_| {
287 LexerError::UnexpectedSymbol {
288 symbol: value.to_string(),
289 span,
290 }
291 })?),
292 span,
293 value,
294 });
295
296 Ok(())
297 }
298
299 fn dot(&mut self) -> LexerResult<()> {
300 let (start, _) = self.next()?;
301 let mut end = start;
302
303 if self.cursor.next_if(|c| c == '.').is_some() {
304 end += 1;
305 }
306
307 let value = &self.source[start..=end];
308 let span = (start as u32, (end + 1) as u32);
309 self.push(Token {
310 kind: TokenKind::Operator(Operator::from_str(value).map_err(|_| {
311 LexerError::UnexpectedSymbol {
312 symbol: value.to_string(),
313 span,
314 }
315 })?),
316 span,
317 value,
318 });
319
320 Ok(())
321 }
322
323 fn cmp_operator(&mut self) -> LexerResult<()> {
324 let (start, _) = self.next()?;
325 let mut end = start;
326
327 if self.cursor.next_if(|c| c == '=').is_some() {
328 end += 1;
329 }
330
331 let value = &self.source[start..=end];
332 self.push(Token {
333 kind: TokenKind::Operator(Operator::from_str(value).map_err(|_| {
334 LexerError::UnexpectedSymbol {
335 symbol: value.to_string(),
336 span: (start as u32, (end + 1) as u32),
337 }
338 })?),
339 span: (start as u32, (end + 1) as u32),
340 value,
341 });
342
343 Ok(())
344 }
345
346 fn semi(&mut self) -> LexerResult<()> {
347 let (start, _) = self.next()?;
348 self.push(Token {
349 kind: TokenKind::Operator(Operator::Semi),
350 span: (start as u32, (start + 1) as u32),
351 value: &self.source[start..=start],
352 });
353
354 Ok(())
355 }
356
357 fn equals(&mut self) -> LexerResult<()> {
358 let (start, _) = self.next()?;
359 let Some((end, _)) = self.cursor.next_if(|c| c == '=') else {
360 self.push(Token {
361 kind: TokenKind::Operator(Operator::Assign),
362 span: (start as u32, (start + 1) as u32),
363 value: &self.source[start..=start],
364 });
365
366 return Ok(());
367 };
368
369 self.push(Token {
370 kind: TokenKind::Operator(Operator::Comparison(ComparisonOperator::Equal)),
371 span: (start as u32, (end + 1) as u32),
372 value: &self.source[start..=end],
373 });
374
375 Ok(())
376 }
377
378 fn question_mark(&mut self) -> LexerResult<()> {
379 let (start, _) = self.next()?;
380 let mut kind = TokenKind::Operator(Operator::QuestionMark);
381 let mut end = start;
382
383 if self.cursor.next_if(|c| c == '?').is_some() {
384 kind = TokenKind::Operator(Operator::Logical(LogicalOperator::NullishCoalescing));
385 end += 1;
386 }
387
388 let value = &self.source[start..=end];
389 self.push(Token {
390 kind,
391 value,
392 span: (start as u32, (end + 1) as u32),
393 });
394
395 Ok(())
396 }
397
398 fn operator(&mut self) -> LexerResult<()> {
399 let (start, _) = self.next()?;
400
401 let value = &self.source[start..=start];
402 let span = (start as u32, (start + 1) as u32);
403 self.push(Token {
404 kind: TokenKind::Operator(Operator::from_str(value).map_err(|_| {
405 LexerError::UnexpectedSymbol {
406 symbol: value.to_string(),
407 span,
408 }
409 })?),
410 span,
411 value,
412 });
413
414 Ok(())
415 }
416
417 fn not(&mut self, start: usize) -> LexerResult<()> {
418 if self.cursor.next_if_is(" in ") {
419 let end = self.cursor.position();
420
421 self.push(Token {
422 kind: TokenKind::Operator(Operator::Comparison(ComparisonOperator::NotIn)),
423 span: (start as u32, (end - 1) as u32),
424 value: "not in",
425 })
426 } else {
427 let end = self.cursor.position();
428
429 self.push(Token {
430 kind: TokenKind::Operator(Operator::Logical(LogicalOperator::Not)),
431 span: (start as u32, end as u32),
432 value: "not",
433 })
434 }
435
436 Ok(())
437 }
438
439 fn identifier(&mut self) -> LexerResult<()> {
440 let (start, _) = self.next()?;
441 let mut end = start;
442
443 while let Some((e, _)) = self.cursor.next_if(|c| is_token_type!(c, "alphanumeric")) {
444 end = e;
445 }
446
447 let value = &self.source[start..=end];
448 match value {
449 "and" => self.push(Token {
450 kind: TokenKind::Operator(Operator::Logical(LogicalOperator::And)),
451 span: (start as u32, (end + 1) as u32),
452 value,
453 }),
454 "or" => self.push(Token {
455 kind: TokenKind::Operator(Operator::Logical(LogicalOperator::Or)),
456 span: (start as u32, (end + 1) as u32),
457 value,
458 }),
459 "in" => self.push(Token {
460 kind: TokenKind::Operator(Operator::Comparison(ComparisonOperator::In)),
461 span: (start as u32, (end + 1) as u32),
462 value,
463 }),
464 "true" => self.push(Token {
465 kind: TokenKind::Boolean(true),
466 span: (start as u32, (end + 1) as u32),
467 value,
468 }),
469 "false" => self.push(Token {
470 kind: TokenKind::Boolean(false),
471 span: (start as u32, (end + 1) as u32),
472 value,
473 }),
474 "not" => self.not(start)?,
475 _ => self.push(Token {
476 kind: Identifier::try_from(value)
477 .map(|identifier| TokenKind::Identifier(identifier))
478 .unwrap_or(TokenKind::Literal),
479 span: (start as u32, (end + 1) as u32),
480 value,
481 }),
482 }
483
484 Ok(())
485 }
486}