qanglang_core/frontend/
tokenizer.rs

1use std::{collections::VecDeque, sync::Arc};
2
3use phf::phf_map;
4
5use crate::SourceMap;
6
7#[derive(PartialEq, Clone, Debug, Copy)]
8#[repr(u8)]
9pub enum TokenType {
10    EqualsEquals,       // ==
11    Equals,             // =
12    Bang,               // !
13    BangEquals,         // !=
14    Less,               // <
15    LessEquals,         // <=
16    GreaterEquals,      // >=
17    Comma,              // ,
18    Dot,                // .
19    Arrow,              // ->
20    Colon,              // :
21    Question,           // ?
22    Plus,               // +
23    Minus,              // -
24    Star,               // *
25    Slash,              // /
26    Modulo,             // %
27    Semicolon,          // ;
28    Greater,            // >
29    LeftParen,          // ()
30    RightParen,         // )
31    LeftBrace,          // {
32    RightBrace,         // }
33    LeftSquareBracket,  // [
34    RightSquareBracket, // [
35    Identifier,         // any identifier that is not covered by the other types
36    String,             // any string literal
37    Number,             // any number literal
38    And,                // and
39    Class,              // class
40    Else,               // else
41    False,              // false
42    For,                // for
43    Fn,                 // fn
44    If,                 // if
45    Nil,                // nil
46    Or,                 // or
47    Return,             // return
48    Super,              // super
49    This,               // this
50    True,               // true
51    Var,                // var
52    While,              // while
53    Break,              // break
54    Continue,           // continue
55    Pipe,               // |>
56    DoubleLeftBrace,    // {{
57    DoubleRightBrace,   // }}
58    Bar,                // |
59    DoubleBar,          // ||
60    OptionalBar,        // ?|
61    OptionalDot,        // ?.
62    Is,                 // is
63    PlusAssign,         // +=
64    MinusAssign,        // -=
65    StarAssign,         // *=
66    SlashAssign,        // /=
67    ModuloAssign,       // %=
68    Mod,                // mod
69    Import,             // import,
70    When,               // when
71    FatArrow,           // =>
72    Error,              // use when an error occurs during tokenization
73    Eof,                // EoF
74}
75
76impl TokenType {
77    pub fn from_keyword(keyword: &str) -> TokenType {
78        *KEYWORDS.get(keyword).unwrap_or(&TokenType::Identifier)
79    }
80}
81
82static KEYWORDS: phf::Map<&'static str, TokenType> = phf_map! {
83    "and" => TokenType::And,
84    "class" => TokenType::Class,
85    "else" => TokenType::Else,
86    "false" => TokenType::False,
87    "for" => TokenType::For,
88    "continue" => TokenType::Continue,
89    "true" => TokenType::True,
90    "var" => TokenType::Var,
91    "while" => TokenType::While,
92    "fn" => TokenType::Fn,
93    "if" => TokenType::If,
94    "nil" => TokenType::Nil,
95    "or" => TokenType::Or,
96    "break" => TokenType::Break,
97    "super" => TokenType::Super,
98    "this" => TokenType::This,
99    "return" => TokenType::Return,
100    "is" => TokenType::Is,
101    "mod" => TokenType::Mod,
102    "import" => TokenType::Import,
103    "when" => TokenType::When,
104};
105
106#[derive(PartialEq, Clone, Debug)]
107pub struct Token {
108    pub token_type: TokenType,
109    pub start: usize,
110    pub end: usize,
111    pub error_message: Option<String>,
112}
113
114impl Token {
115    pub fn lexeme<'a>(&self, source_map: &'a SourceMap) -> &'a [char] {
116        &source_map.get_source()[self.start..self.end]
117    }
118    pub fn line(&self, source_map: &SourceMap) -> u32 {
119        source_map.get_line_number(self.start)
120    }
121
122    pub fn col(&self, source_map: &SourceMap) -> u32 {
123        source_map.get_column_number(self.start)
124    }
125}
126
127#[derive(Debug, Clone)]
128pub struct Tokenizer {
129    source_map: Arc<SourceMap>,
130    line: u32,
131    col: u32,
132    location: usize,
133    is_eof: bool,
134    lookahead_buffer: VecDeque<Token>,
135}
136
137impl Tokenizer {
138    pub fn new(source_map: Arc<SourceMap>) -> Self {
139        Self {
140            source_map,
141            line: 1,
142            col: 1,
143            location: 0,
144            is_eof: false,
145            lookahead_buffer: VecDeque::new(),
146        }
147    }
148
149    pub fn peek_ahead(&mut self, n: usize) -> Option<&Token> {
150        while self.lookahead_buffer.len() <= n {
151            if let Some(token) = self.next_token() {
152                self.lookahead_buffer.push_back(token);
153            } else {
154                break;
155            }
156        }
157        self.lookahead_buffer.get(n)
158    }
159
160    pub fn peek(&mut self) -> Option<&Token> {
161        self.peek_ahead(0)
162    }
163
164    #[allow(dead_code)]
165    pub fn peek_next(&mut self) -> Option<&Token> {
166        self.peek_ahead(1)
167    }
168
169    fn next_token(&mut self) -> Option<Token> {
170        self.skip_whitespace();
171
172        if self.is_at_end() {
173            if self.is_eof {
174                return None;
175            } else {
176                self.is_eof = true;
177                return self.make_token(TokenType::Eof, self.location);
178            }
179        }
180
181        let start = self.location;
182        let c = self.advance();
183
184        match c {
185            '(' => self.make_token(TokenType::LeftParen, start),
186            ')' => self.make_token(TokenType::RightParen, start),
187            '!' if self.match_char('=') => self.make_token(TokenType::BangEquals, start),
188            '!' => self.make_token(TokenType::Bang, start),
189            '<' if self.match_char('=') => self.make_token(TokenType::LessEquals, start),
190            '<' => self.make_token(TokenType::Less, start),
191            '>' if self.match_char('=') => self.make_token(TokenType::GreaterEquals, start),
192            '>' => self.make_token(TokenType::Greater, start),
193            '=' if self.match_char('=') => self.make_token(TokenType::EqualsEquals, start),
194            '=' if self.match_char('>') => self.make_token(TokenType::FatArrow, start),
195            '=' => self.make_token(TokenType::Equals, start),
196            '-' if self.match_char('>') => self.make_token(TokenType::Arrow, start),
197            '-' if self.match_char('=') => self.make_token(TokenType::MinusAssign, start),
198            '-' => self.make_token(TokenType::Minus, start),
199            '*' if self.match_char('=') => self.make_token(TokenType::StarAssign, start),
200            '*' => self.make_token(TokenType::Star, start),
201            '+' if self.match_char('=') => self.make_token(TokenType::PlusAssign, start),
202            '+' => self.make_token(TokenType::Plus, start),
203            '/' if self.match_char('/') => self.single_line_comment(),
204            '/' if self.match_char('*') => self.multi_line_comment(),
205            '/' if self.match_char('=') => self.make_token(TokenType::SlashAssign, start),
206            '/' => self.make_token(TokenType::Slash, start),
207            '%' if self.match_char('=') => self.make_token(TokenType::ModuloAssign, start),
208            '%' => self.make_token(TokenType::Modulo, start),
209            '.' if self.peek_char().is_ascii_digit() => self.number(),
210            '.' => self.make_token(TokenType::Dot, start),
211            ',' => self.make_token(TokenType::Comma, start),
212            ':' => self.make_token(TokenType::Colon, start),
213            '?' if self.match_char('|') => self.make_token(TokenType::OptionalBar, start),
214            '?' if self.match_char('.') => self.make_token(TokenType::OptionalDot, start),
215            '?' => self.make_token(TokenType::Question, start),
216            ';' => self.make_token(TokenType::Semicolon, start),
217            '{' if self.match_char('{') => self.make_token(TokenType::DoubleLeftBrace, start),
218            '}' if self.match_char('}') => self.make_token(TokenType::DoubleRightBrace, start),
219            '{' => self.make_token(TokenType::LeftBrace, start),
220            '}' => self.make_token(TokenType::RightBrace, start),
221            '[' => self.make_token(TokenType::LeftSquareBracket, start),
222            ']' => self.make_token(TokenType::RightSquareBracket, start),
223            '"' => self.string(),
224            '|' if self.match_char('|') => self.make_token(TokenType::DoubleBar, start),
225            '|' if self.match_char('>') => self.make_token(TokenType::Pipe, start),
226            '|' => self.make_token(TokenType::Bar, start),
227            c if c.is_ascii_digit() => self.number(),
228            c if c.is_ascii_alphabetic() || c == '_' => self.identifier(),
229            _ => self.error_token(format!("Unexpected character: '{}'.", c).as_str()),
230        }
231    }
232
233    fn advance(&mut self) -> char {
234        if self.is_at_end() {
235            return '\0';
236        }
237
238        let c = self.source_map.get_source()[self.location];
239        self.location += 1;
240
241        if c == '\n' {
242            self.line += 1;
243            self.col = 1;
244        }
245
246        c
247    }
248
249    // Peeks at the next character.
250    fn peek_char(&self) -> char {
251        if self.is_at_end() {
252            '\0'
253        } else {
254            self.source_map.get_source()[self.location]
255        }
256    }
257
258    fn peek_next_char(&self) -> char {
259        if self.location + 1 >= self.source_map.get_source().len() {
260            '\0'
261        } else {
262            self.source_map.get_source()[self.location + 1]
263        }
264    }
265
266    fn skip_whitespace(&mut self) {
267        loop {
268            match self.peek_char() {
269                ' ' | '\r' | '\t' => {
270                    self.col += 1;
271                    self.advance();
272                }
273                '\n' => {
274                    self.advance();
275                }
276                _ => break,
277            }
278        }
279    }
280
281    fn is_at_end(&self) -> bool {
282        self.is_eof || self.location >= self.source_map.get_source().len()
283    }
284
285    fn match_char(&mut self, expected: char) -> bool {
286        if self.is_at_end() || self.peek_char() != expected {
287            false
288        } else {
289            self.location += expected.len_utf8();
290            // self.advance();
291            true
292        }
293    }
294
295    fn make_token(&mut self, token_type: TokenType, start: usize) -> Option<Token> {
296        let end = self.location;
297
298        Some(Token {
299            token_type,
300            start,
301            end,
302            error_message: None,
303        })
304    }
305
306    fn error_token(&self, message: &str) -> Option<Token> {
307        Some(Token {
308            token_type: TokenType::Error,
309            start: self.location,
310            end: self.location,
311            error_message: Some(message.to_string()),
312        })
313    }
314
315    fn string(&mut self) -> Option<Token> {
316        let start = self.location - 1;
317
318        while self.peek_char() != '"' && !self.is_at_end() {
319            if self.peek_char() == '\n' {
320                // Don't consume the newline - leave it for the next token
321                return Some(Token {
322                    token_type: TokenType::Error,
323                    start,
324                    end: self.location,
325                    error_message: Some("Unterminated string.".to_string()),
326                });
327            }
328
329            if self.peek_char() == '\\' {
330                self.advance_in_string(); // consume backslash
331                if !self.is_at_end() {
332                    let escaped = self.peek_char();
333                    match escaped {
334                        'n' | 't' | 'r' | '\\' | '"' | '\'' | '0' => {
335                            self.advance_in_string(); // valid escape
336                        }
337                        _ => {
338                            return self
339                                .error_token(&format!("Invalid escape sequence: \\{}", escaped));
340                        }
341                    }
342                }
343            } else {
344                self.advance_in_string();
345            }
346        }
347
348        if self.is_at_end() {
349            return self.error_token("Unterminated string.");
350        }
351
352        self.advance(); // Consume closing quote 
353        self.make_token(TokenType::String, start)
354    }
355
356    fn advance_in_string(&mut self) -> char {
357        if self.is_at_end() {
358            return '\0';
359        }
360
361        let c = self.source_map.get_source()[self.location];
362        self.location += 1;
363
364        c
365    }
366
367    fn number(&mut self) -> Option<Token> {
368        let start = self.location - 1; // Account for first digit or dot
369
370        let started_with_dot = self.source_map.get_source()[start] == '.';
371
372        while self.peek_char().is_ascii_digit() {
373            self.advance();
374        }
375
376        if !started_with_dot && self.peek_char() == '.' && self.peek_next_char().is_ascii_digit() {
377            self.advance(); // Consume the '.'
378
379            while self.peek_char().is_ascii_digit() {
380                self.advance();
381            }
382        }
383
384        self.make_token(TokenType::Number, start)
385    }
386
387    fn identifier(&mut self) -> Option<Token> {
388        let start = self.location - 1;
389
390        while self.peek_char().is_ascii_alphanumeric() || self.peek_char() == '_' {
391            self.advance();
392        }
393
394        let keyword: String = self.source_map.get_source()[start..self.location]
395            .iter()
396            .collect();
397        let token_type = TokenType::from_keyword(&keyword);
398
399        self.make_token(token_type, start)
400    }
401
402    fn single_line_comment(&mut self) -> Option<Token> {
403        while self.peek_char() != '\n' && !self.is_at_end() {
404            self.advance();
405        }
406
407        self.next_token()
408    }
409
410    fn multi_line_comment(&mut self) -> Option<Token> {
411        while !self.is_at_end() {
412            if self.peek_char() == '*' && self.peek_next_char() == '/' {
413                self.advance(); // consume *
414                self.advance(); // consume /
415                break;
416            }
417            self.advance();
418        }
419
420        if self.is_at_end()
421            && (self.source_map.get_source().get(self.location - 2) != Some(&'*')
422                || self.source_map.get_source().get(self.location - 1) != Some(&'/'))
423        {
424            return self.error_token("Unterminated comment.");
425        }
426
427        // Continue to next token after consuming comment
428        self.next_token()
429    }
430}
431
432impl Iterator for Tokenizer {
433    type Item = Token;
434
435    fn next(&mut self) -> Option<Self::Item> {
436        if let Some(token) = self.lookahead_buffer.pop_front() {
437            Some(token)
438        } else {
439            self.next_token()
440        }
441    }
442}
qanglang_core/frontend/tokenizer.rs

qanglang_core/frontend/
tokenizer.rs
