monkey_rs/lexer/
parse.rs

1/*!
2A simple character‑based lexer that turns source text into a stream
3of `Token`s, handling identifiers, numbers, operators, and whitespace.
4*/
5
6use crate::token;
7
8/// The lexer to convert source code into tokens representing the source code.
9#[derive(Debug)]
10pub struct Lexer<'a> {
11    /// the input source code to tokenize
12    input: &'a str,
13    /// current position in input (points to current char)
14    position: usize,
15    /// current reading position in input (after current char)
16    read_position: usize,
17    /// current char under examination
18    ch: Option<char>,
19}
20
21impl<'a> Lexer<'a> {
22    /// Create a new lexer over the given input string.
23    ///
24    /// This will initialize the internal state and read the first character,
25    /// so the lexer is ready to produce tokens via `next_token()`.
26    pub fn new(input: &'a str) -> Self {
27        let mut lexer = Self {
28            input,
29            position: 0,
30            read_position: 0,
31            ch: None,
32        };
33        // put the lexer in an initial working state referencing the first
34        // character
35        lexer.read_char();
36        lexer
37    }
38
39    /// Update the lexer state to reflect the next character in the input, if
40    /// any, and advance the position in the input.
41    fn read_char(&mut self) {
42        // check if we have reached end of the input
43        if self.read_position >= self.input.len() {
44            self.ch = None
45        } else {
46            let remainder = &self.input[self.read_position..];
47            if let Some((_, c)) = remainder.char_indices().next() {
48                self.ch = Some(c);
49                self.position = self.read_position;
50                // advance the read position to be a character ahead of the
51                // current character position
52                self.read_position += c.len_utf8();
53                return;
54            }
55        }
56        // reached EOF
57        self.ch = None;
58        self.position = self.read_position;
59    }
60
61    /// Determine and return the next token in the input from the current
62    /// character position.
63    pub fn next_token(&mut self) -> token::Token {
64        // consume character(s) until no whitespace
65        while matches!(self.ch, Some(c) if c.is_whitespace()) {
66            self.read_char();
67        }
68
69        let token = match self.ch {
70            // Single character tokens
71            Some('+') => token::Token::Plus,
72            Some('-') => token::Token::Minus,
73            Some('/') => token::Token::Slash,
74            Some('*') => token::Token::Asterisk,
75            Some('<') => token::Token::Lt,
76            Some('>') => token::Token::Gt,
77            Some(';') => token::Token::Semicolon,
78            Some('(') => token::Token::LParen,
79            Some(')') => token::Token::RParen,
80            Some(',') => token::Token::Comma,
81            Some('{') => token::Token::LBrace,
82            Some('}') => token::Token::RBrace,
83            Some('[') => token::Token::LBracket,
84            Some(']') => token::Token::RBracket,
85            Some(':') => token::Token::Colon,
86            Some('"') => {
87                let str = self.read_string();
88                return token::Token::String(str);
89            }
90
91            // Multi-character tokens (e.g., identifier, integer, etc.)
92            Some(c) if c.is_ascii_alphabetic() => {
93                let ident = self.read_indentifier();
94                return token::lookup_ident(&ident);
95            }
96            Some(c) if c.is_ascii_digit() => {
97                let literal = self.read_number();
98                return token::Token::Int(literal);
99            }
100            Some('=') => {
101                if self.peek_char() == Some('=') {
102                    self.read_char();
103                    self.read_char();
104                    return token::Token::Eq;
105                }
106                self.read_char();
107                return token::Token::Assign;
108            }
109            Some('!') => {
110                if self.peek_char() == Some('=') {
111                    self.read_char();
112                    self.read_char();
113                    return token::Token::NotEq;
114                } else {
115                    self.read_char();
116                    return token::Token::Bang;
117                }
118            }
119
120            // Unknown single character
121            Some(_) => token::Token::Illegal,
122
123            // Reached EOF
124            None => token::Token::Eof,
125        };
126
127        // advance past the consumed character
128        self.read_char();
129        token
130    }
131
132    /// Reads in an identifier and advances the lexer's position until it
133    /// encounters a non-letter character
134    fn read_indentifier(&mut self) -> String {
135        let start = self.position;
136        // identifiers can be alphanumeric separated by underscores
137        while matches!(self.ch, Some(c) if c.is_ascii_alphanumeric() || c == '_') {
138            self.read_char();
139        }
140        self.input[start..self.position].to_string()
141    }
142
143    /// Reads in a number and advances the lexer's position until it encounters
144    /// a non-numeric character. Only supports integer values.
145    fn read_number(&mut self) -> i32 {
146        let start = self.position;
147        while matches!(self.ch, Some(c) if c.is_ascii_digit()) {
148            self.read_char();
149        }
150        self.input[start..self.position]
151            .parse()
152            .expect("Invalid number encountered")
153    }
154
155    /// Read a string value from the opening quotation character.
156    fn read_string(&mut self) -> String {
157        // Skip opening quotation
158        self.read_char();
159        let position = self.position;
160
161        while let Some(ch) = self.ch {
162            if ch == '"' {
163                break;
164            }
165            self.read_char();
166        }
167
168        let str = self.input[position..self.position].to_string();
169
170        // Move past closing quotation
171        self.read_char();
172
173        str
174    }
175
176    /// Peeks the next character from the current position of the lexer.
177    fn peek_char(&self) -> Option<char> {
178        self.input[self.read_position..].chars().next()
179    }
180}
181
182#[cfg(test)]
183mod tests {
184    use super::*;
185
186    /// Verify that the lexer produces the expected token stream from its
187    /// input.
188    fn verify_expected_next_token(expected: &[token::Token], lexer: &mut Lexer) {
189        for (i, expected_tk) in expected.iter().enumerate() {
190            let token: token::Token = lexer.next_token();
191            assert_eq!(
192                token,
193                expected_tk.clone(),
194                "expected[{}] - wrong token. expected={:?}, actual={:?}",
195                i,
196                expected_tk,
197                token
198            );
199        }
200    }
201
202    #[test]
203    fn test_unichar_next_token() {
204        let input = "=+(){},;[]";
205        let mut l = Lexer::new(input);
206
207        let expected: Vec<token::Token> = vec![
208            token::Token::Assign,
209            token::Token::Plus,
210            token::Token::LParen,
211            token::Token::RParen,
212            token::Token::LBrace,
213            token::Token::RBrace,
214            token::Token::Comma,
215            token::Token::Semicolon,
216            token::Token::LBracket,
217            token::Token::RBracket,
218            token::Token::Eof,
219        ];
220
221        verify_expected_next_token(&expected, &mut l);
222    }
223
224    #[test]
225    fn mixed_chars() {
226        let input = r#"let five = 5;
227                     let ten = 10;
228
229                     let add = fn(x, y) {
230                         x + y;
231                     };
232
233            let result = add(five, ten);
234            !-/*5;
235            5 < 10 > 5;
236
237            if (5 < 10) {
238                return true;
239            } else {
240                return false;
241            }
242
243            10 == 10;
244            10 != 9;
245            "foobar"
246            "foo bar"
247            [1, 2];
248            {"foo": "bar"}"#;
249
250        let mut l = Lexer::new(input);
251        let expected: Vec<token::Token> = vec![
252            token::Token::Let,
253            token::Token::Ident("five".to_string()),
254            token::Token::Assign,
255            token::Token::Int(5),
256            token::Token::Semicolon,
257            token::Token::Let,
258            token::Token::Ident("ten".to_string()),
259            token::Token::Assign,
260            token::Token::Int(10),
261            token::Token::Semicolon,
262            token::Token::Let,
263            token::Token::Ident("add".to_string()),
264            token::Token::Assign,
265            token::Token::Function,
266            token::Token::LParen,
267            token::Token::Ident("x".to_string()),
268            token::Token::Comma,
269            token::Token::Ident("y".to_string()),
270            token::Token::RParen,
271            token::Token::LBrace,
272            token::Token::Ident("x".to_string()),
273            token::Token::Plus,
274            token::Token::Ident("y".to_string()),
275            token::Token::Semicolon,
276            token::Token::RBrace,
277            token::Token::Semicolon,
278            token::Token::Let,
279            token::Token::Ident("result".to_string()),
280            token::Token::Assign,
281            token::Token::Ident("add".to_string()),
282            token::Token::LParen,
283            token::Token::Ident("five".to_string()),
284            token::Token::Comma,
285            token::Token::Ident("ten".to_string()),
286            token::Token::RParen,
287            token::Token::Semicolon,
288            token::Token::Bang,
289            token::Token::Minus,
290            token::Token::Slash,
291            token::Token::Asterisk,
292            token::Token::Int(5),
293            token::Token::Semicolon,
294            token::Token::Int(5),
295            token::Token::Lt,
296            token::Token::Int(10),
297            token::Token::Gt,
298            token::Token::Int(5),
299            token::Token::Semicolon,
300            token::Token::If,
301            token::Token::LParen,
302            token::Token::Int(5),
303            token::Token::Lt,
304            token::Token::Int(10),
305            token::Token::RParen,
306            token::Token::LBrace,
307            token::Token::Return,
308            token::Token::True,
309            token::Token::Semicolon,
310            token::Token::RBrace,
311            token::Token::Else,
312            token::Token::LBrace,
313            token::Token::Return,
314            token::Token::False,
315            token::Token::Semicolon,
316            token::Token::RBrace,
317            token::Token::Int(10),
318            token::Token::Eq,
319            token::Token::Int(10),
320            token::Token::Semicolon,
321            token::Token::Int(10),
322            token::Token::NotEq,
323            token::Token::Int(9),
324            token::Token::Semicolon,
325            token::Token::String("foobar".to_string()),
326            token::Token::String("foo bar".to_string()),
327            token::Token::LBracket,
328            token::Token::Int(1),
329            token::Token::Comma,
330            token::Token::Int(2),
331            token::Token::RBracket,
332            token::Token::Semicolon,
333            token::Token::LBrace,
334            token::Token::String("foo".to_string()),
335            token::Token::Colon,
336            token::Token::String("bar".to_string()),
337            token::Token::RBrace,
338            token::Token::Eof,
339        ];
340
341        verify_expected_next_token(&expected, &mut l);
342    }
343}
monkey_rs/lexer/parse.rs

monkey_rs/lexer/
parse.rs
