lexer/
lib.rs

1use crate::token::{lookup_identifier, Span, Token, TokenKind};
2
3mod lexer_test;
4pub mod token;
5
6pub struct Lexer<'a> {
7    input: &'a str,
8    position: usize,
9    read_position: usize,
10    ch: char,
11}
12
13impl<'a> Lexer<'a> {
14    pub fn new(input: &'a str) -> Self {
15        let mut l = Lexer {
16            input,
17            position: 0,
18            read_position: 0,
19            ch: '\0',
20        };
21
22        l.read_char();
23        return l;
24    }
25
26    fn read_char(&mut self) {
27        self.position = self.read_position;
28
29        if self.position >= self.input.len() {
30            self.ch = '\0'
31        } else {
32            self.ch = self.input[self.position..].chars().next().unwrap();
33            self.read_position = self.position + self.ch.len_utf8();
34        }
35    }
36
37    fn peek_char(&self) -> char {
38        if self.read_position >= self.input.len() {
39            '\0'
40        } else {
41            self.input[self.read_position..].chars().next().unwrap()
42        }
43    }
44
45    pub fn next_token(&mut self) -> Token {
46        // println!("self ch {}, position {} read_position {}", self.ch, self.position, self.read_position);
47        // Skip any whitespace and successive line comments before producing a token.
48        self.skip_ignorable();
49        let start = self.position;
50        if self.ch == '\0' {
51            // EOF consumes no source bytes; keep a zero-width span at input.len().
52            return Token {
53                span: Span {
54                    start,
55                    end: start,
56                },
57                kind: TokenKind::EOF,
58            };
59        }
60
61        let t = match self.ch {
62            '=' => {
63                if self.peek_char() == '=' {
64                    self.read_char();
65                    TokenKind::EQ
66                } else {
67                    TokenKind::ASSIGN
68                }
69            }
70            ';' => TokenKind::SEMICOLON,
71            '(' => TokenKind::LPAREN,
72            ')' => TokenKind::RPAREN,
73            ',' => TokenKind::COMMA,
74            '+' => TokenKind::PLUS,
75            '-' => TokenKind::MINUS,
76            '!' => {
77                if self.peek_char() == '=' {
78                    self.read_char();
79                    TokenKind::NotEq
80                } else {
81                    TokenKind::BANG
82                }
83            }
84            '*' => TokenKind::ASTERISK,
85            '/' => TokenKind::SLASH,
86            '<' => TokenKind::LT,
87            '>' => TokenKind::GT,
88            '{' => TokenKind::LBRACE,
89            '}' => TokenKind::RBRACE,
90            '[' => TokenKind::LBRACKET,
91            ':' => TokenKind::COLON,
92            '.' => TokenKind::DOT,
93            ']' => TokenKind::RBRACKET,
94            '"' => {
95                let (start, end, string) = self.read_string();
96                return Token {
97                    span: Span {
98                        start,
99                        end,
100                    },
101                    kind: match string {
102                        Some(value) => TokenKind::STRING(value),
103                        // Unterminated literal: report the whole literal as a
104                        // single ILLEGAL token instead of accepting it.
105                        None => TokenKind::ILLEGAL,
106                    },
107                };
108            }
109            _ => {
110                if is_letter(self.ch) {
111                    let (start, end, identifier) = self.read_identifier();
112                    return Token {
113                        span: Span {
114                            start,
115                            end,
116                        },
117                        kind: lookup_identifier(&identifier),
118                    };
119                } else if is_digit(self.ch) {
120                    let (start, end, num) = self.read_number();
121                    return Token {
122                        span: Span {
123                            start,
124                            end,
125                        },
126                        kind: TokenKind::INT(num),
127                    };
128                } else {
129                    TokenKind::ILLEGAL
130                }
131            }
132        };
133
134        self.read_char();
135        return Token {
136            span: Span {
137                start,
138                end: self.position,
139            },
140            kind: t,
141        };
142    }
143
144    fn skip_whitespace(&mut self) {
145        while self.ch.is_ascii_whitespace() {
146            self.read_char();
147        }
148    }
149
150    fn skip_ignorable(&mut self) {
151        loop {
152            self.skip_whitespace();
153            if self.ch == '/' && self.peek_char() == '/' {
154                self.skip_comments();
155                // Continue the loop, in case there are more comments or whitespace
156                continue;
157            }
158            break;
159        }
160    }
161
162    fn skip_comments(&mut self) {
163        if self.ch == '/' && self.peek_char() == '/' {
164            self.read_char();
165            self.read_char();
166            loop {
167                self.read_char();
168                if self.ch == '\n' || self.ch == '\u{0}' {
169                    // consume the comments end
170                    if self.ch == '\n' {
171                        self.read_char();
172                    }
173                    break;
174                }
175            }
176        }
177    }
178
179    fn read_identifier(&mut self) -> (usize, usize, String) {
180        let pos = self.position;
181        while is_letter(self.ch) || is_digit(self.ch) {
182            self.read_char();
183        }
184
185        let x = self.input[pos..self.position].to_string();
186        return (pos, self.position, x);
187    }
188
189    fn read_number(&mut self) -> (usize, usize, i64) {
190        let pos = self.position;
191        while is_digit(self.ch) {
192            self.read_char();
193        }
194
195        let x = self.input[pos..self.position].parse().unwrap();
196
197        return (pos, self.position, x);
198    }
199
200    /// Reads a string literal starting at the opening `"`. Returns `None` when
201    /// the input ends before a closing quote, in which case the span still
202    /// covers everything scanned so the caller can emit one ILLEGAL token for
203    /// the whole literal.
204    fn read_string(&mut self) -> (usize, usize, Option<String>) {
205        let pos = self.position;
206        loop {
207            self.read_char();
208            if self.ch == '"' || self.ch == '\u{0}' {
209                break;
210            }
211        }
212
213        let value = if self.ch == '"' {
214            let x = self.input[pos + 1..self.position].to_string();
215            // consume the end "
216            self.read_char();
217            Some(x)
218        } else {
219            // The input ended before a closing quote.
220            None
221        };
222
223        return (pos, self.position, value);
224    }
225}
226
227fn is_letter(c: char) -> bool {
228    c.is_ascii_alphabetic() || c == '_'
229}
230
231fn is_digit(c: char) -> bool {
232    c.is_ascii_digit()
233}
lexer/lib.rs

lexer/
lib.rs
