lexer/
lib.rs

1use crate::token::{lookup_identifier, Span, Token, TokenKind};
2
3mod lexer_test;
4pub mod token;
5
6pub struct Lexer<'a> {
7    input: &'a str,
8    position: usize,
9    read_position: usize,
10    ch: char,
11}
12
13impl<'a> Lexer<'a> {
14    pub fn new(input: &'a str) -> Self {
15        let mut l = Lexer {
16            input,
17            position: 0,
18            read_position: 0,
19            ch: '\0',
20        };
21
22        l.read_char();
23        return l;
24    }
25
26    fn read_char(&mut self) {
27        self.position = self.read_position;
28
29        if self.position >= self.input.len() {
30            self.ch = '\0'
31        } else {
32            self.ch = self.input[self.position..].chars().next().unwrap();
33            self.read_position = self.position + self.ch.len_utf8();
34        }
35    }
36
37    fn peek_char(&self) -> char {
38        if self.read_position >= self.input.len() {
39            '\0'
40        } else {
41            self.input[self.read_position..].chars().next().unwrap()
42        }
43    }
44
45    pub fn next_token(&mut self) -> Token {
46        // println!("self ch {}, position {} read_position {}", self.ch, self.position, self.read_position);
47        // Skip any whitespace and successive line comments before producing a token.
48        self.skip_ignorable();
49        let start = self.position;
50        if self.ch == '\0' {
51            // EOF consumes no source bytes; keep a zero-width span at input.len().
52            return Token {
53                span: Span {
54                    start,
55                    end: start,
56                },
57                kind: TokenKind::EOF,
58            };
59        }
60
61        let t = match self.ch {
62            '=' => {
63                if self.peek_char() == '=' {
64                    self.read_char();
65                    TokenKind::EQ
66                } else {
67                    TokenKind::ASSIGN
68                }
69            }
70            ';' => TokenKind::SEMICOLON,
71            '(' => TokenKind::LPAREN,
72            ')' => TokenKind::RPAREN,
73            ',' => TokenKind::COMMA,
74            '+' => TokenKind::PLUS,
75            '-' => TokenKind::MINUS,
76            '!' => {
77                if self.peek_char() == '=' {
78                    self.read_char();
79                    TokenKind::NotEq
80                } else {
81                    TokenKind::BANG
82                }
83            }
84            '*' => TokenKind::ASTERISK,
85            '/' => TokenKind::SLASH,
86            '<' => TokenKind::LT,
87            '>' => TokenKind::GT,
88            '{' => TokenKind::LBRACE,
89            '}' => TokenKind::RBRACE,
90            '[' => TokenKind::LBRACKET,
91            ':' => TokenKind::COLON,
92            '.' => TokenKind::DOT,
93            ']' => TokenKind::RBRACKET,
94            '"' => {
95                let (start, end, string) = self.read_string();
96                return Token {
97                    span: Span {
98                        start,
99                        end,
100                    },
101                    kind: TokenKind::STRING(string),
102                };
103            }
104            _ => {
105                if is_letter(self.ch) {
106                    let (start, end, identifier) = self.read_identifier();
107                    return Token {
108                        span: Span {
109                            start,
110                            end,
111                        },
112                        kind: lookup_identifier(&identifier),
113                    };
114                } else if is_digit(self.ch) {
115                    let (start, end, num) = self.read_number();
116                    return Token {
117                        span: Span {
118                            start,
119                            end,
120                        },
121                        kind: TokenKind::INT(num),
122                    };
123                } else {
124                    TokenKind::ILLEGAL
125                }
126            }
127        };
128
129        self.read_char();
130        return Token {
131            span: Span {
132                start,
133                end: self.position,
134            },
135            kind: t,
136        };
137    }
138
139    fn skip_whitespace(&mut self) {
140        while self.ch.is_ascii_whitespace() {
141            self.read_char();
142        }
143    }
144
145    fn skip_ignorable(&mut self) {
146        loop {
147            self.skip_whitespace();
148            if self.ch == '/' && self.peek_char() == '/' {
149                self.skip_comments();
150                // Continue the loop, in case there are more comments or whitespace
151                continue;
152            }
153            break;
154        }
155    }
156
157    fn skip_comments(&mut self) {
158        if self.ch == '/' && self.peek_char() == '/' {
159            self.read_char();
160            self.read_char();
161            loop {
162                self.read_char();
163                if self.ch == '\n' || self.ch == '\u{0}' {
164                    // consume the comments end
165                    if self.ch == '\n' {
166                        self.read_char();
167                    }
168                    break;
169                }
170            }
171        }
172    }
173
174    fn read_identifier(&mut self) -> (usize, usize, String) {
175        let pos = self.position;
176        while is_letter(self.ch) || is_digit(self.ch) {
177            self.read_char();
178        }
179
180        let x = self.input[pos..self.position].to_string();
181        return (pos, self.position, x);
182    }
183
184    fn read_number(&mut self) -> (usize, usize, i64) {
185        let pos = self.position;
186        while is_digit(self.ch) {
187            self.read_char();
188        }
189
190        let x = self.input[pos..self.position].parse().unwrap();
191
192        return (pos, self.position, x);
193    }
194
195    fn read_string(&mut self) -> (usize, usize, String) {
196        let pos = self.position + 1;
197        loop {
198            self.read_char();
199            if self.ch == '"' || self.ch == '\u{0}' {
200                break;
201            }
202        }
203
204        let x = self.input[pos..self.position].to_string();
205
206        // consume the end "
207        if self.ch == '"' {
208            self.read_char();
209        }
210        return (pos - 1, self.position, x);
211    }
212}
213
214fn is_letter(c: char) -> bool {
215    c.is_ascii_alphabetic() || c == '_'
216}
217
218fn is_digit(c: char) -> bool {
219    c >= '0' && c <= '9'
220}
lexer/lib.rs

lexer/
lib.rs
