lexer/
lib.rs

1use crate::token::{lookup_identifier, Span, Token, TokenKind};
2
3mod lexer_test;
4pub mod token;
5
6pub struct Lexer<'a> {
7    input: &'a str,
8    position: usize,
9    read_position: usize,
10    ch: char,
11}
12
13impl<'a> Lexer<'a> {
14    pub fn new(input: &'a str) -> Self {
15        let mut l = Lexer {
16            input,
17            position: 0,
18            read_position: 0,
19            ch: '\0',
20        };
21
22        l.read_char();
23        return l;
24    }
25
26    fn read_char(&mut self) {
27        self.position = self.read_position;
28
29        if self.position >= self.input.len() {
30            self.ch = '\0'
31        } else {
32            self.ch = self.input[self.position..].chars().next().unwrap();
33            self.read_position = self.position + self.ch.len_utf8();
34        }
35    }
36
37    fn peek_char(&self) -> char {
38        if self.read_position >= self.input.len() {
39            '\0'
40        } else {
41            self.input[self.read_position..].chars().next().unwrap()
42        }
43    }
44
45    pub fn next_token(&mut self) -> Token {
46        // println!("self ch {}, position {} read_position {}", self.ch, self.position, self.read_position);
47        // Skip any whitespace and successive line comments before producing a token.
48        self.skip_ignorable();
49        let start = self.position;
50        if self.ch == '\0' {
51            self.read_char();
52            return Token {
53                span: Span {
54                    start,
55                    end: start + 1,
56                },
57                kind: TokenKind::EOF,
58            };
59        }
60
61        let t = match self.ch {
62            '=' => {
63                if self.peek_char() == '=' {
64                    self.read_char();
65                    TokenKind::EQ
66                } else {
67                    TokenKind::ASSIGN
68                }
69            }
70            ';' => TokenKind::SEMICOLON,
71            '(' => TokenKind::LPAREN,
72            ')' => TokenKind::RPAREN,
73            ',' => TokenKind::COMMA,
74            '+' => TokenKind::PLUS,
75            '-' => TokenKind::MINUS,
76            '!' => {
77                if self.peek_char() == '=' {
78                    self.read_char();
79                    TokenKind::NotEq
80                } else {
81                    TokenKind::BANG
82                }
83            }
84            '*' => TokenKind::ASTERISK,
85            '/' => TokenKind::SLASH,
86            '<' => TokenKind::LT,
87            '>' => TokenKind::GT,
88            '{' => TokenKind::LBRACE,
89            '}' => TokenKind::RBRACE,
90            '[' => TokenKind::LBRACKET,
91            ':' => TokenKind::COLON,
92            ']' => TokenKind::RBRACKET,
93            '"' => {
94                let (start, end, string) = self.read_string();
95                return Token {
96                    span: Span {
97                        start,
98                        end,
99                    },
100                    kind: TokenKind::STRING(string),
101                };
102            }
103            _ => {
104                if is_letter(self.ch) {
105                    let (start, end, identifier) = self.read_identifier();
106                    return Token {
107                        span: Span {
108                            start,
109                            end,
110                        },
111                        kind: lookup_identifier(&identifier),
112                    };
113                } else if is_digit(self.ch) {
114                    let (start, end, num) = self.read_number();
115                    return Token {
116                        span: Span {
117                            start,
118                            end,
119                        },
120                        kind: TokenKind::INT(num),
121                    };
122                } else {
123                    TokenKind::ILLEGAL
124                }
125            }
126        };
127
128        self.read_char();
129        return Token {
130            span: Span {
131                start,
132                end: self.position,
133            },
134            kind: t,
135        };
136    }
137
138    fn skip_whitespace(&mut self) {
139        while self.ch.is_ascii_whitespace() {
140            self.read_char();
141        }
142    }
143
144    fn skip_ignorable(&mut self) {
145        loop {
146            self.skip_whitespace();
147            if self.ch == '/' && self.peek_char() == '/' {
148                self.skip_comments();
149                // Continue the loop, in case there are more comments or whitespace
150                continue;
151            }
152            break;
153        }
154    }
155
156    fn skip_comments(&mut self) {
157        if self.ch == '/' && self.peek_char() == '/' {
158            self.read_char();
159            self.read_char();
160            loop {
161                self.read_char();
162                if self.ch == '\n' || self.ch == '\u{0}' {
163                    // consume the comments end
164                    if self.ch == '\n' {
165                        self.read_char();
166                    }
167                    break;
168                }
169            }
170        }
171    }
172
173    fn read_identifier(&mut self) -> (usize, usize, String) {
174        let pos = self.position;
175        while is_letter(self.ch) {
176            self.read_char();
177        }
178
179        let x = self.input[pos..self.position].to_string();
180        return (pos, self.position, x);
181    }
182
183    fn read_number(&mut self) -> (usize, usize, i64) {
184        let pos = self.position;
185        while is_digit(self.ch) {
186            self.read_char();
187        }
188
189        let x = self.input[pos..self.position].parse().unwrap();
190
191        return (pos, self.position, x);
192    }
193
194    fn read_string(&mut self) -> (usize, usize, String) {
195        let pos = self.position + 1;
196        loop {
197            self.read_char();
198            if self.ch == '"' || self.ch == '\u{0}' {
199                break;
200            }
201        }
202
203        let x = self.input[pos..self.position].to_string();
204
205        // consume the end "
206        if self.ch == '"' {
207            self.read_char();
208        }
209        return (pos - 1, self.position, x);
210    }
211}
212
213fn is_letter(c: char) -> bool {
214    c.is_ascii_alphabetic() || c == '_'
215}
216
217fn is_digit(c: char) -> bool {
218    c >= '0' && c <= '9'
219}
lexer/lib.rs

lexer/
lib.rs
