ocelot_parser/lexer/
lex.rs

1use crate::lexer::token::Token;
2use crate::lexer::token_type::TokenType;
3use ocelot_base::compilation_context::CompilationContext;
4use ocelot_base::diagnostic_level::DiagnosticLevel;
5use ocelot_base::source_annotation::SourceAnnotation;
6use ocelot_base::source_diagnostic::SourceDiagnostic;
7use ocelot_base::source_excerpt::SourceExcerpt;
8use ocelot_base::source_file::SourceFile;
9use ocelot_base::span::Span;
10
11/// Tokenizes a source file into lexical tokens and appends diagnostics to the context.
12pub fn lex(source_file: &SourceFile, context: &mut CompilationContext) -> Vec<Token> {
13    let mut tokens = Vec::new();
14    let source = source_file.source();
15    let bytes = source.as_bytes();
16    let mut index = 0;
17
18    while index < bytes.len() {
19        match bytes[index] {
20            b' ' | b'\t' | b'\n' | b'\r' => {
21                index += 1;
22            }
23            b'(' => {
24                tokens.push(Token::new(TokenType::LeftParen, index, index + 1));
25                index += 1;
26            }
27            b')' => {
28                tokens.push(Token::new(TokenType::RightParen, index, index + 1));
29                index += 1;
30            }
31            b'{' => {
32                tokens.push(Token::new(TokenType::LeftBrace, index, index + 1));
33                index += 1;
34            }
35            b'}' => {
36                tokens.push(Token::new(TokenType::RightBrace, index, index + 1));
37                index += 1;
38            }
39            b';' => {
40                tokens.push(Token::new(TokenType::Semicolon, index, index + 1));
41                index += 1;
42            }
43            b'"' => {
44                let start = index;
45                index += 1;
46
47                while index < bytes.len()
48                    && bytes[index] != b'"'
49                    && bytes[index] != b'\n'
50                    && bytes[index] != b'\r'
51                {
52                    index += 1;
53                }
54
55                if index >= bytes.len() || bytes[index] == b'\n' || bytes[index] == b'\r' {
56                    context.add_diagnostic(unterminated_string_diagnostic(
57                        source_file,
58                        start,
59                        index,
60                    ));
61                    break;
62                }
63
64                index += 1;
65                tokens.push(Token::new(TokenType::String, start, index));
66            }
67            byte if is_identifier_start(byte) => {
68                let start = index;
69                index += 1;
70
71                while index < bytes.len() && is_identifier_continue(bytes[index]) {
72                    index += 1;
73                }
74
75                let token_type = match &source[start..index] {
76                    "test" => TokenType::Test,
77                    _ => TokenType::Identifier,
78                };
79                tokens.push(Token::new(token_type, start, index));
80            }
81            _ => {
82                tokens.push(Token::new(TokenType::Unexpected, index, index + 1));
83                index += 1;
84            }
85        }
86    }
87
88    tokens.push(Token::new(TokenType::EndOfFile, index, index));
89    tokens
90}
91
92fn is_identifier_start(byte: u8) -> bool {
93    byte.is_ascii_alphabetic() || byte == b'_'
94}
95
96fn is_identifier_continue(byte: u8) -> bool {
97    is_identifier_start(byte) || byte.is_ascii_digit()
98}
99
100fn unterminated_string_diagnostic(
101    source_file: &SourceFile,
102    start: usize,
103    end: usize,
104) -> SourceDiagnostic {
105    let (line_number, line_start, line_end) = line_bounds(source_file.source(), start);
106    let excerpt = SourceExcerpt::new(
107        &source_file.path,
108        line_number,
109        &source_file.source()[line_start..line_end],
110    )
111    .with_annotation(SourceAnnotation::new(
112        Span::new(start - line_start, end - line_start),
113        "string is missing a closing quote",
114    ));
115
116    SourceDiagnostic::new(
117        DiagnosticLevel::Error,
118        &source_file.path,
119        "unterminated string literal",
120    )
121    .with_excerpt(excerpt)
122}
123
124fn line_bounds(source: &str, index: usize) -> (usize, usize, usize) {
125    let line_start = source[..index].rfind('\n').map_or(0, |offset| offset + 1);
126    let line_end = source[index..]
127        .find('\n')
128        .map_or(source.len(), |offset| index + offset);
129    let line_number = source[..line_start]
130        .bytes()
131        .filter(|byte| *byte == b'\n')
132        .count()
133        + 1;
134
135    (line_number, line_start, line_end)
136}
137
138#[cfg(test)]
139mod tests {
140    use super::lex;
141    use crate::lexer::token_type::TokenType;
142    use ocelot_base::compilation_context::CompilationContext;
143    use ocelot_base::diagnostic_level::DiagnosticLevel;
144    use ocelot_base::source_file::SourceFile;
145
146    #[test]
147    fn lexes_println_script() {
148        let source_file = SourceFile::new("examples/hello.ocelot", "println(\"hello\");");
149        let mut context = CompilationContext::default();
150        let token_types: Vec<_> = lex(&source_file, &mut context)
151            .into_iter()
152            .map(|token| token.token_type)
153            .collect();
154
155        assert_eq!(
156            token_types,
157            vec![
158                TokenType::Identifier,
159                TokenType::LeftParen,
160                TokenType::String,
161                TokenType::RightParen,
162                TokenType::Semicolon,
163                TokenType::EndOfFile,
164            ]
165        );
166        assert!(!context.has_errors());
167    }
168
169    #[test]
170    fn lexes_test_item_tokens() {
171        let source_file = SourceFile::new(
172            "examples/tests.ocelot",
173            "test \"prints\" { println(\"hello\"); }",
174        );
175        let mut context = CompilationContext::default();
176        let token_types: Vec<_> = lex(&source_file, &mut context)
177            .into_iter()
178            .map(|token| token.token_type)
179            .collect();
180
181        assert_eq!(
182            token_types,
183            vec![
184                TokenType::Test,
185                TokenType::String,
186                TokenType::LeftBrace,
187                TokenType::Identifier,
188                TokenType::LeftParen,
189                TokenType::String,
190                TokenType::RightParen,
191                TokenType::Semicolon,
192                TokenType::RightBrace,
193                TokenType::EndOfFile,
194            ]
195        );
196        assert!(!context.has_errors());
197    }
198
199    #[test]
200    fn skips_whitespace_between_tokens() {
201        let source_file = SourceFile::new("examples/whitespace.ocelot", "println ( \"hello\" ) ;");
202        let mut context = CompilationContext::default();
203        let token_types: Vec<_> = lex(&source_file, &mut context)
204            .into_iter()
205            .map(|token| token.token_type)
206            .collect();
207
208        assert_eq!(
209            token_types,
210            vec![
211                TokenType::Identifier,
212                TokenType::LeftParen,
213                TokenType::String,
214                TokenType::RightParen,
215                TokenType::Semicolon,
216                TokenType::EndOfFile,
217            ]
218        );
219        assert!(!context.has_errors());
220    }
221
222    #[test]
223    fn reports_unterminated_strings_as_source_diagnostics() {
224        let source_file = SourceFile::new("examples/broken.ocelot", "println(\"hello);");
225        let mut context = CompilationContext::default();
226        let token_types: Vec<_> = lex(&source_file, &mut context)
227            .into_iter()
228            .map(|token| token.token_type)
229            .collect();
230
231        assert_eq!(
232            token_types,
233            vec![
234                TokenType::Identifier,
235                TokenType::LeftParen,
236                TokenType::EndOfFile
237            ]
238        );
239        assert!(context.has_errors());
240        assert_eq!(context.source_diagnostics.diagnostics.len(), 1);
241
242        let diagnostic = &context.source_diagnostics.diagnostics[0];
243
244        assert_eq!(diagnostic.level, DiagnosticLevel::Error);
245        assert_eq!(diagnostic.file_path.as_str(), "examples/broken.ocelot");
246        assert_eq!(diagnostic.message, "unterminated string literal");
247        assert_eq!(diagnostic.excerpts.len(), 1);
248        assert_eq!(diagnostic.excerpts[0].line_number, 1);
249        assert_eq!(diagnostic.excerpts[0].source_line, "println(\"hello);");
250        assert_eq!(diagnostic.excerpts[0].annotations.len(), 1);
251        assert_eq!(
252            diagnostic.excerpts[0].annotations[0].span,
253            ocelot_base::span::Span::new(8, 16)
254        );
255        assert_eq!(
256            diagnostic.excerpts[0].annotations[0].message,
257            "string is missing a closing quote"
258        );
259    }
260
261    #[test]
262    fn reports_strings_terminated_by_a_newline_as_source_diagnostics() {
263        let source_file = SourceFile::new("examples/broken.ocelot", "println(\"hello);\n");
264        let mut context = CompilationContext::default();
265        let token_types: Vec<_> = lex(&source_file, &mut context)
266            .into_iter()
267            .map(|token| token.token_type)
268            .collect();
269
270        assert_eq!(
271            token_types,
272            vec![
273                TokenType::Identifier,
274                TokenType::LeftParen,
275                TokenType::EndOfFile
276            ]
277        );
278        assert!(context.has_errors());
279        assert_eq!(context.source_diagnostics.diagnostics.len(), 1);
280
281        let diagnostic = &context.source_diagnostics.diagnostics[0];
282
283        assert_eq!(diagnostic.level, DiagnosticLevel::Error);
284        assert_eq!(diagnostic.file_path.as_str(), "examples/broken.ocelot");
285        assert_eq!(diagnostic.message, "unterminated string literal");
286        assert_eq!(diagnostic.excerpts.len(), 1);
287        assert_eq!(diagnostic.excerpts[0].line_number, 1);
288        assert_eq!(diagnostic.excerpts[0].source_line, "println(\"hello);");
289        assert_eq!(diagnostic.excerpts[0].annotations.len(), 1);
290        assert_eq!(
291            diagnostic.excerpts[0].annotations[0].span,
292            ocelot_base::span::Span::new(8, 16)
293        );
294        assert_eq!(
295            diagnostic.excerpts[0].annotations[0].message,
296            "string is missing a closing quote"
297        );
298    }
299
300    #[test]
301    fn emits_unexpected_tokens_for_unknown_characters() {
302        let source_file = SourceFile::new("examples/unexpected.ocelot", "@");
303        let mut context = CompilationContext::default();
304        let token_types: Vec<_> = lex(&source_file, &mut context)
305            .into_iter()
306            .map(|token| token.token_type)
307            .collect();
308
309        assert_eq!(
310            token_types,
311            vec![TokenType::Unexpected, TokenType::EndOfFile]
312        );
313        assert!(!context.has_errors());
314    }
315}
ocelot_parser/lexer/lex.rs

ocelot_parser/lexer/
lex.rs
