skullbrain-lexer 0.1.0

Lexer for the skullbrain programming language, responsible for tokenizing source code.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

use std::path::Path;

use insta::assert_snapshot;
use serde::Serialize;

use crate::Lexer;

#[test]
fn test_lexer() {
    insta::glob!("inputs/*.skull", |path| actual_test(path))
}

fn actual_test(path: &Path) {
    let source_code = std::fs::read_to_string(path).expect("Failed reading source code file");

    let mut chars_indices = source_code.char_indices().map(|(a, _)| a).peekable();
    let mut start_byte_position = 0usize;

    let formatted_tokens =
        Lexer::new(source_code.chars()).fold(String::new(), |mut string, token| {
            for _ in 0..token.len_repr() {
                chars_indices.next();
            }

            let end_byte_positon = *chars_indices.peek().unwrap_or(&source_code.len());

            let item = format!(
                "{:?}@{}:\"{}\"",
                token.kind(),
                token.len_repr(),
                &source_code[start_byte_position..end_byte_positon]
            );

            string.push_str(item.as_str());
            string.push('\n');
            start_byte_position = end_byte_positon;

            string
        });

    insta::with_settings!({ info => &EscapedString(source_code.as_str()) },{
        assert_snapshot!(formatted_tokens)
    })
}

#[derive(Serialize)]
struct EscapedString<'a>(&'a str);