jayce 12.1.0

jayce is a tokenizer 🌌
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

use jayce::{Duo, Token, Tokenizer};
use std::sync::OnceLock;

const SOURCE: &str = r#"let kind_cat = "I calmly pet my cute cats"
pancake_icecream

very_multiline

"#;

fn duos() -> &'static Vec<Duo<&'static str>> {
    static DUOS: OnceLock<Vec<Duo<&'static str>>> = OnceLock::new();
    DUOS.get_or_init(|| {
        vec![
            Duo::new("whitespace", r"^[^\S\n]+", true),
            Duo::new("comment_line", r"^//(.*)", true),
            Duo::new("comment_block", r"^/\*(.|\n)*?\*/", true),
            Duo::new("newline", r"^\n", true),
            Duo::new("operator", r"^=", true),
            Duo::new("keyword", r"^let", true),
            Duo::new("string", r#"^"[^"]*""#, true),
            Duo::new("identifier", r"^[a-z_]+", true),
        ]
    })
}

const EXPECTED: [Token<&'static str>; 14] = [
    Token {
        kind: &"keyword",
        value: "let",
        pos: (1, 1),
    },
    Token {
        kind: &"whitespace",
        value: " ",
        pos: (1, 4),
    },
    Token {
        kind: &"identifier",
        value: "kind_cat",
        pos: (1, 5),
    },
    Token {
        kind: &"whitespace",
        value: " ",
        pos: (1, 13),
    },
    Token {
        kind: &"operator",
        value: "=",
        pos: (1, 14),
    },
    Token {
        kind: &"whitespace",
        value: " ",
        pos: (1, 15),
    },
    Token {
        kind: &"string",
        value: "\"I calmly pet my cute cats\"",
        pos: (1, 16),
    },
    Token {
        kind: &"newline",
        value: "\n",
        pos: (1, 43),
    },
    Token {
        kind: &"identifier",
        value: "pancake_icecream",
        pos: (2, 1),
    },
    Token {
        kind: &"newline",
        value: "\n",
        pos: (2, 17),
    },
    Token {
        kind: &"newline",
        value: "\n",
        pos: (3, 1),
    },
    Token {
        kind: &"identifier",
        value: "very_multiline",
        pos: (4, 1),
    },
    Token {
        kind: &"newline",
        value: "\n",
        pos: (4, 15),
    },
    Token {
        kind: &"newline",
        value: "\n",
        pos: (5, 1),
    },
];

#[test]
fn multiline() {
    let tokens = Tokenizer::new(SOURCE, duos()).consume_all().unwrap();
    assert_eq!(tokens, EXPECTED);
    assert_eq!(tokens.len(), EXPECTED.len());
}