jayce 12.1.0

jayce is a tokenizer 🌌
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

use jayce::{Duo, Token, Tokenizer};
use std::sync::OnceLock;

const SOURCE: &str = "abc 123 xyz456 // comment";
const SOURCE_PANIC: &str = "🦀";

#[derive(Debug, PartialEq, Clone)]
pub enum Kinds {
    CommentLine,
    Whitespace,

    Alpha,
    Numeric,
}

fn duos() -> &'static Vec<Duo<Kinds>> {
    static DUOS: OnceLock<Vec<Duo<Kinds>>> = OnceLock::new();
    DUOS.get_or_init(|| {
        vec![
            Duo::new(Kinds::CommentLine, r"^//.*$", false),
            Duo::new(Kinds::Whitespace, r"^\s+", false),
            Duo::new(Kinds::Alpha, r"^[a-zA-Z]+", true),
            Duo::new(Kinds::Numeric, r"^\d+", true),
        ]
    })
}

const EXPECTED: [Token<Kinds>; 4] = [
    Token {
        kind: &Kinds::Alpha,
        value: "abc",
        pos: (1, 1),
    },
    Token {
        kind: &Kinds::Numeric,
        value: "123",
        pos: (1, 5),
    },
    Token {
        kind: &Kinds::Alpha,
        value: "xyz",
        pos: (1, 9),
    },
    Token {
        kind: &Kinds::Numeric,
        value: "456",
        pos: (1, 12),
    },
];

#[test]
fn tokenize_all() {
    let tokens = Tokenizer::new(SOURCE, duos()).consume_all().unwrap();
    assert_eq!(tokens, EXPECTED);
    assert_eq!(tokens.len(), EXPECTED.len());
}

#[test]
#[should_panic(expected = "Failed to match at line")]
fn tokenize_all_should_panic() {
    let mut tokenizer = Tokenizer::new(SOURCE_PANIC, duos());
    let _ = tokenizer.consume_all().unwrap();
}