jayce 12.1.0

jayce is a tokenizer 🌌
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

use std::sync::OnceLock;

use crate::Duo;

#[derive(Debug, PartialEq, Clone, Copy)]
pub enum KindsRust {
    Whitespace,
    CommentLine,
    CommentBlock,
    Newline,

    Keyword,
    String,
    Char,
    Lifetime,
    Operator,
    Identifier,
    Integer,
    Float,
    DoubleColon,
    Semicolon,
    OpenBrace,
    CloseBrace,
    OpenParen,
    CloseParen,
    OpenBracket,
    CloseBracket,
    Comma,
    Hash,
    Dot,
    Colon,
    Pipe,
    OpenAngle,
    CloseAngle,
    Caret,
    TempBorrow,
    Question,
    MacroExclamation,
}

pub fn duos_rust() -> &'static Vec<Duo<KindsRust>> {
    static DUOS_RUST: OnceLock<Vec<Duo<KindsRust>>> = OnceLock::new();
    DUOS_RUST.get_or_init(|| {vec![
         Duo::new(KindsRust::Whitespace, r"^[^\S\n]+", false),
        Duo::new(KindsRust::CommentLine, r"^//(.*)", false),
        Duo::new(KindsRust::CommentBlock, r"^/\*(.|\n)*?\*/", false),
        Duo::new(KindsRust::Newline, r"^\n", false),
        Duo::new(
            KindsRust::Keyword,
            r"^(mut|let|if|else|fn|struct|enum|match|use|mod|pub|crate|impl|trait|for|while|loop|break|continue|return|as|const|static|type|where|unsafe|extern|ref|self|super|in|move|dyn|abstract|async|await|become|box|do|final|macro|override|priv|typeof|unsized|virtual|yield)\b",
            true
        ),
        Duo::new(KindsRust::String, r#"^"[^"]*""#, true),
        Duo::new(KindsRust::Char, r"^'(.|\\n)'", true),
        Duo::new(KindsRust::Lifetime, r"^'(?:[a-z_][a-z0-9_]*|static)\b", true),
        Duo::new(KindsRust::Operator, r"^(=|\+|-|\*|/|%)", true),
        Duo::new(KindsRust::Identifier, r"^[a-zA-Z_][a-zA-Z0-9_]*", true),
        Duo::new(KindsRust::Integer, r"^\d+", true),
        Duo::new(KindsRust::Float, r"^\d+\.\d+", true),
        Duo::new(KindsRust::DoubleColon, r"^::", true),
        Duo::new(KindsRust::Semicolon, r"^;", true),
        Duo::new(KindsRust::OpenBrace, r"^\{", true),
        Duo::new(KindsRust::CloseBrace, r"^\}", true),
        Duo::new(KindsRust::OpenParen, r"^\(", true),
        Duo::new(KindsRust::CloseParen, r"^\)", true),
        Duo::new(KindsRust::OpenBracket, r"^\[", true),
        Duo::new(KindsRust::CloseBracket, r"^\]", true),
        Duo::new(KindsRust::Comma, r"^,", true),
        Duo::new(KindsRust::Hash, r"^#", true),
        Duo::new(KindsRust::Dot, r"^\.", true),
        Duo::new(KindsRust::Colon, r"^:", true),
        Duo::new(KindsRust::Pipe, r"^\|", true),
        Duo::new(KindsRust::OpenAngle, r"^<", true),
        Duo::new(KindsRust::CloseAngle, r"^>", true),
        Duo::new(KindsRust::Caret, r"^\^", true),
        Duo::new(KindsRust::TempBorrow, r"^&", true),
        Duo::new(KindsRust::Question, r"^\?", true),
        Duo::new(KindsRust::MacroExclamation, r"^!", true)
    ]})
}

// lazy_static::lazy_static! {
//     pub static ref DUOS_RUST: Vec<Duo<KindsRust>> = vec![
//         Duo::new(KindsRust::Whitespace, r"^[^\S\n]+", false),
//         Duo::new(KindsRust::CommentLine, r"^//(.*)", false),
//         Duo::new(KindsRust::CommentBlock, r"^/\*(.|\n)*?\*/", false),
//         Duo::new(KindsRust::Newline, r"^\n", false),
//         Duo::new(
//             KindsRust::Keyword,
//             r"^(mut|let|if|else|fn|struct|enum|match|use|mod|pub|crate|impl|trait|for|while|loop|break|continue|return|as|const|static|type|where|unsafe|extern|ref|self|super|in|move|dyn|abstract|async|await|become|box|do|final|macro|override|priv|typeof|unsized|virtual|yield)\b",
//             true
//         ),
//         Duo::new(KindsRust::String, r#"^"[^"]*""#, true),
//         Duo::new(KindsRust::Char, r"^'(.|\\n)'", true),
//         Duo::new(KindsRust::Lifetime, r"^'(?:[a-z_][a-z0-9_]*|static)\b", true),
//         Duo::new(KindsRust::Operator, r"^(=|\+|-|\*|/|%)", true),
//         Duo::new(KindsRust::Identifier, r"^[a-zA-Z_][a-zA-Z0-9_]*", true),
//         Duo::new(KindsRust::Integer, r"^\d+", true),
//         Duo::new(KindsRust::Float, r"^\d+\.\d+", true),
//         Duo::new(KindsRust::DoubleColon, r"^::", true),
//         Duo::new(KindsRust::Semicolon, r"^;", true),
//         Duo::new(KindsRust::OpenBrace, r"^\{", true),
//         Duo::new(KindsRust::CloseBrace, r"^\}", true),
//         Duo::new(KindsRust::OpenParen, r"^\(", true),
//         Duo::new(KindsRust::CloseParen, r"^\)", true),
//         Duo::new(KindsRust::OpenBracket, r"^\[", true),
//         Duo::new(KindsRust::CloseBracket, r"^\]", true),
//         Duo::new(KindsRust::Comma, r"^,", true),
//         Duo::new(KindsRust::Hash, r"^#", true),
//         Duo::new(KindsRust::Dot, r"^\.", true),
//         Duo::new(KindsRust::Colon, r"^:", true),
//         Duo::new(KindsRust::Pipe, r"^\|", true),
//         Duo::new(KindsRust::OpenAngle, r"^<", true),
//         Duo::new(KindsRust::CloseAngle, r"^>", true),
//         Duo::new(KindsRust::Caret, r"^\^", true),
//         Duo::new(KindsRust::TempBorrow, r"^&", true),
//         Duo::new(KindsRust::Question, r"^\?", true),
//         Duo::new(KindsRust::MacroExclamation, r"^!", true),
//     ];
// }