lexer/
lexer.rs

1
2use core::ops::Range;
3use splitter::{StrInfo, StrSplitter};
4
5#[derive(Debug, Clone, Copy, PartialEq, Eq, StrInfo)]
6enum TType {
7    #[splitter(" " | "\t" | "\n" | "\r")]
8    Whitespace,
9
10    #[splitter("true" | "false")]
11    Bool,
12
13    #[splitter("||" | "&&")]
14    Operator,
15
16    #[splitter(s if s.chars().all(char::is_alphabetic))]
17    Idenifier,
18
19    #[splitter(_)]
20    Unknown,
21}
22
23#[derive(Debug, Clone, PartialEq, Eq, StrInfo)]
24struct Token<'a> {
25    content: &'a str,
26    token_type: TType,
27    span: Range<usize>,
28}
29
30fn main() {
31    use TType::*;
32
33    let sp = StrSplitter::new(
34        "true || yep && false && unknown5",
35        " ",
36    ).with_info::<Token>();
37
38    let tokens = sp.collect::<Vec<_>>();
39
40    assert_eq!(tokens, vec![
41        Token { content: "true", token_type: Bool, span: 0..4 },
42        Token { content: " ", token_type: Whitespace, span: 4..5 },
43        Token { content: "||", token_type: Operator, span: 5..7 },
44        Token { content: " ", token_type: Whitespace, span: 7..8 },
45        Token { content: "yep", token_type: Idenifier, span: 8..11 },
46        Token { content: " ", token_type: Whitespace, span: 11..12 },
47        Token { content: "&&", token_type: Operator, span: 12..14 },
48        Token { content: " ", token_type: Whitespace, span: 14..15 },
49        Token { content: "false", token_type: Bool, span: 15..20 },
50        Token { content: " ", token_type: Whitespace, span: 20..21 },
51        Token { content: "&&", token_type: Operator, span: 21..23 },
52        Token { content: " ", token_type: Whitespace, span: 23..24 },
53        Token { content: "unknown5", token_type: Unknown, span: 24..32 },
54    ])
55}
lexer/lexer.rs

lexer/
lexer.rs
