lexxor 0.9.2

A fast, extensible, greedy, single-pass text tokenizer for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

use lexxor::input::InputString;
use lexxor::matcher::exact::ExactMatcher;
use lexxor::matcher::float::FloatMatcher;
use lexxor::matcher::integer::IntegerMatcher;
use lexxor::matcher::symbol::SymbolMatcher;
use lexxor::matcher::whitespace::WhitespaceMatcher;
use lexxor::matcher::word::WordMatcher;
use lexxor::token::{
    TOKEN_TYPE_FLOAT, TOKEN_TYPE_INTEGER, TOKEN_TYPE_KEYWORD, TOKEN_TYPE_SYMBOL,
    TOKEN_TYPE_WHITESPACE, TOKEN_TYPE_WORD,
};
use lexxor::{Lexxer, Lexxor};

fn main() {
    // Create a string that resembles programming code
    let code = r#"
fn calculate(x: f32, y: f32) -> f32 {
    let result = x * 3.14 + y;
    if result > 10.0 {
        return result / 2.0;
    } else {
        return result;
    }
}
"#;

    let input = InputString::new(code.to_string());

    // Create a Lexxor tokenizer specifically configured for code
    // Higher precedence for floats and keywords
    let mut lexxor = Lexxor::<512>::new(
        Box::new(input),
        vec![
            Box::new(WhitespaceMatcher {
                index: 0,
                column: 0,
                line: 0,
                precedence: 0,
                running: true,
            }),
            // Use ExactMatcher to recognize keywords with high precedence
            Box::new(ExactMatcher::build_exact_matcher(
                vec!["fn", "let", "if", "else", "return"],
                TOKEN_TYPE_KEYWORD,
                2,
            )),
            Box::new(FloatMatcher {
                index: 0,
                precedence: 1,
                dot: false,
                float: false,
                running: true,
            }),
            Box::new(IntegerMatcher {
                index: 0,
                precedence: 0,
                running: true,
            }),
            Box::new(WordMatcher {
                index: 0,
                precedence: 0,
                running: true,
            }),
            Box::new(SymbolMatcher {
                index: 0,
                precedence: 0,
                running: true,
            }),
        ],
    );

    // Process and display all tokens
    println!("Tokenizing Code Sample:");
    println!("{}", "-".repeat(70));
    println!("{}", code);
    println!("{}", "-".repeat(70));
    println!(
        "{:<15} {:<15} {:<10} {:<10}",
        "TOKEN TYPE", "VALUE", "LINE", "COLUMN"
    );
    println!("{}", "-".repeat(70));

    loop {
        match lexxor.next_token() {
            Ok(Some(token)) => {
                let type_name = match token.token_type {
                    TOKEN_TYPE_WORD => "WORD",
                    TOKEN_TYPE_WHITESPACE => "WHITESPACE",
                    TOKEN_TYPE_SYMBOL => "SYMBOL",
                    TOKEN_TYPE_INTEGER => "INTEGER",
                    TOKEN_TYPE_FLOAT => "FLOAT",
                    TOKEN_TYPE_KEYWORD => "KEYWORD",
                    _ => "OTHER",
                };

                // Skip printing whitespace for cleaner output
                if token.token_type != TOKEN_TYPE_WHITESPACE {
                    println!(
                        "{:<15} {:<15} {:<10} {:<10}",
                        type_name,
                        token.value.replace("\n", "\\n"),
                        token.line,
                        token.column
                    );
                }
            }
            Ok(None) => {
                println!("\nEnd of input reached.");
                break;
            }
            Err(e) => {
                println!("Error during tokenization: {}", e);
                break;
            }
        }
    }
}