lexxor 0.9.2

A fast, extensible, greedy, single-pass text tokenizer for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

use lexxor::input::InputString;
use lexxor::matcher::float::FloatMatcher;
use lexxor::matcher::integer::IntegerMatcher;
use lexxor::matcher::symbol::SymbolMatcher;
use lexxor::matcher::whitespace::WhitespaceMatcher;
use lexxor::matcher::word::WordMatcher;
use lexxor::token::{
    TOKEN_TYPE_FLOAT, TOKEN_TYPE_INTEGER, TOKEN_TYPE_SYMBOL, TOKEN_TYPE_WHITESPACE, TOKEN_TYPE_WORD,
};
use lexxor::{Lexxer, Lexxor};

fn main() {
    // Create a simple input string
    let input_text = "Hello world! This is 42 and 3.14159.";
    let input = InputString::new(input_text.to_string());

    // Create a Lexxor tokenizer with standard matchers
    let mut lexxor = Lexxor::<512>::new(
        Box::new(input),
        vec![
            Box::new(WhitespaceMatcher {
                index: 0,
                column: 0,
                line: 0,
                precedence: 0,
                running: true,
            }),
            Box::new(WordMatcher {
                index: 0,
                precedence: 0,
                running: true,
            }),
            Box::new(IntegerMatcher {
                index: 0,
                precedence: 0,
                running: true,
            }),
            Box::new(FloatMatcher {
                index: 0,
                precedence: 0,
                dot: false,
                float: false,
                running: true,
            }),
            Box::new(SymbolMatcher {
                index: 0,
                precedence: 0,
                running: true,
            }),
        ],
    );

    // Process and display all tokens
    println!("Tokenizing: \"{}\"", input_text);
    println!(
        "{:<15} {:<15} {:<10} {:<10}",
        "TOKEN TYPE", "VALUE", "LINE", "COLUMN"
    );
    println!("{}", "-".repeat(50));

    loop {
        match lexxor.next_token() {
            Ok(Some(token)) => {
                let type_name = match token.token_type {
                    TOKEN_TYPE_WORD => "WORD",
                    TOKEN_TYPE_WHITESPACE => "WHITESPACE",
                    TOKEN_TYPE_SYMBOL => "SYMBOL",
                    TOKEN_TYPE_INTEGER => "INTEGER",
                    TOKEN_TYPE_FLOAT => "FLOAT",
                    _ => "OTHER",
                };

                println!(
                    "{:<15} {:<15} {:<10} {:<10}",
                    type_name,
                    token.value.replace("\n", "\\n"),
                    token.line,
                    token.column
                );
            }
            Ok(None) => {
                println!("\nEnd of input reached.");
                break;
            }
            Err(e) => {
                println!("Error during tokenization: {}", e);
                break;
            }
        }
    }
}