alkale 2.0.0

A simple LL(1) lexer library for Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

//! Sample test from `lib.rs` documentation.

use alkale::{
    format_notification, notification::NotificationSeverity, token::Token, FinalizedLexerResult,
    LexerResult, SourceCodeScanner,
};

type Word<'a> = &'a str;

fn lexer(source: &str) -> FinalizedLexerResult<Word<'_>> {
    let scanner = SourceCodeScanner::new(source);
    let mut result = LexerResult::new();

    while scanner.has_next() {
        // Try to parse out a word.
        if let Some(identifier) = scanner.try_consume_standard_identifier() {
            // We found a word, push it and restart the loop.
            result.push_token(Token::from_spanned(identifier));
            continue;
        }

        // No word was found, consume one character.
        if let Some(char) = scanner.next_span() {
            // If this character wasn't whitespace (i.e. illegal char) then
            // report a notification.
            if !char.is_whitespace() {
                format_notification!("Unrecognized character '{}'", char.data)
                    .span(char.span)
                    .severity(NotificationSeverity::Error)
                    .report(&mut result);
            }
        }
    }

    result.finalize()
}

fn main() {
    let data = "word word another word % that was not a word what!!";

    let result = lexer(data);

    println!("{:#?}", result);
}