alkale 2.0.0

A simple LL(1) lexer library for Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

//! This example demonstrates a very simple way to parse numbers in an expression-like format.

#![allow(missing_docs)]

use alkale::{
    format_notification, map_single_char_token,
    notification::{NotificationBuilder, NotificationSeverity},
    span::Spanned,
    token::Token,
    FinalizedLexerResult, LexerResult, SourceCodeScanner,
};

/// These are the tokens that will be present in the resulting tokens.
#[derive(Debug, Clone)]
pub enum ExprToken<'a> {
    OpenParen,         // (
    CloseParen,        // )
    Add,               // +
    Sub,               // -
    Mul,               // *
    Div,               // /
    Variable(&'a str), // abc | name | ...
    Number(f64),       // 2.3 | 1 | 5e-2 | ...
}

/// Convert an expression string into a [LexerResult] containing tokens
/// and notifications.
fn expression_lexer<'a: 'b, 'b>(source: &'a str) -> FinalizedLexerResult<ExprToken<'b>> {
    // Create a LexerContext from our source code.
    let context = SourceCodeScanner::new(source);
    let mut result = LexerResult::new();

    while context.has_next() {
        // If the next character matches any of these, push the token and its span to the context,
        // consume the character from the source code, and skip the rest of this lexer iteration.
        map_single_char_token!(&context, &mut result,
            '(' => ExprToken::OpenParen,
            ')' => ExprToken::CloseParen,
            '+' => ExprToken::Add,
            '-' => ExprToken::Sub,
            '*' => ExprToken::Mul,
            '/' => ExprToken::Div,
        );

        // If an identifier exists here in the source code, consume it and push it
        // as a token.
        if let Some(Spanned { span, data }) = context.try_consume_standard_identifier() {
            result.push_token(Token::new(ExprToken::Variable(data), span));
            continue;
        }

        // Attempt to parse a number. If one was found and it was valid, push
        // a token for it, otherwise report an error notification.
        if let Some(Spanned { data, span }) = context.try_parse_float() {
            if let Ok(number) = data {
                // The returned number is valid, push a token.
                result.push_token(Token::new(ExprToken::Number(number), span));
            } else {
                // The returned number failed to parse— report a notification.
                NotificationBuilder::new("Floating-point number is malformed")
                    .severity(NotificationSeverity::Error)
                    .span(span)
                    .report(&mut result);
            }

            continue;
        }

        // If more characters exist, take one and throw an error if it's not whitespace.
        if let Some(Spanned { data, span }) = context.next_span() {
            if !data.is_whitespace() {
                format_notification!("Unrecognized character '{data}'")
                    .severity(NotificationSeverity::Error)
                    .span(span)
                    .report(&mut result);
            }
        }
    }

    result.finalize()
}

fn main() {
    // The test program we're going to tokenize.
    let program = "23 * (012 - x) / 1_2_3 + 5e-3";

    // Tokenize and get result
    let result = expression_lexer(program);

    // Print the result— this example should have no notifications.
    println!("{:#?}", result.notifications());
    println!("{:#?}", result.tokens());
}