lualexer 0.1.2

Read Lua code and produce tokens
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

use crate::token::Token;
use crate::utils::{
    ParseResult,
    accumulate_while,
    is_identifier_end,
};

// This function assumes that the next character is a letter or an underscore (`_`).
// It returns a token with a keyword type if the next characters are considered
// as a Lua keyword. Otherwise, it is typed as an Identifier.
pub fn parse_identifier_or_keyword<'a>(input: &'a str) -> ParseResult<'a> {
    let (identifier, next_input) = accumulate_while(input, is_identifier_end);
    let token = match identifier {
        "and" | "break" | "do" | "else" | "elseif" | "end" | "false"
            | "for" | "function" | "if" | "in" | "local" | "nil"
            | "not" | "or" | "repeat" | "return" | "then" | "true"
            | "until" | "while" => Token::new_keyword(identifier),
        _ => Token::new_identifier(identifier),
    };

    Ok((token, next_input))
}

#[cfg(test)]
mod tests {
    use super::*;

    macro_rules! test_identifier {
        ($($input:literal),+) => {
            #[test]
            fn parse_identifiers() {
                $(
                    let (token, rest) = parse_identifier_or_keyword($input).unwrap();
                    assert_eq!(token, Token::new_identifier($input));
                    assert_eq!(rest, "");
                )+
            }
        };
    }

    macro_rules! test_keyword {
        ($($name:ident : $input:literal),+) => {
            $(
                #[test]
                fn $name() {
                    let (token, rest) = parse_identifier_or_keyword($input).unwrap();
                    assert_eq!(token, Token::new_keyword($input));
                    assert_eq!(rest, "");
                }
            )+
        };
    }

    test_identifier!(
        "i",
        "_",
        "identifier",
        "CONSTANT",
        "mixedCASE",
        "snake_case",
        "numbers123",
        "_variable",
        "_123abc"
    );

    test_keyword!(
        keyword_and: "and",
        keyword_break: "break",
        keyword_do: "do",
        keyword_else: "else",
        keyword_elseif: "elseif",
        keyword_end: "end",
        keyword_false: "false",
        keyword_for: "for",
        keyword_function: "function",
        keyword_if: "if",
        keyword_in: "in",
        keyword_local: "local",
        keyword_nil: "nil",
        keyword_not: "not",
        keyword_or: "or",
        keyword_repeat: "repeat",
        keyword_return: "return",
        keyword_then: "then",
        keyword_true: "true",
        keyword_until: "until",
        keyword_while: "while"
    );
}