luau-lexer 0.2.0

A lexer for the luau language.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

//! All `impl` blocks for [`TokenType`]

use crate::{
    prelude::{
        Comment, CompoundOperator, Keyword, Lexable, Lexer, Literal, Operator, ParseError,
        PartialKeyword, Symbol, TokenType,
    },
    utils::is_identifier_start,
};

impl Lexable for TokenType {
    fn try_lex(lexer: &mut Lexer) -> Option<Self> {
        let character = lexer.current_char()?;
        let start = lexer.lexer_position;

        match character {
            '0'..='9' => {
                if let Some(number) = Literal::parse_number(lexer) {
                    return Some(Self::Literal(number));
                }
            }
            '.' => {
                let next_character = lexer.next_char();
                if matches!(next_character, Some('0'..='9')) {
                    if let Some(number) = Literal::parse_number(lexer) {
                        return Some(Self::Literal(number));
                    }
                }
                lexer.consume('.');

                let next_character = lexer.current_char();
                if next_character == Some('.') {
                    lexer.consume('.');

                    if lexer.consume('.') {
                        return Some(Self::Symbol(Symbol::Ellipses));
                    } else {
                        let operator = Operator::Concatenation;

                        return CompoundOperator::try_from_operator(operator, lexer)
                            .map(Self::CompoundOperator)
                            .or(Some(Self::Operator(operator)));
                    }
                } else {
                    return Some(Self::Symbol(Symbol::Dot));
                }
            }
            '\'' | '"' | '`' => {
                if let Some(string) = Literal::parse_string(lexer) {
                    return Some(Self::Literal(string));
                }
            }
            '[' => {
                if matches!(lexer.next_char(), Some('[' | '=')) {
                    if let Some(string) = Literal::parse_string(lexer) {
                        return Some(Self::Literal(string));
                    }
                }
                lexer.consume('[');

                return Some(Self::Symbol(Symbol::OpeningBrackets));
            }
            '>' if lexer.consume_with_next('=') => {
                return Some(Self::CompoundOperator(
                    CompoundOperator::GreaterThanOrEqualTo,
                ));
            }
            '<' if lexer.consume_with_next('=') => {
                return Some(Self::CompoundOperator(CompoundOperator::LessThanOrEqualTo));
            }
            '-' if lexer.consume_with_next('-') => {
                return Comment::try_lex(lexer).map(Self::Comment)
            }
            '-' if lexer.consume_with_next('>') => {
                return Some(Self::Symbol(Symbol::Arrow));
            }
            '=' => {
                lexer.consume('=');

                if lexer.consume('=') {
                    return Some(Self::CompoundOperator(CompoundOperator::EqualEqual));
                } else {
                    return Some(Self::Symbol(Symbol::Equal));
                }
            }
            ':' if lexer.consume_with_next(':') => {
                return Some(Self::Symbol(Symbol::Typecast));
            }
            _ if is_identifier_start(character) => {
                let word = lexer.consume_identifier();

                if let Some(keyword) = Keyword::try_from_str(&word) {
                    return Some(Self::Keyword(keyword));
                } else if let Some(partial_keyword) = PartialKeyword::try_from_str(&word) {
                    return Some(Self::PartialKeyword(partial_keyword));
                }

                match word.as_str() {
                    "true" => return Some(Self::Literal(Literal::Boolean(true))),
                    "false" => return Some(Self::Literal(Literal::Boolean(false))),
                    "and" => return Some(Self::Operator(Operator::And)),
                    "or" => return Some(Self::Operator(Operator::Or)),
                    "not" => return Some(Self::Operator(Operator::Not)),
                    _ => return Some(Self::Identifier(word)),
                }
            }
            _ => {
                if let Some(symbol) = Symbol::try_from_char(character, lexer) {
                    return Some(Self::Symbol(symbol));
                }
                if let Some(operator) = Operator::try_from_chars(character, lexer) {
                    return CompoundOperator::try_from_operator(operator, lexer)
                        .map(Self::CompoundOperator)
                        .or(Some(Self::Operator(operator)));
                }
            }
        }

        lexer.increment_position(1);

        Some(Self::Error(ParseError::new(
            start,
            format!("Unexpected character: {}", character),
            Some(lexer.lexer_position),
        )))
    }
}