lualexer 0.1.2

Read Lua code and produce tokens
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

/// The different type of tokens that will be produced by the lexer
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum TokenType {
    Comment,
    Identifier,
    Keyword,
    Number,
    String,
    Symbol,
    Whitespace,
}

/// A small struct that contains minimal information about a slice of
/// the input. The token does not own the part of the code that it
/// represents, it only keeps a reference to the original input.
#[derive(Debug, Eq, PartialEq)]
pub struct Token<'a> {
    token_type: TokenType,
    slice: &'a str,
}

impl<'a> Token<'a> {
    fn new(token_type: TokenType, slice: &'a str) -> Token<'a> {
        Token {
            token_type,
            slice,
        }
    }

    pub fn new_comment(slice: &'a str) -> Token<'a> {
        Token::new(TokenType::Comment, slice)
    }

    pub fn new_identifier(slice: &'a str) -> Token<'a> {
        Token::new(TokenType::Identifier, slice)
    }

    pub fn new_keyword(slice: &'a str) -> Token<'a> {
        Token::new(TokenType::Keyword, slice)
    }

    pub fn new_number(slice: &'a str) -> Token<'a> {
        Token::new(TokenType::Number, slice)
    }

    pub fn new_string(slice: &'a str) -> Token<'a> {
        Token::new(TokenType::String, slice)
    }

    pub fn new_symbol(slice: &'a str) -> Token<'a> {
        Token::new(TokenType::Symbol, slice)
    }

    pub fn new_whitespace(slice: &'a str) -> Token<'a> {
        Token::new(TokenType::Whitespace, slice)
    }

    pub fn get_type(&self) -> &TokenType {
        &self.token_type
    }

    pub fn get_content(&self) -> &str {
        &self.slice
    }

    pub fn is_type(&self, token_type: TokenType) -> bool {
        self.token_type == token_type
    }

    pub fn is_slice<'b>(&self, compare_with: &'b str) -> bool {
        self.slice == compare_with
    }
}