lualexer 0.1.2

Read Lua code and produce tokens
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

use crate::Lexer;
use crate::token::Token;
use crate::tokenizer::Tokenizer;
use crate::utils::is_whitespace;

#[derive(Debug, Default, Clone)]
pub struct FastTokenizer {}

impl FastTokenizer {
    fn skip_whitespace<'a>(&self, input: &'a str) -> &'a str {
        input.char_indices()
            .find(|(_, character)| !is_whitespace(*character))
            .and_then(|(i, _)| input.get(i..input.len()))
            .unwrap_or("")
    }
}

impl Tokenizer for FastTokenizer {
    fn new() -> FastTokenizer {
        FastTokenizer {}
    }

    fn process_whitespace<'a>(&self, _tokens: &mut Vec<Token<'a>>, input: &'a str) -> &'a str {
        self.skip_whitespace(input)
    }
}

/// A lexer that skips all the information about whitespaces.
pub type FastLexer = Lexer<FastTokenizer>;

#[cfg(test)]
mod tests {
    use super::*;

    fn subject() -> FastTokenizer {
        FastTokenizer {}
    }

    mod skip_whitespace {
        use super::*;

        #[test]
        fn should_return_moved_slice() {
            let input = "   \n\t123";

            assert_eq!("123", subject().skip_whitespace(input));
        }

        #[test]
        fn should_return_same_slice_if_no_whitespace() {
            let input = "123   ";

            assert_eq!(input, subject().skip_whitespace(input));
        }

        #[test]
        fn do_test3() {
            let input = " éàï";

            assert_eq!("éàï", subject().skip_whitespace(input));
        }
    }
}