lualexer 0.1.2

Read Lua code and produce tokens
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

use crate::token::Token;
use crate::lexer::LexerErrorType;
use crate::utils::ParseResult;

// This function assumes that the next character is either a single quote (`'`) or
// a double quote (`"`)
pub fn parse_quoted_string<'a>(input: &'a str) -> ParseResult<'a> {
    let quote = input.chars().next()
        .unwrap_or_else(|| unreachable!("must be called when the next character is a `'` or `\"`"));
    let mut escaped = false;

    input.char_indices()
        .find_map(|(i, character)| {
            if i == 0 {
                None
            } else {
                match character {
                    '\\' => {
                        escaped = !escaped;
                        None
                    },
                    '\n' => {
                        if escaped {
                            escaped = false;
                            None
                        } else {
                            Some(Err(LexerErrorType::UnfinishedString))
                        }
                    },
                    '\r' => None,
                    _ => {
                        if !escaped && character == quote {
                            Some(Ok(i))
                        } else {
                            escaped = false;
                            None
                        }
                    }
                }
            }
        })
        .unwrap_or(Err(LexerErrorType::UnfinishedString))
        .and_then(|i| {
            let (result, rest) = input.split_at(i + 1);

            Ok((Token::new_string(result), rest))
        })
}

#[cfg(test)]
mod tests {
    use super::*;

    mod ok {
        use super::*;

        macro_rules! test_string {
            ($($name:ident : $input:expr),+) => {
                $(
                    #[test]
                    fn $name() {
                        let (token, rest) = parse_quoted_string($input).unwrap();
                        assert_eq!(token, Token::new_string($input));
                        assert_eq!(rest, "");
                    }
                )+
            };
        }

        test_string!(
            double_quoted_word: r#""hello""#,
            single_quote_word: "'hello'",
            double_quoted_with_single_quote: r#""hel'lo""#,
            single_quoted_with_double_quote: r#"'hel"lo'"#,
            escaped_double_quotes: r#""hello \"you\"""#,
            escaped_single_quotes: r#"'hello \'you\''"#,
            escaped_slash_before_double_quote: r#""hello !\\""#,
            escaped_slash_before_single_quote: r#"'hello !\\'"#,
            double_quoted_more_escaped_characters: r#""hello\t !\"\"""#,
            single_quoted_more_escaped_characters: r#"'hello\t !\'\''"#,
            multiline_double_quoted: r#""hello \
multilined""#,
            multiline_single_quoted: r#"'hello \
multilined'"#,
            double_quoted_with_backslash: r#""hey \ you \ ""#,
            single_quoted_with_backslash: r#"'hey \ you \ '"#
        );
    }

    mod err {
        use super::*;

        macro_rules! test_unfinished_string {
            ($($name:ident : $input:expr),+) => {
                $(
                    #[test]
                    fn $name() {
                        let lexer_error = parse_quoted_string($input).unwrap_err();
                        assert_eq!(lexer_error, LexerErrorType::UnfinishedString);
                    }
                )+
            };
        }

        test_unfinished_string!(
            double_quoted_word: r#""hello"#,
            single_quote_word: "'hello",
            double_quoted_with_single_quote: r#""hel'lo"#,
            single_quoted_with_double_quote: r#"'hel"lo"#,
            escaped_double_quotes: r#""hello \"you\""#,
            escaped_slash_before_double_quotes: r#""hello !\\"#,
            double_quoted_more_escaped_characters: r#""hello\t !\"\""#,
            multiline_double_quoted: r#""hello \
multilined"#,
            double_quoted_with_backslash: r#""hey \ you \ "#
        );
    }
}