neyuki 0.6.3

A small Lua-like scripting language implemented in Rust
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

//! Tokens and the tokenizer interface.

#[cfg(test)]
mod tests {
    use super::Lexer;

    #[test]
    fn tokenizes_keywords_numbers_and_strings() {
        let mut lexer = Lexer::new("local x = 42\nif x != 0 then\n  print(\"ok\")\nend");
        let tokens = lexer.tokenize().unwrap();

        assert_eq!(tokens[0].kind, "keyword");
        assert_eq!(tokens[0].value, "local");
        assert_eq!(tokens[1].kind, "name");
        assert_eq!(tokens[1].value, "x");
        assert_eq!(tokens[2].kind, "symbol");
        assert_eq!(tokens[2].value, "=");
        assert_eq!(tokens[3].kind, "number");
        assert_eq!(tokens[3].value, "42");
        assert_eq!(tokens[4].kind, "keyword");
        assert_eq!(tokens[4].value, "if");
        assert_eq!(tokens[5].kind, "name");
        assert_eq!(tokens[5].value, "x");
        assert_eq!(tokens[6].kind, "symbol");
        assert_eq!(tokens[6].value, "!=");
        assert_eq!(tokens[7].kind, "number");
        assert_eq!(tokens[7].value, "0");
        assert_eq!(tokens[8].kind, "keyword");
        assert_eq!(tokens[8].value, "then");
        assert_eq!(tokens[11].kind, "string");
        assert_eq!(tokens[11].value, "ok");
    }

    #[test]
    fn reads_long_bracket_strings() {
        let mut lexer = Lexer::new("local s = [[hello\nworld]]");
        let tokens = lexer.tokenize().unwrap();
        assert_eq!(tokens[3].kind, "string");
        assert_eq!(tokens[3].value, "hello\nworld");
    }

    #[test]
    fn tracks_line_and_column() {
        let mut lexer = Lexer::new("local x = 42\nif");
        let tokens = lexer.tokenize().unwrap();
        assert_eq!(tokens[0].line, 1);
        assert_eq!(tokens[0].col, 1); // "local" starts at col 1
        assert_eq!(tokens[1].col, 7); // "x" starts at col 7
        assert_eq!(tokens[2].col, 9); // "=" at col 9
        assert_eq!(tokens[3].col, 11); // "42" at col 11
        assert_eq!(tokens[4].line, 2);
        assert_eq!(tokens[4].col, 1); // "if" starts at col 1 of line 2
    }

    #[test]
    fn reports_malformed_input_as_errors() {
        let cases = [
            (r#"local s = "abc"#, "unfinished string literal"),
            ("local s = [[abc", "unfinished long bracket"),
            ("local s = `abc", "unfinished interpolated string"),
            (r#"local s = "\u{110000}""#, "not a Unicode scalar value"),
            (r#"local s = "\u{41"#, "unfinished escape"),
            (r#"local s = "\xZZ""#, "expected two hex digits"),
        ];
        for (source, expected) in cases {
            let err = Lexer::new(source).tokenize().unwrap_err();
            assert!(err.contains(expected), "{:?}: {}", source, err);
        }
        let tokens = Lexer::new(r#""\u{1F600}\x41""#).tokenize().unwrap();
        assert_eq!(tokens[0].value, "\u{1F600}A");
    }
}

pub use crate::parser::tokenizer::{DocComment, TokenizerEngine};

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Token {
    pub kind: &'static str,
    pub value: String,
    pub line: usize,
    pub col: usize,
}

pub struct Lexer {
    src: String,
}

impl Lexer {
    pub fn new(source: &str) -> Self {
        Self {
            src: source.to_owned(),
        }
    }

    pub fn tokenize(&mut self) -> Result<Vec<Token>, String> {
        let mut engine = TokenizerEngine::new(&self.src);
        engine.tokenize()
    }

    pub fn tokenize_with_comments(&mut self) -> Result<(Vec<Token>, Vec<DocComment>), String> {
        let mut engine = TokenizerEngine::new(&self.src);
        engine.tokenize_with_comments()
    }
}

pub fn is_keyword(word: &str) -> bool {
    crate::parser::tokenizer::WordEngine::is_keyword(word)
}