lexer/
lexer_test.rs

1#[cfg(test)]
2mod tests {
3    use crate::token::{Token, TokenKind};
4    use crate::Lexer;
5    use insta::*;
6
7    fn test_token_set(l: &mut Lexer) -> Vec<Token> {
8        let mut token_vs: Vec<Token> = vec![];
9        loop {
10            let t = l.next_token();
11            if t.kind == TokenKind::EOF {
12                token_vs.push(t);
13                break;
14            } else {
15                token_vs.push(t);
16            }
17        }
18        token_vs
19    }
20
21    pub fn test_lexer_common(name: &str, input: &str) {
22        let mut l = Lexer::new(input);
23        let token_vs = test_token_set(&mut l);
24
25        assert_snapshot!(name, serde_json::to_string_pretty(&token_vs).unwrap(), input);
26    }
27
28    #[test]
29    fn test_lexer_simple() {
30        test_lexer_common("simple", "=+(){},:;");
31    }
32
33    #[test]
34    fn test_lexer_let() {
35        test_lexer_common("let", "let x=5");
36    }
37
38    #[test]
39    fn test_comments() {
40        test_lexer_common("comments", "// I am comments");
41    }
42
43    #[test]
44    fn test_lexer_let_with_space() {
45        test_lexer_common("let_with_space", "let x = 5");
46    }
47
48    #[test]
49    fn test_lexer_string() {
50        test_lexer_common("string", r#""a""#);
51    }
52
53    #[test]
54    fn test_lexer_array() {
55        test_lexer_common("array", "[3]");
56    }
57
58    #[test]
59    fn test_lexer_hash() {
60        test_lexer_common("hash", r#"{"one": 1, "two": 2, "three": 3}"#);
61    }
62
63    #[test]
64    fn test_lexer_bool() {
65        test_lexer_common("bool", "let y=true");
66    }
67    
68    #[test]
69    fn test_comments_then_blank_line() {
70        // Ensure comments followed by a blank line are skipped correctly
71        // and do not yield ILLEGAL tokens.
72        test_lexer_common("comments_then_blank_line", "// comment\n\nlet x = 5");
73    }
74
75
76
77    #[test]
78    fn test_lexer_complex() {
79        test_lexer_common(
80            "complex",
81            "
82// welcome to monkeylang
83let five = 5;
84let ten = 10;
85
86let add = fn(x, y) {
87  x + y;
88};
89
90let result = add(five, ten);
91!-/*5;
925 < 10 > 5;
93
94if (5 < 10) {
95	return true;
96} else {
97	return false;
98}
99
10010 == 10;
10110 != 9;",
102        );
103    }
104}
lexer/lexer_test.rs

lexer/
lexer_test.rs
