monkey-lexer 1.0.0

#[cfg(test)]
mod tests {
    use crate::token::{Token, TokenKind};
    use crate::Lexer;
    use insta::*;

    fn test_token_set(l: &mut Lexer) -> Vec<Token> {
        let mut token_vs: Vec<Token> = vec![];
        loop {
            let t = l.next_token();
            if t.kind == TokenKind::EOF {
                token_vs.push(t);
                break;
            } else {
                token_vs.push(t);
            }
        }
        token_vs
    }

    pub fn test_lexer_common(name: &str, input: &str) {
        let mut l = Lexer::new(input);
        let token_vs = test_token_set(&mut l);

        assert_snapshot!(name, serde_json::to_string_pretty(&token_vs).unwrap(), input);
    }

    #[test]
    fn test_lexer_simple() {
        test_lexer_common("simple", "=+(){},:;");
    }

    #[test]
    fn test_lexer_let() {
        test_lexer_common("let", "let x=5");
    }

    #[test]
    fn test_comments() {
        test_lexer_common("comments", "// I am comments");
    }

    #[test]
    fn test_lexer_let_with_space() {
        test_lexer_common("let_with_space", "let x = 5");
    }

    #[test]
    fn test_lexer_string() {
        test_lexer_common("string", r#""a""#);
    }

    #[test]
    fn test_lexer_unicode_string() {
        let mut l = Lexer::new(r#""你好""#);
        let token = l.next_token();

        assert_eq!(token.kind, TokenKind::STRING("你好".to_string()));
        assert_eq!(token.span.start, 0);
        assert_eq!(token.span.end, r#""你好""#.len());
    }

    #[test]
    fn test_lexer_array() {
        test_lexer_common("array", "[3]");
    }

    #[test]
    fn test_lexer_hash() {
        test_lexer_common("hash", r#"{"one": 1, "two": 2, "three": 3}"#);
    }

    #[test]
    fn test_lexer_bool() {
        test_lexer_common("bool", "let y=true");
    }

    #[test]
    fn test_lexer_large_input() {
        let input = "let x = 1;".repeat(10_000);
        let mut l = Lexer::new(&input);
        let tokens = test_token_set(&mut l);

        assert_eq!(tokens.len(), 50_001);
        assert_eq!(tokens.last().unwrap().kind, TokenKind::EOF);
    }

    fn assert_eof_span(input: &str) {
        let mut l = Lexer::new(input);
        let tokens = test_token_set(&mut l);
        let eof = tokens.last().unwrap();
        assert_eq!(eof.kind, TokenKind::EOF);
        assert_eq!(eof.span.start, input.len());
        assert_eq!(eof.span.end, input.len());
    }

    #[test]
    fn eof_span_is_zero_width_for_empty_input() {
        assert_eof_span("");
    }

    #[test]
    fn eof_span_is_zero_width_after_trailing_whitespace_and_comments() {
        assert_eof_span("let x = 5;   \n// trailing comment\n\t  ");
    }

    #[test]
    fn eof_span_is_zero_width_for_non_ascii_input() {
        assert_eof_span(r#""你好"; let x = 1;"#);
    }

    #[test]
    fn test_comments_then_blank_line() {
        // Ensure comments followed by a blank line are skipped correctly
        // and do not yield ILLEGAL tokens.
        test_lexer_common("comments_then_blank_line", "// comment\n\nlet x = 5");
    }

    #[test]
    fn test_lexer_complex() {
        test_lexer_common(
            "complex",
            "
// welcome to monkeylang
let five = 5;
let ten = 10;

let add = fn(x, y) {
  x + y;
};

let result = add(five, ten);
!-/*5;
5 < 10 > 5;

if (5 < 10) {
	return true;
} else {
	return false;
}

10 == 10;
10 != 9;",
        );
    }

    #[test]
    fn lexes_class_tokens_and_identifier_digits() {
        let input = "class Node2 { constructor(value1) { this.value1 = new Node2().next; } }";
        let mut lexer = Lexer::new(input);
        let tokens = test_token_set(&mut lexer);
        let kinds = tokens
            .iter()
            .map(|token| token.kind.clone())
            .collect::<Vec<_>>();

        assert_eq!(kinds[0], TokenKind::CLASS);
        assert_eq!(
            kinds[1],
            TokenKind::IDENTIFIER {
                name: "Node2".to_string()
            }
        );
        assert!(kinds.contains(&TokenKind::THIS));
        assert!(kinds.contains(&TokenKind::NEW));
        assert!(kinds.contains(&TokenKind::DOT));
        assert!(kinds.contains(&TokenKind::IDENTIFIER {
            name: "constructor".to_string()
        }));
    }

    #[test]
    fn class_keyword_boundaries_remain_identifiers() {
        let mut lexer = Lexer::new("className newNode thisValue");
        for expected in ["className", "newNode", "thisValue"] {
            assert_eq!(
                lexer.next_token().kind,
                TokenKind::IDENTIFIER {
                    name: expected.to_string(),
                }
            );
        }
    }
}