mango 0.4.0

The Mango programming language (UNDER CONSTRUCTION)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

use ::lazy_static::lazy_static;
use ::regex::Regex;

use crate::lexing::lexer::Lexer;
use crate::lexing::reader::reader::{Reader, ReaderResult};
use crate::token::{ParenthesisCloseToken, ParenthesisOpenToken, Tokens};
use crate::token::collect::{association, identifier, operator, parenthesis_close, parenthesis_open, unlexable};
use crate::util::codeparts::operator::ASSOCIATION_RE;
use crate::util::codeparts::operator::SYMBOL_RE;
use crate::util::strtype::name::IDENTIFIER_RE;

/// Lex an identifier.
pub fn lex_identifier(reader: &mut impl Reader, lexer: &mut impl Lexer) {
    while let ReaderResult::Match(sym) = reader.strip_match(&*IDENTIFIER_RE) {
        lexer.add(identifier(sym.as_str()).unwrap());
    }
}

#[cfg(test)]
mod identifiers {
    use crate::lexing::lexer::Lexer;
    use crate::lexing::tests::create_lexer;
    use crate::token::{IdentifierToken, Tokens};
    use crate::token::collect::identifier;
    use crate::token::tokens::OperatorToken;
    use crate::util::codeparts::Symbol;
    use crate::util::strtype::Name;
    use crate::util::strtype::typ::StrType;

    use super::lex_identifier;
    use std::borrow::Cow;
    use crate::token::collect::token_list::TokenList;

    fn check(input: &str, expected_names: &[&str]) {
        let (source, mut reader, mut lexer) = create_lexer(input);
        lex_identifier(&mut reader, &mut lexer);
        let expected: TokenList = expected_names.iter()
            .map(|n| Tokens::Identifier(IdentifierToken::from_name(Name::new(*n).unwrap())))
            .collect();
        assert_eq!(lexer.tokens(), &expected);
    }

    #[test]
    fn empty() {
        check("", &vec![]);
    }

    #[test]
    fn number_prefix() {
        check("1abc", &vec![]);
        check("0o", &vec![]);
    }

    #[test]
    fn after_mismatch() {
        check("* abc", &vec![]);
        check(". x", &vec![]);
    }

    #[test]
    fn symbol() {
        check("*", &vec![]);
        check("+", &vec![]);
        check(".", &vec![]);
    }

    #[test]
    fn single() {
        check("x", &vec!["x"]);
        check("abc", &vec!["abc"]);
    }

    #[test]
    fn with_numbers() {
        check("x0", &vec!["x0"]);
        check("a1b2c3", &vec!["a1b2c3"]);
    }

    #[test]
    fn underscores() {
        check("_", &vec!["_"]);
        check("_abc", &vec!["_abc"]);
        check("_a_", &vec!["_a_"]);
    }

    #[test]
    fn number_underscore() {
        check("_9", &vec![]);
    }

    #[test]
    fn with_postfix() {
        check("hi?", &vec!["hi"]);
        check("hi!", &vec!["hi"]);
        check("x.y", &vec!["x"]);
        check("a,b", &vec!["a"]);
    }

    #[test]
    fn multiple() {
        check("mangoes are tasty fruits", &vec!["mangoes", "are", "tasty", "fruits"]);
    }
}