mango 0.4.0

The Mango programming language (UNDER CONSTRUCTION)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

use ::lazy_static::lazy_static;
use ::regex::Regex;

use crate::lexing::lexer::Lexer;
use crate::lexing::reader::reader::{Reader, ReaderResult};
use crate::token::{Tokens, ParenthesisOpenToken, ParenthesisCloseToken, UnlexableToken};

lazy_static! {
    static ref SINGLE_RE: Regex = Regex::new(r"(?s)^.").unwrap();
    static ref EMPTY_RE: Regex = Regex::new(r"^[ \t\n\r]*").unwrap();
}

/// Lex a single symbol as unlexable. Should only be used if the lexer is stuck, to unstuck it.
pub fn lex_unlexable(reader: &mut impl Reader, lexer: &mut impl Lexer) {
    let chr = reader.strip_match(&*SINGLE_RE).unwrap().as_str().to_owned();
    lexer.add(Tokens::Unlexable(UnlexableToken::new(chr)));
}

/// Check whether the end of file has been reached (true if EOF).
pub fn lex_eof(reader: &mut impl Reader) -> bool {
    if let ReaderResult::Match(whitespace) = reader.direct_peek(&*EMPTY_RE) {
        debug_assert!(whitespace.len() <= reader.remaining_len());
        if whitespace.len() == reader.remaining_len() {
            // End of file has been reached (the rest is all whitespace)
            return true;
        }
    }
    false
}

#[cfg(test)]
mod unlexable {
    use super::lex_unlexable;
    use crate::lexing::tests::create_lexer;
    use crate::lexing::lexer::Lexer;
    use crate::token::{UnlexableToken, Tokens};

    #[test]
    fn letter() {
        let (source, mut reader, mut lexer) = create_lexer("abc");
        lex_unlexable(&mut reader, &mut lexer);
        assert_eq!(lexer.into_tokens(), vec![Tokens::Unlexable(UnlexableToken::new("a".to_owned()))]);
    }

    #[test]
    fn newline() {
        // Newline is a special case, because normally regex's '.' does not match it.
        let (source, mut reader, mut lexer) = create_lexer("\nabc");
        lex_unlexable(&mut reader, &mut lexer);
        assert_eq!(lexer.into_tokens(), vec![Tokens::Unlexable(UnlexableToken::new("\n".to_owned()))]);
    }
}

#[cfg(test)]
mod eof {
    use super::lex_unlexable;
    use crate::lexing::tests::create_lexer;
    use crate::lexing::lexer::Lexer;
    use crate::token::{UnlexableToken, Tokens};
    use crate::lexing::special::lex_eof;

    #[test]
    fn empty() {
        let (_, mut reader, _) = create_lexer("");
        assert!(lex_eof(&mut reader));
    }

    #[test]
    fn whitespace() {
        let (_, mut reader, _) = create_lexer(" \n\t ");
        assert!(lex_eof(&mut reader));
    }

    #[test]
    fn just_letter() {
        let (_, mut reader, _) = create_lexer("*");
        assert!(!lex_eof(&mut reader));
    }

    #[test]
    fn whitespace_text() {
        let (_, mut reader, _) = create_lexer(" \n\t abc");
        assert!(!lex_eof(&mut reader));
    }
}