mango 0.4.0

The Mango programming language (UNDER CONSTRUCTION)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

use ::lazy_static::lazy_static;
use ::regex::Regex;

use crate::io::source::SourceFile;
use crate::lexing::grouping::lex_grouping;
use crate::lexing::identifier::lex_identifier;
use crate::lexing::indent::lex_indents;
use crate::lexing::lexer::{CodeLexer, Lexer};
use crate::lexing::operator::lex_association;
use crate::lexing::operator::lex_operator;
use crate::lexing::literals::lex_literal;
use crate::lexing::reader::reader::{Reader, ReaderResult};
use crate::lexing::reader::source_reader::SourceReader;
use crate::lexing::separators::lex_separators;
use crate::lexing::special::{lex_eof, lex_unlexable};
use crate::token::{Tokens, UnlexableToken};

//TODO performance: one day maybe use arena allocation

macro_rules! try_lex (
    // Invoke a lexing function, and 'continue' if successful.
    // f: The lexing function
    // r: The Reader implementation
    // l: The Lexer implementation
    ( $f: ident, $r: ident, $l: ident ) => {
        {
            let initial_progress = $l.progress();
            $f(&mut $r, &mut $l);
            if ($l.progress() != initial_progress) {
                continue;
            }
        }
    };
);

/// Lexes a whole source file and returns the tokens.
pub fn lex(source: &SourceFile) -> Vec<Tokens> {
    //TODO performance: does this initial capacity make sense?
    let mut reader = SourceReader::new(source);
    let mut lexer = CodeLexer::new(source.len());
    //TODO: some of these need to be in specific order, like eof/unlexable last, and indent first - but for other the order could be optimized
    loop {
        try_lex!(lex_indents, reader, lexer);
        try_lex!(lex_grouping, reader, lexer);
        try_lex!(lex_operator, reader, lexer);
        try_lex!(lex_association, reader, lexer);
        try_lex!(lex_separators, reader, lexer);
        try_lex!(lex_identifier, reader, lexer);
        try_lex!(lex_literal, reader, lexer);
        if lex_eof(&mut reader) { break }
        try_lex!(lex_unlexable, reader, lexer);
    }
    lexer.into_tokens()
}

#[cfg(test)]
mod try_lex {
    use crate::io::source::SourceFile;
    use crate::lexing::reader::reader::Reader;
    use crate::lexing::reader::source_reader::SourceReader;
    use crate::lexing::tests::create_lexer;
    use crate::token::UnlexableToken;

    use super::*;

    fn lex_fn_match(reader: &mut impl Reader, lexer: &mut impl Lexer) {
        lexer.add(Tokens::Unlexable(UnlexableToken::new("hi".to_owned())))
    }

    fn lex_fn_no_match(reader: &mut impl Reader, lexer: &mut impl Lexer) {}

    #[test]
    fn test_match() {
        let (source, mut reader, mut lexer) = create_lexer("");
        for i in 0..3 {
            try_lex!(lex_fn_match, reader, lexer);
            panic!("Execution should not have reached here, because every iteration matches and should do 'continue'");
        }
    }

    #[test]
    fn test_no_match() {
        let (source, mut reader, mut lexer) = create_lexer("");
        let mut end_of_loop_count = 0;
        for i in 0..3 {
            try_lex!(lex_fn_no_match, reader, lexer);
            end_of_loop_count += 1;
        }
        assert_eq!(end_of_loop_count, 3, "Execution should have reached end of loop, because there is not 'continue' on mismatch");
    }
}