lang_pt 0.1.2

A parser tool to generate recursive descent top down parser.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

use crate::{
    lexeme::{Pattern, ThunkMapper},
    Code,
    ITokenization, Lex, TokenImpl, Tokenizer,
};
use std::{io::BufRead, rc::Rc};

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
enum Token {
    InlineComment,
    MultilineComment,
    EOF,
}
impl TokenImpl for Token {
    fn eof() -> Self {
        Self::EOF
    }

    fn is_structural(&self) -> bool {
        todo!()
    }
}

#[test]
fn f() {
    let comment: Pattern<Token> = Pattern::new(Token::InlineComment, r#"^/\*(.|\n)*?\*/"#).unwrap();

    let comment_variants = ThunkMapper::new(comment, |data, code, _| {
        if code[data.start..data.end].lines().count() > 1 {
            Some(Token::MultilineComment)
        } else {
            None
        }
    });

    let tokenizer = Tokenizer::new(vec![Rc::new(comment_variants)]);
    let inline_comment = "/*This is inline comment*/";
    let inline_comment_tokens = tokenizer.tokenize(&Code::from(inline_comment)).unwrap();
    assert_eq!(
        inline_comment_tokens,
        vec![
            Lex {
                token: Token::InlineComment,
                start: 0,
                end: inline_comment.len()
            },
            Lex {
                token: Token::EOF, // EOF token
                start: inline_comment.len(),
                end: inline_comment.len()
            }
        ]
    );
    let multiline_comment = "/*This is first line\n.Another line comment*/";
    let multiline_comment_tokens = tokenizer.tokenize(&Code::from(multiline_comment)).unwrap();
    assert_eq!(
        multiline_comment_tokens,
        vec![
            Lex {
                token: Token::MultilineComment,
                start: 0,
                end: multiline_comment.len()
            },
            Lex {
                token: Token::EOF, // EOF token
                start: multiline_comment.len(),
                end: multiline_comment.len()
            }
        ]
    );
}