lang_pt 0.1.2

A parser tool to generate recursive descent top down parser.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

use crate::{
    lexeme::{Middleware, Pattern, Punctuations},
    Code,
    ITokenization, Lex, TokenImpl, Tokenizer,
};
use std::rc::Rc;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
enum Token {
    RegexLiteral,
    ID,
    Number,
    Add,
    Mul,
    Div,
    Assign,
    Subtract,
    EOF,
}

impl TokenImpl for Token {
    fn eof() -> Self {
        Self::EOF
    }

    fn is_structural(&self) -> bool {
        todo!()
    }
}

#[test]
fn f() {
    let identifier = Rc::new(Pattern::new(Token::ID, r#"^[_$a-zA-Z][_$\w]*"#).unwrap());
    let number_literal =
        Rc::new(Pattern::new(Token::Number, r"^(0|[\d--0]\d*)(\.\d+)?([eE][+-]?\d+)?").unwrap());

    let punctuations = Rc::new(
        Punctuations::new(vec![
            ("+", Token::Add),
            ("*", Token::Mul),
            ("/", Token::Div),
            ("=", Token::Assign),
            ("-", Token::Subtract),
        ])
        .unwrap(),
    );

    let regex_literal =
        Pattern::new(Token::RegexLiteral, r"^/([^\\/\r\n\[]|\\.|\[[^]]+\])+/").unwrap();

    let validated_regex_literal = Rc::new(Middleware::new(regex_literal, |_, lex_stream| {
        lex_stream.last().map_or(false, |d| match d.token {
            Token::ID | Token::Number => false,
            _ => true,
        })
    }));

    let tokenizer = Tokenizer::new(vec![
        identifier,
        number_literal,
        validated_regex_literal, // Should appear before punctuation so that regex literal is validated before div '/'.
        punctuations,
    ]);

    let lex = tokenizer.tokenize(&Code::from("2/xy/6")).unwrap();
    assert_eq!(
        lex,
        [
            Lex {
                token: Token::Number,
                start: 0,
                end: 1
            },
            Lex {
                token: Token::Div,
                start: 1,
                end: 2
            },
            Lex {
                token: Token::ID,
                start: 2,
                end: 4
            },
            Lex {
                token: Token::Div,
                start: 4,
                end: 5
            },
            Lex {
                token: Token::Number,
                start: 5,
                end: 6
            },
            Lex {
                token: Token::EOF,
                start: 6,
                end: 6
            },
        ]
    );
    let regex_lex = tokenizer.tokenize(&&Code::from("a=/xy/")).unwrap();
    assert_eq!(
        regex_lex,
        [
            Lex {
                token: Token::ID,
                start: 0,
                end: 1
            },
            Lex {
                token: Token::Assign,
                start: 1,
                end: 2
            },
            Lex {
                token: Token::RegexLiteral,
                start: 2,
                end: 6
            },
            Lex {
                token: Token::EOF,
                start: 6,
                end: 6
            },
        ]
    );
}