mini-c-parser 0.1.4

minimal C language lexer & parser & executer from scratch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

use super::character;
use super::identifier;
use super::numeric;
use super::string;
use super::token::Token;
use super::trie;
use rusty_parser as rp;

use rp::IntoParser;

pub fn tokenize(source: String) -> Vec<Token> {
    let cpp_comment = rp::seq!("//", rp::any().not('\n').repeat(0..), '\n'.not_consume()).void(); // c++ comment
    let c_comment = rp::seq!("/*", rp::any().not("*/").repeat(0..), "*/").void();
    // c comment

    let whitespace = rp::or!(' ', '\t', '\n', '\r').void();

    let ignore_all = rp::or!(cpp_comment, c_comment, whitespace).repeat(0..);

    let char_literal = character::char_literal();
    let string_literal = string::string_literal();
    let identifier = identifier::identifier();
    let integer_numeric = numeric::integer_numeric();
    let float_numeric = numeric::float_numeric();
    let trie = trie::build_trie();

    // parse keyword as identifier first
    let one_token = rp::seq!(rp::or!(
        trie,
        identifier,
        string_literal,
        char_literal,
        float_numeric,
        integer_numeric
    ));

    let multiple_tokens = rp::seq!(ignore_all, rp::seq!(one_token, ignore_all).repeat(0..));

    let file = rp::seq!(multiple_tokens, rp::end());

    let res = rp::parse(&file, source.chars());
    if res.output.is_none() {
        panic!("Failed to Tokenize");
    }

    let mut tokens = res.output.unwrap().0;
    let keyword_trie = rp::seq!(trie::build_keyword_trie(), rp::end());

    for token in &mut tokens {
        if let Token::Identifier(ref mut s) = token {
            let keyword_matched = rp::parse(&keyword_trie, s.chars());
            // if it was keyword, change it
            if let Some((t,)) = keyword_matched.output {
                *token = t;
            }
        }
    }

    tokens

    // ignoring unsupported tokens
    // let ignore = rp::or!(
    //     rp::one(Token::Static),
    //     rp::one(Token::Const),
    //     rp::one(Token::Volatile),
    //     rp::one(Token::Auto),
    //     rp::one(Token::Register),
    //     rp::one(Token::Typedef)
    // );
    // tokens
    //     .iter()
    //     .cloned()
    //     .filter(move |t| {
    //         rp::parse(&ignore, Some(t).into_iter().cloned())
    //             .output
    //             .is_none()
    //     })
    //     .collect()
}