chomchom 0.3.3

A simple parser generator for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

type Rstr = std::rc::Rc<str>;

#[derive(Debug)]
pub enum Token {
    Terminal(Rstr),
    Nonterminal(Rstr),
    LBracket,
    RBracket,
    LBrace,
    RBrace,
    LParen,
    RParen,
    Or,
    Equal,
    Plus,
    Semicolon,
    Asterix,
    Epsilon,
}

pub fn tokenize(mut string: &str) -> Vec<Token> {
    string = string.trim();
    let mut ts = Vec::<Token>::new();
    use Token::*;
    while !string.is_empty() {
        if let Some(r) = string.strip_prefix(';') {
            string = r.trim_start();
            ts.push(Semicolon);
        } else if let Some(r) = string.strip_prefix('ε') {
            string = r.trim_start();
            ts.push(Epsilon);
        } else if let Some(r) = string.strip_prefix('+') {
            string = r.trim_start();
            ts.push(Plus);
        } else if let Some(r) = string.strip_prefix('*') {
            string = r.trim_start();
            ts.push(Asterix);
        } else if let Some(r) = string.strip_prefix('=') {
            string = r.trim_start();
            ts.push(Equal);
        } else if let Some(r) = string.strip_prefix('|') {
            string = r.trim_start();
            ts.push(Or);
        } else if let Some(r) = string.strip_prefix('(') {
            string = r.trim_start();
            ts.push(LParen);
        } else if let Some(r) = string.strip_prefix(')') {
            string = r.trim_start();
            ts.push(RParen);
        } else if let Some(r) = string.strip_prefix('{') {
            string = r.trim_start();
            ts.push(LBrace);
        } else if let Some(r) = string.strip_prefix('}') {
            string = r.trim_start();
            ts.push(RBrace);
        } else if let Some(r) = string.strip_prefix('[') {
            string = r.trim_start();
            ts.push(LBracket);
        } else if let Some(r) = string.strip_prefix(']') {
            string = r.trim_start();
            ts.push(RBracket);
        } else if let Some(r) = string.strip_prefix('"') {
            let n = r.find('"').expect("Syntax error");
            let (t, mut r) = r.split_at(n);
            r = r.strip_prefix('"').expect("impossible");
            string = r.trim_start();
            ts.push(Terminal(t.into()));
        } else {
            let n = string
                .find(|c: char| !c.is_alphanumeric() && c != '_')
                .expect("Syntax error");
            if n == 0 {
                ts.push(Epsilon);
                string = string.trim_start();
                continue;
            }
            let (t, r) = string.split_at(n);
            string = r.trim_start();
            ts.push(Nonterminal(t.into()));
        }
    }
    ts
}