earlgrey 0.4.1

A library for parsing context-free grammars using Earley algorithm
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

pub struct EbnfTokenizer<I: Iterator<Item=char>> {
    input: std::iter::Peekable<I>,
    buff: Vec<String>,
}

impl<I: Iterator<Item=char>> EbnfTokenizer<I> {
    pub fn new(input: I) -> Self {
        Self { input: input.peekable(), buff: Vec::new() }
    }

    fn next_result(&mut self) -> Result<Option<String>, String> {
        if self.buff.len() > 0 {
            return Ok(Some(self.buff.remove(0)));
        }
        match self.input.next() {
            // Various single char tokens.
            Some(x) if "[]{}()|;".contains(x) => Ok(Some(x.to_string())),
            // Assignment operator.
            Some(':') => match self.input.next() {
                Some('=') => Ok(Some(":=".to_string())),
                _ => Err("Incomplete := operator".to_string()),
            },
            // Tokenize Strings checking for escapes.
            Some(open) if open == '"' || open == '\'' => {
                self.buff.push(open.to_string());
                let mut quoted_string = String::new();
                let mut escaped = false;
                while let Some(ch) = self.input.next() {
                    // Swallow escape char '\' and prevent string closure
                    if ! escaped && ch == '\\' {
                        escaped = true;
                        continue;
                    }
                    if escaped {
                        quoted_string.push('\\')
                    }
                    // Found close token. Check it wasn't escaped.
                    if ! escaped && open == ch {
                        self.buff.push(quoted_string);
                        self.buff.push(ch.to_string());
                        return self.next_result();
                    }
                    quoted_string.push(ch);
                    escaped = false;
                }
                Err("Unfinished string missing close quote".to_string())
            },
            // Swallow comments until EOL.
            Some('#') => {
                while let Some(nl) = self.input.next() {
                    if nl == '\n' {
                        return self.next_result();
                    }
                }
                Err("Unfinished comment missing EOL".to_string())
            },
            // Tags (starts with '@') and identifiers.
            Some(x) if x.is_ascii_alphabetic() || x == '@' || x == '_' => {
                let mut id = x.to_string();
                while let Some(ch) = self.input.peek() {
                    if ! ch.is_ascii_alphanumeric() && *ch != '_' {
                        break;
                    }
                    id.push(self.input.next().unwrap());
                }
                Ok(Some(id))
            },
            // Swallow whitespace.
            Some(x) if x.is_whitespace() => {
                while let Some(ws) = self.input.peek() {
                    if ! ws.is_whitespace() {
                        break;
                    }
                    self.input.next(); // consume whitespace
                }
                self.next_result()
            },
            Some(ch) => Err(format!("Unexpected char: {}", ch)),
            None => Ok(None),
        }
    }
}

impl<I: Iterator<Item=char>> Iterator for EbnfTokenizer<I> {
    type Item = String;

    fn next(&mut self) -> Option<Self::Item> {
        match self.next_result() {
            Err(e) => {
                eprintln!("EbnfTokenizer error: {}", e);
                None
            },
            Ok(v) => v
        }
    }
}

#[cfg(test)]
mod tests {
    use super::EbnfTokenizer;

    #[test]
    fn simple() {
        let input = r#"
            "hello" world @tag | [ foo ];
            {x} # comment
            a:=(y)
            "escapedstring\""
        "#;
        let expected = vec![
            "\"", "hello", "\"", "world", "@tag", "|", "[", "foo", "]", ";",
            "{", "x", "}", "a", ":=", "(", "y", ")", "\"", "escapedstring\\\"",
            "\""
        ];
        for (idx, token) in EbnfTokenizer::new(input.chars()).enumerate() {
            assert_eq!(token, expected[idx]);
        }
    }
}