lexers 0.1.4

Tools for tokenizing and scanning
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

#![deny(warnings)]

use crate::scanner::Scanner;

pub struct EbnfTokenizer<I: Iterator<Item = char>> {
    input: Scanner<I>,
    lookahead: Vec<String>,
}

impl<I: Iterator<Item = char>> EbnfTokenizer<I> {
    pub fn new(source: I) -> Self {
        EbnfTokenizer {
            input: Scanner::new(source),
            lookahead: Vec::new(),
        }
    }

    pub fn scanner(source: I) -> Scanner<Self> {
        Scanner::new(Self::new(source))
    }
}

impl<I: Iterator<Item = char>> Iterator for EbnfTokenizer<I> {
    type Item = String;
    fn next(&mut self) -> Option<Self::Item> {
        // used for accumulating string parts
        if !self.lookahead.is_empty() {
            return self.lookahead.pop();
        }
        let s = &mut self.input;
        s.scan_whitespace();
        // discard comments starting with '#' until new-line
        if s.accept(&'#').is_some() {
            while let Some(nl) = s.next() {
                if nl == '\n' {
                    s.extract(); // ignore comment
                                 // discard comment and allow more by restarting
                    return self.next();
                }
            }
        }
        if s.accept_any(&['[', ']', '{', '}', '(', ')', '|', ';'])
            .is_some()
        {
            return Some(s.extract_string());
        }
        let backtrack = s.buffer_pos();
        if s.accept(&':').is_some() {
            if s.accept(&'=').is_some() {
                return Some(s.extract_string());
            }
            s.set_buffer_pos(backtrack);
        }
        let backtrack = s.buffer_pos();
        if let Some(q) = s.accept_any(&['"', '\'']) {
            while let Some(n) = s.next() {
                if n == q {
                    // store closing quote
                    self.lookahead.push(n.to_string());
                    // store string content
                    let v = s.extract_string();
                    self.lookahead.push(v[1..v.len() - 1].to_string());
                    // return opening quote
                    return Some(q.to_string());
                }
            }
            s.set_buffer_pos(backtrack);
        }
        let backtrack = s.buffer_pos();
        s.accept(&'@');
        // NOTE: scan_identifier limits the valid options
        if let Some(id) = s.scan_identifier() {
            return Some(id);
        }
        // backtrack possible '@'
        s.set_buffer_pos(backtrack);
        None
    }
}