marpa 0.3.0

Safe bindings to the libmarpa parsing engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

use crate::lexer::token::Token;
use crate::thin::Symbol;
use std::fmt;
use std::str;

#[derive(Default, PartialEq, Eq, PartialOrd, Debug, Copy, Clone)]
pub struct ByteToken(u8);

impl fmt::Display for ByteToken {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let a: [u8; 1] = [self.0];
        match str::from_utf8(&a) {
            Err(_) => write!(f, "{:x}", self.0),
            Ok(s) => write!(f, "'{}'", s),
        }
    }
}

impl From<(Symbol, i32)> for ByteToken {
    fn from((sym, _): (Symbol, i32)) -> Self {
        ByteToken(sym as u8)
    }
}

impl Token for ByteToken {
    fn sym(&self) -> Symbol {
        i32::from(self.0)
    }

    fn value(&self) -> i32 {
        i32::from(self.0) + 1
    }
}

impl ::std::ops::Deref for ByteToken {
    type Target = u8;

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}

pub struct ByteScanner<R: ::std::io::Read> {
    rd: R,
    buffer: Vec<u8>,
    idx: usize,
    end: usize,
}

impl<R: ::std::io::Read> ByteScanner<R> {
    pub fn new(rd: R) -> ByteScanner<R> {
        ByteScanner::with_capacity(rd, 256)
    }

    pub fn with_capacity(rd: R, cap: usize) -> ByteScanner<R> {
        ByteScanner {
            rd,
            buffer: vec![0; cap],
            idx: 0,
            end: 0,
        }
    }

    fn fill_buffer(&mut self) -> ::std::io::Result<usize> {
        self.rd.read(&mut self.buffer)
    }

    fn read_byte(&mut self) -> Option<u8> {
        if self.idx >= self.end {
            self.idx = 0;
            match self.fill_buffer() {
                Ok(size) => {
                    if size == 0 {
                        return None;
                    } else {
                        self.end = size
                    }
                }
                Err(_) => return None,
            }
        }

        self.idx += 1;
        Some(self.buffer[self.idx - 1])
    }
}

impl<R: ::std::io::Read> Iterator for ByteScanner<R> {
    type Item = ByteToken;

    fn next(&mut self) -> Option<ByteToken> {
        match self.read_byte() {
            Some(byte) => Some(ByteToken(byte)),
            None => None,
        }
    }
}

// impl<R: ::std::io::Read> TokenSource for ByteScanner<R> {}

#[cfg(test)]
mod tests {
    use super::super::token_source::TokenSource;
    use super::ByteScanner;
    use super::ByteToken;
    use std::io::Cursor;

    #[test]
    fn test_byte_scanner() {
        let input = Cursor::new("Hello, world!");
        let scanner = ByteScanner::new(input);
        must_compile(&scanner);

        let toks: Vec<ByteToken> = scanner.collect();
        assert!(
            toks == vec![
                ByteToken(72),
                ByteToken(101),
                ByteToken(108),
                ByteToken(108),
                ByteToken(111),
                ByteToken(44),
                ByteToken(32),
                ByteToken(119),
                ByteToken(111),
                ByteToken(114),
                ByteToken(108),
                ByteToken(100),
                ByteToken(33),
            ]
        );
    }

    fn must_compile<T: TokenSource<ByteToken>>(_: &T) {}
}