1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
use std::{
cmp,
io::{Read, Result, Write},
ptr,
};
use super::{Buffer, Token, TokenType};
fn copy_memory(src: &[u8], dst: &mut [u8]) {
let len_src = src.len();
assert!(dst.len() >= len_src);
unsafe {
ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), len_src);
}
}
pub struct TokenReader<'a, I: IntoIterator<Item = Token>> {
iter: I::IntoIter,
src: Option<&'a str>,
buf: Vec<u8>,
ofs: usize,
}
impl<'a, I: IntoIterator<Item = Token>> TokenReader<'a, I> {
pub fn new(iter: I, source: Option<&'a str>) -> TokenReader<'a, I> {
TokenReader {
iter: iter.into_iter(),
src: source,
buf: Vec::with_capacity(128),
ofs: 0,
}
}
}
impl<'a, I: IntoIterator<Item = Token>> Read for TokenReader<'a, I> {
fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
if buf.is_empty() {
return Ok(0);
}
let mut bl = buf.len();
if !self.buf.is_empty() {
let btc = cmp::min(self.buf.len() - self.ofs, buf.len());
copy_memory(&self.buf[self.ofs..self.ofs + btc], buf);
bl -= btc;
self.ofs += btc;
if self.ofs == self.buf.len() {
self.buf.clear();
self.ofs = 0;
}
}
if bl == 0 {
return Ok(buf.len());
}
while bl > 0 {
match self.iter.next() {
None => return Ok(buf.len() - bl),
Some(t) => {
let bytes: &[u8] = match t.kind {
TokenType::String | TokenType::Number => match t.buf {
Buffer::MultiByte(ref b) => &b,
Buffer::Span(ref s) => match self.src {
Some(b) => b[s.first as usize..s.end as usize].as_bytes(),
None => panic!("Must set source if tokens don't provide byter buffers"),
},
},
TokenType::Invalid => b"",
_ => t.kind.as_ref().as_bytes(),
};
let btc = cmp::min(bytes.len(), bl);
let ofs = buf.len() - bl;
copy_memory(&bytes[..btc], &mut buf[ofs..ofs + btc]);
bl -= btc;
if btc < bytes.len() {
debug_assert!(bl == 0);
self.buf.write_all(&bytes[btc..])?;
}
if bl == 0 {
return Ok(buf.len());
}
}
}
}
unreachable!();
}
}