1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
#![deny(warnings)]
pub struct Scanner<I: Iterator>
where
I::Item: Clone,
{
src: I,
buf: Vec<I::Item>,
pos: isize,
}
// Scanners are Iterators
impl<I> Iterator for Scanner<I>
where
I: Iterator,
I::Item: Clone,
{
type Item = I::Item;
fn next(&mut self) -> Option<Self::Item> {
self.pos += 1;
// Check if we need to fill the buffer
let lacking = self.pos - (self.buf.len() as isize) + 1;
if lacking > 0 {
self.buf.extend(self.src.by_ref().take(lacking as usize));
}
// limit the buffer position to the buffer length at most
self.pos = std::cmp::min(self.pos, self.buf.len() as isize);
self.current()
}
}
impl<I> Scanner<I>
where
I: Iterator,
I::Item: Clone,
{
pub fn new(source: I) -> Scanner<I> {
Scanner {
src: source,
buf: Vec::new(),
pos: -1,
}
}
// Allows getting current buffer position to backtrack
pub fn buffer_pos(&self) -> isize {
self.pos
}
// Reset buffer position, normally used for backtracking
// If position is out of bounds set_buffer_pos returns false
pub fn set_buffer_pos(&mut self, pos: isize) -> bool {
if pos < -1 || pos > (self.buf.len() as isize) {
return false;
}
self.pos = pos;
true
}
// Returns the current token on which the scanner is positioned
pub fn current(&self) -> Option<I::Item> {
let pos = self.pos as usize;
if self.pos < 0 || pos >= self.buf.len() {
return None;
}
Some(self.buf[pos].clone())
}
// Steps the scanner back and returns the token at that position
pub fn prev(&mut self) -> Option<I::Item> {
if self.pos >= 0 {
self.pos -= 1;
}
self.current()
}
// Returns the token ahead without actually advancing the scanner
pub fn peek(&mut self) -> Option<I::Item> {
let backtrack = self.pos;
let peeked = self.next();
self.pos = backtrack;
peeked
}
// Returns the previous token without actually backtracking the scanner
pub fn peek_prev(&mut self) -> Option<I::Item> {
let backtrack = self.pos;
let peeked = self.prev();
self.pos = backtrack;
peeked
}
// Returns a view of the current underlying buffer
pub fn view(&self) -> &[I::Item] {
let n = (self.pos + 1) as usize;
&self.buf[..n]
}
// Consumes the buffer into a new token (which can be ignored)
pub fn extract(&mut self) -> Vec<I::Item> {
// Check where to shift buffer
let split_point = std::cmp::min(self.pos + 1, self.buf.len() as isize);
assert!(split_point >= 0);
// Reset buffer cursor
self.pos = -1;
// Split buffer and keep the remainder
let mut remaining = self.buf.split_off(split_point as usize);
std::mem::swap(&mut self.buf, &mut remaining);
remaining
}
}
impl<I> Scanner<I>
where
I: Iterator,
I::Item: Clone + PartialEq,
{
// Advance the scanner only if the next char is the expected one
// self.current() will return the matched char if accept matched
pub fn accept(&mut self, what: &I::Item) -> Option<I::Item> {
let backtrack = self.buffer_pos();
if let Some(next) = self.next() {
if &next == what {
return Some(next);
}
}
self.set_buffer_pos(backtrack);
None
}
// Advance the scanner only if the next char is in the 'any' set,
// self.current() will return the matched char if accept matched any
pub fn accept_any(&mut self, any: &[I::Item]) -> Option<I::Item> {
let backtrack = self.buffer_pos();
if let Some(next) = self.next() {
if any.contains(&next) {
return Some(next);
}
}
self.set_buffer_pos(backtrack);
None
}
// Advance the scanner only if a full match for items form 'what'.
// self.current() will return the last item from 'what'
pub fn accept_all(&mut self, what: impl Iterator<Item=I::Item>) -> bool {
let backtrack = self.buffer_pos();
for item in what {
if self.accept(&item).is_none() {
self.set_buffer_pos(backtrack);
return false;
}
}
true
}
// Skip over the 'over' set, result is if the scanner was advanced,
// self.current() will return the last matching char
pub fn skip_all(&mut self, over: &[I::Item]) -> bool {
let mut advanced = false;
while self.accept_any(over).is_some() {
advanced = true;
}
advanced
}
// Find an element in the 'any' set or EOF, return if the scanner advanced,
// self.current() returns the last non-matching char
pub fn until_any(&mut self, any: &[I::Item]) -> bool {
let mut advanced = false;
while let Some(next) = self.peek() {
if any.contains(&next) {
break;
}
self.next();
advanced = true;
}
advanced
}
}