1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
//a Imports
use crate::{PosnInCharStream, StreamCharSpan};
//tt CharStream
/// The [CharStream] trait allows a stream of [char] to provide extraa methods
///
/// Requires P : PosnInCharStream
pub trait CharStream<P>
where
P: PosnInCharStream,
{
/// Steps along the stream starting at the provided state (and
/// character) while the provided function returns true; the
/// function is provided with the index and character (starting at
/// 0 / ch), and it returns true if the token continues, otherwise
/// false
///
/// If the first invocation of 'f' returns false then the token is
/// said to not match, and 'do_while' returns the stream state and Ok(None).
///
/// If the first N (more than zero) invocations match then the
/// result is the stream state after the matched characters, and
/// Some(initial state, N)
///
/// This can be used to match whitespace (where N is probably
/// discarded), or user 'id' values in a language. The text can be
/// retrieved with the 'get_text' method
fn do_while<F: Fn(usize, char) -> bool>(
&self,
state: P,
ch: char,
f: &F,
) -> (P, Option<(P, usize)>) {
if !f(0, ch) {
return (state, None);
}
let start = state;
let mut n = 1;
let mut state = self.consumed_char(state, ch);
// # Safety
//
// 'ofs' is maintained as a utf8 character point boundary
// within or at the end of the 'str' borrowed by [Self]
while let Some(ch) = self.peek_at(&state) {
if !f(n, ch) {
break;
}
n += 1;
state = self.consumed_char(state, ch);
}
(state, Some((start, n)))
}
/// Steps along the stream starting at the provided state,
/// character and accumulator value while the provided function
/// returns (true, new accumulator); the function is provided with
/// the latest accumulator, index, character (starting at 0 / ch),
/// and it returns true and a new accumulator if the token
/// continues, otherwise false and the final accumulator value
///
/// If the first invocation of 'f' returns false then the token is
/// said to not match, and 'fold' returns the stream state and Ok(None).
///
/// If the first N (more than zero) invocations match then the
/// result is the stream state after the matched characters, and
/// Some(initial state, N, final accumulator)
///
/// This can be used to accumulate significant state about a token
/// as it is parsed, in excess of the simple number of characters.
fn fold<T, F: Fn(&Self, T, &P, usize, char) -> (T, Option<P>)>(
&self,
state: P,
ch: char,
acc: T,
f: &F,
) -> (P, Option<(P, usize, T)>) {
let (mut acc, some_posn) = f(self, acc, &state, 0, ch);
if some_posn.is_none() {
return (state, None);
}
let start = state;
let mut n = 1;
let mut state = some_posn.unwrap();
while let Some(ch) = self.peek_at(&state) {
let (new_acc, more_posn) = f(self, acc, &state, n, ch);
acc = new_acc;
if more_posn.is_none() {
break;
}
n += 1;
state = more_posn.unwrap();
}
(state, Some((start, n, acc)))
}
/// Retrieve a range of bytes from the stream
fn range_as_bytes(&self, ofs: usize, n: usize) -> &[u8];
/// Return true if the content of the stream at 'state' matches
/// the byte slice
fn matches_bytes(&self, state: &P, s: &[u8]) -> bool;
/// Get the text between the start of a span (inclusive) and the
/// end of the span (exclusive).
fn get_text_span(&self, span: &StreamCharSpan<P>) -> &str
where
P: PosnInCharStream;
/// Get the text between the start (inclusive) and the
/// end (exclusive).
fn get_text(&self, start: P, end: P) -> &str;
// Return true if the text at 'pos' matches the string
//
// Waiting for pattern stabiliztion
// fn matches<'call, P:std::str::pattern::Pattern<'call>>(&self, pos: &P, pat: P) -> bool;
/// Match the text at the offset with a str; return true if it matches, else false
fn matches_str(&self, pos: &P, pat: &str) -> bool;
/// Peek at the next character in the stream, returning None if
/// the state is the end of the stream
fn peek_at(&self, state: &P) -> Option<char>;
//cp consumed
/// Move the stream state forward by the specified number of characters
///
/// The characters MUST NOT inclulde newlines
fn consumed(&self, state: P, num_chars: usize) -> P;
//cp consumed_char
/// Get a stream state after consuming the specified character at
/// its current state
fn consumed_char(&self, state: P, ch: char) -> P
where
P: PosnInCharStream,
{
if ch == '\n' {
state.advance_line(1)
} else {
state.advance_cols(ch.len_utf8(), 1)
}
}
//cp consumed_newline
/// Get a stream state after consuming a newline at its current state
///
/// # Safety
///
/// num_bytes *must* correspond to the number of bytes that the
/// newline character consists of, and state *must* point to the
/// bytes offset of that character
unsafe fn consumed_newline(&self, state: P, num_bytes: usize) -> P
where
P: PosnInCharStream,
{
state.advance_line(num_bytes)
}
//cp consumed_ascii_str
/// Get the state after consuming a particular ascii string
/// without newlines
///
/// This is safe as there is no unsafe handling of byte offsets
/// within *state*; however, there is no check that the provided
/// string is ASCII and that it does not contain newlines. If
/// these API rules are broke then the lie and column held by
/// *state* may be incorrect (which is not *unsafe*, but
/// potentially a bug)
fn consumed_ascii_str(&self, state: P, s: &str) -> P
where
P: PosnInCharStream,
{
let n = s.len();
state.advance_cols(n, n)
}
//cp consumed_chars
/// Become the span after consuming a particular string of known character length
///
/// # Safety
///
/// num_bytes *must* correspond to the number of bytes that
/// 'num_chars' indicates start at *state*. If this constraint is
/// not met then the byte offset indicated by the returned value
/// may not correspond to a UTF8 character boundary within the
/// stream.
unsafe fn consumed_chars(&self, state: P, num_bytes: usize, num_chars: usize) -> P
where
P: PosnInCharStream,
{
state.advance_cols(num_bytes, num_chars)
}
//mp commit_consumed
/// Invoked by the Lexer to indicate that the stream has been
/// consumed up to a certain point, and that (for parsing) no
/// state earlier in the stream will be requested in the future
///
/// A truly streaming source can drop earlier data in the stream
/// if this fits the application
fn commit_consumed(&self, _up_to: &P) {}
}