1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
use super::{Token, TokenStream, Tokenizer};
#[derive(Clone)]
pub struct RawTokenizer;
pub struct RawTokenStream {
token: Token,
has_token: bool,
}
impl<'a> Tokenizer<'a> for RawTokenizer {
type TokenStreamImpl = RawTokenStream;
fn token_stream(&self, text: &'a str) -> Self::TokenStreamImpl {
let token = Token {
offset_from: 0,
offset_to: text.len(),
position: 0,
text: text.to_string(),
position_length: 1,
};
RawTokenStream {
token,
has_token: true,
}
}
}
impl TokenStream for RawTokenStream {
fn advance(&mut self) -> bool {
let result = self.has_token;
self.has_token = false;
result
}
fn token(&self) -> &Token {
&self.token
}
fn token_mut(&mut self) -> &mut Token {
&mut self.token
}
}