1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
mod input;
pub use input::LexerInput;
use std::error::Error;
use std::fmt;
#[derive(Debug)]
pub struct Token<K> {
kind: K,
text: String,
}
impl<K> Token<K> {
pub fn new(kind: K, text: String) -> Token<K> {
Token { kind, text }
}
pub fn kind(&self) -> &K {
&self.kind
}
pub fn text(&self) -> &String {
&self.text
}
pub fn into_text(self) -> String {
self.text
}
}
#[derive(Debug)]
pub enum MatchError {
Unexpected(char),
Custom(String),
}
impl fmt::Display for MatchError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
MatchError::Unexpected(c) => writeln!(f, "Unexpected '{}'", c),
MatchError::Custom(ref msg) => msg.fmt(f),
}
}
}
impl Error for MatchError {}
pub type MatchResult<T> = Result<T, MatchError>;
pub trait Matcher<K> {
fn try_match(&self, first_char: char, input: &mut LexerInput) -> MatchResult<K>;
}
impl<F, K> Matcher<K> for F
where F: Fn(char, &mut LexerInput) -> MatchResult<K> {
fn try_match(&self, first_char: char, input: &mut LexerInput) -> MatchResult<K> {
(*self)(first_char, input)
}
}
pub struct Lexer<'a, K> {
input: LexerInput<'a>,
matcher: &'a dyn Matcher<K>,
skip_whitespace: bool,
}
impl<'a, K> Lexer<'a, K> {
pub fn new(input: &'a str, matcher: &'a dyn Matcher<K>, skip_whitespace: bool) -> Lexer<'a, K> {
Lexer {
input: LexerInput::new(input),
matcher,
skip_whitespace,
}
}
}
impl<'a, K> Iterator for Lexer<'a, K> {
type Item = MatchResult<Token<K>>;
fn next(&mut self) -> Option<Self::Item> {
if self.skip_whitespace {
self.input.skip_whitespace();
}
let first_char = match self.input.next() {
Some(byte) => byte,
None => return None,
};
let kind = match self.matcher.try_match(first_char, &mut self.input) {
Ok(kind) => kind,
Err(err) => return Some(Err(err)),
};
Some(Ok(Token::new(kind, self.input.take_buffer())))
}
}