1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
//! How Lexer should produce tokens
use std::borrow::Cow;
use std::fmt::Debug;
use std::marker::PhantomData;
use crate::char_stream::CharStream;
use crate::token::Token;
use crate::token::{CommonToken, OwningToken};
use crate::Arena;
#[inline]
pub fn invalid() -> &'static CommonToken<'static> {
Default::default()
}
/// Trait for creating tokens.
pub trait TokenFactory<'input, 'arena>: Debug + Sized
where
'input: 'arena,
{
/// Type of tokens emitted by this factory.
type Tok: Token + 'input;
fn new(arena: &'arena Arena) -> Self;
/// Creates token either from `sourse` or from pure data in `text`
/// Either `source` or `text` are not None
#[allow(clippy::too_many_arguments)]
fn create<T>(
&self,
source: Option<&mut T>,
ttype: i32,
text: Option<String>,
channel: i32,
start: isize,
stop: isize,
line: u32,
column: i32,
) -> &'arena mut Self::Tok
where
T: CharStream<'input> + ?Sized;
}
/// Default token factory
#[derive(Debug)]
pub struct CommonTokenFactory<'input, 'arena> {
arena: &'arena Arena,
_input: PhantomData<&'input str>,
}
impl<'input, 'arena> CommonTokenFactory<'input, 'arena>
where
'input: 'arena,
{
/// Creates new `CommonTokenFactory`
pub fn new(arena: &'arena Arena) -> Self {
Self {
arena,
_input: PhantomData,
}
}
}
impl<'input, 'arena> TokenFactory<'input, 'arena> for CommonTokenFactory<'input, 'arena>
where
'input: 'arena,
{
type Tok = CommonToken<'input>;
fn new(arena: &'arena Arena) -> Self {
Self::new(arena)
}
#[inline]
fn create<T>(
&self,
source: Option<&mut T>,
ttype: i32,
text: Option<String>,
channel: i32,
start: isize,
stop: isize,
line: u32,
column: i32,
) -> &'arena mut Self::Tok
where
T: CharStream<'input> + ?Sized,
{
let text = match (text, source) {
// Safety: obviously unsafe AF -- we're masquerading a string backed
// by 'arena as one that can live for 'input. The reason this won't blow up, is
// 1) Overriding text is only used by the error recovery mechanism
// in the parser, not the lexer;
// 2) When we're in the parser, the lexer is no longer exposed to the user;
// 3) The parser, along with the AST created by it, never return
// any &'input references to the user, thus any references to
// this text can never be held more than 'arena.
(Some(t), _) => unsafe {
std::mem::transmute::<&'arena str, &'input str>(self.arena.alloc_string(t))
},
(None, Some(x)) => {
if stop >= x.size() || start >= x.size() {
"<EOF>"
} else {
// TODO: change this API to return &'input str directly, so
// we don't have to do this dance with Cow and String --
// force the CharStream implementors to figure out how to
// hold on to the text for at least 'input:
match x.get_text(start, stop) {
Cow::Borrowed(t) => t,
Cow::Owned(t) => unsafe {
// Safety: see 3) in the above comment
std::mem::transmute::<&'arena str, &'input str>(
self.arena.alloc_string(t),
)
},
}
}
}
_ => "",
};
self.arena.alloc_token(CommonToken::new(
ttype,
channel,
start as i32,
stop as i32,
-1,
line,
column,
text,
))
}
}
/// Token factory that creates `OwningToken`s
#[derive(Debug)]
pub struct OwningTokenFactory<'arena>(CommonTokenFactory<'static, 'arena>);
impl<'input, 'arena> TokenFactory<'input, 'arena> for OwningTokenFactory<'arena>
where
'input: 'arena,
{
type Tok = OwningToken;
fn new(arena: &'arena Arena) -> Self {
Self(CommonTokenFactory::new(arena))
}
#[inline]
fn create<T>(
&self,
source: Option<&mut T>,
ttype: i32,
text: Option<String>,
channel: i32,
start: isize,
stop: isize,
line: u32,
column: i32,
) -> &'arena mut Self::Tok
where
T: CharStream<'input> + ?Sized,
{
let text = match (text, source) {
(Some(t), _) => t,
(None, Some(x)) => {
if stop >= x.size() || start >= x.size() {
"<EOF>".to_string()
} else {
x.get_text(start, stop).to_string()
}
}
_ => "".to_string(),
};
self.0.arena.alloc_token(OwningToken::new(
ttype,
channel,
start as i32,
stop as i32,
-1,
line,
column,
text,
))
}
}