1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
use super::super::{Token, TryFromArgumentGroup};
use super::{LoopControl, TokenDeserialize};
use df_ls_diagnostics::DiagnosticsInfo;
use df_ls_lexical_analysis::TreeCursor;
/// Deserialize a list of tokens
impl<T> TokenDeserialize for Vec<T>
where
T: TokenDeserialize + Default + std::fmt::Debug,
{
/// Function is same as `token_deserialize` default function
/// Except for storing the first tokens Reference.
fn deserialize_tokens(
cursor: &mut TreeCursor,
source: &str,
diagnostics: &mut DiagnosticsInfo,
) -> Result<Box<Self>, ()> {
let mut new_self = Box::new(Self::default());
// Start special code
let mut token_name = None;
// End special code
loop {
let node = cursor.node();
match node.kind().as_ref() {
"token" => {
// Start special code
if token_name.is_none() {
// Set `token_name` to the name of the token
let token = Token::deserialize_tokens(cursor, source, diagnostics)?;
// Token does not have to be consumed.
token_name = match token.get_token_name() {
Ok(name) => Some(name.value.clone()),
Err(_) => Some(String::default()),
}
}
// End special code
if let Some(token_name) = &token_name {
match deserialize_general_token_custom::<T>(
cursor,
source,
diagnostics,
*new_self,
token_name,
) {
(LoopControl::DoNothing, new_self_result) => {
// Do nothing
new_self = Box::new(new_self_result);
}
(LoopControl::Break, new_self_result) => {
new_self = Box::new(new_self_result);
break;
}
(LoopControl::Continue, new_self_result) => {
new_self = Box::new(new_self_result);
// Only go to next sibling if there is one, if none: break.
// We have reached the end of the file, so have to go up the stack
let new_node = cursor.node();
if new_node.next_sibling().is_none() {
cursor.goto_parent();
break;
}
// If node did not change: break
// This will prevent infinite loops
if new_node == node {
break;
}
continue;
}
(LoopControl::ErrBreak, _new_self_result) => {
return Err(());
}
}
} else {
unreachable!("token_name not set");
}
}
"comment" => {
// Just consume `comment` and more on to next.
if Token::consume_token(cursor).is_err() {
break;
}
}
"ERROR" => {
// Can safely be ignored. Diagnostics already added by Lexical Analysis.
if Token::consume_token(cursor).is_err() {
break;
}
}
"EOF" => break,
others => {
log::error!("Found an unknown node of kind: {}", others);
break;
}
}
// If node did not change: break
// This will prevent infinite loops
let new_node = cursor.node();
if new_node == node {
break;
}
}
Ok(new_self)
}
fn deserialize_general_token(
_cursor: &mut TreeCursor,
_source: &str,
_diagnostics: &mut DiagnosticsInfo,
new_self: Box<Self>,
) -> (LoopControl, Box<Self>) {
(T::get_vec_loopcontrol(), new_self)
}
fn get_allowed_tokens() -> Option<Vec<String>> {
None
}
}
fn deserialize_general_token_custom<T: TokenDeserialize>(
cursor: &mut TreeCursor,
source: &str,
diagnostics: &mut DiagnosticsInfo,
mut new_self: Vec<T>,
token_name: &String,
) -> (LoopControl, Vec<T>) {
// Needs to check if next token has the right reference
let token = match Token::deserialize_tokens(cursor, source, diagnostics) {
Ok(token) => token,
Err(_err) => {
// When token could not be parsed correctly.
// Token could not be parsed, so we can consume it.
// Because this will always fail.
Token::consume_token(cursor).expect("Token does not have a next sibling");
return (LoopControl::Continue, new_self);
}
};
// This does not consume the token, this is done when parameters are stored.
let current_token_name = match token.get_token_name() {
Ok(name) => &name.value,
Err(_) => {
// The Cursor is at a token that does not start with a Token Name
// This token can not be parsed, go to next
Token::consume_token(cursor).expect("Token does not have a next sibling");
return (LoopControl::Continue, new_self);
}
};
let allowed_tokens = T::get_allowed_tokens();
let mut token_allowed = false;
// Check if token is allowed in this loop
if let Some(allowed_tokens) = &allowed_tokens {
log::debug!(
"Vec<T> check {:?} with allow list: {:?}",
current_token_name,
allowed_tokens
);
// This is used by all `Struct` and `Enum` (but not `enum_value`, see #61) types
for allowed_token in allowed_tokens {
if current_token_name == allowed_token {
token_allowed = true;
break;
}
}
} else {
log::debug!(
"Vec<T> check {:?} with token ref: {:?}",
current_token_name,
token_name
);
// If all tokens allowed, make sure all tokens have the
// same first token reference aka first argument
// This is used for `Vec<Reference>` and similar
if current_token_name == token_name {
token_allowed = true;
}
}
if !token_allowed {
if new_self.is_empty() {
return (LoopControl::ErrBreak, new_self);
} else {
return (LoopControl::Break, new_self);
}
}
let value = TokenDeserialize::deserialize_tokens(cursor, source, diagnostics);
if let Ok(value) = value {
new_self.push(*value);
let loop_control = T::get_vec_loopcontrol();
// String, i32, Tuple and type likes that => DoNothing
// Other type => Continue
(loop_control, new_self)
} else {
// Else message should be already added to diagnostics
// if nothing changed (added), do error break.
if new_self.is_empty() {
(LoopControl::ErrBreak, new_self)
} else {
(LoopControl::Break, new_self)
}
}
}
// ------------------------- Convert a group of arguments to Self -----------------------
/// Parse all left over arguments, no arguments required.
/// This is used to parse infinite amount of arguments
impl<T> TryFromArgumentGroup for Vec<T>
where
T: TryFromArgumentGroup,
{
fn try_from_argument_group(
token: &mut Token,
source: &str,
diagnostics: &mut DiagnosticsInfo,
add_diagnostics_on_err: bool,
) -> Result<Self, ()> {
let mut result = vec![];
while let Some(_arg) = token.get_current_arg_opt() {
result.push(T::try_from_argument_group(
token,
source,
diagnostics,
add_diagnostics_on_err,
)?);
}
Ok(result)
}
}
// ---------------------------- TESTS --------------------------
#[cfg(test)]
mod tests {
use super::*;
use crate::test_tree_structure;
use crate::test_utils::SyntaxTestBuilder;
use df_ls_lexical_analysis::test_utils::LexerTestBuilder;
#[test]
fn test_vec_string_correct() {
let test_builder = SyntaxTestBuilder::from_lexer_test_builder(
LexerTestBuilder::test_source(
"header
[REF:some string]
[REF:some other string]
[REF:short]",
)
.add_test_lexer_diagnostics_codes(vec![])
.add_test_lexer_diagnostics_ranges(vec![]),
)
.add_test_syntax_diagnostics_codes(vec![])
.add_test_syntax_diagnostics_ranges(vec![]);
test_tree_structure!(
test_builder,
[
Vec<String> => vec![
"some string".to_owned(),
"some other string".to_owned(),
"short".to_owned()
],
]
);
}
#[test]
fn test_vec_string_other_ref() {
let test_builder = SyntaxTestBuilder::from_lexer_test_builder(
LexerTestBuilder::test_source(
"header
[REF:some string]
[REF:some other string]
[REFB:short]",
)
.add_test_lexer_diagnostics_codes(vec![])
.add_test_lexer_diagnostics_ranges(vec![]),
)
.add_test_syntax_diagnostics_codes(vec![])
.add_test_syntax_diagnostics_ranges(vec![]);
test_tree_structure!(
test_builder,
[
Vec<String> => vec![
"some string".to_owned(),
"some other string".to_owned(),
],
String => "short".to_owned()
]
);
}
#[test]
fn test_vec_u8_try_from_argument_group_correct() {
let test_builder = SyntaxTestBuilder::from_lexer_test_builder(
LexerTestBuilder::test_source(
"header
[REF:8:5:152:0:26:96]
[REF:5]
[REF]",
)
.add_test_lexer_diagnostics_codes(vec![])
.add_test_lexer_diagnostics_ranges(vec![]),
)
.add_test_syntax_diagnostics_codes(vec![])
.add_test_syntax_diagnostics_ranges(vec![]);
test_tree_structure!(
test_builder,
[
Option<Vec<u8>> => Some(vec![8u8, 5, 152, 0, 26, 96]),
Option<Vec<u8>> => Some(vec![5u8]),
(Vec<u8>,) => (Vec::<u8>::new(),)
]
);
}
}