1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
//! Detects sequences of Punctuation characters, like `;` or `>>=`.
use super::super::lexeme::LexemeKind;
const DETECTED: LexemeKind = LexemeKind::Punctuation;
const UNDETECTED: (LexemeKind, usize) = (LexemeKind::Undetected, 0);
/// Detects sequences of Punctuation characters, like `;` or `>>=`.
///
/// ### Arguments
/// * `orig` The original Rust code, assumed to conform to the 2018 edition
/// * `chr` The character position in `orig` to look at
///
/// ### Returns
/// If `chr` begins a valid looking sequence of Punctuation characters,
/// `detect_punctuation()` returns `LexemeKind::Punctuation` and the character
/// position after it ends.
/// Otherwise, `detect_punctuation()` returns `LexemeKind::Undetected` and `0`.
pub fn detect_punctuation(
orig: &str,
chr: usize,
) -> (
LexemeKind,
usize,
) {
// If the current char is past the last char in `orig`, bail out!
let len = orig.len();
if chr >= len { return UNDETECTED }
// If the current char is not present in PUNCTUATION_1, it is not, and does
// not begin, punctuation. That’s because PUNCTUATION_2 and PUNCTUATION_3
// all start with a PUNCTUATION_1 character.
let c0 = orig.get(chr..chr+1).unwrap_or("~");
if ! PUNCTUATION_1.contains(&c0) { return UNDETECTED }
// If the current char is the last in the code, then it must be punctuation.
if len == chr + 1 { return (DETECTED, len) }
// Get two chars. If they are not a 2-char punctuation, then detect just
// the single-character punctuation.
let c1 = orig.get(chr..chr+2).unwrap_or("~");
if ! PUNCTUATION_2.contains(&c1) { return (DETECTED, chr + 1) }
// If c1 reaches the end of the code, then c0 starts a 2-char punctuation.
if len == chr + 2 { return (DETECTED, len) }
// Get three chars. If they are not a 3-char punctuation, then detect just
// the two-character punctuation.
let c2 = orig.get(chr..chr+3).unwrap_or("~");
if ! PUNCTUATION_3.contains(&c2) { return (DETECTED, chr + 2) }
// `detect_punctuation()` accepts any character at all after finding
// 3-char punctuation. It could also be the end-of-input.
(DETECTED, chr + 3)
}
const PUNCTUATION_1: [&str; 28] = [
"'", // SingleQuote Labels, Lifetimes
"_", // Underscore Wildcard patterns, Inferred types, Unnamed...
"-", // Minus Subtraction, Negation
",", // Comma Various separators
";", // Semi Terminator for situations, Array types
":", // Colon Various separators
"!", // Not Bitwise and Logical NOT, Macro Calls, ...
"?", // Question Question mark operator, Questionably sized, ...
".", // Dot Field access, Tuple index
"(", // OpenParentheses Logic
")", // CloseParentheses Logic
"[", // OpenSquareBraces Arrays
"]", // CloseSquareBraces Arrays
"{", // OpenCurlyBraces Blocks
"}", // CloseCurlyBraces Blocks
"@", // At Subpattern binding
"*", // Star Multiplication, Dereference, Raw Pointers, ...
"/", // Slash Division
"&", // And Bitwise / Logical AND, Borrow, References, ...
"#", // Pound Attributes
"%", // Percent Remainder
"^", // Caret Bitwise and Logical XOR
"+", // Plus Addition, Trait Bounds, Macro Kleene Matcher
"<", // Lt Less than, Generics, Paths
"=", // Eq Assignment, Attributes, Various type definitions
">", // Gt Greater than, Generics, Paths
"|", // Or Bitwise / Logical OR, Closures, if let, ...
"$", // Dollar Macros
];
const PUNCTUATION_2: [&str; 20] = [
"-=", // MinusEq Subtraction assignment
"->", // RArrow Function return type, Closure return type, ...
"::", // PathSep Path separator
"!=", // Ne Not Equal
"..", // DotDot Range, Struct expressions, Patterns
"*=", // StarEq Multiplication assignment
"/=", // SlashEq Division assignment
"&&", // AndAnd Lazy AND, Borrow, References, Reference patterns
"&=", // AndEq Bitwise And assignment
"%=", // PercentEq Remainder assignment
"^=", // CaretEq Bitwise XOR assignment
"+=", // PlusEq Addition assignment
"<<", // Shl Shift Left, Nested Generics
"<=", // Le Less than or equal to
"==", // EqEq Equal
"=>", // FatArrow Match arms, Macros
">=", // Ge Greater than or equal to, Generics
">>", // Shr Shift Right, Nested Generics
"|=", // OrEq Bitwise Or assignment
"||", // OrOr Lazy OR, Closures
];
const PUNCTUATION_3: [&str; 4] = [
"...", // DotDotDot Variadic functions, Range patterns
"..=", // DotDotEq Inclusive Range, Range patterns
"<<=", // ShlEq Shift Left assignment
">>=", // ShrEq Shift Right assignment, Nested Generics
];
#[cfg(test)]
mod tests {
use super::detect_punctuation as detect;
use super::DETECTED as D;
use super::UNDETECTED as U;
#[test]
fn detect_punctuation_correct() {
// Basic.
let orig = "- === 'label ...";
assert_eq!(detect(orig, 0), (D,1)); // -
assert_eq!(detect(orig, 2), (D,4)); // == there is no "===" in Rust
assert_eq!(detect(orig, 3), (D,5)); // == finds the 2nd and 3rd char in ===
assert_eq!(detect(orig, 6), (D,7)); // ' not considered part of the label
assert_eq!(detect(orig, 13), (D,16)); // ...
// Single at end.
assert_eq!(detect(" '", 1), (D,2));
assert_eq!(detect(" _", 1), (D,2));
assert_eq!(detect(" -", 1), (D,2));
assert_eq!(detect(" ,", 1), (D,2));
assert_eq!(detect(" ;", 1), (D,2));
assert_eq!(detect(" :", 1), (D,2));
assert_eq!(detect(" !", 1), (D,2));
assert_eq!(detect(" ?", 1), (D,2));
assert_eq!(detect(" .", 1), (D,2));
assert_eq!(detect(" (", 1), (D,2));
assert_eq!(detect(" )", 1), (D,2));
assert_eq!(detect(" [", 1), (D,2));
assert_eq!(detect(" ]", 1), (D,2));
assert_eq!(detect(" {", 1), (D,2));
assert_eq!(detect(" }", 1), (D,2));
assert_eq!(detect(" @", 1), (D,2));
assert_eq!(detect(" *", 1), (D,2));
assert_eq!(detect(" /", 1), (D,2));
assert_eq!(detect(" &", 1), (D,2));
assert_eq!(detect(" #", 1), (D,2));
assert_eq!(detect(" %", 1), (D,2));
assert_eq!(detect(" ^", 1), (D,2));
assert_eq!(detect(" +", 1), (D,2));
assert_eq!(detect(" <", 1), (D,2));
assert_eq!(detect(" =", 1), (D,2));
assert_eq!(detect(" >", 1), (D,2));
assert_eq!(detect(" |", 1), (D,2));
assert_eq!(detect(" $", 1), (D,2));
// Single then tilde.
assert_eq!(detect(" '~", 1), (D,2));
assert_eq!(detect(" _~", 1), (D,2));
assert_eq!(detect(" -~", 1), (D,2));
assert_eq!(detect(" ,~", 1), (D,2));
assert_eq!(detect(" ;~", 1), (D,2));
assert_eq!(detect(" :~", 1), (D,2));
assert_eq!(detect(" !~", 1), (D,2));
assert_eq!(detect(" ?~", 1), (D,2));
assert_eq!(detect(" .~", 1), (D,2));
assert_eq!(detect(" (~", 1), (D,2));
assert_eq!(detect(" )~", 1), (D,2));
assert_eq!(detect(" [~", 1), (D,2));
assert_eq!(detect(" ]~", 1), (D,2));
assert_eq!(detect(" {~", 1), (D,2));
assert_eq!(detect(" }~", 1), (D,2));
assert_eq!(detect(" @~", 1), (D,2));
assert_eq!(detect(" *~", 1), (D,2));
assert_eq!(detect(" /~", 1), (D,2));
assert_eq!(detect(" &~", 1), (D,2));
assert_eq!(detect(" #~", 1), (D,2));
assert_eq!(detect(" %~", 1), (D,2));
assert_eq!(detect(" ^~", 1), (D,2));
assert_eq!(detect(" +~", 1), (D,2));
assert_eq!(detect(" <~", 1), (D,2));
assert_eq!(detect(" =~", 1), (D,2));
assert_eq!(detect(" >~", 1), (D,2));
assert_eq!(detect(" |~", 1), (D,2));
assert_eq!(detect(" $~", 1), (D,2));
// Single then equals.
// Subset of single-char punctuation which should be terminated by "=".
assert_eq!(detect(" '=", 1), (D,2));
assert_eq!(detect(" _=", 1), (D,2));
assert_eq!(detect(" ,=", 1), (D,2));
assert_eq!(detect(" ;=", 1), (D,2));
assert_eq!(detect(" :=", 1), (D,2));
assert_eq!(detect(" ?=", 1), (D,2));
assert_eq!(detect(" .=", 1), (D,2));
assert_eq!(detect(" (=", 1), (D,2));
assert_eq!(detect(" )=", 1), (D,2));
assert_eq!(detect(" [=", 1), (D,2));
assert_eq!(detect(" ]=", 1), (D,2));
assert_eq!(detect(" {=", 1), (D,2));
assert_eq!(detect(" }=", 1), (D,2));
assert_eq!(detect(" @=", 1), (D,2));
assert_eq!(detect(" #=", 1), (D,2));
assert_eq!(detect(" $=", 1), (D,2));
// Double at end.
assert_eq!(detect(" -=", 1), (D,3));
assert_eq!(detect(" ->", 1), (D,3));
assert_eq!(detect(" ::", 1), (D,3));
assert_eq!(detect(" !=", 1), (D,3));
assert_eq!(detect(" ..", 1), (D,3));
assert_eq!(detect(" *=", 1), (D,3));
assert_eq!(detect(" /=", 1), (D,3));
assert_eq!(detect(" &&", 1), (D,3));
assert_eq!(detect(" &=", 1), (D,3));
assert_eq!(detect(" %=", 1), (D,3));
assert_eq!(detect(" ^=", 1), (D,3));
assert_eq!(detect(" +=", 1), (D,3));
assert_eq!(detect(" <<", 1), (D,3));
assert_eq!(detect(" <=", 1), (D,3));
assert_eq!(detect(" ==", 1), (D,3));
assert_eq!(detect(" =>", 1), (D,3));
assert_eq!(detect(" >=", 1), (D,3));
assert_eq!(detect(" >>", 1), (D,3));
assert_eq!(detect(" |=", 1), (D,3));
assert_eq!(detect(" ||", 1), (D,3));
// Double then tilde.
assert_eq!(detect(" -=~", 1), (D,3));
assert_eq!(detect(" ->~", 1), (D,3));
assert_eq!(detect(" ::~", 1), (D,3));
assert_eq!(detect(" !=~", 1), (D,3));
assert_eq!(detect(" ..~", 1), (D,3));
assert_eq!(detect(" *=~", 1), (D,3));
assert_eq!(detect(" /=~", 1), (D,3));
assert_eq!(detect(" &&~", 1), (D,3));
assert_eq!(detect(" &=~", 1), (D,3));
assert_eq!(detect(" %=~", 1), (D,3));
assert_eq!(detect(" ^=~", 1), (D,3));
assert_eq!(detect(" +=~", 1), (D,3));
assert_eq!(detect(" <<~", 1), (D,3));
assert_eq!(detect(" <=~", 1), (D,3));
assert_eq!(detect(" ==~", 1), (D,3));
assert_eq!(detect(" =>~", 1), (D,3));
assert_eq!(detect(" >=~", 1), (D,3));
assert_eq!(detect(" >>~", 1), (D,3));
assert_eq!(detect(" |=~", 1), (D,3));
assert_eq!(detect(" ||~", 1), (D,3));
// Double then equals.
// Subset of double-char punctuation which should be terminated by "=".
assert_eq!(detect(" -==", 1), (D,3));
assert_eq!(detect(" ->=", 1), (D,3));
assert_eq!(detect(" ::=", 1), (D,3));
assert_eq!(detect(" !==", 1), (D,3));
assert_eq!(detect(" *==", 1), (D,3));
assert_eq!(detect(" /==", 1), (D,3));
assert_eq!(detect(" &&=", 1), (D,3));
assert_eq!(detect(" &==", 1), (D,3));
assert_eq!(detect(" %==", 1), (D,3));
assert_eq!(detect(" ^==", 1), (D,3));
assert_eq!(detect(" +==", 1), (D,3));
assert_eq!(detect(" <==", 1), (D,3));
assert_eq!(detect(" ===", 1), (D,3));
assert_eq!(detect(" =>=", 1), (D,3));
assert_eq!(detect(" >==", 1), (D,3));
assert_eq!(detect(" |==", 1), (D,3));
assert_eq!(detect(" ||=", 1), (D,3));
// Triple at end.
assert_eq!(detect(" ...", 1), (D,4));
assert_eq!(detect(" ..=", 1), (D,4));
assert_eq!(detect(" <<=", 1), (D,4));
assert_eq!(detect(" >>=", 1), (D,4));
// Triple then tilde.
assert_eq!(detect(" ...~", 1), (D,4));
assert_eq!(detect(" ..=~", 1), (D,4));
assert_eq!(detect(" <<=~", 1), (D,4));
assert_eq!(detect(" >>=~", 1), (D,4));
// Triple then equals.
// All triple-char punctuation should be terminated by "=".
assert_eq!(detect(" ...=", 1), (D,4));
assert_eq!(detect(" ..==", 1), (D,4));
assert_eq!(detect(" <<==", 1), (D,4));
assert_eq!(detect(" >>==", 1), (D,4));
}
#[test]
fn detect_punctuation_incorrect() {
let orig = "` =* .:.";
assert_eq!(detect(orig, 0), U); // backtick is not Rust punctuation
assert_eq!(detect(orig, 2), (D, 3)); // the = of =* is accepted
assert_eq!(detect(orig, 5), (D, 6)); // the . of .:. is accepted
}
#[test]
fn detect_punctuation_will_not_panic() {
// Near the end of `orig`.
assert_eq!(detect("", 0), U); // empty string
assert_eq!(detect("~", 0), U); // tilde is not Rust punctuation
assert_eq!(detect(">", 0), (D, 1)); // >
// Invalid `chr`.
assert_eq!(detect("abc", 2), U); // 2 is before "c", so in range
assert_eq!(detect("abc", 3), U); // 3 is after "c", so incorrect
assert_eq!(detect("abc", 4), U); // 4 is out of range
assert_eq!(detect("abc", 100), U); // 100 is way out of range
// Non-ascii.
assert_eq!(detect("€", 1), U); // part way into the three € bytes
assert_eq!(detect(".€", 0), (D,1)); // non-ascii after .
assert_eq!(detect("..€", 0), (D,2)); // non-ascii after ..
assert_eq!(detect("...€", 0), (D,3)); // non-ascii after ...
}
}