1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
pub struct Detokenizer;
impl Detokenizer {
#[inline]
pub fn detokenize<S: AsRef<str>>(tokens: &[S]) -> String {
if tokens.is_empty() {
return String::new();
}
let cap: usize = tokens.iter().map(|t| t.as_ref().len() + 1).sum();
let mut out = String::with_capacity(cap);
let mut no_space_before_next = true;
for tok_ref in tokens {
let tok = tok_ref.as_ref();
if tok.is_empty() {
continue;
}
if matches!(tok, "«" | "(" | "[" | "{") {
if !no_space_before_next {
out.push(' ');
}
out.push_str(tok);
no_space_before_next = true;
} else if matches!(
tok,
"." | "،" | "؛" | ":" | "؟" | "?" | "!" | ")" | "]" | "}" | "»" | "%" | "٪"
) || tok.chars().all(|c| matches!(c, '.' | '؟' | '?' | '!'))
{
out.push_str(tok);
no_space_before_next = false;
} else {
if !no_space_before_next {
out.push(' ');
}
out.push_str(tok);
no_space_before_next = false;
}
}
out
}
}