1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
#[allow(missing_docs)]
pub enum Kind {
Punctuation,
Keyword,
Identifier,
Literal,
Comment,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Token {
pub kind: Kind,
pub raw: Box<[u8]>,
}
impl<'a> From<(Kind, &'a [u8])> for Token {
fn from((kind, value): (Kind, &'a [u8])) -> Token {
Token {
kind,
raw: value.to_owned().into_boxed_slice(),
}
}
}
pub fn remove_comments(v: &mut Vec<Token>) -> &mut Vec<Token> {
v.retain(|t| t.kind != Kind::Comment);
v
}