1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
use crate::Kind;
/// Match a token against one or more [Kinds][Kind].
///
/// Each [Kind] represents the token "type". [KindSet] is a bitmask of all possible [Kinds][Kind]. This is useful for
/// efficiently comparing a token to see if it matches N token [Kinds][Kind].
///
/// # Example
///
/// ```
/// use css_lexer::*;
/// let mut lexer = Lexer::new(&EmptyAtomSet::ATOMS, "width: 1px");
/// // The first token is either an AtKeyword, Ident or Function:
/// assert_eq!(lexer.advance(), KindSet::new(&[Kind::AtKeyword, Kind::Ident, Kind::Function]));
/// ```
#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct KindSet(u64);
impl KindSet {
/// A [KindSet] that matches no [Kinds][Kind].
pub const NONE: KindSet = KindSet::new(&[]);
/// A [KindSet] that matches all trivia; [Kind::Whitespace] and [Kind::Comment].
pub const TRIVIA: KindSet = KindSet::new(&[Kind::Whitespace, Kind::Comment]);
/// A [KindSet] that matches just Whitespace. This is the same as [Kind::Whitespace] but can be useful to apply to
/// functions that expect a [KindSet] rather than [Kind].
pub const WHITESPACE: KindSet = KindSet::new(&[Kind::Whitespace]);
/// A [KindSet] that matches just Whitespace. This is the same as [Kind::Comment] but can be useful to apply to
/// functions that expect a [KindSet] rather than [Kind].
pub const COMMENTS: KindSet = KindSet::new(&[Kind::Comment]);
/// A [KindSet] that matches just Whitespace or Semicolons.
pub const WHITESPACE_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::Whitespace, Kind::Semicolon]);
/// A [KindSet] that matches either [Kind::RightCurly] or [Kind::Semicolon]. This is useful for matching
/// stop-tokens, for example checking the end of a declaration.
pub const RIGHT_CURLY_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::RightCurly, Kind::Semicolon]);
/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::Semicolon]. This is useful for matching
/// stop-tokens, for example checking the end of an at-rule prelude.
pub const LEFT_CURLY_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::LeftCurly, Kind::Semicolon]);
/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::RightParen] or [Kind::Semicolon]. This is useful for
/// matching stop-tokens, for example checking the end of a function.
pub const LEFT_CURLY_RIGHT_PAREN_OR_SEMICOLON: KindSet =
KindSet::new(&[Kind::LeftCurly, Kind::RightParen, Kind::Semicolon]);
/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::RightParen] or [Kind::Comma] or [Kind::Semicolon].
/// This is useful for matching stop-tokens, for example checking the end of a function or Selector.
pub const LEFT_CURLY_RIGHT_PAREN_COMMA_OR_SEMICOLON: KindSet =
KindSet::new(&[Kind::LeftCurly, Kind::RightParen, Kind::Comma, Kind::Semicolon]);
/// A [KindSet] that matches either [Kind::Ident], [Kind::AtKeyword], [Kind::Function], [Kind::Hash].
pub const IDENT_LIKE: KindSet = KindSet::new(&[Kind::Ident, Kind::AtKeyword, Kind::Function, Kind::Hash]);
/// A [KindSet] that matches token kinds which can carry atom string content: [Kind::Ident], [Kind::Function],
/// [Kind::AtKeyword], and [Kind::Dimension]. These are the only kinds for which
/// [Token::atom_bits][crate::Token::atom_bits] may return a non-zero value.
pub const ATOM_LIKE: KindSet = KindSet::new(&[Kind::Ident, Kind::Function, Kind::AtKeyword, Kind::Dimension]);
/// A [KindSet] that matches any single character token, such as [Kind::Delim] or [Kind::Colon] - [Kind::RightCurly].
pub const DELIM_LIKE: KindSet = KindSet::new(&[
Kind::Delim,
Kind::Colon,
Kind::Semicolon,
Kind::Comma,
Kind::LeftSquare,
Kind::RightSquare,
Kind::LeftParen,
Kind::RightParen,
Kind::LeftCurly,
Kind::RightCurly,
]);
/// A [KindSet] that matches either [Kind::LeftCurly], [Kind::LeftParen], [Kind::LeftSquare].
pub const PAIRWISE_START: KindSet = KindSet::new(&[Kind::LeftCurly, Kind::LeftParen, Kind::LeftSquare]);
///
/// A [KindSet] that matches either [Kind::RightCurly], [Kind::RightParen], [Kind::RightSquare].
pub const PAIRWISE_END: KindSet = KindSet::new(&[Kind::RightCurly, Kind::RightParen, Kind::RightSquare]);
/// A [KindSet] that matches _any_ token.
pub const ANY: KindSet = KindSet(u64::MAX);
/// Creates a new [KindSet] with the combination of all given [Kinds][Kind].
///
/// This function is marked `const` to allow creation of const [KindSets][KindSet].
pub const fn new(kinds: &[Kind]) -> Self {
let mut u = 0;
let mut i = 0;
let len = kinds.len();
while i < len {
u |= 1 << (kinds[i] as u8 & 0b111111);
i += 1;
}
Self(u)
}
/// Returns a new [KindSet] with the addition of the supplied [Kind].
///
/// This function is marked `const` to allow creation of const [KindSets][KindSet].
pub const fn add(&self, kind: Kind) -> Self {
Self(self.0 | (1 << (kind as u8 & 0b111111)))
}
/// Returns a new [KindSet] combined with the other [KindSet].
///
/// This function is marked `const` to allow creation of const [KindSets][KindSet].
pub const fn combine(&self, ks: KindSet) -> Self {
Self(self.0 | ks.0)
}
/// Returns a new [KindSet] without the supplied [Kind].
///
/// This function is marked `const` to allow creation of const [KindSets][KindSet].
pub const fn remove(&self, kind: Kind) -> Self {
Self(self.0 ^ (1 << (kind as u8 & 0b111111)))
}
/// Check if a [KindSet] contains the subpplied [Kind].
pub fn contains(&self, kind: Kind) -> bool {
self.0 & (1 << (kind as u8 & 0b111111)) != 0
}
pub(crate) const fn contains_bits(&self, kind_bits: u8) -> bool {
self.0 & (1 << (kind_bits & 0b111111)) != 0
}
}
#[test]
fn test_kindset_contains() {
let set = KindSet::new(&[Kind::Eof, Kind::Whitespace, Kind::Comment]);
assert!(set.contains(Kind::Eof));
assert!(set.contains(Kind::Whitespace));
assert!(set.contains(Kind::Comment));
assert!(!set.contains(Kind::String));
assert!(!set.contains(Kind::Url));
let set = KindSet::new(&[Kind::LeftCurly, Kind::LeftSquare, Kind::LeftParen]);
assert!(set.contains(Kind::LeftCurly));
assert!(!set.contains(Kind::RightCurly));
assert!(set.contains(Kind::LeftSquare));
assert!(!set.contains(Kind::RightSquare));
assert!(set.contains(Kind::LeftParen));
assert!(!set.contains(Kind::RightParen));
assert!(!set.contains(Kind::Ident));
assert!(KindSet::COMMENTS.contains(Kind::Comment));
assert!(!KindSet::COMMENTS.contains(Kind::Delim));
}
#[test]
fn test_kindset_add_remove() {
let k_ident = KindSet::new(&[Kind::Ident]);
let k_ident_eof = k_ident.add(Kind::Eof);
assert!(k_ident.contains(Kind::Ident));
assert!(k_ident_eof.contains(Kind::Ident));
assert!(k_ident_eof.contains(Kind::Eof));
assert!(!k_ident_eof.remove(Kind::Eof).contains(Kind::Eof));
}