1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
use crate::Kind;
/// Match a token against one or more [Kinds][Kind].
///
/// Each [Kind] represents the token "type". [KindSet] is a bitmask of all possible [Kinds][Kind]. This is useful for
/// efficiently comparing a token to see if it matches N token [Kinds][Kind].
///
/// # Example
///
/// ```
/// use css_lexer::*;
/// let mut lexer = Lexer::new(&EmptyAtomSet::ATOMS, "width: 1px");
/// // The first token is either an AtKeyword, Ident or Function:
/// assert_eq!(lexer.advance(), KindSet::new(&[Kind::AtKeyword, Kind::Ident, Kind::Function]));
/// ```
#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct KindSet(u64);
impl KindSet {
/// A [KindSet] that matches no [Kinds][Kind].
pub const NONE: KindSet = KindSet::new(&[]);
/// A [KindSet] that matches all trivia; [Kind::Whitespace] and [Kind::Comment].
pub const TRIVIA: KindSet = KindSet::new(&[Kind::Whitespace, Kind::Comment]);
/// A [KindSet] that matches just Whitespace. This is the same as [Kind::Whitespace] but can be useful to apply to
/// functions that expect a [KindSet] rather than [Kind].
pub const WHITESPACE: KindSet = KindSet::new(&[Kind::Whitespace]);
/// A [KindSet] that matches just Whitespace. This is the same as [Kind::Comment] but can be useful to apply to
/// functions that expect a [KindSet] rather than [Kind].
pub const COMMENTS: KindSet = KindSet::new(&[Kind::Comment]);
/// A [KindSet] that matches just Whitespace or Semicolons.
pub const WHITESPACE_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::Whitespace, Kind::Semicolon]);
/// A [KindSet] that matches either [Kind::RightCurly] or [Kind::Semicolon]. This is useful for matching
/// stop-tokens, for example checking the end of a declaration.
pub const RIGHT_CURLY_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::RightCurly, Kind::Semicolon]);
/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::Semicolon]. This is useful for matching
/// stop-tokens, for example checking the end of an at-rule prelude.
pub const LEFT_CURLY_OR_SEMICOLON: KindSet = KindSet::new(&[Kind::LeftCurly, Kind::Semicolon]);
/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::RightParen] or [Kind::Semicolon]. This is useful for
/// matching stop-tokens, for example checking the end of a function.
pub const LEFT_CURLY_RIGHT_PAREN_OR_SEMICOLON: KindSet =
KindSet::new(&[Kind::LeftCurly, Kind::RightParen, Kind::Semicolon]);
/// A [KindSet] that matches either [Kind::LeftCurly] or [Kind::RightParen] or [Kind::Comma] or [Kind::Semicolon].
/// This is useful for matching stop-tokens, for example checking the end of a function or Selector.
pub const LEFT_CURLY_RIGHT_PAREN_COMMA_OR_SEMICOLON: KindSet =
KindSet::new(&[Kind::LeftCurly, Kind::RightParen, Kind::Comma, Kind::Semicolon]);
/// A [KindSet] that matches either [Kind::Ident], [Kind::AtKeyword], [Kind::Function], [Kind::Hash].
pub const IDENT_LIKE: KindSet = KindSet::new(&[Kind::Ident, Kind::AtKeyword, Kind::Function, Kind::Hash]);
/// A [KindSet] that matches any single character token, such as [Kind::Delim] or [Kind::Colon] - [Kind::RightCurly].
pub const DELIM_LIKE: KindSet = KindSet::new(&[
Kind::Delim,
Kind::Colon,
Kind::Semicolon,
Kind::Comma,
Kind::LeftSquare,
Kind::RightSquare,
Kind::LeftParen,
Kind::RightParen,
Kind::LeftCurly,
Kind::RightCurly,
]);
/// A [KindSet] that matches _any_ token.
pub const ANY: KindSet = KindSet(u64::MAX);
/// Creates a new [KindSet] with the combination of all given [Kinds][Kind].
///
/// This function is marked `const` to allow creation of const [KindSets][KindSet].
pub const fn new(kinds: &[Kind]) -> Self {
let mut u = 0;
let mut i = 0;
let len = kinds.len();
while i < len {
u |= 1 << (kinds[i] as u8 & 0b111111);
i += 1;
}
Self(u)
}
/// Returns a new [KindSet] with the addition of the supplied [Kind].
///
/// This function is marked `const` to allow creation of const [KindSets][KindSet].
pub const fn add(&self, kind: Kind) -> Self {
Self(self.0 | (1 << (kind as u8 & 0b111111)))
}
/// Check if a [KindSet] contains the subpplied [Kind].
pub fn contains(&self, kind: Kind) -> bool {
self.0 & (1 << (kind as u8 & 0b111111)) != 0
}
pub(crate) const fn contains_bits(&self, kind_bits: u8) -> bool {
self.0 & (1 << (kind_bits & 0b111111)) != 0
}
}
#[test]
fn test_kindset_contains() {
let set = KindSet::new(&[Kind::Eof, Kind::Whitespace, Kind::Comment]);
assert!(set.contains(Kind::Eof));
assert!(set.contains(Kind::Whitespace));
assert!(set.contains(Kind::Comment));
assert!(!set.contains(Kind::String));
assert!(!set.contains(Kind::Url));
let set = KindSet::new(&[Kind::LeftCurly, Kind::LeftSquare, Kind::LeftParen]);
assert!(set.contains(Kind::LeftCurly));
assert!(!set.contains(Kind::RightCurly));
assert!(set.contains(Kind::LeftSquare));
assert!(!set.contains(Kind::RightSquare));
assert!(set.contains(Kind::LeftParen));
assert!(!set.contains(Kind::RightParen));
assert!(!set.contains(Kind::Ident));
assert!(KindSet::COMMENTS.contains(Kind::Comment));
assert!(!KindSet::COMMENTS.contains(Kind::Delim));
}