1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
use bitmask_enum::bitmask;
/// A set of runtime feature flags which can be enabled individually or in combination, which will change the way
/// individual [Lexer][crate::Lexer] produces [Tokens][crate::Token].
///
/// To build multiple features, use the bitwise OR operator.
///
/// # Example
///
/// ```
/// use css_lexer::*;
/// let features = Feature::SingleLineComments | Feature::SeparateWhitespace;
/// let mut lexer = Lexer::new_with_features(&EmptyAtomSet::ATOMS, "// foo", features);
/// ```
#[bitmask(u8)]
#[bitmask_config(vec_debug)]
#[derive(Default)]
pub enum Feature {
/// With this flag enabled the [Lexer][crate::Lexer] will produce [Tokens][crate::Token] with
/// [Kind::Comment][crate::Kind::Comment] when it encounters two consecutative SOLIDUS characters (`//`), the
/// [Token][crate::Token] will have a length up to the next newline (`\n`) character. The contents between the two
/// SOLIDUS (`//`) characters and the `\n` will be consumed by this token, so no tokens will be produced for the
/// contents of the comment.
///
/// If this flag is not enabled, encountering something that looks like a single line commet will produce two
/// [Kind::Delim][crate::Kind::Delim] tokens for the two SOLIDUS (`//`) characters, and any number of other tokens
/// depending on the contents of the comment, per the CSS specification.
///
/// A comment with two leading SOLIDUS characters is not valid CSS syntax, but might be considered valid syntax in
/// other CSS-alike languages [for example SASS][1].
///
/// [1]: https://sass-lang.com/documentation/syntax/comments/
///
/// With this feature turned off comments are tokenized per the CSS specification:
///
/// ```md
/// <comment>
/// ╭──────────────────────────────────────────╮
/// │├─ "/*" ─╯-╭─ (anything but "*" followed by "/") ─╮─╰─ "*/" ─┤│
/// ╰──────────────────────────────────────╯
/// ```
///
/// With this feature turned on comments are tokenized with the additional grammar:
///
/// ```md
/// <comment>
/// ╭──────────────────────────────────────────╮
/// │├──╮─ "/*" ─╯-╭─ (anything but "*" followed by "/") ─╮─╰─ "*/" ─╭─┤│
/// │ ╰──────────────────────────────────────╯ │
/// │ ╭───────────────────────────╮ │
/// ├─ "//" ───────╯-╭─ (anything but "\n") ─╮─╰─ "\n" ──────────╯
/// ╰─ ╰───────────────────────╯
/// ```
///
/// # Example
///
/// ```
/// use css_lexer::*;
/// let mut lexer = Lexer::new(&EmptyAtomSet::ATOMS, "// foo");
/// assert_eq!(lexer.advance(), Kind::Delim); // The first `/`
/// assert_eq!(lexer.advance(), Kind::Delim); // The second `/`
/// assert_eq!(lexer.advance(), Kind::Whitespace);
/// assert_eq!(lexer.advance(), Kind::Ident); // The "foo" in the comment
///
/// let mut lexer = Lexer::new_with_features(&EmptyAtomSet::ATOMS, "// foo", Feature::SingleLineComments);
/// let token = lexer.advance();
/// assert_eq!(token, Kind::Comment); // The whole comment "// foo"
/// assert_eq!(token, CommentStyle::Single);
/// ```
SingleLineComments = 0b0001,
/// The CSS Spec mentions that whitespace tokens should be [combined into a single Whitespace token][1]. This means
/// a single whitespace token can contain a cominbation of newlines, tabs, and space characters. This is often fine
/// as whitespace is generally ignored during parsing, however for certain IDE features it might be important to
/// tokenize discrete whitespace [Tokens][crate::Token], each with their own discrete whitespace. Enabling this flag
/// will enforce that the [Lexer][crate::Lexer] outputs these discrete tokens. In other words with this feature
/// enabled, multiple contiguous whitespace tokens may be returned from subsequent calls to
/// [Lexer::advance()][crate::Lexer::advance()], but with this feature off this will never be the case (as whitespace
/// is collapsed into a single [Token][crate::Token]).
///
/// With this feature turned off whitespace-tokens are tokenized per the CSS specification:
///
/// ```md
/// <newline>
/// │├──╮─ "\n" ───╭──┤│
/// ├─ "\r\n" ─┤
/// ├─ "\r" ───┤
/// ╰─ "\f" ───╯
///
/// <whitespace>
/// │├──╮─ " " ───────╭──┤│
/// ├─ "\t" ──────┤
/// ╰─ <newline> ─╯
///
/// <whitespace-token>
/// │├─╭─ <whitespace> ─╮─┤│
/// ╰────────────────╯
/// ```
///
/// With this feature turned on whitespace-tokens are tokenized with the additional grammar:
///
/// ```md
/// <whitespace-token>
/// │├──╮─╭─ " " ───────╮─╭──┤│
/// │ ╰─────────────╯ │
/// ├─╭─ "\t" ──────╮─┤
/// │ ╰─────────────╯ │
/// ╰─╭─ <newline> ─╮─╯
/// ╰─────────────╯
/// ```
///
/// [1]: https://drafts.csswg.org/css-syntax-3/#whitespace-diagram
///
/// # Example
///
/// ```
/// use css_lexer::*;
/// let mut lexer = Lexer::new(&EmptyAtomSet::ATOMS, "\n\thello world");
/// {
/// // This token will be collapsed Whitespace.
/// let token = lexer.advance();
/// assert_eq!(token, Kind::Whitespace);
/// // The Whitespace is comprised of many bits:
/// assert_eq!(token, Whitespace::Newline | Whitespace::Tab);
/// }
/// assert_eq!(lexer.advance(), Kind::Ident);
/// {
/// let token = lexer.advance();
/// assert_eq!(token, Kind::Whitespace);
/// assert_eq!(token, Whitespace::Space);
/// }
/// assert_eq!(lexer.advance(), Kind::Ident);
///
/// let mut lexer = Lexer::new_with_features(&EmptyAtomSet::ATOMS, "\n\thello world", Feature::SeparateWhitespace);
/// {
/// // This token will be discrete Whitespace, just the `\n`.
/// let token = lexer.advance();
/// assert_eq!(token, Kind::Whitespace);
/// // The Whitespace is comprised of a single bit:
/// assert_eq!(token, Whitespace::Newline);
/// }
/// {
/// // This token will be discrete Whitespace, just the `\t`.
/// let token = lexer.advance();
/// assert_eq!(token, Kind::Whitespace);
/// // The Whitespace is comprised of a single bit:
/// assert_eq!(token, Whitespace::Tab);
/// }
/// assert_eq!(lexer.advance(), Kind::Ident);
/// {
/// let token = lexer.advance();
/// assert_eq!(token, Kind::Whitespace);
/// assert_eq!(token, Whitespace::Space);
/// }
/// assert_eq!(lexer.advance(), Kind::Ident);
/// ```
SeparateWhitespace = 0b0010,
/// With this flag enabled the [Lexer][crate::Lexer] will produce [Tokens][crate::Token] with
/// [Kind::UnicodeRange][crate::Kind::UnicodeRange] when it encounters a valid `<unicode-range-token>` as described
/// in the [CSS Syntax specification][1].
///
/// Per the specification, this token is only produced as part of a special re-tokenization of the `unicode-range`
/// descriptor value in `@font-face` rules. Normal tokenization does not produce this token.
///
/// A unicode-range token starts with `U+` or `u+` followed by hex digits and/or `?` wildcards.
///
/// Without this feature enabled, `U+0000-00FF` would be tokenized as multiple separate tokens
/// (Ident, Delim, Number, Delim, Number, Ident, etc).
///
/// [1]: https://drafts.csswg.org/css-syntax/#consume-unicode-range-token
///
/// # Example
///
/// ```
/// use css_lexer::*;
/// let mut lexer = Lexer::new(&EmptyAtomSet::ATOMS, "U+0-7F");
/// assert_eq!(lexer.advance(), Kind::Ident);
///
/// let mut lexer = Lexer::new_with_features(&EmptyAtomSet::ATOMS, "U+0-7F", Feature::UnicodeRange);
/// let token = lexer.advance();
/// assert_eq!(token, Kind::UnicodeRange);
/// ```
UnicodeRange = 0b0100,
}