oxc_css_parser/tokenizer/
token.rs

1//! All supported tokens, and with comments.
2
3use crate::pos::Span;
4#[cfg(feature = "serialize")]
5use serde::Serialize;
6
7#[derive(Clone, Debug, PartialEq)]
8#[cfg_attr(feature = "serialize", derive(Serialize))]
9#[cfg_attr(feature = "serialize", serde(tag = "type", rename_all = "camelCase"))]
10pub struct Comment<'s> {
11    pub content: &'s str,
12    pub kind: CommentKind,
13    pub span: Span,
14}
15
16#[derive(Clone, Debug, PartialEq)]
17#[cfg_attr(feature = "serialize", derive(Serialize))]
18pub enum CommentKind {
19    Block,
20    Line,
21}
22
23#[derive(Clone, Debug, PartialEq)]
24#[cfg_attr(feature = "serialize", derive(Serialize))]
25pub enum Token<'s> {
26    Eof(Eof),
27    Ampersand(Ampersand),
28    BadStr(BadStr<'s>),
29    Asterisk(Asterisk),
30    AsteriskEqual(AsteriskEqual),
31    At(At),
32    AtKeyword(AtKeyword<'s>),
33    AtLBraceVar(AtLBraceVar<'s>),
34    BacktickCode(BacktickCode<'s>),
35    Bar(Bar),
36    BarBar(BarBar),
37    BarEqual(BarEqual),
38    CaretEqual(CaretEqual),
39    Cdc(Cdc),
40    Cdo(Cdo),
41    Colon(Colon),
42    ColonColon(ColonColon),
43    Comma(Comma),
44    Dedent(Dedent),
45    Dimension(Dimension<'s>),
46    DollarEqual(DollarEqual),
47    DollarLBraceVar(DollarLBraceVar<'s>),
48    DollarVar(DollarVar<'s>),
49    Dot(Dot),
50    DotDotDot(DotDotDot),
51    Equal(Equal),
52    EqualEqual(EqualEqual),
53    Exclamation(Exclamation),
54    ExclamationEqual(ExclamationEqual),
55    GreaterThan(GreaterThan),
56    GreaterThanEqual(GreaterThanEqual),
57    Hash(Hash<'s>),
58    HashLBrace(HashLBrace),
59    Ident(Ident<'s>),
60    Indent(Indent),
61    LBrace(LBrace),
62    LBracket(LBracket),
63    LessThan(LessThan),
64    LessThanEqual(LessThanEqual),
65    Linebreak(Linebreak),
66    LParen(LParen),
67    Minus(Minus),
68    Number(Number<'s>),
69    NumberSign(NumberSign),
70    Percent(Percent),
71    Percentage(Percentage<'s>),
72    Placeholder(Placeholder<'s>),
73    Plus(Plus),
74    PlusUnderscore(PlusUnderscore),
75    Question(Question),
76    RBrace(RBrace),
77    RBracket(RBracket),
78    RParen(RParen),
79    Semicolon(Semicolon),
80    Solidus(Solidus),
81    Str(Str<'s>),
82    StrTemplate(StrTemplate<'s>),
83    Tilde(Tilde),
84    TildeEqual(TildeEqual),
85    Unknown(Unknown),
86    UrlRaw(UrlRaw<'s>),
87    UrlTemplate(UrlTemplate<'s>),
88}
89
90#[derive(Clone, Debug, PartialEq)]
91#[cfg_attr(feature = "serialize", derive(Serialize))]
92#[cfg_attr(feature = "serialize", serde(tag = "type", rename_all = "camelCase"))]
93pub struct TokenWithSpan<'s> {
94    pub token: Token<'s>,
95    pub span: Span,
96}
97
98#[derive(Clone, Debug, PartialEq)]
99#[cfg_attr(feature = "serialize", derive(Serialize))]
100#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
101pub struct Ampersand {}
102
103#[derive(Clone, Debug, PartialEq)]
104#[cfg_attr(feature = "serialize", derive(Serialize))]
105#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
106pub struct Asterisk {}
107
108#[derive(Clone, Debug, PartialEq)]
109#[cfg_attr(feature = "serialize", derive(Serialize))]
110#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
111pub struct AsteriskEqual {}
112
113/// CSS Syntax `<bad-string-token>`: a string terminated by a newline or EOF
114/// instead of its quote. Not a tokenizer error in CSS — it is a preserved
115/// token that raw component-value contexts keep verbatim.
116#[derive(Clone, Debug, PartialEq)]
117#[cfg_attr(feature = "serialize", derive(Serialize))]
118#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
119pub struct BadStr<'s> {
120    pub raw: &'s str,
121}
122
123#[derive(Clone, Debug, PartialEq)]
124#[cfg_attr(feature = "serialize", derive(Serialize))]
125#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
126pub struct At {}
127
128#[derive(Clone, Debug, PartialEq)]
129#[cfg_attr(feature = "serialize", derive(Serialize))]
130#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
131pub struct AtKeyword<'s> {
132    pub ident: Ident<'s>,
133}
134
135/// An atomic backtick-delimited template placeholder token (see
136/// [`ParserOptions::template_placeholder`](crate::config::ParserOptions)),
137/// carrying the parsed decimal index and any glued literal suffix.
138#[derive(Clone, Copy, Debug, PartialEq, Eq)]
139#[cfg_attr(feature = "serialize", derive(Serialize))]
140#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
141pub struct Placeholder<'s> {
142    pub index: u32,
143    /// An ident-continuation run glued directly after the placeholder
144    /// (`` `PLACEHOLDER-0`px `` -> index 0, suffix `"px"`), empty when none.
145    /// Mirrors `#{$x}px` being a single identifier rather than two tokens.
146    pub suffix: &'s str,
147}
148
149#[derive(Clone, Debug, PartialEq)]
150#[cfg_attr(feature = "serialize", derive(Serialize))]
151#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
152pub struct AtLBraceVar<'s> {
153    pub ident: Ident<'s>,
154}
155
156#[derive(Clone, Debug, PartialEq)]
157#[cfg_attr(feature = "serialize", derive(Serialize))]
158#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
159pub struct BacktickCode<'s> {
160    pub raw: &'s str,
161}
162
163#[derive(Clone, Debug, PartialEq)]
164#[cfg_attr(feature = "serialize", derive(Serialize))]
165#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
166pub struct Bar {}
167
168#[derive(Clone, Debug, PartialEq)]
169#[cfg_attr(feature = "serialize", derive(Serialize))]
170#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
171pub struct BarBar {}
172
173#[derive(Clone, Debug, PartialEq)]
174#[cfg_attr(feature = "serialize", derive(Serialize))]
175#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
176pub struct BarEqual {}
177
178#[derive(Clone, Debug, PartialEq)]
179#[cfg_attr(feature = "serialize", derive(Serialize))]
180#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
181pub struct CaretEqual {}
182
183/// `-->`
184#[derive(Clone, Debug, PartialEq)]
185#[cfg_attr(feature = "serialize", derive(Serialize))]
186#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
187pub struct Cdc {}
188
189/// `<!--`
190#[derive(Clone, Debug, PartialEq)]
191#[cfg_attr(feature = "serialize", derive(Serialize))]
192#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
193pub struct Cdo {}
194
195#[derive(Clone, Debug, PartialEq)]
196#[cfg_attr(feature = "serialize", derive(Serialize))]
197#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
198pub struct Colon {}
199
200#[derive(Clone, Debug, PartialEq)]
201#[cfg_attr(feature = "serialize", derive(Serialize))]
202#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
203pub struct ColonColon {}
204
205#[derive(Clone, Debug, PartialEq)]
206#[cfg_attr(feature = "serialize", derive(Serialize))]
207#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
208pub struct Comma {}
209
210#[derive(Clone, Debug, PartialEq)]
211#[cfg_attr(feature = "serialize", derive(Serialize))]
212#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
213pub struct Dedent {}
214
215#[derive(Clone, Debug, PartialEq)]
216#[cfg_attr(feature = "serialize", derive(Serialize))]
217#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
218pub struct Dimension<'s> {
219    pub value: Number<'s>,
220    pub unit: Ident<'s>,
221}
222
223#[derive(Clone, Debug, PartialEq)]
224#[cfg_attr(feature = "serialize", derive(Serialize))]
225#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
226pub struct DollarEqual {}
227
228#[derive(Clone, Debug, PartialEq)]
229#[cfg_attr(feature = "serialize", derive(Serialize))]
230#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
231pub struct DollarLBraceVar<'s> {
232    pub ident: Ident<'s>,
233}
234
235#[derive(Clone, Debug, PartialEq)]
236#[cfg_attr(feature = "serialize", derive(Serialize))]
237#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
238pub struct DollarVar<'s> {
239    pub ident: Ident<'s>,
240}
241
242#[derive(Clone, Debug, PartialEq)]
243#[cfg_attr(feature = "serialize", derive(Serialize))]
244#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
245pub struct Dot {}
246
247#[derive(Clone, Debug, PartialEq)]
248#[cfg_attr(feature = "serialize", derive(Serialize))]
249#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
250pub struct DotDotDot {}
251
252#[derive(Clone, Debug, PartialEq)]
253#[cfg_attr(feature = "serialize", derive(Serialize))]
254#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
255pub struct Eof {}
256
257#[derive(Clone, Debug, PartialEq)]
258#[cfg_attr(feature = "serialize", derive(Serialize))]
259#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
260pub struct Equal {}
261
262#[derive(Clone, Debug, PartialEq)]
263#[cfg_attr(feature = "serialize", derive(Serialize))]
264#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
265pub struct EqualEqual {}
266
267#[derive(Clone, Debug, PartialEq)]
268#[cfg_attr(feature = "serialize", derive(Serialize))]
269#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
270pub struct Exclamation {}
271
272#[derive(Clone, Debug, PartialEq)]
273#[cfg_attr(feature = "serialize", derive(Serialize))]
274#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
275pub struct ExclamationEqual {}
276
277#[derive(Clone, Debug, PartialEq)]
278#[cfg_attr(feature = "serialize", derive(Serialize))]
279#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
280pub struct GreaterThan {}
281
282#[derive(Clone, Debug, PartialEq)]
283#[cfg_attr(feature = "serialize", derive(Serialize))]
284#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
285pub struct GreaterThanEqual {}
286
287#[derive(Clone, Debug, PartialEq)]
288#[cfg_attr(feature = "serialize", derive(Serialize))]
289#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
290pub struct Hash<'s> {
291    /// raw string without beginning `#` char
292    pub raw: &'s str,
293    pub escaped: bool,
294}
295
296#[derive(Clone, Debug, PartialEq)]
297#[cfg_attr(feature = "serialize", derive(Serialize))]
298#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
299pub struct HashLBrace {}
300
301#[derive(Clone, Debug, PartialEq)]
302#[cfg_attr(feature = "serialize", derive(Serialize))]
303#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
304pub struct Ident<'s> {
305    pub escaped: bool,
306    pub raw: &'s str,
307}
308
309#[derive(Clone, Debug, PartialEq)]
310#[cfg_attr(feature = "serialize", derive(Serialize))]
311#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
312pub struct Indent {}
313
314#[derive(Clone, Debug, PartialEq)]
315#[cfg_attr(feature = "serialize", derive(Serialize))]
316#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
317pub struct LBrace {}
318
319#[derive(Clone, Debug, PartialEq)]
320#[cfg_attr(feature = "serialize", derive(Serialize))]
321#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
322pub struct LBracket {}
323
324#[derive(Clone, Debug, PartialEq)]
325#[cfg_attr(feature = "serialize", derive(Serialize))]
326#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
327pub struct LessThan {}
328
329#[derive(Clone, Debug, PartialEq)]
330#[cfg_attr(feature = "serialize", derive(Serialize))]
331#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
332pub struct LessThanEqual {}
333
334#[derive(Clone, Debug, PartialEq)]
335#[cfg_attr(feature = "serialize", derive(Serialize))]
336#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
337pub struct Linebreak {}
338
339#[derive(Clone, Debug, PartialEq)]
340#[cfg_attr(feature = "serialize", derive(Serialize))]
341#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
342pub struct LParen {}
343
344#[derive(Clone, Debug, PartialEq)]
345#[cfg_attr(feature = "serialize", derive(Serialize))]
346#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
347pub struct Minus {}
348
349#[derive(Clone, Debug, PartialEq)]
350#[cfg_attr(feature = "serialize", derive(Serialize))]
351#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
352pub struct Number<'s> {
353    pub raw: &'s str,
354}
355
356/// U+0023 `#`
357#[derive(Clone, Debug, PartialEq)]
358#[cfg_attr(feature = "serialize", derive(Serialize))]
359#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
360pub struct NumberSign {}
361
362#[derive(Clone, Debug, PartialEq)]
363#[cfg_attr(feature = "serialize", derive(Serialize))]
364#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
365pub struct Percent {}
366
367#[derive(Clone, Debug, PartialEq)]
368#[cfg_attr(feature = "serialize", derive(Serialize))]
369#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
370pub struct Percentage<'s> {
371    pub value: Number<'s>,
372}
373
374#[derive(Clone, Debug, PartialEq)]
375#[cfg_attr(feature = "serialize", derive(Serialize))]
376#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
377pub struct Plus {}
378
379#[derive(Clone, Debug, PartialEq)]
380#[cfg_attr(feature = "serialize", derive(Serialize))]
381#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
382pub struct PlusUnderscore {}
383
384#[derive(Clone, Debug, PartialEq)]
385#[cfg_attr(feature = "serialize", derive(Serialize))]
386#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
387pub struct Question {}
388
389#[derive(Clone, Debug, PartialEq)]
390#[cfg_attr(feature = "serialize", derive(Serialize))]
391#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
392pub struct RBrace {}
393
394#[derive(Clone, Debug, PartialEq)]
395#[cfg_attr(feature = "serialize", derive(Serialize))]
396#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
397pub struct RBracket {}
398
399#[derive(Clone, Debug, PartialEq)]
400#[cfg_attr(feature = "serialize", derive(Serialize))]
401#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
402pub struct RParen {}
403
404#[derive(Clone, Debug, PartialEq)]
405#[cfg_attr(feature = "serialize", derive(Serialize))]
406#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
407pub struct Semicolon {}
408
409#[derive(Clone, Debug, PartialEq)]
410#[cfg_attr(feature = "serialize", derive(Serialize))]
411#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
412pub struct Solidus {}
413
414#[derive(Clone, Debug, PartialEq)]
415#[cfg_attr(feature = "serialize", derive(Serialize))]
416#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
417pub struct Str<'s> {
418    pub raw: &'s str,
419    pub escaped: bool,
420}
421
422#[derive(Clone, Debug, PartialEq)]
423#[cfg_attr(feature = "serialize", derive(Serialize))]
424#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
425pub struct StrTemplate<'s> {
426    pub raw: &'s str,
427    pub escaped: bool,
428    pub head: bool,
429    pub tail: bool,
430}
431
432#[derive(Clone, Debug, PartialEq)]
433#[cfg_attr(feature = "serialize", derive(Serialize))]
434#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
435pub struct Tilde {}
436
437#[derive(Clone, Debug, PartialEq)]
438#[cfg_attr(feature = "serialize", derive(Serialize))]
439#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
440pub struct TildeEqual {}
441
442/// Any single code point that no other token matches (a stray `^`, a control
443/// character, ...). CSS Syntax calls this a `<delim-token>`: it is not an
444/// error at the tokenizer level, and raw component-value contexts (custom
445/// property values, unparsable declaration values) preserve it verbatim.
446#[derive(Clone, Debug, PartialEq)]
447#[cfg_attr(feature = "serialize", derive(Serialize))]
448#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
449pub struct Unknown {}
450
451#[derive(Clone, Debug, PartialEq)]
452#[cfg_attr(feature = "serialize", derive(Serialize))]
453#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
454pub struct UrlRaw<'s> {
455    pub raw: &'s str,
456    pub escaped: bool,
457}
458
459#[derive(Clone, Debug, PartialEq)]
460#[cfg_attr(feature = "serialize", derive(Serialize))]
461#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
462pub struct UrlTemplate<'s> {
463    pub raw: &'s str,
464    pub escaped: bool,
465    pub tail: bool,
466}
oxc_css_parser/tokenizer/token.rs

oxc_css_parser/tokenizer/
token.rs
