oxc_css_parser/tokenizer/
token.rs

1//! All supported tokens, and with comments.
2
3use crate::pos::Span;
4#[cfg(feature = "serialize")]
5use serde::Serialize;
6
7#[derive(Clone, Copy, Debug, PartialEq, Eq)]
8#[cfg_attr(feature = "serialize", derive(Serialize))]
9#[cfg_attr(feature = "serialize", serde(tag = "type", rename_all = "camelCase"))]
10pub struct Comment<'s> {
11    pub content: &'s str,
12    pub kind: CommentKind,
13    pub span: Span,
14}
15
16#[derive(Clone, Copy, Debug, PartialEq, Eq)]
17#[cfg_attr(feature = "serialize", derive(Serialize))]
18pub enum CommentKind {
19    Block,
20    Line,
21}
22
23#[derive(Clone, Copy, Debug, PartialEq, Eq)]
24#[cfg_attr(feature = "serialize", derive(Serialize))]
25pub enum TokenData {
26    Eof(Eof),
27    Ampersand(Ampersand),
28    BadStr(BadStrMeta),
29    Asterisk(Asterisk),
30    AsteriskEqual(AsteriskEqual),
31    At(At),
32    AtKeyword(IdentMeta),
33    AtLBraceVar(IdentMeta),
34    BacktickCode(BacktickCodeMeta),
35    Bar(Bar),
36    BarBar(BarBar),
37    BarEqual(BarEqual),
38    CaretEqual(CaretEqual),
39    Cdc(Cdc),
40    Cdo(Cdo),
41    Colon(Colon),
42    ColonColon(ColonColon),
43    Comma(Comma),
44    Dedent(Dedent),
45    Dimension(DimensionMeta),
46    DollarEqual(DollarEqual),
47    DollarLBraceVar(IdentMeta),
48    DollarVar(IdentMeta),
49    Dot(Dot),
50    DotDotDot(DotDotDot),
51    Equal(Equal),
52    EqualEqual(EqualEqual),
53    Exclamation(Exclamation),
54    ExclamationEqual(ExclamationEqual),
55    GreaterThan(GreaterThan),
56    GreaterThanEqual(GreaterThanEqual),
57    Hash(HashMeta),
58    HashLBrace(HashLBrace),
59    Ident(IdentMeta),
60    Indent(Indent),
61    LBrace(LBrace),
62    LBracket(LBracket),
63    LessThan(LessThan),
64    LessThanEqual(LessThanEqual),
65    Linebreak(Linebreak),
66    LParen(LParen),
67    Minus(Minus),
68    Number(NumberMeta),
69    NumberSign(NumberSign),
70    Percent(Percent),
71    Percentage(PercentageMeta),
72    Placeholder(PlaceholderMeta),
73    Plus(Plus),
74    PlusUnderscore(PlusUnderscore),
75    Question(Question),
76    RBrace(RBrace),
77    RBracket(RBracket),
78    RParen(RParen),
79    Semicolon(Semicolon),
80    Solidus(Solidus),
81    Str(StrMeta),
82    StrTemplate(StrTemplateMeta),
83    Tilde(Tilde),
84    TildeEqual(TildeEqual),
85    Unknown(Unknown),
86    UrlRaw(UrlRawMeta),
87    UrlTemplate(UrlTemplateMeta),
88}
89
90pub type Token<'s> = TokenData;
91
92#[derive(Clone, Copy, Debug, PartialEq, Eq)]
93#[cfg_attr(feature = "serialize", derive(Serialize))]
94#[cfg_attr(feature = "serialize", serde(tag = "type", rename_all = "camelCase"))]
95pub struct TokenWithSpanData {
96    pub token: TokenData,
97    pub span: Span,
98}
99
100pub type TokenWithSpan<'s> = TokenWithSpanData;
101
102#[derive(Clone, Copy, Debug, PartialEq, Eq)]
103#[cfg_attr(feature = "serialize", derive(Serialize))]
104pub struct BadStrMeta {}
105
106#[derive(Clone, Copy, Debug, PartialEq, Eq)]
107#[cfg_attr(feature = "serialize", derive(Serialize))]
108pub struct BacktickCodeMeta {}
109
110#[derive(Clone, Copy, Debug, PartialEq, Eq)]
111#[cfg_attr(feature = "serialize", derive(Serialize))]
112pub struct IdentMeta {
113    pub escaped: bool,
114}
115
116#[derive(Clone, Copy, Debug, PartialEq, Eq)]
117#[cfg_attr(feature = "serialize", derive(Serialize))]
118pub struct HashMeta {
119    pub escaped: bool,
120}
121
122#[derive(Clone, Copy, Debug, PartialEq, Eq)]
123#[cfg_attr(feature = "serialize", derive(Serialize))]
124pub struct NumberMeta {}
125
126#[derive(Clone, Copy, Debug, PartialEq, Eq)]
127#[cfg_attr(feature = "serialize", derive(Serialize))]
128pub struct DimensionMeta {
129    pub number_end: u32,
130    pub unit_escaped: bool,
131}
132
133#[derive(Clone, Copy, Debug, PartialEq, Eq)]
134#[cfg_attr(feature = "serialize", derive(Serialize))]
135pub struct PercentageMeta {}
136
137#[derive(Clone, Copy, Debug, PartialEq, Eq)]
138#[cfg_attr(feature = "serialize", derive(Serialize))]
139pub struct PlaceholderMeta {
140    pub index: u32,
141    pub suffix_start: u32,
142}
143
144#[derive(Clone, Copy, Debug, PartialEq, Eq)]
145#[cfg_attr(feature = "serialize", derive(Serialize))]
146pub struct StrMeta {
147    pub escaped: bool,
148}
149
150#[derive(Clone, Copy, Debug, PartialEq, Eq)]
151#[cfg_attr(feature = "serialize", derive(Serialize))]
152pub struct StrTemplateMeta {
153    pub escaped: bool,
154    pub head: bool,
155    pub tail: bool,
156}
157
158#[derive(Clone, Copy, Debug, PartialEq, Eq)]
159#[cfg_attr(feature = "serialize", derive(Serialize))]
160pub struct UrlMeta {
161    pub escaped: bool,
162}
163
164pub type UrlRawMeta = UrlMeta;
165
166#[derive(Clone, Copy, Debug, PartialEq, Eq)]
167#[cfg_attr(feature = "serialize", derive(Serialize))]
168pub struct UrlTemplateMeta {
169    pub escaped: bool,
170    pub tail: bool,
171}
172
173impl TokenWithSpanData {
174    #[inline]
175    pub fn raw<'s>(&self, source: &'s str) -> &'s str {
176        &source[self.span.start..self.span.end]
177    }
178
179    #[inline]
180    pub fn bad_str<'s>(&self, source: &'s str) -> Option<BadStr<'s>> {
181        matches!(self.token, TokenData::BadStr(..)).then(|| BadStr { raw: self.raw(source) })
182    }
183
184    #[inline]
185    pub fn at_keyword<'s>(&self, source: &'s str) -> Option<AtKeyword<'s>> {
186        match self.token {
187            TokenData::AtKeyword(meta) => Some(AtKeyword {
188                ident: Ident {
189                    raw: &source[self.span.start + 1..self.span.end],
190                    escaped: meta.escaped,
191                },
192            }),
193            _ => None,
194        }
195    }
196
197    #[inline]
198    pub fn at_l_brace_var<'s>(&self, source: &'s str) -> Option<AtLBraceVar<'s>> {
199        match self.token {
200            TokenData::AtLBraceVar(meta) => Some(AtLBraceVar {
201                ident: Ident {
202                    raw: &source[self.span.start + 2..self.span.end - 1],
203                    escaped: meta.escaped,
204                },
205            }),
206            _ => None,
207        }
208    }
209
210    #[inline]
211    pub fn backtick_code<'s>(&self, source: &'s str) -> Option<BacktickCode<'s>> {
212        matches!(self.token, TokenData::BacktickCode(..))
213            .then(|| BacktickCode { raw: self.raw(source) })
214    }
215
216    #[inline]
217    pub fn dimension<'s>(&self, source: &'s str) -> Option<Dimension<'s>> {
218        match self.token {
219            TokenData::Dimension(meta) => {
220                let number_end = meta.number_end as usize;
221                Some(Dimension {
222                    value: Number { raw: &source[self.span.start..number_end] },
223                    unit: Ident {
224                        raw: &source[number_end..self.span.end],
225                        escaped: meta.unit_escaped,
226                    },
227                })
228            }
229            _ => None,
230        }
231    }
232
233    #[inline]
234    pub fn dimension_value_raw<'s>(&self, source: &'s str) -> Option<&'s str> {
235        match self.token {
236            TokenData::Dimension(meta) => Some(&source[self.span.start..meta.number_end as usize]),
237            _ => None,
238        }
239    }
240
241    #[inline]
242    pub fn dollar_l_brace_var<'s>(&self, source: &'s str) -> Option<DollarLBraceVar<'s>> {
243        match self.token {
244            TokenData::DollarLBraceVar(meta) => Some(DollarLBraceVar {
245                ident: Ident {
246                    raw: &source[self.span.start + 2..self.span.end - 1],
247                    escaped: meta.escaped,
248                },
249            }),
250            _ => None,
251        }
252    }
253
254    #[inline]
255    pub fn dollar_var<'s>(&self, source: &'s str) -> Option<DollarVar<'s>> {
256        match self.token {
257            TokenData::DollarVar(meta) => Some(DollarVar {
258                ident: Ident {
259                    raw: &source[self.span.start + 1..self.span.end],
260                    escaped: meta.escaped,
261                },
262            }),
263            _ => None,
264        }
265    }
266
267    #[inline]
268    pub fn hash<'s>(&self, source: &'s str) -> Option<Hash<'s>> {
269        match self.token {
270            TokenData::Hash(meta) => Some(Hash {
271                raw: &source[self.span.start + 1..self.span.end],
272                escaped: meta.escaped,
273            }),
274            _ => None,
275        }
276    }
277
278    #[inline]
279    pub fn hash_raw<'s>(&self, source: &'s str) -> Option<&'s str> {
280        matches!(self.token, TokenData::Hash(..))
281            .then(|| &source[self.span.start + 1..self.span.end])
282    }
283
284    #[inline]
285    pub fn ident<'s>(&self, source: &'s str) -> Option<Ident<'s>> {
286        match self.token {
287            TokenData::Ident(meta) => Some(Ident { raw: self.raw(source), escaped: meta.escaped }),
288            _ => None,
289        }
290    }
291
292    #[inline]
293    pub fn ident_raw<'s>(&self, source: &'s str) -> Option<&'s str> {
294        matches!(self.token, TokenData::Ident(..)).then(|| self.raw(source))
295    }
296
297    #[inline]
298    pub fn is_ident_raw(&self, source: &str, expected: &str) -> bool {
299        self.ident_raw(source).is_some_and(|raw| raw == expected)
300    }
301
302    #[inline]
303    pub fn is_ident_raw_starts_with(&self, source: &str, prefix: &str) -> bool {
304        self.ident_raw(source).is_some_and(|raw| raw.starts_with(prefix))
305    }
306
307    #[inline]
308    pub fn is_ident_name_eq_ignore_ascii_case(&self, source: &str, expected: &str) -> bool {
309        self.ident(source).is_some_and(|ident| ident.name().eq_ignore_ascii_case(expected))
310    }
311
312    #[inline]
313    pub fn number<'s>(&self, source: &'s str) -> Option<Number<'s>> {
314        matches!(self.token, TokenData::Number(..)).then(|| Number { raw: self.raw(source) })
315    }
316
317    #[inline]
318    pub fn number_raw<'s>(&self, source: &'s str) -> Option<&'s str> {
319        matches!(self.token, TokenData::Number(..)).then(|| self.raw(source))
320    }
321
322    #[inline]
323    pub fn percentage<'s>(&self, source: &'s str) -> Option<Percentage<'s>> {
324        matches!(self.token, TokenData::Percentage(..)).then(|| Percentage {
325            value: Number { raw: &source[self.span.start..self.span.end - 1] },
326        })
327    }
328
329    #[inline]
330    pub fn placeholder<'s>(&self, source: &'s str) -> Option<Placeholder<'s>> {
331        match self.token {
332            TokenData::Placeholder(meta) => Some(Placeholder {
333                index: meta.index,
334                suffix: &source[meta.suffix_start as usize..self.span.end],
335            }),
336            _ => None,
337        }
338    }
339
340    #[inline]
341    pub fn str<'s>(&self, source: &'s str) -> Option<Str<'s>> {
342        match self.token {
343            TokenData::Str(meta) => Some(Str { raw: self.raw(source), escaped: meta.escaped }),
344            _ => None,
345        }
346    }
347
348    #[inline]
349    pub fn str_template<'s>(&self, source: &'s str) -> Option<StrTemplate<'s>> {
350        match self.token {
351            TokenData::StrTemplate(meta) => Some(StrTemplate {
352                raw: self.raw(source),
353                escaped: meta.escaped,
354                head: meta.head,
355                tail: meta.tail,
356            }),
357            _ => None,
358        }
359    }
360
361    #[inline]
362    pub fn url_raw<'s>(&self, source: &'s str) -> Option<UrlRaw<'s>> {
363        match self.token {
364            TokenData::UrlRaw(meta) => {
365                Some(UrlRaw { raw: self.raw(source), escaped: meta.escaped })
366            }
367            _ => None,
368        }
369    }
370
371    #[inline]
372    pub fn url_template<'s>(&self, source: &'s str) -> Option<UrlTemplate<'s>> {
373        match self.token {
374            TokenData::UrlTemplate(meta) => {
375                Some(UrlTemplate { raw: self.raw(source), escaped: meta.escaped, tail: meta.tail })
376            }
377            _ => None,
378        }
379    }
380}
381
382#[derive(Clone, Copy, Debug, PartialEq, Eq)]
383#[cfg_attr(feature = "serialize", derive(Serialize))]
384#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
385pub struct Ampersand {}
386
387#[derive(Clone, Copy, Debug, PartialEq, Eq)]
388#[cfg_attr(feature = "serialize", derive(Serialize))]
389#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
390pub struct Asterisk {}
391
392#[derive(Clone, Copy, Debug, PartialEq, Eq)]
393#[cfg_attr(feature = "serialize", derive(Serialize))]
394#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
395pub struct AsteriskEqual {}
396
397/// CSS Syntax `<bad-string-token>`: a string terminated by a newline or EOF
398/// instead of its quote. Not a tokenizer error in CSS — it is a preserved
399/// token that raw component-value contexts keep verbatim.
400#[derive(Clone, Copy, Debug, PartialEq, Eq)]
401#[cfg_attr(feature = "serialize", derive(Serialize))]
402#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
403pub struct BadStr<'s> {
404    pub raw: &'s str,
405}
406
407#[derive(Clone, Copy, Debug, PartialEq, Eq)]
408#[cfg_attr(feature = "serialize", derive(Serialize))]
409#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
410pub struct At {}
411
412#[derive(Clone, Copy, Debug, PartialEq, Eq)]
413#[cfg_attr(feature = "serialize", derive(Serialize))]
414#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
415pub struct AtKeyword<'s> {
416    pub ident: Ident<'s>,
417}
418
419/// An atomic backtick-delimited template placeholder token (see
420/// [`ParserOptions::template_placeholder`](crate::config::ParserOptions)),
421/// carrying the parsed decimal index and any glued literal suffix.
422#[derive(Clone, Copy, Debug, PartialEq, Eq)]
423#[cfg_attr(feature = "serialize", derive(Serialize))]
424#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
425pub struct Placeholder<'s> {
426    pub index: u32,
427    /// An ident-continuation run glued directly after the placeholder
428    /// (`` `PLACEHOLDER-0`px `` -> index 0, suffix `"px"`), empty when none.
429    /// Mirrors `#{$x}px` being a single identifier rather than two tokens.
430    pub suffix: &'s str,
431}
432
433#[derive(Clone, Copy, Debug, PartialEq, Eq)]
434#[cfg_attr(feature = "serialize", derive(Serialize))]
435#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
436pub struct AtLBraceVar<'s> {
437    pub ident: Ident<'s>,
438}
439
440#[derive(Clone, Copy, Debug, PartialEq, Eq)]
441#[cfg_attr(feature = "serialize", derive(Serialize))]
442#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
443pub struct BacktickCode<'s> {
444    pub raw: &'s str,
445}
446
447#[derive(Clone, Copy, Debug, PartialEq, Eq)]
448#[cfg_attr(feature = "serialize", derive(Serialize))]
449#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
450pub struct Bar {}
451
452#[derive(Clone, Copy, Debug, PartialEq, Eq)]
453#[cfg_attr(feature = "serialize", derive(Serialize))]
454#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
455pub struct BarBar {}
456
457#[derive(Clone, Copy, Debug, PartialEq, Eq)]
458#[cfg_attr(feature = "serialize", derive(Serialize))]
459#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
460pub struct BarEqual {}
461
462#[derive(Clone, Copy, Debug, PartialEq, Eq)]
463#[cfg_attr(feature = "serialize", derive(Serialize))]
464#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
465pub struct CaretEqual {}
466
467/// `-->`
468#[derive(Clone, Copy, Debug, PartialEq, Eq)]
469#[cfg_attr(feature = "serialize", derive(Serialize))]
470#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
471pub struct Cdc {}
472
473/// `<!--`
474#[derive(Clone, Copy, Debug, PartialEq, Eq)]
475#[cfg_attr(feature = "serialize", derive(Serialize))]
476#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
477pub struct Cdo {}
478
479#[derive(Clone, Copy, Debug, PartialEq, Eq)]
480#[cfg_attr(feature = "serialize", derive(Serialize))]
481#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
482pub struct Colon {}
483
484#[derive(Clone, Copy, Debug, PartialEq, Eq)]
485#[cfg_attr(feature = "serialize", derive(Serialize))]
486#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
487pub struct ColonColon {}
488
489#[derive(Clone, Copy, Debug, PartialEq, Eq)]
490#[cfg_attr(feature = "serialize", derive(Serialize))]
491#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
492pub struct Comma {}
493
494#[derive(Clone, Copy, Debug, PartialEq, Eq)]
495#[cfg_attr(feature = "serialize", derive(Serialize))]
496#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
497pub struct Dedent {}
498
499#[derive(Clone, Copy, Debug, PartialEq, Eq)]
500#[cfg_attr(feature = "serialize", derive(Serialize))]
501#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
502pub struct Dimension<'s> {
503    pub value: Number<'s>,
504    pub unit: Ident<'s>,
505}
506
507#[derive(Clone, Copy, Debug, PartialEq, Eq)]
508#[cfg_attr(feature = "serialize", derive(Serialize))]
509#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
510pub struct DollarEqual {}
511
512#[derive(Clone, Copy, Debug, PartialEq, Eq)]
513#[cfg_attr(feature = "serialize", derive(Serialize))]
514#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
515pub struct DollarLBraceVar<'s> {
516    pub ident: Ident<'s>,
517}
518
519#[derive(Clone, Copy, Debug, PartialEq, Eq)]
520#[cfg_attr(feature = "serialize", derive(Serialize))]
521#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
522pub struct DollarVar<'s> {
523    pub ident: Ident<'s>,
524}
525
526#[derive(Clone, Copy, Debug, PartialEq, Eq)]
527#[cfg_attr(feature = "serialize", derive(Serialize))]
528#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
529pub struct Dot {}
530
531#[derive(Clone, Copy, Debug, PartialEq, Eq)]
532#[cfg_attr(feature = "serialize", derive(Serialize))]
533#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
534pub struct DotDotDot {}
535
536#[derive(Clone, Copy, Debug, PartialEq, Eq)]
537#[cfg_attr(feature = "serialize", derive(Serialize))]
538#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
539pub struct Eof {}
540
541#[derive(Clone, Copy, Debug, PartialEq, Eq)]
542#[cfg_attr(feature = "serialize", derive(Serialize))]
543#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
544pub struct Equal {}
545
546#[derive(Clone, Copy, Debug, PartialEq, Eq)]
547#[cfg_attr(feature = "serialize", derive(Serialize))]
548#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
549pub struct EqualEqual {}
550
551#[derive(Clone, Copy, Debug, PartialEq, Eq)]
552#[cfg_attr(feature = "serialize", derive(Serialize))]
553#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
554pub struct Exclamation {}
555
556#[derive(Clone, Copy, Debug, PartialEq, Eq)]
557#[cfg_attr(feature = "serialize", derive(Serialize))]
558#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
559pub struct ExclamationEqual {}
560
561#[derive(Clone, Copy, Debug, PartialEq, Eq)]
562#[cfg_attr(feature = "serialize", derive(Serialize))]
563#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
564pub struct GreaterThan {}
565
566#[derive(Clone, Copy, Debug, PartialEq, Eq)]
567#[cfg_attr(feature = "serialize", derive(Serialize))]
568#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
569pub struct GreaterThanEqual {}
570
571#[derive(Clone, Copy, Debug, PartialEq, Eq)]
572#[cfg_attr(feature = "serialize", derive(Serialize))]
573#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
574pub struct Hash<'s> {
575    /// raw string without beginning `#` char
576    pub raw: &'s str,
577    pub escaped: bool,
578}
579
580#[derive(Clone, Copy, Debug, PartialEq, Eq)]
581#[cfg_attr(feature = "serialize", derive(Serialize))]
582#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
583pub struct HashLBrace {}
584
585#[derive(Clone, Copy, Debug, PartialEq, Eq)]
586#[cfg_attr(feature = "serialize", derive(Serialize))]
587#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
588pub struct Ident<'s> {
589    pub escaped: bool,
590    pub raw: &'s str,
591}
592
593#[derive(Clone, Copy, Debug, PartialEq, Eq)]
594#[cfg_attr(feature = "serialize", derive(Serialize))]
595#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
596pub struct Indent {}
597
598#[derive(Clone, Copy, Debug, PartialEq, Eq)]
599#[cfg_attr(feature = "serialize", derive(Serialize))]
600#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
601pub struct LBrace {}
602
603#[derive(Clone, Copy, Debug, PartialEq, Eq)]
604#[cfg_attr(feature = "serialize", derive(Serialize))]
605#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
606pub struct LBracket {}
607
608#[derive(Clone, Copy, Debug, PartialEq, Eq)]
609#[cfg_attr(feature = "serialize", derive(Serialize))]
610#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
611pub struct LessThan {}
612
613#[derive(Clone, Copy, Debug, PartialEq, Eq)]
614#[cfg_attr(feature = "serialize", derive(Serialize))]
615#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
616pub struct LessThanEqual {}
617
618#[derive(Clone, Copy, Debug, PartialEq, Eq)]
619#[cfg_attr(feature = "serialize", derive(Serialize))]
620#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
621pub struct Linebreak {}
622
623#[derive(Clone, Copy, Debug, PartialEq, Eq)]
624#[cfg_attr(feature = "serialize", derive(Serialize))]
625#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
626pub struct LParen {}
627
628#[derive(Clone, Copy, Debug, PartialEq, Eq)]
629#[cfg_attr(feature = "serialize", derive(Serialize))]
630#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
631pub struct Minus {}
632
633#[derive(Clone, Copy, Debug, PartialEq, Eq)]
634#[cfg_attr(feature = "serialize", derive(Serialize))]
635#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
636pub struct Number<'s> {
637    pub raw: &'s str,
638}
639
640/// U+0023 `#`
641#[derive(Clone, Copy, Debug, PartialEq, Eq)]
642#[cfg_attr(feature = "serialize", derive(Serialize))]
643#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
644pub struct NumberSign {}
645
646#[derive(Clone, Copy, Debug, PartialEq, Eq)]
647#[cfg_attr(feature = "serialize", derive(Serialize))]
648#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
649pub struct Percent {}
650
651#[derive(Clone, Copy, Debug, PartialEq, Eq)]
652#[cfg_attr(feature = "serialize", derive(Serialize))]
653#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
654pub struct Percentage<'s> {
655    pub value: Number<'s>,
656}
657
658#[derive(Clone, Copy, Debug, PartialEq, Eq)]
659#[cfg_attr(feature = "serialize", derive(Serialize))]
660#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
661pub struct Plus {}
662
663#[derive(Clone, Copy, Debug, PartialEq, Eq)]
664#[cfg_attr(feature = "serialize", derive(Serialize))]
665#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
666pub struct PlusUnderscore {}
667
668#[derive(Clone, Copy, Debug, PartialEq, Eq)]
669#[cfg_attr(feature = "serialize", derive(Serialize))]
670#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
671pub struct Question {}
672
673#[derive(Clone, Copy, Debug, PartialEq, Eq)]
674#[cfg_attr(feature = "serialize", derive(Serialize))]
675#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
676pub struct RBrace {}
677
678#[derive(Clone, Copy, Debug, PartialEq, Eq)]
679#[cfg_attr(feature = "serialize", derive(Serialize))]
680#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
681pub struct RBracket {}
682
683#[derive(Clone, Copy, Debug, PartialEq, Eq)]
684#[cfg_attr(feature = "serialize", derive(Serialize))]
685#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
686pub struct RParen {}
687
688#[derive(Clone, Copy, Debug, PartialEq, Eq)]
689#[cfg_attr(feature = "serialize", derive(Serialize))]
690#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
691pub struct Semicolon {}
692
693#[derive(Clone, Copy, Debug, PartialEq, Eq)]
694#[cfg_attr(feature = "serialize", derive(Serialize))]
695#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
696pub struct Solidus {}
697
698#[derive(Clone, Copy, Debug, PartialEq, Eq)]
699#[cfg_attr(feature = "serialize", derive(Serialize))]
700#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
701pub struct Str<'s> {
702    pub raw: &'s str,
703    pub escaped: bool,
704}
705
706#[derive(Clone, Copy, Debug, PartialEq, Eq)]
707#[cfg_attr(feature = "serialize", derive(Serialize))]
708#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
709pub struct StrTemplate<'s> {
710    pub raw: &'s str,
711    pub escaped: bool,
712    pub head: bool,
713    pub tail: bool,
714}
715
716#[derive(Clone, Copy, Debug, PartialEq, Eq)]
717#[cfg_attr(feature = "serialize", derive(Serialize))]
718#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
719pub struct Tilde {}
720
721#[derive(Clone, Copy, Debug, PartialEq, Eq)]
722#[cfg_attr(feature = "serialize", derive(Serialize))]
723#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
724pub struct TildeEqual {}
725
726/// Any single code point that no other token matches (a stray `^`, a control
727/// character, ...). CSS Syntax calls this a `<delim-token>`: it is not an
728/// error at the tokenizer level, and raw component-value contexts (custom
729/// property values, unparsable declaration values) preserve it verbatim.
730#[derive(Clone, Copy, Debug, PartialEq, Eq)]
731#[cfg_attr(feature = "serialize", derive(Serialize))]
732#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
733pub struct Unknown {}
734
735#[derive(Clone, Copy, Debug, PartialEq, Eq)]
736#[cfg_attr(feature = "serialize", derive(Serialize))]
737#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
738pub struct UrlRaw<'s> {
739    pub raw: &'s str,
740    pub escaped: bool,
741}
742
743#[derive(Clone, Copy, Debug, PartialEq, Eq)]
744#[cfg_attr(feature = "serialize", derive(Serialize))]
745#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
746pub struct UrlTemplate<'s> {
747    pub raw: &'s str,
748    pub escaped: bool,
749    pub tail: bool,
750}
751
752#[cfg(test)]
753mod test {
754    use super::{TokenData, TokenWithSpanData};
755
756    const _: () = assert!(size_of::<TokenData>() <= 12);
757    const _: () = assert!(size_of::<TokenWithSpanData>() <= 32);
758}
oxc_css_parser/tokenizer/token.rs

oxc_css_parser/tokenizer/
token.rs
