raffia/tokenizer/
token.rs

1//! All supported tokens, and with comments.
2
3use crate::pos::Span;
4use raffia_macro::{EnumAsIs, SpanIgnoredEq, Spanned};
5#[cfg(feature = "serialize")]
6use serde::Serialize;
7
8#[derive(Clone, Debug, Spanned, PartialEq, SpanIgnoredEq)]
9#[cfg_attr(feature = "serialize", derive(Serialize))]
10#[cfg_attr(feature = "serialize", serde(tag = "type", rename_all = "camelCase"))]
11pub struct Comment<'s> {
12    pub content: &'s str,
13    pub kind: CommentKind,
14    pub span: Span,
15}
16
17#[derive(Clone, Debug, Spanned, PartialEq, SpanIgnoredEq, EnumAsIs)]
18#[cfg_attr(feature = "serialize", derive(Serialize))]
19pub enum CommentKind {
20    Block,
21    Line,
22}
23
24#[derive(Clone, Debug, PartialEq, EnumAsIs)]
25#[cfg_attr(feature = "serialize", derive(Serialize))]
26pub enum Token<'s> {
27    Eof(Eof),
28    Ampersand(Ampersand),
29    Asterisk(Asterisk),
30    AsteriskEqual(AsteriskEqual),
31    At(At),
32    AtKeyword(AtKeyword<'s>),
33    AtLBraceVar(AtLBraceVar<'s>),
34    BacktickCode(BacktickCode<'s>),
35    Bar(Bar),
36    BarBar(BarBar),
37    BarEqual(BarEqual),
38    CaretEqual(CaretEqual),
39    Cdc(Cdc),
40    Cdo(Cdo),
41    Colon(Colon),
42    ColonColon(ColonColon),
43    Comma(Comma),
44    Dedent(Dedent),
45    Dimension(Dimension<'s>),
46    DollarEqual(DollarEqual),
47    DollarLBraceVar(DollarLBraceVar<'s>),
48    DollarVar(DollarVar<'s>),
49    Dot(Dot),
50    DotDotDot(DotDotDot),
51    Equal(Equal),
52    EqualEqual(EqualEqual),
53    Exclamation(Exclamation),
54    ExclamationEqual(ExclamationEqual),
55    GreaterThan(GreaterThan),
56    GreaterThanEqual(GreaterThanEqual),
57    Hash(Hash<'s>),
58    HashLBrace(HashLBrace),
59    Ident(Ident<'s>),
60    Indent(Indent),
61    LBrace(LBrace),
62    LBracket(LBracket),
63    LessThan(LessThan),
64    LessThanEqual(LessThanEqual),
65    Linebreak(Linebreak),
66    LParen(LParen),
67    Minus(Minus),
68    Number(Number<'s>),
69    NumberSign(NumberSign),
70    Percent(Percent),
71    Percentage(Percentage<'s>),
72    Plus(Plus),
73    PlusUnderscore(PlusUnderscore),
74    Question(Question),
75    RBrace(RBrace),
76    RBracket(RBracket),
77    RParen(RParen),
78    Semicolon(Semicolon),
79    Solidus(Solidus),
80    Str(Str<'s>),
81    StrTemplate(StrTemplate<'s>),
82    Tilde(Tilde),
83    TildeEqual(TildeEqual),
84    UrlRaw(UrlRaw<'s>),
85    UrlTemplate(UrlTemplate<'s>),
86}
87
88#[derive(Clone, Debug, Spanned, PartialEq)]
89#[cfg_attr(feature = "serialize", derive(Serialize))]
90#[cfg_attr(feature = "serialize", serde(tag = "type", rename_all = "camelCase"))]
91pub struct TokenWithSpan<'s> {
92    pub token: Token<'s>,
93    pub span: Span,
94}
95
96#[derive(Clone, Debug, PartialEq)]
97#[cfg_attr(feature = "serialize", derive(Serialize))]
98#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
99pub struct Ampersand {}
100
101#[derive(Clone, Debug, PartialEq)]
102#[cfg_attr(feature = "serialize", derive(Serialize))]
103#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
104pub struct Asterisk {}
105
106#[derive(Clone, Debug, PartialEq)]
107#[cfg_attr(feature = "serialize", derive(Serialize))]
108#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
109pub struct AsteriskEqual {}
110
111#[derive(Clone, Debug, PartialEq)]
112#[cfg_attr(feature = "serialize", derive(Serialize))]
113#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
114pub struct At {}
115
116#[derive(Clone, Debug, PartialEq)]
117#[cfg_attr(feature = "serialize", derive(Serialize))]
118#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
119pub struct AtKeyword<'s> {
120    pub ident: Ident<'s>,
121}
122
123#[derive(Clone, Debug, PartialEq)]
124#[cfg_attr(feature = "serialize", derive(Serialize))]
125#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
126pub struct AtLBraceVar<'s> {
127    pub ident: Ident<'s>,
128}
129
130#[derive(Clone, Debug, PartialEq)]
131#[cfg_attr(feature = "serialize", derive(Serialize))]
132#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
133pub struct BacktickCode<'s> {
134    pub raw: &'s str,
135}
136
137#[derive(Clone, Debug, PartialEq)]
138#[cfg_attr(feature = "serialize", derive(Serialize))]
139#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
140pub struct Bar {}
141
142#[derive(Clone, Debug, PartialEq)]
143#[cfg_attr(feature = "serialize", derive(Serialize))]
144#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
145pub struct BarBar {}
146
147#[derive(Clone, Debug, PartialEq)]
148#[cfg_attr(feature = "serialize", derive(Serialize))]
149#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
150pub struct BarEqual {}
151
152#[derive(Clone, Debug, PartialEq)]
153#[cfg_attr(feature = "serialize", derive(Serialize))]
154#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
155pub struct CaretEqual {}
156
157/// `-->`
158#[derive(Clone, Debug, PartialEq)]
159#[cfg_attr(feature = "serialize", derive(Serialize))]
160#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
161pub struct Cdc {}
162
163/// `<!--`
164#[derive(Clone, Debug, PartialEq)]
165#[cfg_attr(feature = "serialize", derive(Serialize))]
166#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
167pub struct Cdo {}
168
169#[derive(Clone, Debug, PartialEq)]
170#[cfg_attr(feature = "serialize", derive(Serialize))]
171#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
172pub struct Colon {}
173
174#[derive(Clone, Debug, PartialEq)]
175#[cfg_attr(feature = "serialize", derive(Serialize))]
176#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
177pub struct ColonColon {}
178
179#[derive(Clone, Debug, PartialEq)]
180#[cfg_attr(feature = "serialize", derive(Serialize))]
181#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
182pub struct Comma {}
183
184#[derive(Clone, Debug, PartialEq)]
185#[cfg_attr(feature = "serialize", derive(Serialize))]
186#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
187pub struct Dedent {}
188
189#[derive(Clone, Debug, PartialEq)]
190#[cfg_attr(feature = "serialize", derive(Serialize))]
191#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
192pub struct Dimension<'s> {
193    pub value: Number<'s>,
194    pub unit: Ident<'s>,
195}
196
197#[derive(Clone, Debug, PartialEq)]
198#[cfg_attr(feature = "serialize", derive(Serialize))]
199#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
200pub struct DollarEqual {}
201
202#[derive(Clone, Debug, PartialEq)]
203#[cfg_attr(feature = "serialize", derive(Serialize))]
204#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
205pub struct DollarLBraceVar<'s> {
206    pub ident: Ident<'s>,
207}
208
209#[derive(Clone, Debug, PartialEq)]
210#[cfg_attr(feature = "serialize", derive(Serialize))]
211#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
212pub struct DollarVar<'s> {
213    pub ident: Ident<'s>,
214}
215
216#[derive(Clone, Debug, PartialEq)]
217#[cfg_attr(feature = "serialize", derive(Serialize))]
218#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
219pub struct Dot {}
220
221#[derive(Clone, Debug, PartialEq)]
222#[cfg_attr(feature = "serialize", derive(Serialize))]
223#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
224pub struct DotDotDot {}
225
226#[derive(Clone, Debug, PartialEq)]
227#[cfg_attr(feature = "serialize", derive(Serialize))]
228#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
229pub struct Eof {}
230
231#[derive(Clone, Debug, PartialEq)]
232#[cfg_attr(feature = "serialize", derive(Serialize))]
233#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
234pub struct Equal {}
235
236#[derive(Clone, Debug, PartialEq)]
237#[cfg_attr(feature = "serialize", derive(Serialize))]
238#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
239pub struct EqualEqual {}
240
241#[derive(Clone, Debug, PartialEq)]
242#[cfg_attr(feature = "serialize", derive(Serialize))]
243#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
244pub struct Exclamation {}
245
246#[derive(Clone, Debug, PartialEq)]
247#[cfg_attr(feature = "serialize", derive(Serialize))]
248#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
249pub struct ExclamationEqual {}
250
251#[derive(Clone, Debug, PartialEq)]
252#[cfg_attr(feature = "serialize", derive(Serialize))]
253#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
254pub struct GreaterThan {}
255
256#[derive(Clone, Debug, PartialEq)]
257#[cfg_attr(feature = "serialize", derive(Serialize))]
258#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
259pub struct GreaterThanEqual {}
260
261#[derive(Clone, Debug, PartialEq)]
262#[cfg_attr(feature = "serialize", derive(Serialize))]
263#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
264pub struct Hash<'s> {
265    /// raw string without beginning `#` char
266    pub raw: &'s str,
267    pub escaped: bool,
268}
269
270#[derive(Clone, Debug, PartialEq)]
271#[cfg_attr(feature = "serialize", derive(Serialize))]
272#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
273pub struct HashLBrace {}
274
275#[derive(Clone, Debug, PartialEq)]
276#[cfg_attr(feature = "serialize", derive(Serialize))]
277#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
278pub struct Ident<'s> {
279    pub escaped: bool,
280    pub raw: &'s str,
281}
282
283#[derive(Clone, Debug, PartialEq)]
284#[cfg_attr(feature = "serialize", derive(Serialize))]
285#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
286pub struct Indent {}
287
288#[derive(Clone, Debug, PartialEq)]
289#[cfg_attr(feature = "serialize", derive(Serialize))]
290#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
291pub struct LBrace {}
292
293#[derive(Clone, Debug, PartialEq)]
294#[cfg_attr(feature = "serialize", derive(Serialize))]
295#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
296pub struct LBracket {}
297
298#[derive(Clone, Debug, PartialEq)]
299#[cfg_attr(feature = "serialize", derive(Serialize))]
300#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
301pub struct LessThan {}
302
303#[derive(Clone, Debug, PartialEq)]
304#[cfg_attr(feature = "serialize", derive(Serialize))]
305#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
306pub struct LessThanEqual {}
307
308#[derive(Clone, Debug, PartialEq)]
309#[cfg_attr(feature = "serialize", derive(Serialize))]
310#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
311pub struct Linebreak {}
312
313#[derive(Clone, Debug, PartialEq)]
314#[cfg_attr(feature = "serialize", derive(Serialize))]
315#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
316pub struct LParen {}
317
318#[derive(Clone, Debug, PartialEq)]
319#[cfg_attr(feature = "serialize", derive(Serialize))]
320#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
321pub struct Minus {}
322
323#[derive(Clone, Debug, PartialEq)]
324#[cfg_attr(feature = "serialize", derive(Serialize))]
325#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
326pub struct Number<'s> {
327    pub raw: &'s str,
328}
329
330/// U+0023 `#`
331#[derive(Clone, Debug, PartialEq)]
332#[cfg_attr(feature = "serialize", derive(Serialize))]
333#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
334pub struct NumberSign {}
335
336#[derive(Clone, Debug, PartialEq)]
337#[cfg_attr(feature = "serialize", derive(Serialize))]
338#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
339pub struct Percent {}
340
341#[derive(Clone, Debug, PartialEq)]
342#[cfg_attr(feature = "serialize", derive(Serialize))]
343#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
344pub struct Percentage<'s> {
345    pub value: Number<'s>,
346}
347
348#[derive(Clone, Debug, PartialEq)]
349#[cfg_attr(feature = "serialize", derive(Serialize))]
350#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
351pub struct Plus {}
352
353#[derive(Clone, Debug, PartialEq)]
354#[cfg_attr(feature = "serialize", derive(Serialize))]
355#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
356pub struct PlusUnderscore {}
357
358#[derive(Clone, Debug, PartialEq)]
359#[cfg_attr(feature = "serialize", derive(Serialize))]
360#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
361pub struct Question {}
362
363#[derive(Clone, Debug, PartialEq)]
364#[cfg_attr(feature = "serialize", derive(Serialize))]
365#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
366pub struct RBrace {}
367
368#[derive(Clone, Debug, PartialEq)]
369#[cfg_attr(feature = "serialize", derive(Serialize))]
370#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
371pub struct RBracket {}
372
373#[derive(Clone, Debug, PartialEq)]
374#[cfg_attr(feature = "serialize", derive(Serialize))]
375#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
376pub struct RParen {}
377
378#[derive(Clone, Debug, PartialEq)]
379#[cfg_attr(feature = "serialize", derive(Serialize))]
380#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
381pub struct Semicolon {}
382
383#[derive(Clone, Debug, PartialEq)]
384#[cfg_attr(feature = "serialize", derive(Serialize))]
385#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
386pub struct Solidus {}
387
388#[derive(Clone, Debug, PartialEq)]
389#[cfg_attr(feature = "serialize", derive(Serialize))]
390#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
391pub struct Str<'s> {
392    pub raw: &'s str,
393    pub escaped: bool,
394}
395
396#[derive(Clone, Debug, PartialEq)]
397#[cfg_attr(feature = "serialize", derive(Serialize))]
398#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
399pub struct StrTemplate<'s> {
400    pub raw: &'s str,
401    pub escaped: bool,
402    pub head: bool,
403    pub tail: bool,
404}
405
406#[derive(Clone, Debug, PartialEq)]
407#[cfg_attr(feature = "serialize", derive(Serialize))]
408#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
409pub struct Tilde {}
410
411#[derive(Clone, Debug, PartialEq)]
412#[cfg_attr(feature = "serialize", derive(Serialize))]
413#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
414pub struct TildeEqual {}
415
416#[derive(Clone, Debug, PartialEq)]
417#[cfg_attr(feature = "serialize", derive(Serialize))]
418#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
419pub struct UrlRaw<'s> {
420    pub raw: &'s str,
421    pub escaped: bool,
422}
423
424#[derive(Clone, Debug, PartialEq)]
425#[cfg_attr(feature = "serialize", derive(Serialize))]
426#[cfg_attr(feature = "serialize", serde(tag = "kind", rename_all = "camelCase"))]
427pub struct UrlTemplate<'s> {
428    pub raw: &'s str,
429    pub escaped: bool,
430    pub tail: bool,
431}
raffia/tokenizer/token.rs

raffia/tokenizer/
token.rs
