1use std::{iter::Rev, str::Chars};
2
3use crate::Range;
4
5pub const C_LINE_FEED: char = '\n';
6pub const C_CARRIAGE_RETURN: char = '\r';
7pub const C_FORM_FEED: char = '\u{c}';
8
9pub const C_TAB: char = '\t';
10pub const C_SPACE: char = ' ';
11
12pub const C_SOLIDUS: char = '/';
13pub const C_REVERSE_SOLIDUS: char = '\\';
14pub const C_ASTERISK: char = '*';
15
16pub const C_LEFT_PARENTHESIS: char = '(';
17pub const C_RIGHT_PARENTHESIS: char = ')';
18pub const C_LEFT_CURLY: char = '{';
19pub const C_RIGHT_CURLY: char = '}';
20pub const C_LEFT_SQUARE: char = '[';
21pub const C_RIGHT_SQUARE: char = ']';
22
23pub const C_QUOTATION_MARK: char = '"';
24pub const C_APOSTROPHE: char = '\'';
25
26pub const C_FULL_STOP: char = '.';
27pub const C_COLON: char = ':';
28pub const C_SEMICOLON: char = ';';
29pub const C_COMMA: char = ',';
30pub const C_PERCENTAGE: char = '%';
31pub const C_AT_SIGN: char = '@';
32
33pub const C_LOW_LINE: char = '_';
34pub const C_LOWER_A: char = 'a';
35pub const C_LOWER_E: char = 'e';
36pub const C_LOWER_F: char = 'f';
37pub const C_LOWER_Z: char = 'z';
38pub const C_UPPER_A: char = 'A';
39pub const C_UPPER_E: char = 'E';
40pub const C_UPPER_F: char = 'F';
41pub const C_UPPER_Z: char = 'Z';
42pub const C_0: char = '0';
43pub const C_9: char = '9';
44
45pub const C_NUMBER_SIGN: char = '#';
46pub const C_PLUS_SIGN: char = '+';
47pub const C_HYPHEN_MINUS: char = '-';
48
49pub const C_LESS_THAN_SIGN: char = '<';
50pub const C_GREATER_THAN_SIGN: char = '>';
51
52pub type Pos = u32;
53
54pub trait Visitor<'s> {
55 fn comment(&mut self, _: &mut Lexer<'s>, _: Pos, _: Pos) -> Option<()> {
56 Some(())
57 }
58 fn function(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
59 fn ident(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
60 fn url(
61 &mut self,
62 lexer: &mut Lexer<'s>,
63 start: Pos,
64 end: Pos,
65 content_start: Pos,
66 content_end: Pos,
67 ) -> Option<()>;
68 fn string(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
69 fn is_selector(&mut self, lexer: &mut Lexer<'s>) -> Option<bool>;
70 fn id(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
71 fn left_parenthesis(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
72 fn right_parenthesis(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
73 fn comma(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
74 fn class(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
75 fn pseudo_function(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
76 fn pseudo_class(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
77 fn semicolon(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
78 fn at_keyword(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
79 fn left_curly_bracket(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
80 fn right_curly_bracket(&mut self, lexer: &mut Lexer<'s>, start: Pos, end: Pos) -> Option<()>;
81}
82
83#[derive(Debug, Clone)]
84pub struct Lexer<'s, I: Iterator<Item = char> = Chars<'s>> {
85 value: &'s str,
86 iter: I,
87 cur_pos: Option<Pos>,
88 cur: Option<char>,
89 peek: Option<char>,
90 peek2: Option<char>,
91}
92
93impl<'s> Lexer<'s> {
94 pub fn new(value: &'s str) -> Self {
95 let mut iter = value.chars();
96 let peek = iter.next();
97 let peek2 = iter.next();
98 Self {
99 value,
100 iter,
101 cur_pos: None,
102 cur: None,
103 peek,
104 peek2,
105 }
106 }
107
108 pub fn turn_back(self, end: Pos) -> Lexer<'s, Rev<Chars<'s>>> {
109 let value = self.slice(0, end).unwrap();
110 let mut iter = value.chars().rev();
111 let peek = iter.next();
112 let peek2 = iter.next();
113 Lexer {
114 value,
115 iter,
116 cur_pos: None,
117 cur: None,
118 peek,
119 peek2,
120 }
121 }
122
123 pub fn slice(&self, start: Pos, end: Pos) -> Option<&'s str> {
124 Self::slice_range(self.value, &Range::new(start, end))
125 }
126
127 pub fn slice_range<'a>(input: &'a str, range: &Range) -> Option<&'a str> {
128 input.get(range.start as usize..range.end as usize)
129 }
130}
131
132impl<'s, I: Iterator<Item = char>> Lexer<'s, I> {
133 pub fn consume(&mut self) {
134 self.cur_pos = self.peek_pos();
135 self.cur = self.peek;
136 self.peek = self.peek2;
137 self.peek2 = self.iter.next();
138 }
139
140 pub fn cur_pos(&self) -> Option<Pos> {
141 self.cur_pos
142 }
143
144 pub fn cur(&self) -> Option<char> {
145 self.cur
146 }
147
148 pub fn peek_pos(&self) -> Option<Pos> {
149 if let Some(pos) = self.cur_pos() {
150 self.cur().map(|c| pos + c.len_utf8() as u32)
151 } else {
152 Some(0)
153 }
154 }
155
156 pub fn peek(&self) -> Option<char> {
157 self.peek
158 }
159
160 pub fn peek2_pos(&self) -> Option<Pos> {
161 self.peek_pos()
162 .and_then(|pos| self.peek().map(|c| pos + c.len_utf8() as u32))
163 }
164
165 pub fn peek2(&self) -> Option<char> {
166 self.peek2
167 }
168}
169
170impl<'s> Lexer<'s> {
171 pub fn lex<T: Visitor<'s>>(&mut self, visitor: &mut T) {
172 self.lex_impl(visitor);
173 }
174
175 fn lex_impl<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
176 self.consume();
177 while self.cur().is_some() {
178 self.consume_comments_with_visitor(visitor)?;
179 match self.cur()? {
181 c if is_white_space(c) => self.consume_space()?,
182 C_QUOTATION_MARK => self.consume_string(visitor, C_QUOTATION_MARK)?,
183 C_NUMBER_SIGN => self.consume_number_sign(visitor)?,
184 C_APOSTROPHE => self.consume_string(visitor, C_APOSTROPHE)?,
185 C_LEFT_PARENTHESIS => self.consume_left_parenthesis(visitor)?,
186 C_RIGHT_PARENTHESIS => self.consume_right_parenthesis(visitor)?,
187 C_PLUS_SIGN => self.consume_plus_sign()?,
188 C_COMMA => self.consume_comma(visitor)?,
189 C_HYPHEN_MINUS => self.consume_minus(visitor)?,
190 C_FULL_STOP => self.consume_full_stop(visitor)?,
191 C_COLON => self.consume_potential_pseudo(visitor)?,
192 C_SEMICOLON => self.consume_semicolon(visitor)?,
193 C_LESS_THAN_SIGN => self.consume_less_than_sign()?,
194 C_AT_SIGN => self.consume_at_sign(visitor)?,
195 C_LEFT_SQUARE => self.consume_delim(),
196 C_REVERSE_SOLIDUS => self.consume_reverse_solidus(visitor)?,
197 C_RIGHT_SQUARE => self.consume_delim(),
198 C_LEFT_CURLY => self.consume_left_curly(visitor)?,
199 C_RIGHT_CURLY => self.consume_right_curly(visitor)?,
200 c if is_digit(c) => self.consume_numeric_token()?,
201 c if is_ident_start(c) => self.consume_ident_like(visitor)?,
202 _ => self.consume_delim(),
203 }
204 }
205 Some(())
206 }
207
208 pub fn consume_comments_with_visitor<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
209 if self.cur()? == C_SOLIDUS && self.peek()? == C_ASTERISK {
210 let start = self.cur_pos()?;
211 self.consume();
212 loop {
213 self.consume();
214 let c = self.cur()?;
215 if c == C_ASTERISK && self.peek()? == C_SOLIDUS {
216 self.consume();
217 self.consume();
218 break;
219 }
220 }
221 let end = self.cur_pos()?;
222 visitor.comment(self, start, end)?;
223 }
224 Some(())
225 }
226
227 pub fn consume_white_space_and_comments_with_visitor<T: Visitor<'s>>(
228 &mut self,
229 visitor: &mut T,
230 ) -> Option<()> {
231 loop {
232 self.consume_comments_with_visitor(visitor)?;
233 if is_white_space(self.cur()?) {
234 self.consume_space()?;
235 } else {
236 break;
237 }
238 }
239 Some(())
240 }
241
242 pub fn consume_delim(&mut self) {
243 self.consume();
244 }
245
246 pub fn consume_numeric_token(&mut self) -> Option<()> {
247 self.consume_number()?;
248 let c = self.cur()?;
249 if start_ident_sequence(c, self.peek()?, self.peek2()?) {
250 return self.consume_ident_sequence();
251 }
252 if c == C_PERCENTAGE {
253 self.consume();
254 }
255 Some(())
256 }
257
258 pub fn consume_number(&mut self) -> Option<()> {
259 self.consume();
260 while is_digit(self.cur()?) {
261 self.consume();
262 }
263 if self.cur()? == C_FULL_STOP && is_digit(self.peek()?) {
264 self.consume();
265 self.consume();
266 while is_digit(self.cur()?) {
267 self.consume();
268 }
269 }
270 let c = self.cur()?;
271 if c == C_LOWER_E || c == C_UPPER_E {
272 let c = self.peek()?;
273 if is_digit(c) {
274 self.consume();
275 } else if c == C_HYPHEN_MINUS || c == C_PLUS_SIGN {
276 let c = self.peek2()?;
277 if is_digit(c) {
278 self.consume();
279 self.consume();
280 } else {
281 return Some(());
282 }
283 } else {
284 return Some(());
285 }
286 } else {
287 return Some(());
288 }
289 self.consume();
290 while is_digit(self.cur()?) {
291 self.consume();
292 }
293 Some(())
294 }
295
296 pub fn consume_ident_sequence(&mut self) -> Option<()> {
297 loop {
298 let c = self.cur()?;
299 if maybe_valid_escape(c) {
300 self.consume();
301 self.consume_escaped()?;
302 } else if is_ident(c) {
303 self.consume();
304 } else {
305 return Some(());
306 }
307 }
308 }
309
310 pub fn consume_escaped(&mut self) -> Option<()> {
311 if is_hex_digit(self.cur()?) {
312 for _ in 1..5 {
313 self.consume();
314 if !is_hex_digit(self.cur()?) {
315 break;
316 }
317 }
318 if is_white_space(self.cur()?) {
319 self.consume();
320 }
321 } else {
322 self.consume();
323 }
324 Some(())
325 }
326
327 pub fn consume_ident_like<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
328 let start = self.cur_pos()?;
329 self.consume_ident_sequence()?;
330 let peek_pos = self.peek_pos()?;
331 if self.cur_pos()? == start + 3 && self.slice(start, peek_pos)?.eq_ignore_ascii_case("url(")
332 {
333 self.consume();
334 while is_white_space(self.cur()?) {
335 self.consume();
336 }
337 let c = self.cur()?;
338 if c == C_QUOTATION_MARK || c == C_APOSTROPHE {
339 visitor.function(self, start, peek_pos)
340 } else {
341 self.consume_url(visitor, start)
342 }
343 } else if self.cur()? == C_LEFT_PARENTHESIS {
344 self.consume();
345 visitor.function(self, start, self.cur_pos()?)
346 } else {
347 visitor.ident(self, start, self.cur_pos()?)
348 }
349 }
350
351 pub fn consume_url<T: Visitor<'s>>(
352 self: &mut Lexer<'s>,
353 visitor: &mut T,
354 start: Pos,
355 ) -> Option<()> {
356 let content_start = self.cur_pos()?;
357 loop {
358 let c = self.cur()?;
359 if maybe_valid_escape(c) {
360 self.consume();
361 self.consume_escaped()?;
362 } else if is_white_space(c) {
363 let content_end = self.cur_pos()?;
364 self.consume();
365 while is_white_space(self.cur()?) {
366 self.consume();
367 }
368 if self.cur()? != C_RIGHT_PARENTHESIS {
369 return Some(());
370 }
371 self.consume();
372 return visitor.url(self, start, self.cur_pos()?, content_start, content_end);
373 } else if c == C_RIGHT_PARENTHESIS {
374 let content_end = self.cur_pos()?;
375 self.consume();
376 return visitor.url(self, start, self.cur_pos()?, content_start, content_end);
377 } else if c == C_LEFT_PARENTHESIS {
378 return Some(());
379 } else {
380 self.consume();
381 }
382 }
383 }
384
385 pub fn consume_string<T: Visitor<'s>>(&mut self, visitor: &mut T, end: char) -> Option<()> {
386 let start = self.cur_pos()?;
387 self.consume();
388 loop {
389 let c = self.cur()?;
390 if c == end {
391 self.consume();
392 break;
393 } else if is_new_line(c) {
394 break;
395 } else if c == C_REVERSE_SOLIDUS {
396 self.consume();
397 let c2 = self.cur()?;
398 if is_new_line(c2) {
399 self.consume();
400 } else if are_valid_escape(c, c2) {
401 self.consume_escaped()?;
402 }
403 } else {
404 self.consume();
405 }
406 }
407 visitor.string(self, start, self.cur_pos()?)
408 }
409
410 pub fn consume_number_sign<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
411 let c2 = self.peek()?;
412 let start = self.cur_pos()?;
413 if is_ident(c2) || are_valid_escape(c2, self.peek2()?) {
414 self.consume();
415 if !visitor.is_selector(self)? {
416 return Some(());
417 }
418 if !start_ident_sequence(self.cur()?, self.peek()?, self.peek2()?) {
419 return visitor.id(self, start, self.cur_pos()?);
420 }
421 self.consume_ident_sequence()?;
422 visitor.id(self, start, self.cur_pos()?)
423 } else {
424 self.consume_delim();
425 visitor.id(self, start, self.cur_pos()?)
426 }
427 }
428
429 pub fn consume_left_parenthesis<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
430 self.consume();
431 let end = self.cur_pos()?;
432 visitor.left_parenthesis(self, end - 1, end)
433 }
434
435 pub fn consume_right_parenthesis<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
436 self.consume();
437 let end = self.cur_pos()?;
438 visitor.right_parenthesis(self, end - 1, end)
439 }
440
441 pub fn consume_plus_sign(&mut self) -> Option<()> {
442 if start_number(self.cur()?, self.peek()?, self.peek2()?) {
443 self.consume_numeric_token()?;
444 } else {
445 self.consume_delim();
446 }
447 Some(())
448 }
449
450 pub fn consume_comma<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
451 self.consume();
452 let end = self.cur_pos()?;
453 visitor.comma(self, end - 1, end)
454 }
455
456 pub fn consume_minus<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
457 let c = self.cur()?;
458 let c2 = self.peek()?;
459 let c3 = self.peek2()?;
460 if start_number(c, c2, c3) {
461 self.consume_numeric_token()?;
462 } else if c2 == C_HYPHEN_MINUS && c3 == C_GREATER_THAN_SIGN {
463 self.consume();
464 self.consume();
465 } else if start_ident_sequence(c, c2, c3) {
466 self.consume_ident_like(visitor)?;
467 } else {
468 self.consume_delim();
469 }
470 Some(())
471 }
472
473 pub fn consume_full_stop<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
474 let c = self.cur()?;
475 let c2 = self.peek()?;
476 let c3 = self.peek2()?;
477 if start_number(c, c2, c3) {
478 return self.consume_numeric_token();
479 }
480 let start = self.cur_pos()?;
481 self.consume();
482 if !visitor.is_selector(self)? {
483 return Some(());
484 }
485 if !start_ident_sequence(c2, c3, self.peek2()?) {
486 return visitor.class(self, start, self.cur_pos()?);
487 }
488 self.consume_ident_sequence()?;
489 visitor.class(self, start, self.cur_pos()?)
490 }
491
492 pub fn consume_potential_pseudo<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
493 let start = self.cur_pos()?;
494 self.consume();
495 if !visitor.is_selector(self)?
496 || !start_ident_sequence(self.cur()?, self.peek()?, self.peek2()?)
497 {
498 return Some(());
499 }
500 self.consume_ident_sequence()?;
501 if self.cur()? == C_LEFT_PARENTHESIS {
502 self.consume();
503 visitor.pseudo_function(self, start, self.cur_pos()?)
504 } else {
505 visitor.pseudo_class(self, start, self.cur_pos()?)
506 }
507 }
508
509 pub fn consume_semicolon<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
510 self.consume();
511 let end = self.cur_pos()?;
512 visitor.semicolon(self, end - 1, end)
513 }
514
515 pub fn consume_less_than_sign(&mut self) -> Option<()> {
516 self.consume();
517 if self.cur()? == '!' && self.peek()? == '-' && self.peek2()? == '-' {
518 self.consume();
519 self.consume();
520 self.consume();
521 }
522 Some(())
523 }
524
525 pub fn consume_at_sign<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
526 let start = self.cur_pos()?;
527 self.consume();
528 if start_ident_sequence(self.cur()?, self.peek()?, self.peek2()?) {
529 self.consume_ident_sequence()?;
530 return visitor.at_keyword(self, start, self.cur_pos()?);
531 }
532 Some(())
533 }
534
535 pub fn consume_reverse_solidus<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
536 if are_valid_escape(self.cur()?, self.peek()?) {
537 self.consume_ident_like(visitor)?;
538 } else {
539 self.consume_delim();
540 }
541 Some(())
542 }
543
544 pub fn consume_left_curly<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
545 self.consume();
546 let end = self.cur_pos()?;
547 visitor.left_curly_bracket(self, end - 1, end)
548 }
549
550 pub fn consume_right_curly<T: Visitor<'s>>(&mut self, visitor: &mut T) -> Option<()> {
551 self.consume();
552 let end = self.cur_pos()?;
553 visitor.right_curly_bracket(self, end - 1, end)
554 }
555}
556
557impl<'s, I: Iterator<Item = char>> Lexer<'s, I> {
558 pub fn consume_comments(&mut self) -> Option<()> {
559 if self.cur()? == C_SOLIDUS && self.peek()? == C_ASTERISK {
560 self.consume();
561 loop {
562 self.consume();
563 let c = self.cur()?;
564 if c == C_ASTERISK && self.peek()? == C_SOLIDUS {
565 self.consume();
566 self.consume();
567 break;
568 }
569 }
570 }
571 Some(())
572 }
573
574 pub fn consume_space(&mut self) -> Option<()> {
575 self.consume();
576 while is_white_space(self.cur()?) {
577 self.consume();
578 }
579 Some(())
580 }
581
582 pub fn consume_white_space_and_comments(&mut self) -> Option<()> {
583 loop {
584 self.consume_comments()?;
585 if is_white_space(self.cur()?) {
586 self.consume_space()?;
587 } else {
588 break;
589 }
590 }
591 Some(())
592 }
593}
594
595pub fn is_new_line(c: char) -> bool {
596 c == C_LINE_FEED || c == C_CARRIAGE_RETURN || c == C_FORM_FEED
597}
598
599pub fn is_space(c: char) -> bool {
600 c == C_TAB || c == C_SPACE
601}
602
603pub fn is_white_space(c: char) -> bool {
604 is_new_line(c) || is_space(c)
605}
606
607pub fn is_digit(c: char) -> bool {
608 c >= C_0 && c <= C_9
609}
610
611pub fn is_hex_digit(c: char) -> bool {
612 is_digit(c) || (c >= C_UPPER_A && c <= C_UPPER_F) || (c >= C_LOWER_A && c <= C_LOWER_F)
613}
614
615pub fn is_ident_start(c: char) -> bool {
616 c == C_LOW_LINE
617 || (c >= C_LOWER_A && c <= C_LOWER_Z)
618 || (c >= C_UPPER_A && c <= C_UPPER_Z)
619 || c > '\u{80}'
620}
621
622pub fn is_ident(c: char) -> bool {
623 is_ident_start(c) || is_digit(c) || c == C_HYPHEN_MINUS
624}
625
626pub fn start_ident_sequence(c1: char, c2: char, c3: char) -> bool {
627 if c1 == C_HYPHEN_MINUS {
628 is_ident_start(c2) || c2 == C_HYPHEN_MINUS || are_valid_escape(c2, c3)
629 } else {
630 is_ident_start(c1) || are_valid_escape(c1, c2)
631 }
632}
633
634pub fn maybe_valid_escape(c: char) -> bool {
635 c == C_REVERSE_SOLIDUS
636}
637
638pub fn are_valid_escape(c1: char, c2: char) -> bool {
639 c1 == C_REVERSE_SOLIDUS && !is_new_line(c2)
640}
641
642pub fn start_number(c1: char, c2: char, c3: char) -> bool {
643 if c1 == C_PLUS_SIGN || c1 == C_HYPHEN_MINUS {
644 is_digit(c2) || (c2 == C_FULL_STOP && is_digit(c3))
645 } else {
646 is_digit(c1) || (c1 == C_FULL_STOP && is_digit(c2))
647 }
648}
649
650#[cfg(test)]
651mod tests {
652 use super::*;
653 use indoc::indoc;
654 use std::fmt::Write as _;
655
656 fn assert_lexer_state<I: Iterator<Item = char>>(
657 lexer: &Lexer<'_, I>,
658 cur: Option<char>,
659 cur_pos: Option<Pos>,
660 peek: Option<char>,
661 peek_pos: Option<Pos>,
662 peek2: Option<char>,
663 peek2_pos: Option<Pos>,
664 ) {
665 assert_eq!(lexer.cur(), cur);
666 assert_eq!(lexer.cur_pos(), cur_pos);
667 assert_eq!(lexer.peek(), peek);
668 assert_eq!(lexer.peek_pos(), peek_pos);
669 assert_eq!(lexer.peek2(), peek2);
670 assert_eq!(lexer.peek2_pos(), peek2_pos);
671 }
672
673 #[derive(Default)]
674 struct Snapshot {
675 results: Vec<(String, String)>,
676 }
677
678 impl Snapshot {
679 pub fn add(&mut self, key: &str, value: &str) {
680 self.results.push((key.to_string(), value.to_string()))
681 }
682
683 pub fn snapshot(&self) -> String {
684 self.results
685 .iter()
686 .fold(String::new(), |mut output, (key, value)| {
687 writeln!(output, "{key}: {value}").expect("writing to String cannot fail");
688 output
689 })
690 }
691 }
692
693 impl Visitor<'_> for Snapshot {
694 fn function(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
695 self.add("function", lexer.slice(start, end)?);
696 Some(())
697 }
698
699 fn ident(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
700 self.add("ident", lexer.slice(start, end)?);
701 Some(())
702 }
703
704 fn url(
705 &mut self,
706 lexer: &mut Lexer,
707 _: Pos,
708 _: Pos,
709 content_start: Pos,
710 content_end: Pos,
711 ) -> Option<()> {
712 self.add("url", lexer.slice(content_start, content_end)?);
713 Some(())
714 }
715
716 fn string(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
717 self.add("string", lexer.slice(start, end)?);
718 Some(())
719 }
720
721 fn is_selector(&mut self, _: &mut Lexer) -> Option<bool> {
722 Some(true)
723 }
724
725 fn id(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
726 self.add("id", lexer.slice(start, end)?);
727 Some(())
728 }
729
730 fn left_parenthesis(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
731 self.add("left_parenthesis", lexer.slice(start, end)?);
732 Some(())
733 }
734
735 fn right_parenthesis(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
736 self.add("right_parenthesis", lexer.slice(start, end)?);
737 Some(())
738 }
739
740 fn comma(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
741 self.add("comma", lexer.slice(start, end)?);
742 Some(())
743 }
744
745 fn class(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
746 self.add("class", lexer.slice(start, end)?);
747 Some(())
748 }
749
750 fn pseudo_function(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
751 self.add("pseudo_function", lexer.slice(start, end)?);
752 Some(())
753 }
754
755 fn pseudo_class(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
756 self.add("pseudo_class", lexer.slice(start, end)?);
757 Some(())
758 }
759
760 fn semicolon(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
761 self.add("semicolon", lexer.slice(start, end)?);
762 Some(())
763 }
764
765 fn at_keyword(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
766 self.add("at_keyword", lexer.slice(start, end)?);
767 Some(())
768 }
769
770 fn left_curly_bracket(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
771 self.add("left_curly", lexer.slice(start, end)?);
772 Some(())
773 }
774
775 fn right_curly_bracket(&mut self, lexer: &mut Lexer, start: Pos, end: Pos) -> Option<()> {
776 self.add("right_curly", lexer.slice(start, end)?);
777 Some(())
778 }
779 }
780
781 fn assert_lexer_snapshot(input: &str, snapshot: &str) {
782 let mut s = Snapshot::default();
783 let mut l = Lexer::new(input);
784 l.lex(&mut s);
785 assert!(l.cur().is_none());
786 similar_asserts::assert_eq!(s.snapshot(), snapshot);
787 }
788
789 #[test]
790 fn lexer_state_1() {
791 let mut l = Lexer::new("");
792 assert_lexer_state(&l, None, None, None, Some(0), None, None);
793 l.consume();
794 assert_eq!(l.cur(), None);
795 assert_lexer_state(&l, None, Some(0), None, None, None, None);
796 l.consume();
797 assert_eq!(l.cur(), None);
798 }
799
800 #[test]
801 fn lexer_state_2() {
802 let mut l = Lexer::new("0壹👂삼");
803 assert_lexer_state(&l, None, None, Some('0'), Some(0), Some('壹'), Some(1));
804 l.consume();
805 assert_eq!(l.cur(), Some('0'));
806 assert_lexer_state(
807 &l,
808 Some('0'),
809 Some(0),
810 Some('壹'),
811 Some(1),
812 Some('👂'),
813 Some(4),
814 );
815 l.consume();
816 assert_eq!(l.cur(), Some('壹'));
817 assert_lexer_state(
818 &l,
819 Some('壹'),
820 Some(1),
821 Some('👂'),
822 Some(4),
823 Some('삼'),
824 Some(8),
825 );
826 l.consume();
827 assert_eq!(l.cur(), Some('👂'));
828 assert_lexer_state(&l, Some('👂'), Some(4), Some('삼'), Some(8), None, Some(11));
829 l.consume();
830 assert_eq!(l.cur(), Some('삼'));
831 assert_lexer_state(&l, Some('삼'), Some(8), None, Some(11), None, None);
832 l.consume();
833 assert_eq!(l.cur(), None);
834 assert_lexer_state(&l, None, Some(11), None, None, None, None);
835 l.consume();
836 assert_eq!(l.cur(), None);
837 }
838
839 #[test]
840 fn lexer_state_3() {
841 let l = Lexer::new("");
842 let mut l = l.turn_back(0);
843 assert_lexer_state(&l, None, None, None, Some(0), None, None);
844 l.consume();
845 assert_lexer_state(&l, None, Some(0), None, None, None, None);
846 }
847
848 #[test]
849 fn parse_urls() {
850 assert_lexer_snapshot(
851 indoc! {r#"
852 body {
853 background: url(
854 https://example\2f4a8f.com\
855 /image.png
856 )
857 }
858 --element\ name.class\ name#_id {
859 background: url( "https://example.com/some url \"with\" 'spaces'.png" ) url('https://example.com/\'"quotes"\'.png');
860 }
861 "#},
862 indoc! {r#"
863 ident: body
864 left_curly: {
865 ident: background
866 url: https://example\2f4a8f.com\
867 /image.png
868 right_curly: }
869 ident: --element\ name
870 class: .class\ name
871 id: #_id
872 left_curly: {
873 ident: background
874 function: url(
875 string: "https://example.com/some url \"with\" 'spaces'.png"
876 right_parenthesis: )
877 function: url(
878 string: 'https://example.com/\'"quotes"\'.png'
879 right_parenthesis: )
880 semicolon: ;
881 right_curly: }
882 "#},
883 );
884 }
885
886 #[test]
887 fn parse_pseudo_functions() {
888 assert_lexer_snapshot(
889 indoc! {r#"
890 :local(.class#id, .class:not(*:hover)) { color: red; }
891 :import(something from ":somewhere") {}
892 "#},
893 indoc! {r#"
894 pseudo_function: :local(
895 class: .class
896 id: #id
897 comma: ,
898 class: .class
899 pseudo_function: :not(
900 pseudo_class: :hover
901 right_parenthesis: )
902 right_parenthesis: )
903 left_curly: {
904 ident: color
905 ident: red
906 semicolon: ;
907 right_curly: }
908 pseudo_function: :import(
909 ident: something
910 ident: from
911 string: ":somewhere"
912 right_parenthesis: )
913 left_curly: {
914 right_curly: }
915 "#},
916 );
917 }
918
919 #[test]
920 fn parse_at_rules() {
921 assert_lexer_snapshot(
922 indoc! {r#"
923 @media (max-size: 100px) {
924 @import "external.css";
925 body { color: red; }
926 }
927 "#},
928 indoc! {r#"
929 at_keyword: @media
930 left_parenthesis: (
931 ident: max-size
932 right_parenthesis: )
933 left_curly: {
934 at_keyword: @import
935 string: "external.css"
936 semicolon: ;
937 ident: body
938 left_curly: {
939 ident: color
940 ident: red
941 semicolon: ;
942 right_curly: }
943 right_curly: }
944 "#},
945 );
946 }
947
948 #[test]
949 fn parse_escape() {
950 assert_lexer_snapshot(
951 indoc! {r#"
952 body {
953 a\
954 a: \
955 url(https://example\2f4a8f.com\
956 /image.png)
957 b: url(#\
958 hash)
959 }
960 "#},
961 indoc! {r#"
962 ident: body
963 left_curly: {
964 ident: a\
965 a
966 url: https://example\2f4a8f.com\
967 /image.png
968 ident: b
969 url: #\
970 hash
971 right_curly: }
972 "#},
973 );
974 }
975
976 #[test]
977 fn parse_pseudo_elements() {
978 assert_lexer_snapshot(
979 indoc! {r#"
980 a::after {
981 content: ' (' attr(href) ')';
982 }
983 "#},
984 indoc! {r#"
985 ident: a
986 pseudo_class: :after
987 left_curly: {
988 ident: content
989 string: ' ('
990 function: attr(
991 ident: href
992 right_parenthesis: )
993 string: ')'
994 semicolon: ;
995 right_curly: }
996 "#},
997 );
998 }
999
1000 #[test]
1001 fn parse_minimized_urls() {
1002 assert_lexer_snapshot(
1003 "body{background:url(./image.png)}",
1004 indoc! {r#"
1005 ident: body
1006 left_curly: {
1007 ident: background
1008 pseudo_function: :url(
1009 class: .
1010 ident: image
1011 class: .png
1012 right_parenthesis: )
1013 right_curly: }
1014 "#},
1015 );
1016 }
1017}