1use std::iter::{Enumerate, Peekable};
2use std::str::Chars;
3
4#[derive(Debug, Clone, PartialEq)]
5pub enum Token {
6 And,
7 AndAnd,
8 Caret,
9 Colon,
10 Dot,
11 DotDotDot,
12 Eq,
13 EqEq,
14 EqEqEq,
15 EqGt,
16 Gt,
17 GtEq,
18 GtGt,
19 GtGtGt,
20 Lt,
21 LtEq,
22 LtLt,
23 Minus,
24 MinusEq,
25 MinusMinus,
26 Not,
27 NotEq,
28 NotEqEq,
29 Percent,
30 PercentEq,
31 Pipe,
32 PipePipe,
33 Plus,
34 PlusEq,
35 PlusPlus,
36 Question,
37 QuestionDot,
38 QuestionQuestion,
39 Slash,
40 SlashEq,
41 Star,
42 StarEq,
43 StarStar,
44 StarStarEq,
45 Tilde,
46 LParen,
47 RParen,
48 LBracket,
49 RBracket,
50 LBrace,
51 RBrace,
52 Comma,
53 Semicolon,
54 Backtick,
55 DollarBrace,
56 Await,
57 Delete,
58 False,
59 In,
60 InstanceOf,
61 New,
62 Null,
63 Super,
64 This,
65 True,
66 Typeof,
67 Undefined,
68 Void,
69 Identifier(String),
70 Number(f64),
71 BigInt(num_bigint::BigInt),
72 String(String),
73 Regex(String, Option<String>),
74 Eof,
75}
76
77#[derive(Debug)]
78pub enum LexerError {
79 UnexpectedCharacter { position: usize, character: char },
80 UnterminatedString { position: usize },
81 UnterminatedRegex { position: usize },
82 UnexpectedEndOfInput,
83 UnicodeOutOfRange,
84}
85
86fn ws(ch: char) -> bool {
87 matches!(
88 ch,
89 ' ' | '\t'
90 | '\u{000b}'
91 | '\u{000c}'
92 | '\u{00a0}'
93 | '\u{1680}'
94 | '\u{2000}'
95 | '\u{2001}'
96 | '\u{2002}'
97 | '\u{2003}'
98 | '\u{2004}'
99 | '\u{2005}'
100 | '\u{2006}'
101 | '\u{2007}'
102 | '\u{2008}'
103 | '\u{2009}'
104 | '\u{200a}'
105 | '\u{202f}'
106 | '\u{205f}'
107 | '\u{3000}'
108 | '\u{feff}'
109 )
110}
111
112fn lt(ch: char) -> bool {
113 matches!(ch, '\n' | '\r' | '\u{2028}' | '\u{2029}')
114}
115
116fn id_start(ch: char) -> bool {
117 match ch {
118 '$' | '_' | ('a'..='z') | ('A'..='Z') | ('\u{80}'..='\u{10FFFF}') => true,
119 _ => ch > '\u{80}' && unicode_id_start::is_id_start_unicode(ch),
120 }
121}
122
123fn id_continue(ch: char) -> bool {
124 match ch {
125 '$' | '_' | 'A'..='Z' | 'a'..='z' | '0'..='9' => true,
126 _ => ch > '\u{80}' && unicode_id_start::is_id_continue_unicode(ch),
127 }
128}
129
130fn keyword(id: &str) -> Option<Token> {
131 match id.len() {
132 2 if id == "in" => Some(Token::In),
133 3 if id == "new" => Some(Token::New),
134 4 if id == "true" => Some(Token::True),
135 4 if id == "null" => Some(Token::Null),
136 4 if id == "this" => Some(Token::This),
137 4 if id == "void" => Some(Token::Void),
138 5 if id == "false" => Some(Token::False),
139 5 if id == "super" => Some(Token::Super),
140 5 if id == "await" => Some(Token::Await),
141 6 if id == "typeof" => Some(Token::Typeof),
142 6 if id == "delete" => Some(Token::Delete),
143 9 if id == "undefined" => Some(Token::Undefined),
144 10 if id == "instanceof" => Some(Token::InstanceOf),
145 _ => None,
146 }
147}
148
149fn regex_flag(ch: char) -> bool {
150 matches!(ch, 'g' | 'i' | 'm' | 's' | 'u' | 'y')
151}
152
153macro_rules! greedy_match {
154 ($chars:expr, $token:expr, $default:expr, $($char:expr, $next:expr),+) => {
155 $token = $default;
156 $(
157 if let Some((_, la)) = $chars.peek() {
158 if *la == $char {
159 $chars.next();
160 $token = $next;
161 }
162 }
163 )+
164 }
165}
166
167macro_rules! template_expr {
168 (
169$chars:expr,
170$token:expr,
171$tokens:expr,
172$in_template:expr,
173$template_expr_stack:expr,
174$end_token:expr
175) => {
176 if $in_template[$in_template.len() - 1] {
177 $tokens.push($end_token);
178
179 let (str, dollar_brace) = lex_string(&mut $chars, '`')?;
180
181 if dollar_brace {
182 $template_expr_stack.push(true);
183 $in_template.push(false);
184 $tokens.push(str);
185 $token = Token::DollarBrace;
186 } else {
187 $token = str;
188 }
189 } else {
190 $token = $end_token;
191 }
192 };
193}
194
195pub fn lex(expr: &str) -> Result<Vec<Token>, LexerError> {
197 if expr.trim().is_empty() {
198 return Err(LexerError::UnexpectedEndOfInput);
199 }
200
201 let mut tokens = Vec::new();
202 let mut chars = expr.chars().enumerate().peekable();
203 let mut token: Token;
204 let mut in_template = vec![false];
205 let mut template_expr_stack: Vec<bool> = Vec::new();
206
207 while let Some((i, ch)) = chars.next() {
208 if ws(ch) || lt(ch) {
209 continue;
210 }
211
212 if id_start(ch) {
213 let mut j = i;
214
215 while let Some((_, ch)) = chars.peek() {
216 if id_continue(*ch) {
217 chars.next();
218 j += 1;
219 } else {
220 break;
221 }
222 }
223
224 let value = &expr[i..=j];
225
226 token = keyword(value).unwrap_or_else(|| Token::Identifier(value.to_string()))
227 } else if ch == '\'' || ch == '"' {
228 token = lex_string(&mut chars, ch)?.0;
229 } else if ch.is_ascii_digit() {
230 token = lex_number(&mut chars, ch);
231 } else {
232 match ch {
233 '(' => {
234 token = Token::LParen;
235 }
236
237 ')' => {
238 token = Token::RParen;
239 }
240
241 ';' => {
242 token = Token::Semicolon;
243 }
244
245 ',' => {
246 token = Token::Comma;
247 }
248
249 '.' => {
250 token = Token::Dot;
251
252 if let Some((_, la)) = chars.peek() {
253 if *la == '.' {
254 chars.next();
255 if let Some((_, la)) = chars.peek() {
256 if *la == '.' {
257 chars.next();
258 token = Token::DotDotDot;
259 }
260 }
261 } else if (*la).is_ascii_digit() {
262 token = lex_number(&mut chars, ch);
263 }
264 }
265 }
266
267 '`' => {
268 let last = in_template.len() - 1;
269 in_template[last] = !in_template[last];
270
271 template_expr!(
272 chars,
273 token,
274 tokens,
275 in_template,
276 template_expr_stack,
277 Token::Backtick
278 );
279 }
280
281 '[' => {
282 token = Token::LBracket;
283 }
284
285 ']' => {
286 token = Token::RBracket;
287 }
288
289 '{' => {
290 template_expr_stack.push(false);
291 token = Token::LBrace
292 }
293
294 '}' => {
295 let end_expr = template_expr_stack.pop().unwrap_or(false);
296
297 if end_expr {
298 in_template.pop();
299 }
300
301 template_expr!(
302 chars,
303 token,
304 tokens,
305 in_template,
306 template_expr_stack,
307 Token::RBrace
308 );
309 }
310
311 '?' => {
312 greedy_match!(
313 chars,
314 token,
315 Token::Question,
316 '?',
317 Token::QuestionQuestion,
318 '.',
319 Token::QuestionDot
320 );
321 }
322
323 ':' => {
324 token = Token::Colon;
325 }
326
327 '=' => {
328 greedy_match!(
329 chars,
330 token,
331 Token::Eq,
332 '=',
333 Token::EqEq,
334 '=',
335 Token::EqEqEq,
336 '>',
337 Token::EqGt
338 );
339 }
340
341 '!' => {
342 greedy_match!(
343 chars,
344 token,
345 Token::Not,
346 '=',
347 Token::NotEq,
348 '=',
349 Token::NotEqEq
350 );
351 }
352
353 '&' => {
354 greedy_match!(chars, token, Token::And, '&', Token::AndAnd);
355 }
356
357 '|' => {
358 greedy_match!(chars, token, Token::Pipe, '|', Token::PipePipe);
359 }
360
361 '<' => {
362 greedy_match!(chars, token, Token::Lt, '=', Token::LtEq, '<', Token::LtLt);
363 }
364
365 '>' => {
366 greedy_match!(
367 chars,
368 token,
369 Token::Gt,
370 '=',
371 Token::GtEq,
372 '>',
373 Token::GtGt,
374 '>',
375 Token::GtGtGt
376 );
377 }
378
379 '+' => {
380 greedy_match!(
381 chars,
382 token,
383 Token::Plus,
384 '=',
385 Token::PlusEq,
386 '+',
387 Token::PlusPlus
388 );
389 }
390
391 '-' => {
392 greedy_match!(
393 chars,
394 token,
395 Token::Minus,
396 '=',
397 Token::MinusEq,
398 '-',
399 Token::MinusMinus
400 );
401 }
402
403 '*' => {
404 greedy_match!(
405 chars,
406 token,
407 Token::Star,
408 '=',
409 Token::StarEq,
410 '*',
411 Token::StarStar,
412 '=',
413 Token::StarStarEq
414 );
415 }
416
417 '/' => {
418 token = Token::Slash;
419
420 if let Some((_, la)) = chars.peek() {
421 if *la == '/' {
422 chars.next();
424 for (_, ch) in chars.by_ref() {
425 if lt(ch) {
426 break;
427 }
428 }
429 continue;
430 } else if *la == '*' {
431 chars.next();
433 while let Some((_, ch)) = chars.next() {
434 if ch == '*' {
435 if let Some((_, ch)) = chars.peek() {
436 if *ch == '/' {
437 chars.next();
438 break;
439 }
440 }
441 }
442 }
443 continue;
444 } else if *la == '=' {
445 chars.next();
446 token = Token::SlashEq;
447 } else if tokens.is_empty()
448 || tokens.last().map_or(false, |t| {
449 matches!(
450 *t,
451 Token::LParen
452 | Token::Comma
453 | Token::Eq
454 | Token::Colon
455 | Token::LBracket
456 | Token::Not
457 | Token::And
458 | Token::Pipe
459 | Token::Question
460 | Token::LBrace
461 | Token::RBrace
462 | Token::Semicolon
463 )
464 })
465 {
466 let mut value = String::new();
467 let mut in_set = false;
468
469 loop {
470 if let Some((i, ch)) = chars.next() {
471 if ch == '\n' || ch == '\r' {
472 return Err(LexerError::UnterminatedRegex { position: i });
473 }
474
475 match ch {
476 '/' if !in_set => break,
477 '\\' => {
478 value.push(ch);
479 if let Some((_, ch)) = chars.next() {
480 value.push(ch);
481 } else {
482 return Err(LexerError::UnexpectedEndOfInput);
483 }
484 continue;
485 }
486 '[' => {
487 in_set = true;
488 }
489 ']' => {
490 in_set = false;
491 }
492 _ => {}
493 }
494
495 value.push(ch);
496 } else {
497 return Err(LexerError::UnexpectedEndOfInput);
498 }
499 }
500
501 let mut flags = String::new();
502
503 while let Some((_, ch)) = chars.peek() {
504 if regex_flag(*ch) {
505 flags.push(*ch);
506 chars.next();
507 } else {
508 break;
509 }
510 }
511
512 token =
513 Token::Regex(value, Option::from(flags).filter(|s| !s.is_empty()))
514 }
515 }
516 }
517
518 '%' => {
519 greedy_match!(chars, token, Token::Percent, '=', Token::PercentEq);
520 }
521
522 '~' => {
523 token = Token::Tilde;
524 }
525
526 '^' => {
527 token = Token::Caret;
528 }
529
530 _ => {
531 return Err(LexerError::UnexpectedCharacter {
532 position: i,
533 character: ch,
534 });
535 }
536 }
537 }
538
539 tokens.push(token);
540 }
541
542 tokens.push(Token::Eof);
543
544 Ok(tokens)
545}
546
547macro_rules! hex_to_char {
548 ($s:expr, $hex:expr) => {
549 let point = u32::from_str_radix($hex, 16).unwrap();
550
551 if (point > 0x10ffff) {
552 return Err(LexerError::UnicodeOutOfRange);
553 }
554
555 $s.push(char::from_u32(u32::from_str_radix($hex, 16).unwrap()).unwrap());
556 };
557}
558
559macro_rules! invalid_hex {
560 ($position:expr, $char:expr) => {
561 return Err(LexerError::UnexpectedCharacter {
562 position: $position,
563 character: $char,
564 });
565 };
566}
567
568macro_rules! match_hex {
569 ($chars:expr, $count:expr, $hex:expr) => {
570 for _ in 0..$count {
571 if let Some((i, ch)) = $chars.peek() {
572 if ch.is_digit(16) {
573 $hex.push(*ch);
574 $chars.next();
575 } else {
576 invalid_hex!(*i, *ch);
577 }
578 } else {
579 return Err(LexerError::UnexpectedEndOfInput);
580 }
581 }
582 };
583}
584
585fn lex_string(
586 chars: &mut Peekable<Enumerate<Chars>>,
587 delim: char,
588) -> Result<(Token, bool), LexerError> {
589 let mut s = String::new();
590 let template = delim == '`';
591 let mut dollar_brace = false;
592
593 loop {
594 if let Some((i, ch)) = chars.peek() {
595 if *ch == '\n' || *ch == '\r' {
596 return Err(LexerError::UnterminatedString { position: *i });
597 }
598
599 if *ch == delim {
600 if !template {
601 chars.next();
602 }
603 break;
604 }
605
606 if *ch == '\\' {
607 chars.next();
608
609 if let Some((_, ch)) = chars.peek() {
610 match *ch {
611 'n' => s.push('\n'),
612 't' => s.push('\t'),
613 'r' => s.push('\r'),
614 '0' => s.push('\0'),
615 'b' => s.push('\u{0008}'),
616 'f' => s.push('\u{000C}'),
617 'v' => s.push('\u{000B}'),
618
619 '\n' | '\r' | '\u{2028}' | '\u{2029}' => {}
621
622 'x' => {
623 chars.next();
624 let mut hex = String::new();
625 match_hex!(chars, 2, hex);
626 hex_to_char!(s, &hex);
627 continue;
628 }
629
630 'u' => {
631 chars.next();
632 let mut hex = String::new();
633
634 if let Some((i, ch)) = chars.peek() {
635 if ch.is_ascii_hexdigit() {
636 hex.push(*ch);
637 chars.next();
638 match_hex!(chars, 3, hex);
639 } else if *ch == '{' {
640 chars.next();
641
642 while let Some((i, ch)) = chars.peek() {
643 if ch.is_ascii_hexdigit() {
644 hex.push(*ch);
645 chars.next();
646 } else if *ch == '}' {
647 chars.next();
648 break;
649 } else {
650 invalid_hex!(*i, *ch);
651 }
652 }
653 } else {
654 invalid_hex!(*i, *ch);
655 }
656 }
657
658 hex_to_char!(s, &hex);
659 continue;
660 }
661
662 _ => s.push(*ch),
663 }
664
665 chars.next();
666 }
667 } else if template && *ch == '$' {
668 chars.next();
669
670 if let Some((_, '{')) = chars.peek() {
671 chars.next();
672 dollar_brace = true;
673 break;
674 }
675
676 s.push('$');
677 } else {
678 s.push(*ch);
679 chars.next();
680 }
681 } else {
682 return Err(LexerError::UnexpectedEndOfInput);
683 }
684 }
685
686 Ok((Token::String(s), dollar_brace))
687}
688
689fn lex_number(chars: &mut Peekable<Enumerate<Chars>>, ch: char) -> Token {
690 if ch == '0' {
691 if let Some((_, la)) = chars.peek() {
692 if *la == 'b' || *la == 'B' {
693 return based(chars, 2);
694 }
695
696 if *la == 'o' || *la == 'O' {
697 return based(chars, 8);
698 }
699
700 if *la == 'x' || *la == 'X' {
701 return based(chars, 16);
702 }
703 }
704 }
705
706 let mut value = String::from(ch);
707 let mut seen_dot = false;
708 let mut seen_e = false;
709 let mut sep = ch != '.';
710
711 while let Some((_, ch)) = chars.peek() {
712 let res = {
713 let ch = *ch;
714 ch.is_ascii_digit()
715 };
716 if res {
717 value.push(*ch);
718 chars.next();
719 sep = true;
720 } else if !seen_dot && !seen_e && *ch == '.' {
721 seen_dot = true;
722 value.push(*ch);
723 chars.next();
724 } else if *ch == '+' || *ch == '-' {
725 value.push(*ch);
726 chars.next();
727 } else if !seen_e && (*ch == 'e' || *ch == 'E') {
728 seen_e = true;
729 value.push(*ch);
730 chars.next();
731 } else if *ch == 'n' {
732 chars.next();
733 return Token::BigInt(value.parse().unwrap());
734 } else if sep && *ch == '_' {
735 chars.next();
736 sep = false;
737 } else {
738 break;
739 }
740 }
741
742 Token::Number(value.parse().unwrap())
743}
744
745fn based(chars: &mut Peekable<Enumerate<Chars>>, radix: u32) -> Token {
746 chars.next();
747
748 let mut value = String::new();
749 let mut sep = false;
750
751 while let Some((_, ch)) = chars.peek() {
752 if char::is_digit(*ch, radix) {
753 value.push(*ch);
754 chars.next();
755 sep = true;
756 } else if sep && *ch == '_' {
757 chars.next();
758 sep = false;
759 } else if *ch == 'n' {
760 chars.next();
761 return Token::BigInt(u64::from_str_radix(&value, radix).unwrap().into());
762 } else {
763 break;
764 }
765 }
766
767 Token::Number(u64::from_str_radix(&value, radix).unwrap() as f64)
768}
769
770#[cfg(test)]
771mod tests {
772 use insta::assert_debug_snapshot;
773
774 use super::*;
775
776 #[test]
777 fn arithmetic_operators() {
778 assert_debug_snapshot!(lex("+ - * / ** %"));
779 }
780
781 #[test]
782 fn assignment_operators() {
783 assert_debug_snapshot!(lex("= += -= *= /= %= **="));
784 }
785
786 #[test]
787 fn comparison_operators() {
788 assert_debug_snapshot!(lex("== === != !== < > <= >="));
789 }
790
791 #[test]
792 fn ternary_operator() {
793 assert_debug_snapshot!(lex("? :"));
794 }
795
796 #[test]
797 fn logical_operators() {
798 assert_debug_snapshot!(lex("&& ||"));
799 }
800
801 #[test]
802 fn bitwise_operators() {
803 assert_debug_snapshot!(lex("~ & | ^ << >> >>>"));
804 }
805 #[test]
806 fn unary_operators() {
807 assert_debug_snapshot!(lex("++ -- !"));
808 }
809
810 #[test]
811 fn nullish_coalescing_operator() {
812 assert_debug_snapshot!(lex("??"));
813 }
814
815 #[test]
816 fn other_punctuation() {
817 assert_debug_snapshot!(lex(", ; ( ) [ ] { } . ?. => ..."));
818 }
819
820 #[test]
821 fn keywords() {
822 assert_debug_snapshot!(lex(
823 "true false null undefined typeof void in instanceof new super await this"
824 ));
825 }
826
827 #[test]
828 fn identifiers() {
829 assert_debug_snapshot!(lex("foo $bar _baz $123 _456 $ _ abc123"));
830 }
831
832 #[test]
833 fn whitespace() {
834 assert_debug_snapshot!(lex("a \t\n\rb"));
835 }
836
837 #[test]
838 fn strings() {
839 assert_debug_snapshot!(lex(
840 r#" "" '' "foo" 'bar' "foo\"bar" 'bar\'foo' "foo\nbar" 'bar\tfoo' "\x00" '\x58' "😀"
841 '\u{1F600}' "\u1234" "\u{0}" "\u{0}" "#
842 ));
843 }
844
845 #[test]
846 fn templates() {
847 assert_debug_snapshot!(lex(r#" ${ `` `$` `\${` `}` `foo` `${bar}` `abc${bar}def`
848 `${ `a${b}c` }` `\u{1F600}` sql`select` `${{ $id }}`
849 `${ { id: { } } }` sql``.f { a: `${a.title}!` } "#));
850 }
851
852 #[test]
853 fn long_strings() {
854 assert_debug_snapshot!(lex("'abc \\\n def'"));
855 }
856
857 #[test]
858 fn numbers() {
859 assert_debug_snapshot!(lex(
860 "123 -123.456 .456 0.456 123e45 123e+45 123e-45 1. +42.4 -69e12
861 .34e-5 123456789123456789 123456789123456789n -123n"
862 ));
863 }
864
865 #[test]
866 fn binary_numbers() {
867 assert_debug_snapshot!(lex("0b0 0b1 0B1111 -0b01111 +0b10101010 0b0101n"));
868 }
869
870 #[test]
871 fn octal_numbers() {
872 assert_debug_snapshot!(lex("0o0 0o1 0O7777 -0o7777 +0o7777 0o123n"));
873 }
874
875 #[test]
876 fn hexadecimal_numbers() {
877 assert_debug_snapshot!(lex("0x0 0x1 0XFfFF -0xFFFF +0xAaAA 0x123n"));
878 }
879
880 #[test]
881 fn separators() {
882 assert_debug_snapshot!(lex("0b1_0_0 1_000_000 1.0e1_1 1_2.3_4 .55_5 0o7_7 0xf_f"));
883 }
884
885 #[test]
886 fn line_comments() {
887 assert_debug_snapshot!(lex("// foo\n42 // bar\r// baz qux"));
888 }
889
890 #[test]
891 fn block_comments() {
892 assert_debug_snapshot!(lex("1 /* foo */ 2 /* bar\n\nabc */ 3 /*** baz qux **/
893 4 /*\\*/ 5 /*/*/ 6 /**/ 7 /* /* \\*\\/ */ 8"));
894 }
895
896 #[test]
897 fn regexes() {
898 assert_debug_snapshot!(lex(r#" /foo/, /o\/o/, /fo\[[/a\]]a\/]o\\/, /abc/gimsuy "#));
899 }
900
901 #[test]
902 fn lexer_errors() {
903 assert!(matches!(lex(""), Err(LexerError::UnexpectedEndOfInput)));
904 assert!(matches!(lex(" "), Err(LexerError::UnexpectedEndOfInput)));
905 assert!(matches!(lex("\""), Err(LexerError::UnexpectedEndOfInput)));
906 assert!(matches!(lex("'"), Err(LexerError::UnexpectedEndOfInput)));
907 assert!(matches!(
908 lex("'\n'"),
909 Err(LexerError::UnterminatedString { position: 1 })
910 ));
911 assert!(matches!(
912 lex("'\r'"),
913 Err(LexerError::UnterminatedString { position: 1 })
914 ));
915 assert!(matches!(
916 lex("'\\' 12"),
917 Err(LexerError::UnexpectedEndOfInput)
918 ));
919 assert!(matches!(
920 lex("`${` 12"),
921 Err(LexerError::UnexpectedEndOfInput)
922 ));
923 assert!(matches!(
924 lex("'\\x'"),
925 Err(LexerError::UnexpectedCharacter {
926 position: 3,
927 character: '\''
928 })
929 ));
930 assert!(matches!(
931 lex("'\\x0'"),
932 Err(LexerError::UnexpectedCharacter {
933 position: 4,
934 character: '\''
935 })
936 ));
937 assert!(matches!(
938 lex("'\\u0'"),
939 Err(LexerError::UnexpectedCharacter {
940 position: 4,
941 character: '\''
942 })
943 ));
944 assert!(matches!(
945 lex("'\\u01'"),
946 Err(LexerError::UnexpectedCharacter {
947 position: 5,
948 character: '\''
949 })
950 ));
951 assert!(matches!(
952 lex("'\\u012'"),
953 Err(LexerError::UnexpectedCharacter {
954 position: 6,
955 character: '\''
956 })
957 ));
958 assert!(matches!(
959 lex("'\\u012z'"),
960 Err(LexerError::UnexpectedCharacter {
961 position: 6,
962 character: 'z'
963 })
964 ));
965 assert!(matches!(
966 lex("'\\u{1F600A}'"),
967 Err(LexerError::UnicodeOutOfRange)
968 ));
969 assert!(matches!(
970 lex("'\\u{1F600A'"),
971 Err(LexerError::UnexpectedCharacter {
972 position: 10,
973 character: '\''
974 })
975 ));
976 assert!(matches!(
977 lex("'\\u{1F600AAAAA'"),
978 Err(LexerError::UnexpectedCharacter {
979 position: 14,
980 character: '\''
981 })
982 ));
983 assert!(matches!(
984 lex("'\\u{1FW}'"),
985 Err(LexerError::UnexpectedCharacter {
986 position: 6,
987 character: 'W'
988 })
989 ));
990 assert!(matches!(lex("/abc"), Err(LexerError::UnexpectedEndOfInput)));
991 }
992}