1use super::*;
2
3#[derive(Clone, Copy, Debug, PartialEq)]
5pub enum Token<'src> {
6 Code { contents: &'src str },
7 CodeLine { closed: bool, contents: &'src str },
8 Interpolation { contents: &'src str },
9 InterpolationLine { closed: bool, contents: &'src str },
10 Text { contents: &'src str, index: usize },
11}
12
13impl Display for Token<'_> {
14 fn fmt(&self, f: &mut Formatter) -> fmt::Result {
15 let block = self.block();
16
17 if let Some(block) = block {
18 write!(f, "{}", block.open_delimiter())?;
19 }
20
21 write!(f, "{}", self.contents())?;
22
23 match self {
24 Self::CodeLine { closed, .. } | Self::InterpolationLine { closed, .. } if !closed => {}
25 _ => {
26 if let Some(block) = block {
27 write!(f, "{}", block.close_delimiter())?;
28 }
29 }
30 }
31
32 Ok(())
33 }
34}
35
36impl<'src> Token<'src> {
37 pub fn parse(src: &'src str) -> Result<Vec<Self>, Error> {
38 let mut tokens = Vec::new();
39 let mut i = 0;
40 let mut j = 0;
41 let mut index = 0;
42 while j < src.len() {
43 let rest = &src[j..];
44
45 let Some(block) = Block::from_rest(rest) else {
46 j += rest.chars().next().unwrap().len_utf8();
47 continue;
48 };
49
50 let before_open = j;
51 let after_open = before_open + block.open_delimiter().len();
52
53 let (before_close, closed) = match src[after_open..].find(block.close_delimiter()) {
54 Some(before_close) => (after_open + before_close, true),
55 None if block.is_line() => (src.len(), false),
56 None => return Err(Error::Unclosed(block)),
57 };
58
59 let after_close = if closed {
60 before_close + block.close_delimiter().len()
61 } else {
62 before_close
63 };
64
65 let emit_empty = if cfg!(feature = "reload") {
66 let previous_is_code = matches!(
67 tokens.last(),
68 Some(Token::Code { .. } | Token::CodeLine { .. }),
69 );
70
71 let current_is_code = matches! {
72 block,
73 Block::Code | Block::CodeLine,
74 };
75
76 tokens.is_empty() || !(previous_is_code && current_is_code)
77 } else {
78 false
79 };
80
81 if i != j || emit_empty {
82 tokens.push(Self::Text {
83 contents: &src[i..j],
84 index,
85 });
86 index += 1;
87 }
88
89 tokens.push(block.token(&src[after_open..before_close], closed));
90
91 j = after_close;
92 i = after_close;
93 }
94
95 let emit_empty = if cfg!(feature = "reload") {
96 tokens.is_empty() || !matches!(tokens.last(), Some(Token::Text { .. }))
97 } else {
98 false
99 };
100
101 if i != j || emit_empty {
102 tokens.push(Self::Text {
103 contents: &src[i..j],
104 index,
105 });
106 }
107
108 Ok(tokens)
109 }
110
111 fn code(self) -> Option<&'src str> {
112 match self {
113 Self::Code { .. }
114 | Self::CodeLine { .. }
115 | Self::Interpolation { .. }
116 | Self::InterpolationLine { .. } => Some(self.contents().trim()),
117 Self::Text { .. } => None,
118 }
119 }
120
121 fn contents(self) -> &'src str {
122 match self {
123 Self::Code { contents }
124 | Self::CodeLine { contents, .. }
125 | Self::Interpolation { contents }
126 | Self::InterpolationLine { contents, .. }
127 | Self::Text { contents, .. } => contents,
128 }
129 }
130
131 fn block(self) -> Option<Block> {
132 match self {
133 Self::Code { .. } => Some(Block::Code),
134 Self::CodeLine { .. } => Some(Block::CodeLine),
135 Self::Interpolation { .. } => Some(Block::Interpolation),
136 Self::InterpolationLine { .. } => Some(Block::InterpolationLine),
137 Self::Text { .. } => None,
138 }
139 }
140
141 #[must_use]
142 pub fn is_compatible_with(self, other: Self) -> bool {
143 if self.code() != other.code() {
144 return false;
145 }
146
147 if self.block() != other.block() {
148 for token in [self, other] {
149 if !matches!(token, Self::Code { .. } | Self::CodeLine { .. }) {
150 return false;
151 }
152 }
153 }
154
155 if let Self::InterpolationLine { closed, .. } = self {
156 if let Self::InterpolationLine { closed: other, .. } = other {
157 if closed != other {
158 return false;
159 }
160 }
161 }
162
163 true
164 }
165
166 #[must_use]
167 pub fn text(self) -> Option<&'src str> {
168 if let Self::Text { contents, .. } = self {
169 Some(contents)
170 } else {
171 None
172 }
173 }
174}
175
176#[cfg(test)]
177mod tests {
178 use {super::*, pretty_assertions::assert_eq, Token::*};
179
180 #[test]
181 fn compatibility() {
182 #[track_caller]
183 fn case(a: Token, b: Token) {
184 assert!(a.is_compatible_with(b));
185 }
186 case(
187 Text {
188 contents: "foo",
189 index: 0,
190 },
191 Text {
192 contents: "bar",
193 index: 1,
194 },
195 );
196 case(Code { contents: "foo" }, Code { contents: "foo" });
197 case(Code { contents: " foo" }, Code { contents: "foo" });
198 case(Code { contents: "foo " }, Code { contents: "foo" });
199 case(
200 CodeLine {
201 contents: "foo",
202 closed: true,
203 },
204 CodeLine {
205 contents: "foo",
206 closed: true,
207 },
208 );
209 case(
210 CodeLine {
211 contents: "foo",
212 closed: false,
213 },
214 CodeLine {
215 contents: "foo",
216 closed: false,
217 },
218 );
219 case(
220 CodeLine {
221 contents: "foo",
222 closed: true,
223 },
224 CodeLine {
225 contents: "foo",
226 closed: false,
227 },
228 );
229 case(
230 CodeLine {
231 contents: "foo",
232 closed: false,
233 },
234 CodeLine {
235 contents: "foo",
236 closed: true,
237 },
238 );
239 case(
240 Code { contents: "foo" },
241 CodeLine {
242 contents: "foo",
243 closed: true,
244 },
245 );
246 case(
247 CodeLine {
248 contents: "foo",
249 closed: true,
250 },
251 Code { contents: "foo" },
252 );
253 case(
254 Interpolation { contents: "foo" },
255 Interpolation { contents: "foo" },
256 );
257 case(
258 InterpolationLine {
259 contents: "foo",
260 closed: true,
261 },
262 InterpolationLine {
263 contents: "foo",
264 closed: true,
265 },
266 );
267 case(
268 InterpolationLine {
269 contents: "foo",
270 closed: false,
271 },
272 InterpolationLine {
273 contents: "foo",
274 closed: false,
275 },
276 );
277 }
278
279 #[test]
280 fn incompatibility() {
281 #[track_caller]
282 fn case(a: Token, b: Token) {
283 assert!(!a.is_compatible_with(b));
284 }
285 case(
286 Text {
287 contents: "foo",
288 index: 0,
289 },
290 Code { contents: "bar" },
291 );
292 case(Code { contents: "foo" }, Interpolation { contents: "bar" });
293 case(
294 Interpolation { contents: "foo" },
295 InterpolationLine {
296 contents: "bar",
297 closed: false,
298 },
299 );
300 case(
301 InterpolationLine {
302 contents: "foo",
303 closed: true,
304 },
305 InterpolationLine {
306 contents: "bar",
307 closed: true,
308 },
309 );
310 case(
311 InterpolationLine {
312 contents: "foo",
313 closed: true,
314 },
315 InterpolationLine {
316 contents: "foo",
317 closed: false,
318 },
319 );
320 }
321
322 #[track_caller]
323 fn assert_parse(expected: &str, expected_tokens: &[Token]) {
324 let actual_tokens = Token::parse(expected).unwrap();
325 assert_eq!(actual_tokens, expected_tokens);
326 let actual = actual_tokens
327 .iter()
328 .map(ToString::to_string)
329 .collect::<String>();
330 assert_eq!(actual, expected);
331 }
332
333 #[test]
334 fn empty() {
335 assert_parse(
336 "",
337 &[
338 #[cfg(feature = "reload")]
339 Text {
340 contents: "",
341 index: 0,
342 },
343 ],
344 );
345 }
346
347 #[test]
348 fn text() {
349 assert_parse(
350 "foo",
351 &[Text {
352 contents: "foo",
353 index: 0,
354 }],
355 );
356 }
357
358 #[test]
359 fn code() {
360 assert_parse(
361 "{% foo %}",
362 &[
363 #[cfg(feature = "reload")]
364 Text {
365 contents: "",
366 index: 0,
367 },
368 Code { contents: " foo " },
369 #[cfg(feature = "reload")]
370 Text {
371 contents: "",
372 index: 1,
373 },
374 ],
375 );
376 assert_parse(
377 "{%%}",
378 &[
379 #[cfg(feature = "reload")]
380 Text {
381 contents: "",
382 index: 0,
383 },
384 Code { contents: "" },
385 #[cfg(feature = "reload")]
386 Text {
387 contents: "",
388 index: 1,
389 },
390 ],
391 );
392 }
393
394 #[test]
395 fn code_line() {
396 assert_parse(
397 "%% foo\n",
398 &[
399 #[cfg(feature = "reload")]
400 Text {
401 contents: "",
402 index: 0,
403 },
404 CodeLine {
405 contents: " foo",
406 closed: true,
407 },
408 #[cfg(feature = "reload")]
409 Text {
410 contents: "",
411 index: 1,
412 },
413 ],
414 );
415 assert_parse(
416 "%% foo",
417 &[
418 #[cfg(feature = "reload")]
419 Text {
420 contents: "",
421 index: 0,
422 },
423 CodeLine {
424 contents: " foo",
425 closed: false,
426 },
427 #[cfg(feature = "reload")]
428 Text {
429 contents: "",
430 index: 1,
431 },
432 ],
433 );
434 assert_parse(
435 "%%\n",
436 &[
437 #[cfg(feature = "reload")]
438 Text {
439 contents: "",
440 index: 0,
441 },
442 CodeLine {
443 contents: "",
444 closed: true,
445 },
446 #[cfg(feature = "reload")]
447 Text {
448 contents: "",
449 index: 1,
450 },
451 ],
452 );
453 assert_parse(
454 "%%",
455 &[
456 #[cfg(feature = "reload")]
457 Text {
458 contents: "",
459 index: 0,
460 },
461 CodeLine {
462 contents: "",
463 closed: false,
464 },
465 #[cfg(feature = "reload")]
466 Text {
467 contents: "",
468 index: 1,
469 },
470 ],
471 );
472 }
473
474 #[test]
475 fn interpolation() {
476 assert_parse(
477 "{{ foo }}",
478 &[
479 #[cfg(feature = "reload")]
480 Text {
481 contents: "",
482 index: 0,
483 },
484 Interpolation { contents: " foo " },
485 #[cfg(feature = "reload")]
486 Text {
487 contents: "",
488 index: 1,
489 },
490 ],
491 );
492 assert_parse(
493 "{{foo}}",
494 &[
495 #[cfg(feature = "reload")]
496 Text {
497 contents: "",
498 index: 0,
499 },
500 Interpolation { contents: "foo" },
501 #[cfg(feature = "reload")]
502 Text {
503 contents: "",
504 index: 1,
505 },
506 ],
507 );
508 assert_parse(
509 "{{ }}",
510 &[
511 #[cfg(feature = "reload")]
512 Text {
513 contents: "",
514 index: 0,
515 },
516 Interpolation { contents: " " },
517 #[cfg(feature = "reload")]
518 Text {
519 contents: "",
520 index: 1,
521 },
522 ],
523 );
524 assert_parse(
525 "{{}}",
526 &[
527 #[cfg(feature = "reload")]
528 Text {
529 contents: "",
530 index: 0,
531 },
532 Interpolation { contents: "" },
533 #[cfg(feature = "reload")]
534 Text {
535 contents: "",
536 index: 1,
537 },
538 ],
539 );
540 }
541
542 #[test]
543 fn interpolation_line() {
544 assert_parse(
545 "$$ foo\n",
546 &[
547 #[cfg(feature = "reload")]
548 Text {
549 contents: "",
550 index: 0,
551 },
552 InterpolationLine {
553 contents: " foo",
554 closed: true,
555 },
556 #[cfg(feature = "reload")]
557 Text {
558 contents: "",
559 index: 1,
560 },
561 ],
562 );
563 assert_parse(
564 "$$ foo",
565 &[
566 #[cfg(feature = "reload")]
567 Text {
568 contents: "",
569 index: 0,
570 },
571 InterpolationLine {
572 contents: " foo",
573 closed: false,
574 },
575 #[cfg(feature = "reload")]
576 Text {
577 contents: "",
578 index: 1,
579 },
580 ],
581 );
582 assert_parse(
583 "$$\n",
584 &[
585 #[cfg(feature = "reload")]
586 Text {
587 contents: "",
588 index: 0,
589 },
590 InterpolationLine {
591 contents: "",
592 closed: true,
593 },
594 #[cfg(feature = "reload")]
595 Text {
596 contents: "",
597 index: 1,
598 },
599 ],
600 );
601 assert_parse(
602 "$$",
603 &[
604 #[cfg(feature = "reload")]
605 Text {
606 contents: "",
607 index: 0,
608 },
609 InterpolationLine {
610 contents: "",
611 closed: false,
612 },
613 #[cfg(feature = "reload")]
614 Text {
615 contents: "",
616 index: 1,
617 },
618 ],
619 );
620 }
621
622 #[test]
623 fn mixed() {
624 assert_parse(
625 "foo {% bar %} baz",
626 &[
627 Text {
628 contents: "foo ",
629 index: 0,
630 },
631 Code { contents: " bar " },
632 Text {
633 contents: " baz",
634 index: 1,
635 },
636 ],
637 );
638 assert_parse(
639 "{{ foo }} bar {% baz %} bob",
640 &[
641 #[cfg(feature = "reload")]
642 Text {
643 contents: "",
644 index: 0,
645 },
646 Interpolation { contents: " foo " },
647 Text {
648 contents: " bar ",
649 index: cfg!(feature = "reload").into(),
650 },
651 Code { contents: " baz " },
652 Text {
653 contents: " bob",
654 index: if cfg!(feature = "reload") { 2 } else { 1 },
655 },
656 ],
657 );
658 assert_parse(
659 "foo %% bar\nbaz",
660 &[
661 Text {
662 contents: "foo ",
663 index: 0,
664 },
665 CodeLine {
666 contents: " bar",
667 closed: true,
668 },
669 Text {
670 contents: "baz",
671 index: 1,
672 },
673 ],
674 );
675 assert_parse(
676 "foo $$ bar\nbaz",
677 &[
678 Text {
679 contents: "foo ",
680 index: 0,
681 },
682 InterpolationLine {
683 contents: " bar",
684 closed: true,
685 },
686 Text {
687 contents: "baz",
688 index: 1,
689 },
690 ],
691 );
692 assert_parse(
693 "{{ foo }}{{ bar }}{{ baz }}",
694 &[
695 #[cfg(feature = "reload")]
696 Text {
697 contents: "",
698 index: 0,
699 },
700 Interpolation { contents: " foo " },
701 #[cfg(feature = "reload")]
702 Text {
703 contents: "",
704 index: 1,
705 },
706 Interpolation { contents: " bar " },
707 #[cfg(feature = "reload")]
708 Text {
709 contents: "",
710 index: 2,
711 },
712 Interpolation { contents: " baz " },
713 #[cfg(feature = "reload")]
714 Text {
715 contents: "",
716 index: 3,
717 },
718 ],
719 );
720 assert_parse(
721 "a {{ b }} c {{ d }} e",
722 &[
723 Text {
724 contents: "a ",
725 index: 0,
726 },
727 Interpolation { contents: " b " },
728 Text {
729 contents: " c ",
730 index: 1,
731 },
732 Interpolation { contents: " d " },
733 Text {
734 contents: " e",
735 index: 2,
736 },
737 ],
738 );
739 assert_parse(
740 "foo {% bar %} baz {% bob %} bill",
741 &[
742 Text {
743 contents: "foo ",
744 index: 0,
745 },
746 Code { contents: " bar " },
747 Text {
748 contents: " baz ",
749 index: 1,
750 },
751 Code { contents: " bob " },
752 Text {
753 contents: " bill",
754 index: 2,
755 },
756 ],
757 );
758 assert_parse(
759 "foo %% bar\nbaz %% bob\nbill",
760 &[
761 Text {
762 contents: "foo ",
763 index: 0,
764 },
765 CodeLine {
766 contents: " bar",
767 closed: true,
768 },
769 Text {
770 contents: "baz ",
771 index: 1,
772 },
773 CodeLine {
774 contents: " bob",
775 closed: true,
776 },
777 Text {
778 contents: "bill",
779 index: 2,
780 },
781 ],
782 );
783 assert_parse(
784 "text {{ interp }} more {% code %} text %% line\n$$ value\nend",
785 &[
786 Text {
787 contents: "text ",
788 index: 0,
789 },
790 Interpolation {
791 contents: " interp ",
792 },
793 Text {
794 contents: " more ",
795 index: 1,
796 },
797 Code { contents: " code " },
798 Text {
799 contents: " text ",
800 index: 2,
801 },
802 CodeLine {
803 contents: " line",
804 closed: true,
805 },
806 #[cfg(feature = "reload")]
807 Text {
808 contents: "",
809 index: 3,
810 },
811 InterpolationLine {
812 contents: " value",
813 closed: true,
814 },
815 Text {
816 contents: "end",
817 index: if cfg!(feature = "reload") { 4 } else { 3 },
818 },
819 ],
820 );
821 }
822
823 #[test]
824 fn delimiters() {
825 assert_parse(
826 "{ } % $ %} }}",
827 &[Text {
828 contents: "{ } % $ %} }}",
829 index: 0,
830 }],
831 );
832 assert_parse(
833 "%}",
834 &[Text {
835 contents: "%}",
836 index: 0,
837 }],
838 );
839 assert_parse(
840 "}}",
841 &[Text {
842 contents: "}}",
843 index: 0,
844 }],
845 );
846 }
847
848 #[test]
849 fn nesting() {
850 assert_parse(
851 "{{ foo {{ bar }}",
852 &[
853 #[cfg(feature = "reload")]
854 Text {
855 contents: "",
856 index: 0,
857 },
858 Interpolation {
859 contents: " foo {{ bar ",
860 },
861 #[cfg(feature = "reload")]
862 Text {
863 contents: "",
864 index: 1,
865 },
866 ],
867 );
868 assert_parse(
869 "{% foo {% bar %}",
870 &[
871 #[cfg(feature = "reload")]
872 Text {
873 contents: "",
874 index: 0,
875 },
876 Code {
877 contents: " foo {% bar ",
878 },
879 #[cfg(feature = "reload")]
880 Text {
881 contents: "",
882 index: 1,
883 },
884 ],
885 );
886 }
887
888 #[test]
889 fn unicode() {
890 assert_parse(
891 "Hello 世界",
892 &[Text {
893 contents: "Hello 世界",
894 index: 0,
895 }],
896 );
897 assert_parse(
898 "{{ 日本語 }}",
899 &[
900 #[cfg(feature = "reload")]
901 Text {
902 contents: "",
903 index: 0,
904 },
905 Interpolation {
906 contents: " 日本語 ",
907 },
908 #[cfg(feature = "reload")]
909 Text {
910 contents: "",
911 index: 1,
912 },
913 ],
914 );
915 assert_parse(
916 "{% émoji 🚀 %}",
917 &[
918 #[cfg(feature = "reload")]
919 Text {
920 contents: "",
921 index: 0,
922 },
923 Code {
924 contents: " émoji 🚀 ",
925 },
926 #[cfg(feature = "reload")]
927 Text {
928 contents: "",
929 index: 1,
930 },
931 ],
932 );
933 assert_parse(
934 "%% unicode line 中文\n",
935 &[
936 #[cfg(feature = "reload")]
937 Text {
938 contents: "",
939 index: 0,
940 },
941 CodeLine {
942 contents: " unicode line 中文",
943 closed: true,
944 },
945 #[cfg(feature = "reload")]
946 Text {
947 contents: "",
948 index: 1,
949 },
950 ],
951 );
952 assert_parse(
953 "$$ emoji 🎉\n",
954 &[
955 #[cfg(feature = "reload")]
956 Text {
957 contents: "",
958 index: 0,
959 },
960 InterpolationLine {
961 contents: " emoji 🎉",
962 closed: true,
963 },
964 #[cfg(feature = "reload")]
965 Text {
966 contents: "",
967 index: 1,
968 },
969 ],
970 );
971 }
972
973 #[test]
974 fn whitespace() {
975 assert_parse(
976 " foo",
977 &[Text {
978 contents: " foo",
979 index: 0,
980 }],
981 );
982 assert_parse(
983 "foo ",
984 &[Text {
985 contents: "foo ",
986 index: 0,
987 }],
988 );
989 assert_parse(
990 " {{ foo }} ",
991 &[
992 Text {
993 contents: " ",
994 index: 0,
995 },
996 Interpolation {
997 contents: " foo ",
998 },
999 Text {
1000 contents: " ",
1001 index: 1,
1002 },
1003 ],
1004 );
1005 assert_parse(
1006 "\t\tfoo\t\t",
1007 &[Text {
1008 contents: "\t\tfoo\t\t",
1009 index: 0,
1010 }],
1011 );
1012 assert_parse(
1013 "\n\nfoo\n\n",
1014 &[Text {
1015 contents: "\n\nfoo\n\n",
1016 index: 0,
1017 }],
1018 );
1019 }
1020
1021 #[test]
1022 fn complex() {
1023 assert_parse(
1024 "Hello {{ name }}!
1025{% for item in items { %}
1026Item: {{ item }}
1027{% } %}
1028Done.",
1029 &[
1030 Text {
1031 contents: "Hello ",
1032 index: 0,
1033 },
1034 Interpolation { contents: " name " },
1035 Text {
1036 contents: "!\n",
1037 index: 1,
1038 },
1039 Code {
1040 contents: " for item in items { ",
1041 },
1042 Text {
1043 contents: "\nItem: ",
1044 index: 2,
1045 },
1046 Interpolation { contents: " item " },
1047 Text {
1048 contents: "\n",
1049 index: 3,
1050 },
1051 Code { contents: " } " },
1052 Text {
1053 contents: "\nDone.",
1054 index: 4,
1055 },
1056 ],
1057 );
1058 }
1059
1060 #[test]
1061 fn unclosed() {
1062 assert_eq!(Token::parse("{%"), Err(Error::Unclosed(Block::Code)),);
1063 assert_eq!(
1064 Token::parse("{{"),
1065 Err(Error::Unclosed(Block::Interpolation)),
1066 );
1067 }
1068}