1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3
4#[derive(Debug, Clone, Default, PartialEq, Eq)]
10pub struct ParsedImage {
11 pub src: String,
12 pub alt: String,
13 pub width: i64,
19 pub height: i64,
20}
21
22#[derive(Debug, Clone, Default)]
29pub struct ParsedSpan {
30 pub text: String,
31 pub bold: bool,
32 pub italic: bool,
33 pub underline: bool,
34 pub strikeout: bool,
35 pub code: bool,
36 pub superscript: bool,
38 pub subscript: bool,
40 pub link_href: Option<String>,
41 pub image: Option<ParsedImage>,
43 pub footnote_ref: Option<String>,
46}
47
48#[derive(Debug, Clone)]
50pub struct ParsedTableCell {
51 pub spans: Vec<ParsedSpan>,
52}
53
54#[derive(Debug, Clone)]
56pub struct ParsedTable {
57 pub header_rows: usize,
59 pub rows: Vec<Vec<ParsedTableCell>>,
61 pub blockquote_depth: u32,
64 pub blockquote_opens: u32,
67}
68
69impl ParsedTable {
70 pub fn column_count(&self) -> usize {
80 self.rows.iter().map(Vec::len).max().unwrap_or(0)
81 }
82}
83
84const SQUARED_CELLS_ALWAYS: usize = 4096;
88
89const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
99
100fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
111 let squared = table.rows.len().saturating_mul(table.column_count());
112 let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
113 if squared <= allowed {
114 elements.push(ParsedElement::Table(table));
115 return;
116 }
117 elements.extend(
118 ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
119 .into_iter()
120 .filter(|block| spans_carry_content(&block.spans))
121 .map(ParsedElement::Block),
122 );
123}
124
125fn mark_quotation_opened(elements: &mut [ParsedElement], first: usize) {
129 let Some(rest) = elements.get_mut(first..) else {
130 return;
131 };
132 for element in rest {
133 match element {
134 ParsedElement::Block(block) => {
135 block.blockquote_opens += 1;
136 return;
137 }
138 ParsedElement::Table(table) => {
139 table.blockquote_opens += 1;
140 return;
141 }
142 ParsedElement::FootnoteDefinition { .. } => {}
143 }
144 }
145}
146
147fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
149 rows.iter().map(Vec::len).sum()
150}
151
152#[derive(Debug, Clone)]
154pub enum ParsedElement {
155 Block(ParsedBlock),
156 Table(ParsedTable),
157 FootnoteDefinition {
164 label: String,
165 blocks: Vec<ParsedBlock>,
166 },
167}
168
169impl ParsedElement {
170 pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
173 let mut blocks = Vec::new();
174 for elem in elements {
175 match elem {
176 ParsedElement::Block(b) => blocks.push(b),
177 ParsedElement::FootnoteDefinition { .. } => {}
182 ParsedElement::Table(t) => {
183 for row in t.rows {
184 for cell in row {
185 blocks.push(ParsedBlock {
186 spans: cell.spans,
187 heading_level: None,
188 list_style: None,
189 list_indent: 0,
190 list_prefix: String::new(),
191 list_suffix: String::new(),
192 list_start: None,
193 marker: None,
194 is_code_block: false,
195 code_language: None,
196 blockquote_depth: t.blockquote_depth,
197 blockquote_opens: 0,
198 line_height: None,
199 non_breakable_lines: None,
200 page_break_before: None,
201 direction: None,
202 background_color: None,
203 alignment: None,
204 top_margin: None,
205 text_indent: None,
206 semantic_role: None,
207 });
208 }
209 }
210 }
211 }
212 }
213 if blocks.is_empty() {
214 blocks.push(ParsedBlock {
215 spans: vec![ParsedSpan {
216 text: String::new(),
217 ..Default::default()
218 }],
219 heading_level: None,
220 list_style: None,
221 list_indent: 0,
222 list_prefix: String::new(),
223 list_suffix: String::new(),
224 list_start: None,
225 marker: None,
226 is_code_block: false,
227 code_language: None,
228 blockquote_depth: 0,
229 blockquote_opens: 0,
230 line_height: None,
231 non_breakable_lines: None,
232 page_break_before: None,
233 direction: None,
234 background_color: None,
235 alignment: None,
236 top_margin: None,
237 text_indent: None,
238 semantic_role: None,
239 });
240 }
241 blocks
242 }
243}
244
245#[derive(Debug, Clone, Default)]
252pub struct ParsedBlock {
253 pub spans: Vec<ParsedSpan>,
254 pub heading_level: Option<i64>,
255 pub list_style: Option<ListStyle>,
256 pub list_indent: u32,
257 pub list_prefix: String,
260 pub list_suffix: String,
263 pub list_start: Option<i64>,
267 pub marker: Option<MarkerType>,
270 pub is_code_block: bool,
271 pub code_language: Option<String>,
272 pub blockquote_depth: u32,
273 pub blockquote_opens: u32,
280 pub line_height: Option<i64>,
281 pub non_breakable_lines: Option<bool>,
282 pub page_break_before: Option<bool>,
285 pub direction: Option<TextDirection>,
286 pub background_color: Option<String>,
287 pub alignment: Option<Alignment>,
290 pub top_margin: Option<i64>,
294 pub text_indent: Option<i64>,
298 pub semantic_role: Option<SemanticRole>,
302}
303
304impl ParsedBlock {
305 pub fn is_inline_only(&self) -> bool {
308 self.heading_level.is_none()
309 && self.list_style.is_none()
310 && !self.is_code_block
311 && self.blockquote_depth == 0
312 && self.line_height.is_none()
313 && self.non_breakable_lines.is_none()
314 && self.page_break_before.is_none()
315 && self.direction.is_none()
316 && self.background_color.is_none()
317 && self.alignment.is_none()
318 && self.top_margin.is_none()
319 && self.text_indent.is_none()
320 }
321}
322
323fn dangling_footnote_labels(
352 markdown: &str,
353 options: pulldown_cmark::Options,
354) -> std::collections::BTreeSet<String> {
355 use pulldown_cmark::{Event, Parser, Tag};
356
357 let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
358 for event in Parser::new_ext(markdown, options) {
359 if let Event::Start(Tag::FootnoteDefinition(label)) = event {
360 defined.insert(label.to_string());
361 }
362 }
363
364 let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
365 let bytes = markdown.as_bytes();
366 let mut search_from = 0usize;
367 while let Some(rel) = markdown[search_from..].find("[^") {
368 let open = search_from + rel;
369 let label_start = open + 2;
370 let Some(close_rel) = markdown[label_start..].find(']') else {
371 break;
372 };
373 let close = label_start + close_rel;
374 let label = &markdown[label_start..close];
375 let looks_like_definition = bytes.get(close + 1) == Some(&b':');
380 if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
381 referenced.insert(label.to_string());
382 }
383 search_from = close + 1;
384 }
385
386 referenced.difference(&defined).cloned().collect()
387}
388
389pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
397 use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
398
399 let options = Options::ENABLE_STRIKETHROUGH
410 | Options::ENABLE_TABLES
411 | Options::ENABLE_TASKLISTS
412 | Options::ENABLE_FOOTNOTES;
413
414 let dangling = dangling_footnote_labels(markdown, options);
421 let augmented_owner;
422 let source: &str = if dangling.is_empty() {
423 markdown
424 } else {
425 augmented_owner = dangling
426 .iter()
427 .fold(markdown.to_string(), |mut acc, label| {
428 acc.push_str("\n\n[^");
429 acc.push_str(label);
430 acc.push_str("]:\n");
431 acc
432 });
433 &augmented_owner
434 };
435 let parser = Parser::new_ext(source, options);
436
437 let mut elements: Vec<ParsedElement> = Vec::new();
438 let mut current_spans: Vec<ParsedSpan> = Vec::new();
439 let mut current_heading: Option<i64> = None;
440 let mut current_list_style: Option<ListStyle> = None;
441 let mut current_list_start: Option<i64> = None;
442 let mut is_code_block = false;
443 let mut code_language: Option<String> = None;
444 let mut blockquote_depth: u32 = 0;
445 let mut quotations_open: Vec<usize> = Vec::new();
447 let mut in_block = false;
448
449 let mut bold = false;
451 let mut italic = false;
452 let mut strikeout = false;
453 let mut link_href: Option<String> = None;
454 let mut pending_image: Option<ParsedImage> = None;
456
457 let mut footnote_open: Option<(String, usize)> = None;
464
465 let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
468 let mut list_start_stack: Vec<Option<i64>> = Vec::new();
469 let mut current_list_indent: u32 = 0;
470
471 let mut in_table = false;
473 let mut in_table_head = false;
474 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
475 let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
476 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
477 let mut table_header_rows: usize = 0;
478
479 for event in parser {
480 match event {
481 Event::Start(Tag::Paragraph) => {
482 in_block = true;
483 current_heading = None;
484 is_code_block = false;
485 }
486 Event::End(TagEnd::Paragraph) => {
487 if !current_spans.is_empty() || in_block {
488 elements.push(ParsedElement::Block(ParsedBlock {
489 spans: std::mem::take(&mut current_spans),
490 heading_level: current_heading.take(),
491 list_style: current_list_style.clone(),
492 list_indent: current_list_indent,
493 list_prefix: String::new(),
494 list_suffix: String::new(),
495 list_start: current_list_start,
496 marker: None,
497 is_code_block: false,
498 code_language: None,
499 blockquote_depth,
500 blockquote_opens: 0,
501 line_height: None,
502 non_breakable_lines: None,
503 page_break_before: None,
504 direction: None,
505 background_color: None,
506 alignment: None,
507 top_margin: None,
508 text_indent: None,
509 semantic_role: None,
510 }));
511 }
512 in_block = false;
513 current_list_style = None;
514 current_list_start = None;
515 }
516 Event::Start(Tag::Heading { level, .. }) => {
517 in_block = true;
518 current_heading = Some(heading_level_to_i64(level));
519 is_code_block = false;
520 }
521 Event::End(TagEnd::Heading(_)) => {
522 elements.push(ParsedElement::Block(ParsedBlock {
523 spans: std::mem::take(&mut current_spans),
524 heading_level: current_heading.take(),
525 list_style: None,
526 list_indent: 0,
527 list_prefix: String::new(),
528 list_suffix: String::new(),
529 list_start: None,
530 marker: None,
531 is_code_block: false,
532 code_language: None,
533 blockquote_depth,
534 blockquote_opens: 0,
535 line_height: None,
536 non_breakable_lines: None,
537 page_break_before: None,
538 direction: None,
539 background_color: None,
540 alignment: None,
541 top_margin: None,
542 text_indent: None,
543 semantic_role: None,
544 }));
545 in_block = false;
546 }
547 Event::Start(Tag::List(ordered)) => {
548 let style = if ordered.is_some() {
549 Some(ListStyle::Decimal)
550 } else {
551 Some(ListStyle::Disc)
552 };
553 list_stack.push(style);
554 list_start_stack.push(ordered.and_then(list_start_of));
555 }
556 Event::End(TagEnd::List(_)) => {
557 list_stack.pop();
558 list_start_stack.pop();
559 }
560 Event::Start(Tag::Item) => {
561 if !current_spans.is_empty() {
564 elements.push(ParsedElement::Block(ParsedBlock {
565 spans: std::mem::take(&mut current_spans),
566 heading_level: None,
567 list_style: current_list_style.clone(),
568 list_indent: current_list_indent,
569 list_prefix: String::new(),
570 list_suffix: String::new(),
571 list_start: current_list_start,
572 marker: None,
573 is_code_block: false,
574 code_language: None,
575 blockquote_depth,
576 blockquote_opens: 0,
577 line_height: None,
578 non_breakable_lines: None,
579 page_break_before: None,
580 direction: None,
581 background_color: None,
582 alignment: None,
583 top_margin: None,
584 text_indent: None,
585 semantic_role: None,
586 }));
587 }
588 in_block = true;
589 current_list_style = list_stack.last().cloned().flatten();
590 current_list_start = list_start_stack.last().copied().flatten();
591 current_list_indent = if list_stack.is_empty() {
592 0
593 } else {
594 (list_stack.len() - 1) as u32
595 };
596 }
597 Event::End(TagEnd::Item) => {
598 if !current_spans.is_empty() {
601 elements.push(ParsedElement::Block(ParsedBlock {
602 spans: std::mem::take(&mut current_spans),
603 heading_level: None,
604 list_style: current_list_style.clone(),
605 list_indent: current_list_indent,
606 list_prefix: String::new(),
607 list_suffix: String::new(),
608 list_start: current_list_start,
609 marker: None,
610 is_code_block: false,
611 code_language: None,
612 blockquote_depth,
613 blockquote_opens: 0,
614 line_height: None,
615 non_breakable_lines: None,
616 page_break_before: None,
617 direction: None,
618 background_color: None,
619 alignment: None,
620 top_margin: None,
621 text_indent: None,
622 semantic_role: None,
623 }));
624 }
625 in_block = false;
626 current_list_style = None;
627 current_list_start = None;
628 }
629 Event::Start(Tag::CodeBlock(kind)) => {
630 in_block = true;
631 is_code_block = true;
632 code_language = match &kind {
633 pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
634 Some(lang.to_string())
635 }
636 _ => None,
637 };
638 }
639 Event::End(TagEnd::CodeBlock) => {
640 if let Some(last) = current_spans.last_mut()
642 && last.text.ends_with('\n')
643 {
644 last.text.truncate(last.text.len() - 1);
645 }
646 elements.push(ParsedElement::Block(ParsedBlock {
647 spans: std::mem::take(&mut current_spans),
648 heading_level: None,
649 list_style: None,
650 list_indent: 0,
651 list_prefix: String::new(),
652 list_suffix: String::new(),
653 list_start: None,
654 marker: None,
655 is_code_block: true,
656 code_language: code_language.take(),
657 blockquote_depth,
658 blockquote_opens: 0,
659 line_height: None,
660 non_breakable_lines: None,
661 page_break_before: None,
662 direction: None,
663 background_color: None,
664 alignment: None,
665 top_margin: None,
666 text_indent: None,
667 semantic_role: None,
668 }));
669 in_block = false;
670 is_code_block = false;
671 }
672 Event::Start(Tag::Table(_)) => {
674 in_table = true;
675 in_table_head = false;
676 table_rows.clear();
677 current_row_cells.clear();
678 current_cell_spans.clear();
679 table_header_rows = 0;
680 }
681 Event::End(TagEnd::Table) => {
682 let own_cells = cells_in_rows(&table_rows);
683 push_table(
684 &mut elements,
685 ParsedTable {
686 header_rows: table_header_rows,
687 rows: std::mem::take(&mut table_rows),
688 blockquote_depth,
689 blockquote_opens: 0,
690 },
691 own_cells,
692 );
693 in_table = false;
694 }
695 Event::Start(Tag::TableHead) => {
696 in_table_head = true;
697 current_row_cells.clear();
698 }
699 Event::End(TagEnd::TableHead) => {
700 table_rows.push(std::mem::take(&mut current_row_cells));
702 table_header_rows += 1;
703 in_table_head = false;
704 }
705 Event::Start(Tag::TableRow) => {
706 current_row_cells.clear();
707 }
708 Event::End(TagEnd::TableRow) if !in_table_head => {
709 table_rows.push(std::mem::take(&mut current_row_cells));
711 }
712 Event::Start(Tag::TableCell) => {
713 current_cell_spans.clear();
714 }
715 Event::End(TagEnd::TableCell) => {
716 current_row_cells.push(ParsedTableCell {
717 spans: std::mem::take(&mut current_cell_spans),
718 });
719 }
720 Event::Start(Tag::Emphasis) => {
722 italic = true;
723 }
724 Event::End(TagEnd::Emphasis) => {
725 italic = false;
726 }
727 Event::Start(Tag::Strong) => {
728 bold = true;
729 }
730 Event::End(TagEnd::Strong) => {
731 bold = false;
732 }
733 Event::Start(Tag::Strikethrough) => {
734 strikeout = true;
735 }
736 Event::End(TagEnd::Strikethrough) => {
737 strikeout = false;
738 }
739 Event::Start(Tag::Link { dest_url, .. }) => {
740 link_href = Some(dest_url.to_string());
741 }
742 Event::End(TagEnd::Link) => {
743 link_href = None;
744 }
745 Event::Start(Tag::Image { dest_url, .. }) => {
748 pending_image = Some(ParsedImage {
749 src: dest_url.to_string(),
750 alt: String::new(),
751 width: 0,
752 height: 0,
753 });
754 }
755 Event::End(TagEnd::Image) => {
756 if let Some(image) = pending_image.take() {
757 let span = ParsedSpan {
758 text: String::new(),
759 bold,
760 italic,
761 underline: false,
762 strikeout,
763 code: false,
764 superscript: false,
765 subscript: false,
766 link_href: link_href.clone(),
767 image: Some(image),
768 footnote_ref: None,
769 };
770 if in_table {
771 current_cell_spans.push(span);
772 } else {
773 if !in_block {
774 in_block = true;
775 }
776 current_spans.push(span);
777 }
778 }
779 }
780 Event::Text(text) => {
781 if let Some(img) = pending_image.as_mut() {
786 img.alt.push_str(&text);
787 continue;
788 }
789 let span = ParsedSpan {
790 text: text.to_string(),
791 bold,
792 italic,
793 underline: false,
794 strikeout,
795 code: is_code_block,
796 superscript: false,
797 subscript: false,
798 link_href: link_href.clone(),
799 image: None,
800 footnote_ref: None,
801 };
802 if in_table {
803 current_cell_spans.push(span);
804 } else {
805 if !in_block {
806 in_block = true;
807 }
808 current_spans.push(span);
809 }
810 }
811 Event::Code(text) => {
812 let span = ParsedSpan {
813 text: text.to_string(),
814 bold,
815 italic,
816 underline: false,
817 strikeout,
818 code: true,
819 superscript: false,
820 subscript: false,
821 link_href: link_href.clone(),
822 image: None,
823 footnote_ref: None,
824 };
825 if in_table {
826 current_cell_spans.push(span);
827 } else {
828 if !in_block {
829 in_block = true;
830 }
831 current_spans.push(span);
832 }
833 }
834 Event::SoftBreak => {
835 let span = ParsedSpan {
836 text: " ".to_string(),
837 bold,
838 italic,
839 underline: false,
840 strikeout,
841 code: false,
842 superscript: false,
843 subscript: false,
844 link_href: link_href.clone(),
845 image: None,
846 footnote_ref: None,
847 };
848 if in_table {
849 current_cell_spans.push(span);
850 } else {
851 current_spans.push(span);
852 }
853 }
854 Event::HardBreak if !current_spans.is_empty() || in_block => {
855 elements.push(ParsedElement::Block(ParsedBlock {
857 spans: std::mem::take(&mut current_spans),
858 heading_level: current_heading.take(),
859 list_style: current_list_style.clone(),
860 list_indent: current_list_indent,
861 list_prefix: String::new(),
862 list_suffix: String::new(),
863 list_start: current_list_start,
864 marker: None,
865 is_code_block,
866 code_language: code_language.clone(),
867 blockquote_depth,
868 blockquote_opens: 0,
869 line_height: None,
870 non_breakable_lines: None,
871 page_break_before: None,
872 direction: None,
873 background_color: None,
874 alignment: None,
875 top_margin: None,
876 text_indent: None,
877 semantic_role: None,
878 }));
879 }
880 Event::Start(Tag::BlockQuote(_)) => {
881 blockquote_depth += 1;
882 quotations_open.push(elements.len());
883 }
884 Event::End(TagEnd::BlockQuote(_)) => {
885 blockquote_depth = blockquote_depth.saturating_sub(1);
886 if let Some(first) = quotations_open.pop() {
887 mark_quotation_opened(&mut elements, first);
888 }
889 }
890 Event::Start(Tag::FootnoteDefinition(label)) => {
901 if !current_spans.is_empty() {
902 elements.push(ParsedElement::Block(ParsedBlock {
903 spans: std::mem::take(&mut current_spans),
904 heading_level: current_heading.take(),
905 list_style: current_list_style.clone(),
906 list_indent: current_list_indent,
907 list_prefix: String::new(),
908 list_suffix: String::new(),
909 list_start: current_list_start,
910 marker: None,
911 is_code_block: false,
912 code_language: None,
913 blockquote_depth,
914 blockquote_opens: 0,
915 line_height: None,
916 non_breakable_lines: None,
917 page_break_before: None,
918 direction: None,
919 background_color: None,
920 alignment: None,
921 top_margin: None,
922 text_indent: None,
923 semantic_role: None,
924 }));
925 }
926 footnote_open = Some((label.to_string(), elements.len()));
927 }
928 Event::End(TagEnd::FootnoteDefinition) => {
929 if !current_spans.is_empty() {
930 elements.push(ParsedElement::Block(ParsedBlock {
931 spans: std::mem::take(&mut current_spans),
932 heading_level: current_heading.take(),
933 list_style: current_list_style.clone(),
934 list_indent: current_list_indent,
935 list_prefix: String::new(),
936 list_suffix: String::new(),
937 list_start: current_list_start,
938 marker: None,
939 is_code_block: false,
940 code_language: None,
941 blockquote_depth,
942 blockquote_opens: 0,
943 line_height: None,
944 non_breakable_lines: None,
945 page_break_before: None,
946 direction: None,
947 background_color: None,
948 alignment: None,
949 top_margin: None,
950 text_indent: None,
951 semantic_role: None,
952 }));
953 }
954 if let Some((label, start)) = footnote_open.take() {
955 let blocks: Vec<ParsedBlock> = elements
956 .drain(start..)
957 .filter_map(|e| match e {
958 ParsedElement::Block(b) => Some(b),
959 _ => None,
962 })
963 .collect();
964 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
965 }
966 }
967 Event::FootnoteReference(label) => {
973 let span = ParsedSpan {
974 text: String::new(),
975 bold,
976 italic,
977 underline: false,
978 strikeout,
979 code: false,
980 superscript: false,
981 subscript: false,
982 link_href: link_href.clone(),
983 image: None,
984 footnote_ref: Some(label.to_string()),
985 };
986 if in_table {
987 current_cell_spans.push(span);
988 } else {
989 if !in_block {
990 in_block = true;
991 }
992 current_spans.push(span);
993 }
994 }
995 _ => {}
996 }
997 }
998
999 if !current_spans.is_empty() {
1001 elements.push(ParsedElement::Block(ParsedBlock {
1002 spans: std::mem::take(&mut current_spans),
1003 heading_level: current_heading,
1004 list_style: current_list_style,
1005 list_indent: current_list_indent,
1006 list_prefix: String::new(),
1007 list_suffix: String::new(),
1008 list_start: current_list_start,
1009 marker: None,
1010 is_code_block,
1011 code_language: code_language.take(),
1012 blockquote_depth,
1013 blockquote_opens: 0,
1014 line_height: None,
1015 non_breakable_lines: None,
1016 page_break_before: None,
1017 direction: None,
1018 background_color: None,
1019 alignment: None,
1020 top_margin: None,
1021 text_indent: None,
1022 semantic_role: None,
1023 }));
1024 }
1025
1026 if !dangling.is_empty() {
1033 elements.retain(
1034 |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
1035 );
1036 }
1037
1038 if elements.is_empty() {
1040 elements.push(ParsedElement::Block(ParsedBlock {
1041 spans: vec![ParsedSpan {
1042 text: String::new(),
1043 ..Default::default()
1044 }],
1045 heading_level: None,
1046 list_style: None,
1047 list_indent: 0,
1048 list_prefix: String::new(),
1049 list_suffix: String::new(),
1050 list_start: None,
1051 marker: None,
1052 is_code_block: false,
1053 code_language: None,
1054 blockquote_depth: 0,
1055 blockquote_opens: 0,
1056 line_height: None,
1057 non_breakable_lines: None,
1058 page_break_before: None,
1059 direction: None,
1060 background_color: None,
1061 alignment: None,
1062 top_margin: None,
1063 text_indent: None,
1064 semantic_role: None,
1065 }));
1066 }
1067
1068 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1071 elements
1072}
1073
1074fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
1075 use pulldown_cmark::HeadingLevel;
1076 match level {
1077 HeadingLevel::H1 => 1,
1078 HeadingLevel::H2 => 2,
1079 HeadingLevel::H3 => 3,
1080 HeadingLevel::H4 => 4,
1081 HeadingLevel::H5 => 5,
1082 HeadingLevel::H6 => 6,
1083 }
1084}
1085
1086use scraper::Node;
1089
1090#[derive(Debug, Clone, Default)]
1092struct BlockStyles {
1093 line_height: Option<i64>,
1094 non_breakable_lines: Option<bool>,
1095 page_break_before: Option<bool>,
1096 direction: Option<TextDirection>,
1097 background_color: Option<String>,
1098 preserve_whitespace: Option<bool>,
1103}
1104
1105fn parse_block_styles(style: &str) -> BlockStyles {
1109 let mut result = BlockStyles::default();
1110 for part in style.split(';') {
1111 let part = part.trim();
1112 if let Some((prop, val)) = part.split_once(':') {
1113 let prop = prop.trim().to_ascii_lowercase();
1114 let val = val.trim();
1115 match prop.as_str() {
1116 "line-height" => {
1117 if let Ok(v) = val.parse::<f64>() {
1119 result.line_height = Some((v * 1000.0) as i64);
1120 }
1121 }
1122 "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1123 result.non_breakable_lines = Some(true);
1124 if val != "nowrap" {
1125 result.preserve_whitespace = Some(true);
1126 }
1127 }
1128 "break-before" | "page-break-before" => {
1132 result.page_break_before = match val.to_ascii_lowercase().as_str() {
1133 "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1134 "avoid" | "auto" => Some(false),
1135 _ => None,
1136 };
1137 }
1138 "direction" => {
1139 if val.eq_ignore_ascii_case("rtl") {
1140 result.direction = Some(TextDirection::RightToLeft);
1141 } else if val.eq_ignore_ascii_case("ltr") {
1142 result.direction = Some(TextDirection::LeftToRight);
1143 }
1144 }
1145 "background-color" | "background" => {
1146 result.background_color = Some(val.to_string());
1147 }
1148 _ => {}
1149 }
1150 }
1151 }
1152 result
1153}
1154
1155fn is_html_space(ch: char) -> bool {
1162 matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1163}
1164
1165fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1167 spans
1168 .iter()
1169 .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1170}
1171
1172fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1186 fn is_replaced(span: &ParsedSpan) -> bool {
1187 span.image.is_some() || span.footnote_ref.is_some()
1188 }
1189
1190 for span in spans.iter_mut() {
1194 if is_replaced(span) {
1195 continue;
1196 }
1197 let mut out = String::with_capacity(span.text.len());
1198 let mut in_run = false;
1199 for ch in span.text.chars() {
1200 if is_html_space(ch) {
1201 if !in_run {
1202 out.push(' ');
1203 in_run = true;
1204 }
1205 } else {
1206 out.push(ch);
1207 in_run = false;
1208 }
1209 }
1210 span.text = out;
1211 }
1212
1213 let mut prev_ends_with_space = true;
1216 for span in spans.iter_mut() {
1217 if is_replaced(span) {
1218 prev_ends_with_space = false;
1219 continue;
1220 }
1221 if span.text.is_empty() {
1222 prev_ends_with_space = false;
1226 continue;
1227 }
1228 if prev_ends_with_space && span.text.starts_with(' ') {
1229 span.text.remove(0);
1230 }
1231 if !span.text.is_empty() {
1234 prev_ends_with_space = span.text.ends_with(' ');
1235 }
1236 }
1237
1238 for span in spans.iter_mut().rev() {
1242 if is_replaced(span) {
1243 break;
1244 }
1245 if span.text.is_empty() {
1246 continue;
1247 }
1248 if span.text.ends_with(' ') {
1249 span.text.pop();
1250 }
1251 break;
1252 }
1253}
1254
1255pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
1276 let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
1277 if !block.spans.iter().any(breaks) {
1278 return vec![block];
1279 }
1280 let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
1281 for mut span in std::mem::take(&mut block.spans) {
1282 if !breaks(&span) {
1283 push_to_last_line(&mut lines, span);
1284 continue;
1285 }
1286 let text = std::mem::take(&mut span.text);
1287 for (i, piece) in text.split('\n').enumerate() {
1288 if i > 0 {
1289 lines.push(Vec::new());
1290 }
1291 if !piece.is_empty() {
1292 push_to_last_line(
1293 &mut lines,
1294 ParsedSpan {
1295 text: piece.to_string(),
1296 ..span.clone()
1297 },
1298 );
1299 }
1300 }
1301 }
1302 if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
1303 lines.pop();
1304 }
1305 let later_line = ParsedBlock {
1306 page_break_before: None,
1307 blockquote_opens: 0,
1308 ..block.clone()
1309 };
1310 let mut first_line = Some(block);
1311 lines
1312 .into_iter()
1313 .map(|spans| {
1314 let spans = if spans.is_empty() {
1315 vec![ParsedSpan::default()]
1316 } else {
1317 spans
1318 };
1319 match first_line.take() {
1320 Some(first) => ParsedBlock { spans, ..first },
1321 None => ParsedBlock {
1322 spans,
1323 ..later_line.clone()
1324 },
1325 }
1326 })
1327 .collect()
1328}
1329
1330fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
1333 match lines.last_mut() {
1334 Some(line) => line.push(span),
1335 None => lines.push(vec![span]),
1336 }
1337}
1338
1339pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
1340 ParsedElement::flatten_to_blocks(parse_html_elements(html))
1341}
1342
1343pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
1357
1358fn html_footnote_span(
1361 el: &scraper::node::Element,
1362 link_href: Option<String>,
1363) -> Option<ParsedSpan> {
1364 let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
1365 if label.is_empty() {
1366 return None;
1367 }
1368 Some(ParsedSpan {
1369 text: String::new(),
1370 link_href,
1371 footnote_ref: Some(label.to_string()),
1372 ..Default::default()
1373 })
1374}
1375
1376fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
1384 let src = el.attr("src")?;
1385 if src.is_empty() {
1386 return None;
1387 }
1388 let dim = |name: &str| -> i64 {
1389 el.attr(name)
1390 .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
1391 .filter(|n| *n > 0)
1392 .unwrap_or(0)
1393 };
1394 Some(ParsedSpan {
1395 text: String::new(),
1396 link_href,
1397 image: Some(ParsedImage {
1398 src: src.to_string(),
1399 alt: el.attr("alt").unwrap_or_default().to_string(),
1400 width: dim("width"),
1401 height: dim("height"),
1402 }),
1403 ..Default::default()
1404 })
1405}
1406
1407fn is_html_list(node: &Node) -> bool {
1409 node.as_element()
1410 .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
1411}
1412
1413fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
1426 use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
1427 use ego_tree::iter::Edge;
1428
1429 let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
1431 let mut depth = 0usize;
1432 for edge in tree.root().traverse() {
1433 match edge {
1434 Edge::Open(node) if is_html_list(node.value()) => {
1435 if depth == MAX_LIST_LEVELS {
1436 too_deep.push(node.id());
1437 }
1438 depth += 1;
1439 }
1440 Edge::Close(node) if is_html_list(node.value()) => {
1441 depth = depth.saturating_sub(1);
1442 }
1443 _ => {}
1444 }
1445 }
1446
1447 for list in too_deep {
1448 let Some(list_ref) = tree.get(list) else {
1452 continue;
1453 };
1454 let mut holder = None;
1455 let mut current = list_ref;
1456 while let Some(parent) = current.parent() {
1457 if is_html_list(parent.value()) {
1458 holder = Some(current.id());
1459 break;
1460 }
1461 current = parent;
1462 }
1463 let Some(holder) = holder else {
1464 continue;
1465 };
1466 let mut after = holder;
1467 let items: Vec<ego_tree::NodeId> = list_ref
1468 .descendants()
1469 .filter(|node| {
1470 node.value()
1471 .as_element()
1472 .is_some_and(|el| el.name() == "li")
1473 })
1474 .map(|node| node.id())
1475 .collect();
1476 for item in items {
1477 if tree.get(after).and_then(|node| node.parent()).is_none() {
1480 break;
1481 }
1482 if let Some(mut node) = tree.get_mut(after) {
1483 node.insert_id_after(item);
1484 after = item;
1485 }
1486 }
1487 let holds_text = tree.get(list).is_some_and(|node| {
1492 node.descendants()
1493 .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
1494 });
1495 if !holds_text {
1496 if let Some(mut node) = tree.get_mut(list) {
1497 node.detach();
1498 }
1499 } else if holder != list
1500 && tree.get(after).and_then(|node| node.parent()).is_some()
1501 && let Some(mut node) = tree.get_mut(after)
1502 {
1503 node.insert_id_after(list);
1504 }
1505 }
1506}
1507
1508const HTML_DEPTH_ON_THE_CALLERS_STACK: usize = 32;
1512
1513const MAX_HTML_READ_DEPTH: usize = 256;
1516
1517const HTML_READER_STACK: usize = 2 << 20;
1520
1521const HTML_STACK_PER_LEVEL: usize = 16 << 10;
1524
1525fn html_integer(value: &str) -> Option<i64> {
1531 let value = value.trim_start_matches(|c: char| c.is_ascii_whitespace());
1532 let (negative, digits) = match value.as_bytes().first() {
1533 Some(b'-') => (true, &value[1..]),
1534 Some(b'+') => (false, &value[1..]),
1535 _ => (false, value),
1536 };
1537 let digits: &str = digits
1538 .split(|c: char| !c.is_ascii_digit())
1539 .next()
1540 .unwrap_or("");
1541 if digits.is_empty() {
1542 return None;
1543 }
1544 let magnitude = digits.bytes().fold(0i64, |number, digit| {
1545 number
1546 .saturating_mul(10)
1547 .saturating_add(i64::from(digit - b'0'))
1548 });
1549 Some(if negative { -magnitude } else { magnitude })
1550}
1551
1552fn html_tree_depth(tree: &ego_tree::Tree<Node>) -> usize {
1555 use ego_tree::iter::Edge;
1556 let mut depth = 0usize;
1557 let mut deepest = 0usize;
1558 for edge in tree.root().traverse() {
1559 match edge {
1560 Edge::Open(node) if node.value().is_element() => {
1561 depth += 1;
1562 deepest = deepest.max(depth);
1563 }
1564 Edge::Close(node) if node.value().is_element() => depth = depth.saturating_sub(1),
1565 _ => {}
1566 }
1567 }
1568 deepest
1569}
1570
1571pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
1583 use scraper::Html;
1584
1585 let read = |depth_limit: usize, html: &str| {
1586 let mut fragment = Html::parse_fragment(html);
1587 flatten_deep_html_lists(&mut fragment.tree);
1588 let mut elements = read_html_tree(&fragment, depth_limit);
1589 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1590 elements
1591 };
1592 let mut fragment = Html::parse_fragment(html);
1593 flatten_deep_html_lists(&mut fragment.tree);
1594 let depth = html_tree_depth(&fragment.tree);
1595 if depth <= HTML_DEPTH_ON_THE_CALLERS_STACK {
1596 let mut elements = read_html_tree(&fragment, MAX_HTML_READ_DEPTH);
1597 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1598 return elements;
1599 }
1600 drop(fragment);
1601 let stack = HTML_READER_STACK + depth.min(MAX_HTML_READ_DEPTH + 1) * HTML_STACK_PER_LEVEL;
1602 let owned = html.to_owned();
1603 let on_its_own_thread = std::thread::Builder::new()
1604 .stack_size(stack)
1605 .spawn(move || read(MAX_HTML_READ_DEPTH, &owned))
1606 .ok()
1607 .and_then(|reader| reader.join().ok());
1608 on_its_own_thread.unwrap_or_else(|| read(HTML_DEPTH_ON_THE_CALLERS_STACK, html))
1609}
1610
1611fn read_html_tree(fragment: &scraper::Html, depth_limit: usize) -> Vec<ParsedElement> {
1615 let mut elements: Vec<ParsedElement> = Vec::new();
1616
1617 let root = fragment.root_element();
1619
1620 #[derive(Clone, Default)]
1621 struct FmtState {
1622 bold: bool,
1623 italic: bool,
1624 underline: bool,
1625 strikeout: bool,
1626 code: bool,
1627 superscript: bool,
1628 subscript: bool,
1629 link_href: Option<String>,
1630 }
1631
1632 type HtmlList = (Option<ListStyle>, Option<i64>);
1635
1636 fn is_metadata_tag(tag: &str) -> bool {
1646 matches!(
1647 tag,
1648 "head"
1649 | "style"
1650 | "script"
1651 | "title"
1652 | "meta"
1653 | "link"
1654 | "base"
1655 | "noscript"
1656 | "template"
1657 )
1658 }
1659
1660 fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
1669 use ego_tree::iter::Edge;
1670 let mut text = String::new();
1671 let mut hidden = 0usize;
1673 for edge in node.traverse() {
1674 match edge {
1675 Edge::Open(open) => match open.value() {
1676 Node::Element(el) => {
1677 if hidden > 0 || is_metadata_tag(el.name()) {
1678 hidden += 1;
1679 } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
1680 text.push(' ');
1681 }
1682 }
1683 Node::Text(t) if hidden == 0 => text.push_str(t),
1684 _ => {}
1685 },
1686 Edge::Close(close) => {
1687 if hidden > 0 && close.value().is_element() {
1688 hidden -= 1;
1689 }
1690 }
1691 }
1692 }
1693 text
1694 }
1695
1696 fn text_span(text: String, state: &FmtState) -> ParsedSpan {
1698 ParsedSpan {
1699 text,
1700 bold: state.bold,
1701 italic: state.italic,
1702 underline: state.underline,
1703 strikeout: state.strikeout,
1704 code: state.code,
1705 superscript: state.superscript,
1706 subscript: state.subscript,
1707 link_href: state.link_href.clone(),
1708 image: None,
1709 footnote_ref: None,
1710 }
1711 }
1712
1713 fn collect_cell_spans(
1715 node: ego_tree::NodeRef<Node>,
1716 state: &FmtState,
1717 spans: &mut Vec<ParsedSpan>,
1718 depth: usize,
1719 limit: usize,
1720 ) {
1721 if depth > limit {
1722 let text = subtree_text(node);
1723 if !text.is_empty() {
1724 spans.push(text_span(text, state));
1725 }
1726 return;
1727 }
1728 for child in node.children() {
1729 match child.value() {
1730 Node::Text(text) => {
1731 let t = text.text.to_string();
1732 if !t.is_empty() {
1733 spans.push(ParsedSpan {
1734 text: t,
1735 bold: state.bold,
1736 italic: state.italic,
1737 underline: state.underline,
1738 strikeout: state.strikeout,
1739 code: state.code,
1740 superscript: state.superscript,
1741 subscript: state.subscript,
1742 link_href: state.link_href.clone(),
1743 image: None,
1744 footnote_ref: None,
1745 });
1746 }
1747 }
1748 Node::Element(el) => {
1749 let tag = el.name();
1750 if is_metadata_tag(tag) {
1751 continue;
1752 }
1753 let mut new_state = state.clone();
1754 match tag {
1755 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
1759 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
1760 {
1761 spans.push(span);
1762 }
1763 continue;
1764 }
1765 "b" | "strong" => new_state.bold = true,
1766 "i" | "em" => new_state.italic = true,
1767 "u" | "ins" => new_state.underline = true,
1768 "s" | "del" | "strike" => new_state.strikeout = true,
1769 "code" => new_state.code = true,
1770 "sup" => new_state.superscript = true,
1771 "sub" => new_state.subscript = true,
1772 "a" => {
1773 if let Some(href) = el.attr("href") {
1774 new_state.link_href = Some(href.to_string());
1775 }
1776 }
1777 "img" => {
1778 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
1779 spans.push(span);
1780 }
1781 continue;
1782 }
1783 "br" => {
1789 spans.push(text_span(" ".to_string(), state));
1790 continue;
1791 }
1792 "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
1793 | "h3" | "h4" | "h5" | "h6" => {
1794 spans.push(text_span(" ".to_string(), state));
1795 }
1796 _ => {}
1797 }
1798 collect_cell_spans(child, &new_state, spans, depth + 1, limit);
1799 }
1800 _ => {}
1801 }
1802 }
1803 }
1804
1805 const MAX_HTML_COLSPAN: usize = 1000;
1808
1809 fn parse_table_element(
1813 table_node: ego_tree::NodeRef<Node>,
1814 depth: usize,
1815 limit: usize,
1816 ) -> (ParsedTable, usize) {
1817 let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
1818 let mut header_rows: usize = 0;
1819 let mut own_cells: usize = 0;
1820
1821 fn collect_rows(
1823 node: ego_tree::NodeRef<Node>,
1824 rows: &mut Vec<Vec<ParsedTableCell>>,
1825 header_rows: &mut usize,
1826 own_cells: &mut usize,
1827 in_thead: bool,
1828 walk: (usize, usize),
1829 ) {
1830 let (depth, limit) = walk;
1831 for child in node.children() {
1832 if let Node::Element(el) = child.value() {
1833 match el.name() {
1834 "thead" => collect_rows(
1835 child,
1836 rows,
1837 header_rows,
1838 own_cells,
1839 true,
1840 (depth + 1, limit),
1841 ),
1842 "tbody" | "tfoot" => collect_rows(
1843 child,
1844 rows,
1845 header_rows,
1846 own_cells,
1847 false,
1848 (depth + 1, limit),
1849 ),
1850 "tr" => {
1851 let mut cells: Vec<ParsedTableCell> = Vec::new();
1852 let mut own_end = 0;
1855 for td in child.children() {
1856 if let Node::Element(td_el) = td.value()
1857 && matches!(td_el.name(), "td" | "th")
1858 {
1859 let mut spans = Vec::new();
1860 let state = FmtState::default();
1861 collect_cell_spans(td, &state, &mut spans, depth + 2, limit);
1862 collapse_inline_whitespace(&mut spans);
1863 if spans.is_empty() {
1864 spans.push(ParsedSpan::default());
1865 }
1866 cells.push(ParsedTableCell { spans });
1867 *own_cells += 1;
1868 own_end = cells.len();
1869 let span = td_el
1874 .attr("colspan")
1875 .and_then(|value| value.trim().parse::<usize>().ok())
1876 .unwrap_or(1)
1877 .clamp(1, MAX_HTML_COLSPAN);
1878 for _ in 1..span {
1879 if cells.len() >= MAX_HTML_COLSPAN {
1880 break;
1881 }
1882 cells.push(ParsedTableCell {
1883 spans: vec![ParsedSpan::default()],
1884 });
1885 }
1886 }
1887 }
1888 cells.truncate(own_end);
1892 if !cells.is_empty() {
1893 rows.push(cells);
1894 if in_thead {
1895 *header_rows += 1;
1896 }
1897 }
1898 }
1899 _ => {}
1900 }
1901 }
1902 }
1903 }
1904
1905 collect_rows(
1906 table_node,
1907 &mut rows,
1908 &mut header_rows,
1909 &mut own_cells,
1910 false,
1911 (depth, limit),
1912 );
1913
1914 if header_rows == 0 && !rows.is_empty() {
1916 header_rows = 1;
1917 }
1918
1919 (
1920 ParsedTable {
1921 header_rows,
1922 rows,
1923 blockquote_depth: 0,
1926 blockquote_opens: 0,
1927 },
1928 own_cells,
1929 )
1930 }
1931
1932 #[allow(clippy::too_many_arguments)]
1933 fn walk_node(
1934 node: ego_tree::NodeRef<Node>,
1935 state: &FmtState,
1936 elements: &mut Vec<ParsedElement>,
1937 current_list: &HtmlList,
1938 blockquote_depth: u32,
1939 list_depth: u32,
1940 depth: usize,
1941 limit: usize,
1942 ) {
1943 if depth > limit {
1944 let text = subtree_text(node);
1945 if !text.trim().is_empty() {
1946 let mut spans = vec![text_span(text, state)];
1947 collapse_inline_whitespace(&mut spans);
1948 elements.push(ParsedElement::Block(ParsedBlock {
1949 spans,
1950 blockquote_depth,
1951 blockquote_opens: 0,
1952 ..ParsedBlock::default()
1953 }));
1954 }
1955 return;
1956 }
1957 match node.value() {
1958 Node::Element(el) => {
1959 let tag = el.name();
1960 if is_metadata_tag(tag) {
1961 return;
1962 }
1963 let mut new_state = state.clone();
1964 let mut new_list = current_list.clone();
1965 let mut bq_depth = blockquote_depth;
1966 let mut new_list_depth = list_depth;
1967
1968 let is_block_tag = matches!(
1970 tag,
1971 "p" | "div"
1972 | "h1"
1973 | "h2"
1974 | "h3"
1975 | "h4"
1976 | "h5"
1977 | "h6"
1978 | "li"
1979 | "pre"
1980 | "br"
1981 | "blockquote"
1982 | "body"
1983 | "html"
1984 );
1985
1986 match tag {
1988 "b" | "strong" => new_state.bold = true,
1989 "i" | "em" => new_state.italic = true,
1990 "u" | "ins" => new_state.underline = true,
1991 "s" | "del" | "strike" => new_state.strikeout = true,
1992 "code" => new_state.code = true,
1993 "sup" => new_state.superscript = true,
1994 "sub" => new_state.subscript = true,
1995 "a" => {
1996 if let Some(href) = el.attr("href") {
1997 new_state.link_href = Some(href.to_string());
1998 }
1999 }
2000 "ul" => {
2001 new_list = (Some(ListStyle::Disc), None);
2002 new_list_depth = list_depth + 1;
2003 }
2004 "ol" => {
2005 let start = el
2009 .attr("start")
2010 .and_then(html_integer)
2011 .and_then(crate::parser_tools::list_start::list_start);
2012 new_list = (Some(ListStyle::Decimal), start);
2013 new_list_depth = list_depth + 1;
2014 }
2015 "blockquote" => {
2016 bq_depth += 1;
2017 }
2018 _ => {}
2019 }
2020
2021 let heading_level = match tag {
2023 "h1" => Some(1),
2024 "h2" => Some(2),
2025 "h3" => Some(3),
2026 "h4" => Some(4),
2027 "h5" => Some(5),
2028 "h6" => Some(6),
2029 _ => None,
2030 };
2031
2032 let is_code_block = tag == "pre";
2033
2034 let code_language = if is_code_block {
2036 node.children().find_map(|child| {
2037 if let Node::Element(cel) = child.value()
2038 && cel.name() == "code"
2039 && let Some(cls) = cel.attr("class")
2040 {
2041 return cls
2042 .split_whitespace()
2043 .find_map(|c| c.strip_prefix("language-"))
2044 .map(|l| l.to_string());
2045 }
2046 None
2047 })
2048 } else {
2049 None
2050 };
2051
2052 let css = if is_block_tag {
2054 el.attr("style").map(parse_block_styles).unwrap_or_default()
2055 } else {
2056 BlockStyles::default()
2057 };
2058
2059 if tag == "table" {
2060 let (mut parsed_table, own_cells) = parse_table_element(node, depth, limit);
2062 if !parsed_table.rows.is_empty() {
2063 parsed_table.blockquote_depth = bq_depth;
2064 push_table(elements, parsed_table, own_cells);
2065 }
2066 return;
2067 }
2068
2069 if tag == "br" {
2070 elements.push(ParsedElement::Block(ParsedBlock {
2072 spans: vec![ParsedSpan {
2073 text: String::new(),
2074 ..Default::default()
2075 }],
2076 heading_level: None,
2077 list_style: None,
2078 list_indent: 0,
2079 list_prefix: String::new(),
2080 list_suffix: String::new(),
2081 list_start: None,
2082 marker: None,
2083 is_code_block: false,
2084 code_language: None,
2085 blockquote_depth: bq_depth,
2086 blockquote_opens: 0,
2087 line_height: None,
2088 non_breakable_lines: None,
2089 page_break_before: None,
2090 direction: None,
2091 background_color: None,
2092 alignment: None,
2093 top_margin: None,
2094 text_indent: None,
2095 semantic_role: None,
2096 }));
2097 return;
2098 }
2099
2100 if tag == "blockquote" {
2101 let first = elements.len();
2103 for child in node.children() {
2104 walk_node(
2105 child,
2106 &new_state,
2107 elements,
2108 &new_list,
2109 bq_depth,
2110 new_list_depth,
2111 depth + 1,
2112 limit,
2113 );
2114 }
2115 mark_quotation_opened(elements, first);
2116 } else if is_block_tag && tag != "br" {
2117 let mut spans: Vec<ParsedSpan> = Vec::new();
2122 let mut nested_elements: Vec<ParsedElement> = Vec::new();
2123 collect_inline_spans(
2124 node,
2125 &new_state,
2126 &mut spans,
2127 &new_list,
2128 &mut nested_elements,
2129 bq_depth,
2130 new_list_depth,
2131 depth + 1,
2132 limit,
2133 );
2134
2135 let (list_style_for_block, list_start_for_block) = if tag == "li" {
2136 new_list.clone()
2137 } else {
2138 (None, None)
2139 };
2140
2141 let list_indent_for_block = if tag == "li" {
2142 new_list_depth.saturating_sub(1)
2143 } else {
2144 0
2145 };
2146
2147 if !(is_code_block || css.preserve_whitespace == Some(true)) {
2148 collapse_inline_whitespace(&mut spans);
2149 }
2150
2151 let own_run_is_content =
2159 spans_carry_content(&spans) || nested_elements.is_empty();
2160
2161 if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
2162 let is_code_block = is_code_block
2166 && !spans
2167 .iter()
2168 .any(|span| span.image.is_some() || span.footnote_ref.is_some());
2169 let block = ParsedBlock {
2170 spans,
2171 heading_level,
2172 list_style: list_style_for_block,
2173 list_indent: list_indent_for_block,
2174 list_prefix: String::new(),
2175 list_suffix: String::new(),
2176 list_start: list_start_for_block,
2177 marker: None,
2178 is_code_block,
2179 code_language: code_language.filter(|_| is_code_block),
2180 blockquote_depth: bq_depth,
2181 blockquote_opens: 0,
2182 line_height: css.line_height,
2183 non_breakable_lines: css.non_breakable_lines,
2184 page_break_before: css.page_break_before,
2185 direction: css.direction,
2186 background_color: css.background_color,
2187 alignment: None,
2188 top_margin: None,
2189 text_indent: None,
2190 semantic_role: None,
2191 };
2192 if is_code_block {
2196 elements.push(ParsedElement::Block(block));
2197 } else {
2198 elements.extend(
2199 split_block_at_line_breaks(block)
2200 .into_iter()
2201 .map(ParsedElement::Block),
2202 );
2203 }
2204 }
2205 elements.append(&mut nested_elements);
2207 } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
2208 for child in node.children() {
2210 walk_node(
2211 child,
2212 &new_state,
2213 elements,
2214 &new_list,
2215 bq_depth,
2216 new_list_depth,
2217 depth + 1,
2218 limit,
2219 );
2220 }
2221 } else {
2222 for child in node.children() {
2224 walk_node(
2225 child,
2226 &new_state,
2227 elements,
2228 current_list,
2229 bq_depth,
2230 list_depth,
2231 depth + 1,
2232 limit,
2233 );
2234 }
2235 }
2236 }
2237 Node::Text(text) => {
2238 let t = text.text.to_string();
2239 let trimmed = t.trim();
2240 if !trimmed.is_empty() {
2241 elements.push(ParsedElement::Block(ParsedBlock {
2243 spans: vec![ParsedSpan {
2244 text: trimmed.to_string(),
2245 bold: state.bold,
2246 italic: state.italic,
2247 underline: state.underline,
2248 strikeout: state.strikeout,
2249 code: state.code,
2250 superscript: state.superscript,
2251 subscript: state.subscript,
2252 link_href: state.link_href.clone(),
2253 image: None,
2254 footnote_ref: None,
2255 }],
2256 heading_level: None,
2257 list_style: None,
2258 list_indent: 0,
2259 list_prefix: String::new(),
2260 list_suffix: String::new(),
2261 list_start: None,
2262 marker: None,
2263 is_code_block: false,
2264 code_language: None,
2265 blockquote_depth,
2266 blockquote_opens: 0,
2267 line_height: None,
2268 non_breakable_lines: None,
2269 page_break_before: None,
2270 direction: None,
2271 background_color: None,
2272 alignment: None,
2273 top_margin: None,
2274 text_indent: None,
2275 semantic_role: None,
2276 }));
2277 }
2278 }
2279 _ => {
2280 for child in node.children() {
2282 walk_node(
2283 child,
2284 state,
2285 elements,
2286 current_list,
2287 blockquote_depth,
2288 list_depth,
2289 depth + 1,
2290 limit,
2291 );
2292 }
2293 }
2294 }
2295 }
2296
2297 #[allow(clippy::too_many_arguments)]
2301 fn collect_inline_spans(
2302 node: ego_tree::NodeRef<Node>,
2303 state: &FmtState,
2304 spans: &mut Vec<ParsedSpan>,
2305 current_list: &HtmlList,
2306 elements: &mut Vec<ParsedElement>,
2307 blockquote_depth: u32,
2308 list_depth: u32,
2309 depth: usize,
2310 limit: usize,
2311 ) {
2312 if depth > limit {
2313 let text = subtree_text(node);
2314 if !text.is_empty() {
2315 spans.push(text_span(text, state));
2316 }
2317 return;
2318 }
2319 for child in node.children() {
2320 match child.value() {
2321 Node::Text(text) => {
2322 let t = text.text.to_string();
2323 if !t.is_empty() {
2324 spans.push(ParsedSpan {
2325 text: t,
2326 bold: state.bold,
2327 italic: state.italic,
2328 underline: state.underline,
2329 strikeout: state.strikeout,
2330 code: state.code,
2331 superscript: state.superscript,
2332 subscript: state.subscript,
2333 link_href: state.link_href.clone(),
2334 image: None,
2335 footnote_ref: None,
2336 });
2337 }
2338 }
2339 Node::Element(el) => {
2340 let tag = el.name();
2341 if is_metadata_tag(tag) {
2342 continue;
2343 }
2344 let mut new_state = state.clone();
2345
2346 match tag {
2347 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2351 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2352 {
2353 spans.push(span);
2354 }
2355 continue;
2356 }
2357 "b" | "strong" => new_state.bold = true,
2358 "i" | "em" => new_state.italic = true,
2359 "u" | "ins" => new_state.underline = true,
2360 "s" | "del" | "strike" => new_state.strikeout = true,
2361 "code" => new_state.code = true,
2362 "sup" => new_state.superscript = true,
2363 "sub" => new_state.subscript = true,
2364 "a" => {
2365 if let Some(href) = el.attr("href") {
2366 new_state.link_href = Some(href.to_string());
2367 }
2368 }
2369 "img" => {
2370 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2371 spans.push(span);
2372 }
2373 continue;
2374 }
2375 _ => {}
2376 }
2377
2378 let nested_block = matches!(
2380 tag,
2381 "p" | "div"
2382 | "h1"
2383 | "h2"
2384 | "h3"
2385 | "h4"
2386 | "h5"
2387 | "h6"
2388 | "li"
2389 | "pre"
2390 | "blockquote"
2391 | "ul"
2392 | "ol"
2393 );
2394
2395 if tag == "br" {
2396 spans.push(ParsedSpan {
2399 text: String::new(),
2400 ..Default::default()
2401 });
2402 } else if nested_block || tag == "table" {
2403 walk_node(
2405 child,
2406 &new_state,
2407 elements,
2408 current_list,
2409 blockquote_depth,
2410 list_depth,
2411 depth + 1,
2412 limit,
2413 );
2414 } else {
2415 collect_inline_spans(
2417 child,
2418 &new_state,
2419 spans,
2420 current_list,
2421 elements,
2422 blockquote_depth,
2423 list_depth,
2424 depth + 1,
2425 limit,
2426 );
2427 }
2428 }
2429 _ => {}
2430 }
2431 }
2432 }
2433
2434 let initial_state = FmtState::default();
2435 let mut root_spans: Vec<ParsedSpan> = Vec::new();
2439 collect_inline_spans(
2440 *root,
2441 &initial_state,
2442 &mut root_spans,
2443 &(None, None),
2444 &mut elements,
2445 0,
2446 0,
2447 0,
2448 depth_limit,
2449 );
2450 collapse_inline_whitespace(&mut root_spans);
2451 if spans_carry_content(&root_spans) {
2454 elements.push(ParsedElement::Block(ParsedBlock {
2455 spans: root_spans,
2456 heading_level: None,
2457 list_style: None,
2458 list_indent: 0,
2459 list_prefix: String::new(),
2460 list_suffix: String::new(),
2461 list_start: None,
2462 marker: None,
2463 is_code_block: false,
2464 code_language: None,
2465 blockquote_depth: 0,
2466 blockquote_opens: 0,
2467 line_height: None,
2468 non_breakable_lines: None,
2469 page_break_before: None,
2470 direction: None,
2471 background_color: None,
2472 alignment: None,
2473 top_margin: None,
2474 text_indent: None,
2475 semantic_role: None,
2476 }));
2477 }
2478
2479 if elements.is_empty() {
2481 elements.push(ParsedElement::Block(ParsedBlock {
2482 spans: vec![ParsedSpan {
2483 text: String::new(),
2484 ..Default::default()
2485 }],
2486 heading_level: None,
2487 list_style: None,
2488 list_indent: 0,
2489 list_prefix: String::new(),
2490 list_suffix: String::new(),
2491 list_start: None,
2492 marker: None,
2493 is_code_block: false,
2494 code_language: None,
2495 blockquote_depth: 0,
2496 blockquote_opens: 0,
2497 line_height: None,
2498 non_breakable_lines: None,
2499 page_break_before: None,
2500 direction: None,
2501 background_color: None,
2502 alignment: None,
2503 top_margin: None,
2504 text_indent: None,
2505 semantic_role: None,
2506 }));
2507 }
2508
2509 elements
2510}
2511
2512pub fn character_format_from_span(
2516 span: &ParsedSpan,
2517 is_code_block: bool,
2518) -> crate::format_runs::CharacterFormat {
2519 use crate::entities::CharVerticalAlignment;
2520 crate::format_runs::CharacterFormat {
2521 font_bold: if span.bold { Some(true) } else { None },
2522 font_italic: if span.italic { Some(true) } else { None },
2523 font_underline: if span.underline { Some(true) } else { None },
2524 font_strikeout: if span.strikeout { Some(true) } else { None },
2525 font_family: if span.code || is_code_block {
2526 Some("monospace".to_string())
2527 } else {
2528 None
2529 },
2530 anchor_href: span.link_href.clone(),
2531 is_anchor: if span.link_href.is_some() {
2532 Some(true)
2533 } else {
2534 None
2535 },
2536 vertical_alignment: if span.superscript {
2537 Some(CharVerticalAlignment::SuperScript)
2538 } else if span.subscript {
2539 Some(CharVerticalAlignment::SubScript)
2540 } else {
2541 None
2542 },
2543 ..Default::default()
2544 }
2545}
2546
2547pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
2559 use crate::format_runs::{
2560 CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
2561 };
2562
2563 let mut plain_text = String::new();
2564 let mut runs: Vec<FormatRun> = Vec::new();
2565 let mut images: Vec<ImageAnchor> = Vec::new();
2566 let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
2567 let default = CharacterFormat::default();
2568
2569 for span in spans {
2570 let byte_start = plain_text.len() as u32;
2571
2572 if let Some(label) = &span.footnote_ref {
2573 plain_text.push('\u{FFFC}');
2576 let mut format = character_format_from_span(span, is_code_block);
2591 format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
2592 footnote_refs.push(FootnoteRefAnchor {
2593 byte_offset: byte_start,
2594 label: label.clone(),
2595 format,
2596 });
2597 continue;
2598 }
2599
2600 if let Some(image) = &span.image {
2601 plain_text.push('\u{FFFC}');
2605 images.push(ImageAnchor {
2606 byte_offset: byte_start,
2607 name: image.src.clone(),
2608 alt: image.alt.clone(),
2609 width: image.width,
2610 height: image.height,
2611 quality: 100,
2612 format: character_format_from_span(span, is_code_block),
2613 });
2614 continue;
2615 }
2616
2617 plain_text.push_str(&span.text);
2618 let byte_end = plain_text.len() as u32;
2619 if byte_start == byte_end {
2620 continue;
2621 }
2622 let format = character_format_from_span(span, is_code_block);
2623 if format == default {
2624 continue;
2625 }
2626 runs.push(FormatRun {
2627 byte_start,
2628 byte_end,
2629 format,
2630 });
2631 }
2632 coalesce_in_place(&mut runs);
2633 ParsedInline {
2634 plain_text,
2635 runs,
2636 images,
2637 footnote_refs,
2638 }
2639}
2640
2641#[derive(Debug, Clone, Default)]
2648pub struct ParsedInline {
2649 pub plain_text: String,
2650 pub runs: Vec<crate::format_runs::FormatRun>,
2651 pub images: Vec<crate::format_runs::ImageAnchor>,
2652 pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
2653}
2654
2655fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
2663 use jotdown::ListBulletType as B;
2664 match b {
2665 B::Dash => ListStyle::Disc,
2666 B::Star => ListStyle::Circle,
2667 B::Plus => ListStyle::Square,
2668 }
2669}
2670
2671fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
2673 use jotdown::OrderedListNumbering as N;
2674 match n {
2675 N::Decimal => ListStyle::Decimal,
2676 N::AlphaLower => ListStyle::LowerAlpha,
2677 N::AlphaUpper => ListStyle::UpperAlpha,
2678 N::RomanLower => ListStyle::LowerRoman,
2679 N::RomanUpper => ListStyle::UpperRoman,
2680 }
2681}
2682
2683fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
2687 use jotdown::OrderedListStyle as S;
2688 match style {
2689 S::Period => (String::new(), ".".to_string()),
2690 S::Paren => (String::new(), ")".to_string()),
2691 S::ParenParen => ("(".to_string(), ")".to_string()),
2692 }
2693}
2694
2695#[derive(Debug, Clone, Default)]
2699struct DjotBlockStyle {
2700 alignment: Option<Alignment>,
2701 line_height: Option<i64>,
2702 non_breakable_lines: Option<bool>,
2703 page_break_before: Option<bool>,
2704 direction: Option<TextDirection>,
2705 background_color: Option<String>,
2706 top_margin: Option<i64>,
2707 text_indent: Option<i64>,
2708 semantic_role: Option<SemanticRole>,
2709}
2710
2711impl DjotBlockStyle {
2712 fn merge_from(&mut self, other: DjotBlockStyle) {
2716 if other.alignment.is_some() {
2717 self.alignment = other.alignment;
2718 }
2719 if other.line_height.is_some() {
2720 self.line_height = other.line_height;
2721 }
2722 if other.non_breakable_lines.is_some() {
2723 self.non_breakable_lines = other.non_breakable_lines;
2724 }
2725 if other.page_break_before.is_some() {
2726 self.page_break_before = other.page_break_before;
2727 }
2728 if other.direction.is_some() {
2729 self.direction = other.direction;
2730 }
2731 if other.background_color.is_some() {
2732 self.background_color = other.background_color;
2733 }
2734 if other.top_margin.is_some() {
2735 self.top_margin = other.top_margin;
2736 }
2737 if other.text_indent.is_some() {
2738 self.text_indent = other.text_indent;
2739 }
2740 if other.semantic_role.is_some() {
2741 self.semantic_role = other.semantic_role.clone();
2742 }
2743 }
2744}
2745
2746fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
2752 let mut style = DjotBlockStyle::default();
2753
2754 if opts.alignment
2755 && let Some(v) = attrs.get_value("alignment")
2756 {
2757 style.alignment = match v.to_string().as_str() {
2758 "left" => Some(Alignment::Left),
2759 "right" => Some(Alignment::Right),
2760 "center" => Some(Alignment::Center),
2761 "justify" => Some(Alignment::Justify),
2762 _ => None,
2763 };
2764 }
2765 if opts.line_height
2766 && let Some(v) = attrs.get_value("line_height")
2767 {
2768 style.line_height = v.to_string().parse::<i64>().ok();
2769 }
2770 if opts.direction
2771 && let Some(v) = attrs.get_value("direction")
2772 {
2773 style.direction = match v.to_string().as_str() {
2774 "ltr" => Some(TextDirection::LeftToRight),
2775 "rtl" => Some(TextDirection::RightToLeft),
2776 _ => None,
2777 };
2778 }
2779 if opts.non_breakable_lines
2780 && let Some(v) = attrs.get_value("non_breakable_lines")
2781 {
2782 style.non_breakable_lines = match v.to_string().as_str() {
2783 "true" => Some(true),
2784 "false" => Some(false),
2785 _ => None,
2786 };
2787 }
2788 if opts.page_break_before
2789 && let Some(v) = attrs.get_value("page_break_before")
2790 {
2791 style.page_break_before = match v.to_string().as_str() {
2792 "true" => Some(true),
2793 "false" => Some(false),
2794 _ => None,
2795 };
2796 }
2797 if opts.background_color
2798 && let Some(v) = attrs.get_value("background_color")
2799 {
2800 style.background_color = Some(v.to_string());
2801 }
2802 if opts.top_margin
2803 && let Some(v) = attrs.get_value("top_margin")
2804 {
2805 style.top_margin = v.to_string().parse::<i64>().ok();
2806 }
2807 if opts.text_indent
2808 && let Some(v) = attrs.get_value("text_indent")
2809 {
2810 style.text_indent = v.to_string().parse::<i64>().ok();
2811 }
2812 if opts.semantic_role
2813 && let Some(v) = attrs.get_value("semantic_role")
2814 {
2815 style.semantic_role = match v.to_string().as_str() {
2816 "epigraph" => Some(SemanticRole::Epigraph),
2817 _ => None,
2821 };
2822 }
2823
2824 style
2825}
2826
2827fn list_start_of(start: u64) -> Option<i64> {
2831 crate::parser_tools::list_start::list_start_from_unsigned(start)
2832}
2833
2834#[allow(clippy::too_many_arguments)]
2837fn djot_push_block(
2838 elements: &mut Vec<ParsedElement>,
2839 spans: Vec<ParsedSpan>,
2840 heading_level: Option<i64>,
2841 list_style: Option<ListStyle>,
2842 list_indent: u32,
2843 list_prefix: String,
2844 list_suffix: String,
2845 list_start: Option<i64>,
2846 marker: Option<MarkerType>,
2847 is_code_block: bool,
2848 code_language: Option<String>,
2849 blockquote_depth: u32,
2850 style: DjotBlockStyle,
2851) {
2852 elements.push(ParsedElement::Block(ParsedBlock {
2853 spans,
2854 heading_level,
2855 list_style,
2856 list_indent,
2857 list_prefix,
2858 list_suffix,
2859 list_start,
2860 marker,
2861 is_code_block,
2862 code_language,
2863 blockquote_depth,
2864 blockquote_opens: 0,
2865 line_height: style.line_height,
2866 non_breakable_lines: style.non_breakable_lines,
2867 page_break_before: style.page_break_before,
2868 direction: style.direction,
2869 background_color: style.background_color,
2870 alignment: style.alignment,
2871 top_margin: style.top_margin,
2872 text_indent: style.text_indent,
2873 semantic_role: style.semantic_role.clone(),
2874 }));
2875}
2876
2877fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
2890 djot.lines()
2891 .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
2892 .filter(|line| !line.is_empty())
2893 .map(|line| {
2894 ParsedElement::Block(ParsedBlock {
2895 spans: vec![ParsedSpan {
2896 text: line.to_string(),
2897 ..Default::default()
2898 }],
2899 ..Default::default()
2900 })
2901 })
2902 .collect()
2903}
2904
2905pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
2926 use jotdown::{Container as C, Event as E, ListKind};
2927
2928 let parsable = crate::parser_tools::djot_depth::parsable(djot);
2954 let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
2955 return djot_source_lines(djot);
2956 };
2957
2958 let mut elements: Vec<ParsedElement> = Vec::new();
2959 let mut current_spans: Vec<ParsedSpan> = Vec::new();
2960 let mut current_heading: Option<i64> = None;
2961 let mut is_code_block = false;
2962 let mut code_language: Option<String> = None;
2963 let mut blockquote_depth: u32 = 0;
2964 let mut quotations_open: Vec<usize> = Vec::new();
2966 let mut pending_style = DjotBlockStyle::default();
2969
2970 let mut bold = false;
2972 let mut italic = false;
2973 let mut underline = false;
2974 let mut strikeout = false;
2975 let mut code = false;
2976 let mut superscript = false;
2977 let mut subscript = false;
2978 let mut link_href: Option<String> = None;
2979 let mut pending_image: Option<ParsedImage> = None;
2982
2983 let mut list_stack: Vec<(ListStyle, String, String, Option<i64>)> = Vec::new();
2985 let mut cur_list_style: Option<ListStyle> = None;
2987 let mut cur_list_prefix = String::new();
2988 let mut cur_list_suffix = String::new();
2989 let mut cur_list_start: Option<i64> = None;
2990 let mut cur_list_indent: u32 = 0;
2991 let mut cur_marker: Option<MarkerType> = None;
2992
2993 let mut in_table_cell = false;
2995 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
2996 let mut current_row: Vec<ParsedTableCell> = Vec::new();
2997 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
2998 let mut table_header_rows: usize = 0;
2999 let mut row_is_head = false;
3000
3001 let mut skip_depth: u32 = 0;
3005
3006 let mut footnote_open: Option<(String, usize)> = None;
3010
3011 macro_rules! push_text {
3016 ($t:expr) => {{
3017 if let Some(img) = pending_image.as_mut() {
3022 img.alt.push_str(($t).as_ref());
3023 } else {
3024 let sp = ParsedSpan {
3025 text: ($t).to_string(),
3026 bold,
3027 italic,
3028 underline,
3029 strikeout,
3030 code,
3031 superscript,
3032 subscript,
3033 link_href: link_href.clone(),
3034 image: None,
3035 footnote_ref: None,
3036 };
3037 if in_table_cell {
3038 current_cell_spans.push(sp);
3039 } else {
3040 current_spans.push(sp);
3041 }
3042 }
3043 }};
3044 }
3045
3046 macro_rules! push_image {
3048 ($img:expr) => {{
3049 let sp = ParsedSpan {
3050 text: String::new(),
3051 bold,
3052 italic,
3053 underline,
3054 strikeout,
3055 code,
3056 superscript,
3057 subscript,
3058 link_href: link_href.clone(),
3059 image: Some($img),
3060 footnote_ref: None,
3061 };
3062 if in_table_cell {
3063 current_cell_spans.push(sp);
3064 } else {
3065 current_spans.push(sp);
3066 }
3067 }};
3068 }
3069
3070 macro_rules! enter_item {
3073 ($marker:expr) => {{
3074 if !current_spans.is_empty() {
3075 djot_push_block(
3076 &mut elements,
3077 std::mem::take(&mut current_spans),
3078 None,
3079 cur_list_style.clone(),
3080 cur_list_indent,
3081 cur_list_prefix.clone(),
3082 cur_list_suffix.clone(),
3083 cur_list_start,
3084 cur_marker.clone(),
3085 false,
3086 None,
3087 blockquote_depth,
3088 DjotBlockStyle::default(),
3089 );
3090 }
3091 let (style, prefix, suffix, start) = list_stack.last().cloned().unwrap_or((
3092 ListStyle::Disc,
3093 String::new(),
3094 String::new(),
3095 None,
3096 ));
3097 cur_list_style = Some(style);
3098 cur_list_prefix = prefix;
3099 cur_list_suffix = suffix;
3100 cur_list_start = start;
3101 cur_list_indent = list_stack.len().saturating_sub(1) as u32;
3102 cur_marker = $marker;
3103 }};
3104 }
3105
3106 macro_rules! line_break {
3113 () => {{
3114 if in_table_cell {
3115 push_text!(" ");
3116 } else if !current_spans.is_empty() && current_heading.is_some() {
3117 djot_push_block(
3122 &mut elements,
3123 std::mem::take(&mut current_spans),
3124 current_heading,
3125 None,
3126 0,
3127 String::new(),
3128 String::new(),
3129 None,
3130 None,
3131 false,
3132 None,
3133 blockquote_depth,
3134 pending_style.clone(),
3135 );
3136 pending_style.page_break_before = None;
3137 } else if !current_spans.is_empty() {
3138 djot_push_block(
3141 &mut elements,
3142 std::mem::take(&mut current_spans),
3143 None,
3144 cur_list_style.clone(),
3145 cur_list_indent,
3146 cur_list_prefix.clone(),
3147 cur_list_suffix.clone(),
3148 cur_list_start,
3149 cur_marker.clone(),
3150 is_code_block,
3151 code_language.clone(),
3152 blockquote_depth,
3153 pending_style.clone(),
3154 );
3155 pending_style.page_break_before = None;
3156 }
3157 }};
3158 }
3159
3160 for event in events {
3161 if skip_depth > 0 {
3162 match event {
3163 E::Start(..) => skip_depth += 1,
3164 E::End(_) => skip_depth -= 1,
3165 _ => {}
3166 }
3167 continue;
3168 }
3169
3170 match event {
3171 E::Start(C::Document, _) | E::End(C::Document) => {}
3173 E::Start(C::Section { .. }, attrs) => {
3174 if list_stack.is_empty() {
3177 pending_style.merge_from(block_attrs_to_style(&attrs, options));
3178 }
3179 }
3180 E::End(C::Section { .. }) => {}
3181 E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
3182
3183 E::Start(C::Blockquote, _) => {
3185 blockquote_depth += 1;
3186 quotations_open.push(elements.len());
3187 }
3188 E::End(C::Blockquote) => {
3189 blockquote_depth = blockquote_depth.saturating_sub(1);
3190 if let Some(first) = quotations_open.pop() {
3191 mark_quotation_opened(&mut elements, first);
3192 }
3193 }
3194
3195 E::Start(C::List { kind, .. }, _) => {
3197 let (style, prefix, suffix, start) = match kind {
3198 ListKind::Unordered(b) | ListKind::Task(b) => {
3199 (djot_bullet_style(b), String::new(), String::new(), None)
3200 }
3201 ListKind::Ordered {
3202 numbering,
3203 style,
3204 start,
3205 } => {
3206 let (p, s) = djot_ordered_affixes(style);
3207 (djot_ordered_style(numbering), p, s, list_start_of(start))
3208 }
3209 };
3210 list_stack.push((style, prefix, suffix, start));
3211 }
3212 E::End(C::List { .. }) => {
3213 list_stack.pop();
3214 cur_list_style = None;
3215 cur_list_start = None;
3216 cur_marker = None;
3217 }
3218 E::Start(C::ListItem, _) => enter_item!(None),
3219 E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
3220 MarkerType::Checked
3221 } else {
3222 MarkerType::Unchecked
3223 })),
3224 E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
3225 if !current_spans.is_empty() {
3227 djot_push_block(
3228 &mut elements,
3229 std::mem::take(&mut current_spans),
3230 None,
3231 cur_list_style.clone(),
3232 cur_list_indent,
3233 cur_list_prefix.clone(),
3234 cur_list_suffix.clone(),
3235 cur_list_start,
3236 cur_marker.clone(),
3237 false,
3238 None,
3239 blockquote_depth,
3240 DjotBlockStyle::default(),
3241 );
3242 }
3243 cur_list_style = None;
3244 cur_list_start = None;
3245 cur_marker = None;
3246 }
3247
3248 E::Start(C::Heading { level, .. }, attrs) => {
3250 current_heading = Some(level as i64);
3251 pending_style.merge_from(block_attrs_to_style(&attrs, options));
3254 }
3255 E::End(C::Heading { .. }) => {
3256 djot_push_block(
3257 &mut elements,
3258 std::mem::take(&mut current_spans),
3259 current_heading.take(),
3260 None,
3261 0,
3262 String::new(),
3263 String::new(),
3264 None,
3265 None,
3266 false,
3267 None,
3268 blockquote_depth,
3269 std::mem::take(&mut pending_style),
3270 );
3271 }
3272 E::Start(C::Paragraph, attrs) => {
3273 current_heading = None;
3274 pending_style = if list_stack.is_empty() {
3278 block_attrs_to_style(&attrs, options)
3279 } else {
3280 DjotBlockStyle::default()
3281 };
3282 }
3283 E::End(C::Paragraph) => {
3284 if !current_spans.is_empty() {
3285 djot_push_block(
3286 &mut elements,
3287 std::mem::take(&mut current_spans),
3288 None,
3289 cur_list_style.clone(),
3290 cur_list_indent,
3291 cur_list_prefix.clone(),
3292 cur_list_suffix.clone(),
3293 cur_list_start,
3294 cur_marker.clone(),
3295 false,
3296 None,
3297 blockquote_depth,
3298 std::mem::take(&mut pending_style),
3299 );
3300 }
3301 cur_list_style = None;
3302 cur_list_start = None;
3303 cur_marker = None;
3304 }
3305 E::Start(C::CodeBlock { language }, _) => {
3306 is_code_block = true;
3307 code_language = if language.is_empty() {
3308 None
3309 } else {
3310 Some(language.to_string())
3311 };
3312 }
3313 E::End(C::CodeBlock { .. }) => {
3314 if let Some(last) = current_spans.last_mut()
3316 && last.text.ends_with('\n')
3317 {
3318 last.text.pop();
3319 }
3320 djot_push_block(
3321 &mut elements,
3322 std::mem::take(&mut current_spans),
3323 None,
3324 None,
3325 0,
3326 String::new(),
3327 String::new(),
3328 None,
3329 None,
3330 true,
3331 code_language.take(),
3332 blockquote_depth,
3333 DjotBlockStyle::default(),
3334 );
3335 is_code_block = false;
3336 }
3337
3338 E::Start(C::Table, _) => {
3340 table_rows.clear();
3341 current_row.clear();
3342 current_cell_spans.clear();
3343 table_header_rows = 0;
3344 }
3345 E::End(C::Table) => {
3346 let own_cells = cells_in_rows(&table_rows);
3347 push_table(
3348 &mut elements,
3349 ParsedTable {
3350 header_rows: table_header_rows,
3351 rows: std::mem::take(&mut table_rows),
3352 blockquote_depth,
3353 blockquote_opens: 0,
3354 },
3355 own_cells,
3356 );
3357 }
3358 E::Start(C::TableRow { head }, _) => {
3359 row_is_head = head;
3360 current_row.clear();
3361 }
3362 E::End(C::TableRow { .. }) => {
3363 if row_is_head {
3364 table_header_rows += 1;
3365 }
3366 table_rows.push(std::mem::take(&mut current_row));
3367 }
3368 E::Start(C::TableCell { .. }, _) => {
3369 in_table_cell = true;
3370 current_cell_spans.clear();
3371 }
3372 E::End(C::TableCell { .. }) => {
3373 in_table_cell = false;
3374 current_row.push(ParsedTableCell {
3375 spans: std::mem::take(&mut current_cell_spans),
3376 });
3377 }
3378
3379 E::Start(C::Strong, _) => bold = true,
3381 E::End(C::Strong) => bold = false,
3382 E::Start(C::Emphasis, _) => italic = true,
3383 E::End(C::Emphasis) => italic = false,
3384 E::Start(C::Verbatim, _) => code = true,
3385 E::End(C::Verbatim) => code = false,
3386 E::Start(C::Superscript, _) => superscript = true,
3387 E::End(C::Superscript) => superscript = false,
3388 E::Start(C::Subscript, _) => subscript = true,
3389 E::End(C::Subscript) => subscript = false,
3390 E::Start(C::Insert, _) => underline = true,
3391 E::End(C::Insert) => underline = false,
3392 E::Start(C::Delete, _) => strikeout = true,
3393 E::End(C::Delete) => strikeout = false,
3394 E::Start(C::Mark, _) | E::End(C::Mark) => {}
3396 E::Start(C::Span, _) | E::End(C::Span) => {}
3397 E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
3398 E::End(C::Link(..)) => link_href = None,
3399 E::Start(C::Image(src, _), attrs) => {
3404 let attr_num = |key: &str| -> i64 {
3405 attrs
3406 .get_value(key)
3407 .map(|v| v.to_string())
3408 .and_then(|v| v.trim().parse::<i64>().ok())
3409 .filter(|n| *n > 0)
3410 .unwrap_or(0)
3411 };
3412 pending_image = Some(ParsedImage {
3413 src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
3416 alt: String::new(),
3417 width: attr_num("width"),
3418 height: attr_num("height"),
3419 });
3420 }
3421 E::End(C::Image(..)) => {
3422 if let Some(img) = pending_image.take() {
3423 push_image!(img);
3424 }
3425 }
3426
3427 E::Start(C::Footnote { label }, _) => {
3436 if !current_spans.is_empty() {
3437 djot_push_block(
3438 &mut elements,
3439 std::mem::take(&mut current_spans),
3440 None,
3441 cur_list_style.clone(),
3442 cur_list_indent,
3443 cur_list_prefix.clone(),
3444 cur_list_suffix.clone(),
3445 cur_list_start,
3446 cur_marker.clone(),
3447 false,
3448 None,
3449 blockquote_depth,
3450 DjotBlockStyle::default(),
3451 );
3452 }
3453 footnote_open = Some((label.to_string(), elements.len()));
3454 }
3455 E::End(C::Footnote { .. }) => {
3456 if !current_spans.is_empty() {
3457 djot_push_block(
3458 &mut elements,
3459 std::mem::take(&mut current_spans),
3460 None,
3461 cur_list_style.clone(),
3462 cur_list_indent,
3463 cur_list_prefix.clone(),
3464 cur_list_suffix.clone(),
3465 cur_list_start,
3466 cur_marker.clone(),
3467 false,
3468 None,
3469 blockquote_depth,
3470 DjotBlockStyle::default(),
3471 );
3472 }
3473 if let Some((label, start)) = footnote_open.take() {
3474 let blocks: Vec<ParsedBlock> = elements
3475 .drain(start..)
3476 .filter_map(|e| match e {
3477 ParsedElement::Block(b) => Some(b),
3478 _ => None,
3482 })
3483 .collect();
3484 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
3485 }
3486 }
3487
3488 E::Start(
3490 C::Math { .. }
3491 | C::RawBlock { .. }
3492 | C::RawInline { .. }
3493 | C::DescriptionList
3494 | C::DescriptionDetails
3495 | C::DescriptionTerm
3496 | C::Caption
3497 | C::LinkDefinition { .. },
3498 _,
3499 ) => skip_depth = 1,
3500
3501 E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
3513 for (i, piece) in s.split('\n').enumerate() {
3514 if i > 0 {
3515 line_break!();
3516 }
3517 if !piece.is_empty() {
3518 push_text!(piece);
3519 }
3520 }
3521 }
3522 E::Str(s) => push_text!(s.as_ref()),
3523 E::Softbreak => push_text!(" "),
3524 E::LeftSingleQuote => push_text!("\u{2018}"),
3525 E::RightSingleQuote => push_text!("\u{2019}"),
3526 E::LeftDoubleQuote => push_text!("\u{201C}"),
3527 E::RightDoubleQuote => push_text!("\u{201D}"),
3528 E::Ellipsis => push_text!("\u{2026}"),
3529 E::EnDash => push_text!("\u{2013}"),
3530 E::EmDash => push_text!("\u{2014}"),
3531 E::NonBreakingSpace => push_text!("\u{00A0}"),
3532 E::Hardbreak => line_break!(),
3533 E::FootnoteReference(label) => {
3539 let sp = ParsedSpan {
3540 text: String::new(),
3541 bold,
3542 italic,
3543 underline,
3544 strikeout,
3545 code,
3546 superscript,
3547 subscript,
3548 link_href: link_href.clone(),
3549 image: None,
3550 footnote_ref: Some(label.to_string()),
3551 };
3552 if in_table_cell {
3553 current_cell_spans.push(sp);
3554 } else {
3555 current_spans.push(sp);
3556 }
3557 }
3558 E::Symbol(_) => {}
3561 E::Escape | E::Blankline => {}
3562 E::ThematicBreak(_) | E::Attributes(_) => {}
3563
3564 _ => {}
3567 }
3568 }
3569
3570 if !current_spans.is_empty() {
3572 djot_push_block(
3573 &mut elements,
3574 std::mem::take(&mut current_spans),
3575 current_heading.take(),
3576 cur_list_style.clone(),
3577 cur_list_indent,
3578 cur_list_prefix.clone(),
3579 cur_list_suffix.clone(),
3580 cur_list_start,
3581 cur_marker.clone(),
3582 is_code_block,
3583 code_language.take(),
3584 blockquote_depth,
3585 std::mem::take(&mut pending_style),
3586 );
3587 }
3588
3589 if elements.is_empty() {
3592 djot_push_block(
3593 &mut elements,
3594 vec![ParsedSpan {
3595 text: String::new(),
3596 ..Default::default()
3597 }],
3598 None,
3599 None,
3600 0,
3601 String::new(),
3602 String::new(),
3603 None,
3604 None,
3605 false,
3606 None,
3607 0,
3608 DjotBlockStyle::default(),
3609 );
3610 }
3611
3612 elements
3613}
3614
3615#[cfg(test)]
3616mod tests {
3617 use super::*;
3618
3619 fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
3621 ParsedElement::flatten_to_blocks(parse_markdown(md))
3622 }
3623
3624 #[test]
3625 fn test_parse_markdown_simple_paragraph() {
3626 let blocks = parse_markdown_blocks("Hello **world**");
3627 assert_eq!(blocks.len(), 1);
3628 assert!(blocks[0].spans.len() >= 2);
3629 let plain_span = blocks[0]
3631 .spans
3632 .iter()
3633 .find(|s| s.text.contains("Hello"))
3634 .unwrap();
3635 assert!(!plain_span.bold);
3636 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3637 assert!(bold_span.bold);
3638 }
3639
3640 #[test]
3641 fn test_parse_markdown_heading() {
3642 let blocks = parse_markdown_blocks("# Title");
3643 assert_eq!(blocks.len(), 1);
3644 assert_eq!(blocks[0].heading_level, Some(1));
3645 assert_eq!(blocks[0].spans[0].text, "Title");
3646 }
3647
3648 #[test]
3649 fn test_parse_markdown_list() {
3650 let blocks = parse_markdown_blocks("- item1\n- item2");
3651 assert!(blocks.len() >= 2);
3652 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3653 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3654 }
3655
3656 fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
3658 elements
3659 .iter()
3660 .map(|e| match e {
3661 ParsedElement::Block(b) => (false, b.blockquote_depth),
3662 ParsedElement::Table(t) => (true, t.blockquote_depth),
3663 ParsedElement::FootnoteDefinition { .. } => (false, 0),
3666 })
3667 .collect()
3668 }
3669
3670 #[test]
3671 fn test_parse_markdown_table_in_blockquote_records_depth() {
3672 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3673 assert_eq!(element_depths(&elements), vec![(true, 1)]);
3674 }
3675
3676 #[test]
3677 fn test_parse_markdown_text_then_table_in_blockquote() {
3678 let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
3679 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
3680 }
3681
3682 #[test]
3683 fn test_parse_markdown_table_after_blockquote_closes() {
3684 let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
3685 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
3686 }
3687
3688 #[test]
3689 fn test_parse_markdown_table_in_nested_blockquote() {
3690 let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
3691 assert_eq!(element_depths(&elements), vec![(true, 2)]);
3692 }
3693
3694 #[test]
3695 fn test_parse_markdown_list_in_blockquote_records_depth() {
3696 let elements = parse_markdown("> - item1\n> - item2");
3697 let depths = element_depths(&elements);
3698 assert_eq!(depths, vec![(false, 1), (false, 1)]);
3699 for e in &elements {
3700 if let ParsedElement::Block(b) = e {
3701 assert_eq!(b.list_style, Some(ListStyle::Disc));
3702 }
3703 }
3704 }
3705
3706 #[test]
3707 fn test_parse_html_table_in_blockquote_records_depth() {
3708 let elements = parse_html_elements(
3709 "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
3710 );
3711 assert_eq!(element_depths(&elements), vec![(true, 1)]);
3712 }
3713
3714 #[test]
3715 fn test_parse_html_table_after_blockquote() {
3716 let elements = parse_html_elements(
3717 "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
3718 );
3719 let depths = element_depths(&elements);
3720 assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
3722 assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
3723 }
3724
3725 #[test]
3726 fn test_flatten_to_blocks_propagates_blockquote_depth() {
3727 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3728 let blocks = ParsedElement::flatten_to_blocks(elements);
3729 assert!(!blocks.is_empty());
3730 for b in &blocks {
3731 assert_eq!(b.blockquote_depth, 1);
3732 }
3733 }
3734
3735 #[test]
3736 fn test_parse_html_simple() {
3737 let blocks = parse_html("<p>Hello <b>world</b></p>");
3738 assert_eq!(blocks.len(), 1);
3739 assert!(blocks[0].spans.len() >= 2);
3740 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3741 assert!(bold_span.bold);
3742 }
3743
3744 #[test]
3745 fn test_parse_html_multiple_paragraphs() {
3746 let blocks = parse_html("<p>A</p><p>B</p>");
3747 assert_eq!(blocks.len(), 2);
3748 }
3749
3750 #[test]
3751 fn test_parse_html_heading() {
3752 let blocks = parse_html("<h2>Subtitle</h2>");
3753 assert_eq!(blocks.len(), 1);
3754 assert_eq!(blocks[0].heading_level, Some(2));
3755 }
3756
3757 #[test]
3758 fn test_parse_html_list() {
3759 let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
3760 assert!(blocks.len() >= 2);
3761 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3762 }
3763
3764 #[test]
3765 fn test_parse_markdown_code_block() {
3766 let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
3767 assert_eq!(blocks.len(), 1);
3768 assert!(blocks[0].is_code_block);
3769 assert!(blocks[0].spans[0].code);
3770 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3772 assert_eq!(
3773 text, "fn main() {}",
3774 "code block text should not have trailing newline"
3775 );
3776 }
3777
3778 #[test]
3779 fn test_parse_markdown_nested_formatting() {
3780 let blocks = parse_markdown_blocks("***bold italic***");
3781 assert_eq!(blocks.len(), 1);
3782 let span = &blocks[0].spans[0];
3783 assert!(span.bold);
3784 assert!(span.italic);
3785 }
3786
3787 #[test]
3788 fn test_parse_markdown_link() {
3789 let blocks = parse_markdown_blocks("[click](http://example.com)");
3790 assert_eq!(blocks.len(), 1);
3791 let span = &blocks[0].spans[0];
3792 assert_eq!(span.text, "click");
3793 assert_eq!(span.link_href, Some("http://example.com".to_string()));
3794 }
3795
3796 #[test]
3797 fn test_parse_markdown_empty() {
3798 let blocks = parse_markdown_blocks("");
3799 assert_eq!(blocks.len(), 1);
3800 assert!(blocks[0].spans[0].text.is_empty());
3801 }
3802
3803 #[test]
3804 fn test_parse_html_empty() {
3805 let blocks = parse_html("");
3806 assert_eq!(blocks.len(), 1);
3807 assert!(blocks[0].spans[0].text.is_empty());
3808 }
3809
3810 #[test]
3811 fn test_parse_html_nested_formatting() {
3812 let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
3813 assert_eq!(blocks.len(), 1);
3814 let span = &blocks[0].spans[0];
3815 assert!(span.bold);
3816 assert!(span.italic);
3817 }
3818
3819 #[test]
3820 fn test_parse_html_link() {
3821 let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
3822 assert_eq!(blocks.len(), 1);
3823 let span = &blocks[0].spans[0];
3824 assert_eq!(span.text, "click");
3825 assert_eq!(span.link_href, Some("http://example.com".to_string()));
3826 }
3827
3828 #[test]
3829 fn test_parse_html_ordered_list() {
3830 let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
3831 assert!(blocks.len() >= 2);
3832 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3833 }
3834
3835 #[test]
3836 fn test_parse_markdown_ordered_list() {
3837 let blocks = parse_markdown_blocks("1. first\n2. second");
3838 assert!(blocks.len() >= 2);
3839 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3840 }
3841
3842 #[test]
3843 fn test_parse_html_blockquote_nested() {
3844 let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
3845 assert!(blocks.len() >= 3);
3846 }
3847
3848 #[test]
3849 fn test_parse_block_styles_line_height() {
3850 let styles = parse_block_styles("line-height: 1.5");
3851 assert_eq!(styles.line_height, Some(1500));
3852 }
3853
3854 #[test]
3855 fn test_parse_block_styles_direction_rtl() {
3856 let styles = parse_block_styles("direction: rtl");
3857 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3858 }
3859
3860 #[test]
3861 fn test_parse_block_styles_background_color() {
3862 let styles = parse_block_styles("background-color: #ff0000");
3863 assert_eq!(styles.background_color, Some("#ff0000".to_string()));
3864 }
3865
3866 #[test]
3867 fn test_parse_block_styles_white_space_pre() {
3868 let styles = parse_block_styles("white-space: pre");
3869 assert_eq!(styles.non_breakable_lines, Some(true));
3870 }
3871
3872 #[test]
3873 fn test_parse_block_styles_multiple() {
3874 let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
3875 assert_eq!(styles.line_height, Some(2000));
3876 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3877 assert_eq!(styles.background_color, Some("blue".to_string()));
3878 }
3879
3880 #[test]
3881 fn test_parse_html_block_styles_extracted() {
3882 let blocks = parse_html(
3883 r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
3884 );
3885 assert_eq!(blocks.len(), 1);
3886 assert_eq!(blocks[0].line_height, Some(1500));
3887 assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
3888 assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
3889 }
3890
3891 #[test]
3895 fn test_parse_html_collapses_exporter_line_wrapping() {
3896 let blocks = parse_html(
3897 "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
3898 <p>\nsecond para</p>\n</body>",
3899 );
3900 let texts: Vec<String> = blocks
3901 .iter()
3902 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
3903 .collect();
3904 assert_eq!(texts, vec!["first line wrapped here", "second para"]);
3905 }
3906
3907 #[test]
3911 fn test_parse_html_drops_whitespace_between_blocks() {
3912 let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
3913 assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
3914 assert!(
3915 blocks
3916 .iter()
3917 .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
3918 "no block keeps a literal newline"
3919 );
3920 }
3921
3922 #[test]
3925 fn test_parse_html_collapses_across_span_boundaries() {
3926 let blocks = parse_html("<p> <b>bold</b>\n <i>italic</i> </p>");
3927 assert_eq!(blocks.len(), 1);
3928 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3929 assert_eq!(text, "bold italic");
3930 }
3931
3932 #[test]
3935 fn test_parse_html_keeps_no_break_space() {
3936 let blocks = parse_html("<p>\nAttention : ici</p>");
3937 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3938 assert_eq!(text, "Attention\u{a0}: ici");
3939 }
3940
3941 #[test]
3944 fn test_parse_html_keeps_empty_paragraph() {
3945 let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
3946 assert_eq!(blocks.len(), 3);
3947 let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
3948 assert_eq!(text, "");
3949 }
3950
3951 #[test]
3955 fn test_parse_html_preserves_whitespace_in_pre() {
3956 let blocks = parse_html("<pre>\nfn main() {\n let x = 1;\n}</pre>");
3959 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3960 assert_eq!(text, "fn main() {\n let x = 1;\n}");
3961
3962 let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n b</p>");
3965 let texts: Vec<String> = styled
3966 .iter()
3967 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
3968 .collect();
3969 assert_eq!(texts, ["a", " b"]);
3970
3971 let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n b</p>");
3972 let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
3973 assert_eq!(text, "a b");
3974 }
3975
3976 #[test]
3979 fn test_parse_html_collapses_table_cell_whitespace() {
3980 let elements = parse_html_elements(
3981 "<table>\n<tr>\n<td>\n one\n</td>\n<td>\n two\n</td>\n</tr>\n</table>",
3982 );
3983 let table = elements
3984 .iter()
3985 .find_map(|e| match e {
3986 ParsedElement::Table(t) => Some(t),
3987 _ => None,
3988 })
3989 .expect("a table");
3990 let cells: Vec<String> = table.rows[0]
3991 .iter()
3992 .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
3993 .collect();
3994 assert_eq!(cells, vec!["one", "two"]);
3995 }
3996
3997 #[test]
3998 fn test_parse_html_white_space_pre() {
3999 let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
4000 assert_eq!(blocks.len(), 1);
4001 assert_eq!(blocks[0].non_breakable_lines, Some(true));
4002 }
4003
4004 #[test]
4005 fn test_parse_html_no_styles_returns_none() {
4006 let blocks = parse_html("<p>plain</p>");
4007 assert_eq!(blocks.len(), 1);
4008 assert_eq!(blocks[0].line_height, None);
4009 assert_eq!(blocks[0].direction, None);
4010 assert_eq!(blocks[0].background_color, None);
4011 assert_eq!(blocks[0].non_breakable_lines, None);
4012 }
4013
4014 #[test]
4015 fn test_parse_markdown_nested_list_indent() {
4016 let md = "- top\n - nested\n - deep";
4017 let blocks = parse_markdown_blocks(md);
4018 assert_eq!(blocks.len(), 3);
4019 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4020 assert_eq!(blocks[0].list_indent, 0);
4021 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4022 assert_eq!(blocks[1].list_indent, 1);
4023 assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
4024 assert_eq!(blocks[2].list_indent, 2);
4025 }
4026
4027 #[test]
4028 fn test_parse_markdown_nested_ordered_list_indent() {
4029 let md = "1. first\n 1. nested\n 2. nested2";
4030 let blocks = parse_markdown_blocks(md);
4031 assert_eq!(blocks.len(), 3);
4032 assert_eq!(blocks[0].list_indent, 0);
4033 assert_eq!(blocks[1].list_indent, 1);
4034 assert_eq!(blocks[2].list_indent, 1);
4035 }
4036
4037 #[test]
4038 fn test_parse_html_nested_list_indent() {
4039 let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
4040 let blocks = parse_html(html);
4041 assert!(blocks.len() >= 2);
4042 assert_eq!(blocks[0].list_indent, 0);
4043 assert_eq!(blocks[1].list_indent, 1);
4044 }
4045
4046 #[test]
4047 fn test_parse_markdown_table() {
4048 let md = "| A | B |\n|---|---|\n| 1 | 2 |";
4049 let elements = parse_markdown(md);
4050 assert_eq!(elements.len(), 1);
4051 match &elements[0] {
4052 ParsedElement::Table(table) => {
4053 assert_eq!(table.header_rows, 1);
4054 assert_eq!(table.rows.len(), 2); assert_eq!(table.rows[0].len(), 2);
4057 assert_eq!(table.rows[0][0].spans[0].text, "A");
4058 assert_eq!(table.rows[0][1].spans[0].text, "B");
4059 assert_eq!(table.rows[1].len(), 2);
4061 assert_eq!(table.rows[1][0].spans[0].text, "1");
4062 assert_eq!(table.rows[1][1].spans[0].text, "2");
4063 }
4064 _ => panic!("Expected ParsedElement::Table"),
4065 }
4066 }
4067
4068 #[test]
4069 fn test_parse_markdown_table_with_formatting() {
4070 let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
4071 let elements = parse_markdown(md);
4072 assert_eq!(elements.len(), 1);
4073 match &elements[0] {
4074 ParsedElement::Table(table) => {
4075 assert_eq!(table.rows.len(), 2);
4076 assert!(table.rows[0][0].spans[0].bold);
4078 assert!(table.rows[0][1].spans[0].code);
4080 assert!(table.rows[0][2].spans[0].italic);
4082 assert!(table.rows[1][0].spans[0].strikeout);
4084 assert_eq!(
4086 table.rows[1][2].spans[0].link_href,
4087 Some("http://x.com".to_string())
4088 );
4089 }
4090 _ => panic!("Expected ParsedElement::Table"),
4091 }
4092 }
4093
4094 #[test]
4095 fn test_parse_markdown_mixed_content_with_table() {
4096 let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
4097 let elements = parse_markdown(md);
4098 assert_eq!(elements.len(), 3);
4099 assert!(matches!(&elements[0], ParsedElement::Block(_)));
4100 assert!(matches!(&elements[1], ParsedElement::Table(_)));
4101 assert!(matches!(&elements[2], ParsedElement::Block(_)));
4102 }
4103}
4104
4105#[cfg(test)]
4106mod djot_tests {
4107 use super::*;
4108 use crate::entities::MarkerType;
4109
4110 fn blocks(d: &str) -> Vec<ParsedBlock> {
4111 ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
4112 }
4113
4114 fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
4115 b.spans.iter().find(|s| pred(s)).expect("span not found")
4116 }
4117
4118 #[test]
4119 fn paragraph_bold_italic() {
4120 let b = blocks("normal *bold* _italic_");
4121 assert_eq!(b.len(), 1);
4122 assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
4123 assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
4124 }
4125
4126 #[test]
4127 fn heading_levels() {
4128 assert_eq!(blocks("# H1")[0].heading_level, Some(1));
4129 assert_eq!(blocks("### H3")[0].heading_level, Some(3));
4130 assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
4131 }
4132
4133 #[test]
4134 fn unordered_bullet_styles_are_distinct() {
4135 assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
4136 assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
4137 assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
4138 }
4139
4140 #[test]
4141 fn ordered_delimiters() {
4142 let period = blocks("1. a");
4143 assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
4144 assert_eq!(period[0].list_prefix, "");
4145 assert_eq!(period[0].list_suffix, ".");
4146
4147 let paren = blocks("1) a");
4148 assert_eq!(paren[0].list_suffix, ")");
4149 assert_eq!(paren[0].list_prefix, "");
4150
4151 let paren_paren = blocks("(1) a");
4152 assert_eq!(paren_paren[0].list_prefix, "(");
4153 assert_eq!(paren_paren[0].list_suffix, ")");
4154 }
4155
4156 #[test]
4157 fn task_list_markers() {
4158 let b = blocks("- [ ] a\n- [x] b");
4159 assert_eq!(b.len(), 2);
4160 assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
4161 assert_eq!(b[1].marker, Some(MarkerType::Checked));
4162 }
4163
4164 #[test]
4165 fn code_block_with_language() {
4166 let b = blocks("```rust\nfn main() {}\n```");
4167 assert_eq!(b.len(), 1);
4168 assert!(b[0].is_code_block);
4169 assert_eq!(b[0].code_language.as_deref(), Some("rust"));
4170 let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
4171 assert_eq!(text, "fn main() {}");
4172 }
4173
4174 #[test]
4175 fn link_href() {
4176 let b = blocks("[text](http://example.com)");
4177 let s = first_span_with(&b[0], |s| s.text == "text");
4178 assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
4179 }
4180
4181 #[test]
4182 fn superscript_subscript() {
4183 assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
4184 assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
4185 }
4186
4187 #[test]
4188 fn delete_insert_verbatim() {
4189 assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
4190 assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
4191 assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
4192 }
4193
4194 #[test]
4195 fn blockquote_depth() {
4196 let els = parse_djot("> quoted", &DjotImportOptions::default());
4197 match &els[0] {
4198 ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
4199 _ => panic!("expected block"),
4200 }
4201 }
4202
4203 #[test]
4204 fn nested_list_indent() {
4205 let b = blocks("- a\n\n - b\n\n - c");
4210 assert_eq!(b.len(), 3);
4211 assert_eq!(b[0].list_indent, 0);
4212 assert_eq!(b[1].list_indent, 1);
4213 assert_eq!(b[2].list_indent, 2);
4214 }
4215
4216 #[test]
4217 fn table_parsed_as_table() {
4218 let els = parse_djot(
4219 "| a | b |\n|---|---|\n| c | d |",
4220 &DjotImportOptions::default(),
4221 );
4222 assert_eq!(els.len(), 1);
4223 match &els[0] {
4224 ParsedElement::Table(t) => {
4225 assert_eq!(t.header_rows, 1);
4226 assert_eq!(t.rows.len(), 2);
4227 assert_eq!(t.rows[0][0].spans[0].text, "a");
4228 assert_eq!(t.rows[1][1].spans[0].text, "d");
4229 }
4230 _ => panic!("expected table"),
4231 }
4232 }
4233
4234 #[test]
4235 fn smart_punctuation_normalised_to_unicode() {
4236 let text: String = blocks("a... b---c")[0]
4237 .spans
4238 .iter()
4239 .map(|s| s.text.as_str())
4240 .collect();
4241 assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
4242 assert!(text.contains('\u{2014}'), "em dash: {text:?}");
4243 }
4244
4245 #[test]
4246 fn unrepresentable_constructs_dropped_without_leaking_text() {
4247 let b = blocks("para1\n\n---\n\npara2");
4249 assert_eq!(b.len(), 2);
4250 assert_eq!(
4251 b[0].spans
4252 .iter()
4253 .map(|s| s.text.as_str())
4254 .collect::<String>(),
4255 "para1"
4256 );
4257 assert_eq!(
4258 b[1].spans
4259 .iter()
4260 .map(|s| s.text.as_str())
4261 .collect::<String>(),
4262 "para2"
4263 );
4264
4265 let d = blocks(":::\ninside\n:::");
4267 let joined: String = d
4268 .iter()
4269 .flat_map(|b| b.spans.iter())
4270 .map(|s| s.text.as_str())
4271 .collect();
4272 assert_eq!(joined, "inside");
4273
4274 let m = blocks("before $`E=mc^2` after");
4276 let joined: String = m
4277 .iter()
4278 .flat_map(|b| b.spans.iter())
4279 .map(|s| s.text.as_str())
4280 .collect();
4281 assert!(joined.contains("before"), "{joined:?}");
4282 assert!(joined.contains("after"), "{joined:?}");
4283 assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
4284 }
4285
4286 #[test]
4287 fn empty_document_yields_one_empty_block() {
4288 let b = blocks("");
4289 assert_eq!(b.len(), 1);
4290 assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
4291 }
4292
4293 #[test]
4294 fn block_attributes_parse_into_block() {
4295 let b = blocks(
4296 "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
4297 );
4298 assert_eq!(b.len(), 1);
4299 assert_eq!(b[0].alignment, Some(Alignment::Center));
4300 assert_eq!(b[0].line_height, Some(1500));
4301 assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
4302 assert_eq!(b[0].non_breakable_lines, Some(true));
4303 assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
4304 }
4305
4306 #[test]
4307 fn spacing_block_attributes_parse_into_block() {
4308 let b = blocks("{top_margin=24 text_indent=0}\nhello");
4312 assert_eq!(b.len(), 1);
4313 assert_eq!(b[0].top_margin, Some(24));
4314 assert_eq!(b[0].text_indent, Some(0));
4315 }
4316
4317 #[test]
4318 fn a_zero_text_indent_is_distinct_from_an_absent_one() {
4319 let explicit = blocks("{text_indent=0}\nhello");
4323 let absent = blocks("hello");
4324 assert_eq!(explicit[0].text_indent, Some(0));
4325 assert_eq!(absent[0].text_indent, None);
4326 assert!(!explicit[0].is_inline_only());
4327 assert!(absent[0].is_inline_only());
4328 }
4329
4330 #[test]
4331 fn spacing_block_attributes_respect_import_options() {
4332 let src = "{top_margin=24 text_indent=0}\nhello";
4333 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4334 assert_eq!(b[0].top_margin, None);
4335 assert_eq!(b[0].text_indent, None);
4336 }
4337
4338 #[test]
4339 fn block_attributes_on_heading() {
4340 let b = blocks("{alignment=right}\n# Title");
4341 assert_eq!(b[0].heading_level, Some(1));
4342 assert_eq!(b[0].alignment, Some(Alignment::Right));
4343 }
4344
4345 #[test]
4346 fn block_attributes_respect_import_options() {
4347 let src = "{alignment=center line_height=1500}\nhello";
4350 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4351 assert_eq!(b[0].alignment, None);
4352 assert_eq!(b[0].line_height, None);
4353 assert_eq!(
4354 b[0].spans
4355 .iter()
4356 .map(|s| s.text.as_str())
4357 .collect::<String>(),
4358 "hello"
4359 );
4360 }
4361
4362 #[test]
4363 fn list_item_block_attributes_are_dropped() {
4364 let b = blocks("{alignment=center}\n- item");
4367 assert!(b.iter().all(|blk| blk.alignment.is_none()));
4368 }
4369
4370 #[test]
4371 fn unknown_alignment_value_is_ignored() {
4372 let b = blocks("{alignment=sideways}\nhello");
4373 assert_eq!(b[0].alignment, None);
4374 }
4375}
4376
4377pub const TABLE_ANCHOR: &str = "\u{FFFC}";
4389
4390fn span_prose(span: &ParsedSpan, out: &mut String) {
4398 if span.image.is_some() || span.footnote_ref.is_some() {
4399 out.push('\u{FFFC}');
4400 return;
4401 }
4402 out.push_str(&span.text);
4403}
4404
4405fn block_prose(block: &ParsedBlock) -> String {
4406 let mut prose = String::new();
4407 for span in &block.spans {
4408 span_prose(span, &mut prose);
4409 }
4410 prose
4411}
4412
4413fn cell_prose(cell: &ParsedTableCell) -> String {
4414 let mut prose = String::new();
4415 for span in &cell.spans {
4416 span_prose(span, &mut prose);
4417 }
4418 prose
4419}
4420
4421pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
4472 let elements = parse_djot(djot, options);
4476
4477 let mut out = String::with_capacity(djot.len());
4480
4481 let mut first = true;
4486 let push = |text: &str, out: &mut String, first: &mut bool| {
4487 if *first {
4488 *first = false;
4489 } else {
4490 out.push('\n');
4491 }
4492 out.push_str(text);
4493 };
4494
4495 for element in &elements {
4496 match element {
4497 ParsedElement::Block(block) => {
4498 push(&block_prose(block), &mut out, &mut first);
4499 }
4500 ParsedElement::FootnoteDefinition { .. } => {}
4510 ParsedElement::Table(table) => {
4511 push(TABLE_ANCHOR, &mut out, &mut first);
4518 let columns = table.column_count();
4521 for row in &table.rows {
4522 for column in 0..columns {
4523 let prose = row.get(column).map(cell_prose).unwrap_or_default();
4524 push(&prose, &mut out, &mut first);
4525 }
4526 }
4527 }
4528 }
4529 }
4530 out
4531}
4532
4533#[cfg(test)]
4534mod html_footnote_tests {
4535 use super::*;
4536
4537 #[test]
4541 fn a_literal_reference_in_text_would_be_escaped_into_prose() {
4542 let blocks = parse_html("<p>The ferry.[^1]</p>");
4543 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4544 assert!(
4545 text.contains("[^1]"),
4546 "the parser keeps it as characters: {text:?}"
4547 );
4548 assert!(
4549 blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
4550 "text is text — it must not become a reference by accident"
4551 );
4552 }
4553
4554 #[test]
4555 fn an_attributed_element_becomes_a_real_footnote_reference() {
4556 let blocks = parse_html(
4557 r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
4558 );
4559 let refs: Vec<&str> = blocks
4560 .iter()
4561 .flat_map(|b| b.spans.iter())
4562 .filter_map(|s| s.footnote_ref.as_deref())
4563 .collect();
4564 assert_eq!(refs, vec!["3"]);
4565
4566 let marker = blocks[0]
4569 .spans
4570 .iter()
4571 .find(|s| s.footnote_ref.is_some())
4572 .expect("the reference span");
4573 assert!(marker.text.is_empty(), "{marker:?}");
4574 }
4575
4576 #[test]
4579 fn any_element_carrying_the_attribute_works() {
4580 for html in [
4581 r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
4582 r#"<p>a<span data-footnote-ref="1"></span></p>"#,
4583 r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
4584 ] {
4585 let blocks = parse_html(html);
4586 assert_eq!(
4587 blocks
4588 .iter()
4589 .flat_map(|b| b.spans.iter())
4590 .filter_map(|s| s.footnote_ref.as_deref())
4591 .collect::<Vec<_>>(),
4592 vec!["1"],
4593 "failed for {html}"
4594 );
4595 }
4596 }
4597
4598 #[test]
4599 fn an_empty_or_absent_label_is_not_a_reference() {
4600 for html in [
4601 r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
4602 r#"<p>a<sup></sup></p>"#,
4603 ] {
4604 let blocks = parse_html(html);
4605 assert!(
4606 blocks
4607 .iter()
4608 .flat_map(|b| b.spans.iter())
4609 .all(|s| s.footnote_ref.is_none()),
4610 "failed for {html}"
4611 );
4612 }
4613 }
4614}
4615
4616#[cfg(test)]
4617mod html_depth_tests {
4618 use super::*;
4621
4622 fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
4624 blocks
4625 .iter()
4626 .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
4627 .collect()
4628 }
4629
4630 #[test]
4635 fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
4636 use crate::parser_tools::list_depth::MAX_LIST_INDENT;
4637 let depth = 300;
4638 let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
4639 html.push_str(&"</li></ol>".repeat(depth));
4640 let blocks = parse_html(&html);
4641 let items: Vec<(String, u32)> = blocks
4642 .iter()
4643 .filter(|block| block.list_style.is_some())
4644 .map(|block| {
4645 let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
4646 (text, block.list_indent)
4647 })
4648 .collect();
4649 let expected: Vec<(String, u32)> = (0..depth)
4650 .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
4651 .collect();
4652 assert_eq!(items, expected);
4653 }
4654
4655 #[test]
4657 fn text_in_a_deep_html_list_outside_its_items_is_kept() {
4658 let depth = 40;
4659 let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
4660 html.push_str("<ul>loose words<li>last</li></ul>");
4661 html.push_str(&"</li></ul>".repeat(depth));
4662 let texts = block_texts(&parse_html(&html)).join(" ");
4663 assert!(texts.contains("loose words"), "{texts:?}");
4664 assert!(texts.contains("last"), "{texts:?}");
4665 assert!(texts.contains("item 39"), "{texts:?}");
4666 }
4667
4668 #[test]
4671 fn html_nested_past_the_reader_depth_keeps_its_words() {
4672 for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
4673 let html = format!(
4674 "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
4675 open.repeat(300),
4676 close.repeat(300)
4677 );
4678 let texts = block_texts(&parse_html(&html)).join(" ");
4679 let words: Vec<&str> = texts.split_whitespace().collect();
4680 assert_eq!(
4681 words,
4682 ["Top.", "Deep", "words", "here.", "End."],
4683 "for {open}"
4684 );
4685 }
4686 }
4687
4688 #[test]
4690 fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
4691 let html = format!(
4692 "<table><tr><td>{}deep words{}</td></tr></table>",
4693 "<span>".repeat(300),
4694 "</span>".repeat(300)
4695 );
4696 let elements = parse_html_elements(&html);
4697 let Some(ParsedElement::Table(table)) = elements.first() else {
4698 panic!("a table: {elements:?}");
4699 };
4700 let text: String = table.rows[0][0]
4701 .spans
4702 .iter()
4703 .map(|span| span.text.as_str())
4704 .collect();
4705 assert_eq!(text, "deep words");
4706 }
4707
4708 #[test]
4712 fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
4713 for cell in [
4714 "One<br/>Two",
4715 "<p>One</p><p>Two</p>",
4716 "<ul><li>One</li><li>Two</li></ul>",
4717 ] {
4718 let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
4719 let Some(ParsedElement::Table(table)) = elements.first() else {
4720 panic!("a table: {elements:?}");
4721 };
4722 let text: String = table.rows[0][0]
4723 .spans
4724 .iter()
4725 .map(|span| span.text.as_str())
4726 .collect();
4727 assert_eq!(text, "One Two", "for {cell}");
4728 }
4729 }
4730}