1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3
4#[derive(Debug, Clone, Default, PartialEq, Eq)]
10pub struct ParsedImage {
11 pub src: String,
12 pub alt: String,
13 pub width: i64,
19 pub height: i64,
20}
21
22#[derive(Debug, Clone, Default)]
29pub struct ParsedSpan {
30 pub text: String,
31 pub bold: bool,
32 pub italic: bool,
33 pub underline: bool,
34 pub strikeout: bool,
35 pub code: bool,
36 pub superscript: bool,
38 pub subscript: bool,
40 pub link_href: Option<String>,
41 pub image: Option<ParsedImage>,
43 pub footnote_ref: Option<String>,
46}
47
48#[derive(Debug, Clone)]
50pub struct ParsedTableCell {
51 pub spans: Vec<ParsedSpan>,
52}
53
54#[derive(Debug, Clone)]
56pub struct ParsedTable {
57 pub header_rows: usize,
59 pub rows: Vec<Vec<ParsedTableCell>>,
61 pub blockquote_depth: u32,
64 pub blockquote_opens: u32,
67}
68
69impl ParsedTable {
70 pub fn column_count(&self) -> usize {
80 self.rows.iter().map(Vec::len).max().unwrap_or(0)
81 }
82}
83
84const SQUARED_CELLS_ALWAYS: usize = 4096;
88
89const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
99
100fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
111 let squared = table.rows.len().saturating_mul(table.column_count());
112 let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
113 if squared <= allowed {
114 elements.push(ParsedElement::Table(table));
115 return;
116 }
117 elements.extend(
118 ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
119 .into_iter()
120 .filter(|block| spans_carry_content(&block.spans))
121 .map(ParsedElement::Block),
122 );
123}
124
125fn mark_quotation_opened(elements: &mut [ParsedElement], first: usize) {
129 let Some(rest) = elements.get_mut(first..) else {
130 return;
131 };
132 for element in rest {
133 match element {
134 ParsedElement::Block(block) => {
135 block.blockquote_opens += 1;
136 return;
137 }
138 ParsedElement::Table(table) => {
139 table.blockquote_opens += 1;
140 return;
141 }
142 ParsedElement::FootnoteDefinition { .. } => {}
143 }
144 }
145}
146
147fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
149 rows.iter().map(Vec::len).sum()
150}
151
152#[derive(Debug, Clone)]
154pub enum ParsedElement {
155 Block(ParsedBlock),
156 Table(ParsedTable),
157 FootnoteDefinition {
164 label: String,
165 blocks: Vec<ParsedBlock>,
166 },
167}
168
169impl ParsedElement {
170 pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
173 let mut blocks = Vec::new();
174 for elem in elements {
175 match elem {
176 ParsedElement::Block(b) => blocks.push(b),
177 ParsedElement::FootnoteDefinition { .. } => {}
182 ParsedElement::Table(t) => {
183 for row in t.rows {
184 for cell in row {
185 blocks.push(ParsedBlock {
186 spans: cell.spans,
187 heading_level: None,
188 list_style: None,
189 list_indent: 0,
190 list_prefix: String::new(),
191 list_suffix: String::new(),
192 list_start: None,
193 marker: None,
194 is_code_block: false,
195 code_language: None,
196 blockquote_depth: t.blockquote_depth,
197 blockquote_opens: 0,
198 line_height: None,
199 non_breakable_lines: None,
200 page_break_before: None,
201 front_matter: None,
202 direction: None,
203 background_color: None,
204 alignment: None,
205 top_margin: None,
206 text_indent: None,
207 semantic_role: None,
208 });
209 }
210 }
211 }
212 }
213 }
214 if blocks.is_empty() {
215 blocks.push(ParsedBlock {
216 spans: vec![ParsedSpan {
217 text: String::new(),
218 ..Default::default()
219 }],
220 heading_level: None,
221 list_style: None,
222 list_indent: 0,
223 list_prefix: String::new(),
224 list_suffix: String::new(),
225 list_start: None,
226 marker: None,
227 is_code_block: false,
228 code_language: None,
229 blockquote_depth: 0,
230 blockquote_opens: 0,
231 line_height: None,
232 non_breakable_lines: None,
233 page_break_before: None,
234 front_matter: None,
235 direction: None,
236 background_color: None,
237 alignment: None,
238 top_margin: None,
239 text_indent: None,
240 semantic_role: None,
241 });
242 }
243 blocks
244 }
245}
246
247#[derive(Debug, Clone, Default)]
254pub struct ParsedBlock {
255 pub spans: Vec<ParsedSpan>,
256 pub heading_level: Option<i64>,
257 pub list_style: Option<ListStyle>,
258 pub list_indent: u32,
259 pub list_prefix: String,
262 pub list_suffix: String,
265 pub list_start: Option<i64>,
269 pub marker: Option<MarkerType>,
272 pub is_code_block: bool,
273 pub code_language: Option<String>,
274 pub blockquote_depth: u32,
275 pub blockquote_opens: u32,
282 pub line_height: Option<i64>,
283 pub non_breakable_lines: Option<bool>,
284 pub page_break_before: Option<bool>,
287 pub front_matter: Option<bool>,
290 pub direction: Option<TextDirection>,
291 pub background_color: Option<String>,
292 pub alignment: Option<Alignment>,
295 pub top_margin: Option<i64>,
299 pub text_indent: Option<i64>,
303 pub semantic_role: Option<SemanticRole>,
307}
308
309impl ParsedBlock {
310 pub fn is_inline_only(&self) -> bool {
313 self.heading_level.is_none()
314 && self.list_style.is_none()
315 && !self.is_code_block
316 && self.blockquote_depth == 0
317 && self.line_height.is_none()
318 && self.non_breakable_lines.is_none()
319 && self.page_break_before.is_none()
320 && self.front_matter.is_none()
321 && self.direction.is_none()
322 && self.background_color.is_none()
323 && self.alignment.is_none()
324 && self.top_margin.is_none()
325 && self.text_indent.is_none()
326 }
327}
328
329fn dangling_footnote_labels(
358 markdown: &str,
359 options: pulldown_cmark::Options,
360) -> std::collections::BTreeSet<String> {
361 use pulldown_cmark::{Event, Parser, Tag};
362
363 let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
364 for event in Parser::new_ext(markdown, options) {
365 if let Event::Start(Tag::FootnoteDefinition(label)) = event {
366 defined.insert(label.to_string());
367 }
368 }
369
370 let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
371 let bytes = markdown.as_bytes();
372 let mut search_from = 0usize;
373 while let Some(rel) = markdown[search_from..].find("[^") {
374 let open = search_from + rel;
375 let label_start = open + 2;
376 let Some(close_rel) = markdown[label_start..].find(']') else {
377 break;
378 };
379 let close = label_start + close_rel;
380 let label = &markdown[label_start..close];
381 let looks_like_definition = bytes.get(close + 1) == Some(&b':');
386 if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
387 referenced.insert(label.to_string());
388 }
389 search_from = close + 1;
390 }
391
392 referenced.difference(&defined).cloned().collect()
393}
394
395pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
403 use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
404
405 let options = Options::ENABLE_STRIKETHROUGH
416 | Options::ENABLE_TABLES
417 | Options::ENABLE_TASKLISTS
418 | Options::ENABLE_FOOTNOTES;
419
420 let dangling = dangling_footnote_labels(markdown, options);
427 let augmented_owner;
428 let source: &str = if dangling.is_empty() {
429 markdown
430 } else {
431 augmented_owner = dangling
432 .iter()
433 .fold(markdown.to_string(), |mut acc, label| {
434 acc.push_str("\n\n[^");
435 acc.push_str(label);
436 acc.push_str("]:\n");
437 acc
438 });
439 &augmented_owner
440 };
441 let parser = Parser::new_ext(source, options);
442
443 let mut elements: Vec<ParsedElement> = Vec::new();
444 let mut current_spans: Vec<ParsedSpan> = Vec::new();
445 let mut current_heading: Option<i64> = None;
446 let mut current_list_style: Option<ListStyle> = None;
447 let mut current_list_start: Option<i64> = None;
448 let mut is_code_block = false;
449 let mut code_language: Option<String> = None;
450 let mut blockquote_depth: u32 = 0;
451 let mut quotations_open: Vec<usize> = Vec::new();
453 let mut in_block = false;
454
455 let mut bold = false;
457 let mut italic = false;
458 let mut strikeout = false;
459 let mut link_href: Option<String> = None;
460 let mut pending_image: Option<ParsedImage> = None;
462
463 let mut footnote_open: Option<(String, usize)> = None;
470
471 let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
474 let mut list_start_stack: Vec<Option<i64>> = Vec::new();
475 let mut current_list_indent: u32 = 0;
476
477 let mut in_table = false;
479 let mut in_table_head = false;
480 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
481 let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
482 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
483 let mut table_header_rows: usize = 0;
484
485 for event in parser {
486 match event {
487 Event::Start(Tag::Paragraph) => {
488 in_block = true;
489 current_heading = None;
490 is_code_block = false;
491 }
492 Event::End(TagEnd::Paragraph) => {
493 if !current_spans.is_empty() || in_block {
494 elements.push(ParsedElement::Block(ParsedBlock {
495 spans: std::mem::take(&mut current_spans),
496 heading_level: current_heading.take(),
497 list_style: current_list_style.clone(),
498 list_indent: current_list_indent,
499 list_prefix: String::new(),
500 list_suffix: String::new(),
501 list_start: current_list_start,
502 marker: None,
503 is_code_block: false,
504 code_language: None,
505 blockquote_depth,
506 blockquote_opens: 0,
507 line_height: None,
508 non_breakable_lines: None,
509 page_break_before: None,
510 front_matter: None,
511 direction: None,
512 background_color: None,
513 alignment: None,
514 top_margin: None,
515 text_indent: None,
516 semantic_role: None,
517 }));
518 }
519 in_block = false;
520 current_list_style = None;
521 current_list_start = None;
522 }
523 Event::Start(Tag::Heading { level, .. }) => {
524 in_block = true;
525 current_heading = Some(heading_level_to_i64(level));
526 is_code_block = false;
527 }
528 Event::End(TagEnd::Heading(_)) => {
529 elements.push(ParsedElement::Block(ParsedBlock {
530 spans: std::mem::take(&mut current_spans),
531 heading_level: current_heading.take(),
532 list_style: None,
533 list_indent: 0,
534 list_prefix: String::new(),
535 list_suffix: String::new(),
536 list_start: None,
537 marker: None,
538 is_code_block: false,
539 code_language: None,
540 blockquote_depth,
541 blockquote_opens: 0,
542 line_height: None,
543 non_breakable_lines: None,
544 page_break_before: None,
545 front_matter: None,
546 direction: None,
547 background_color: None,
548 alignment: None,
549 top_margin: None,
550 text_indent: None,
551 semantic_role: None,
552 }));
553 in_block = false;
554 }
555 Event::Start(Tag::List(ordered)) => {
556 let style = if ordered.is_some() {
557 Some(ListStyle::Decimal)
558 } else {
559 Some(ListStyle::Disc)
560 };
561 list_stack.push(style);
562 list_start_stack.push(ordered.and_then(list_start_of));
563 }
564 Event::End(TagEnd::List(_)) => {
565 list_stack.pop();
566 list_start_stack.pop();
567 }
568 Event::Start(Tag::Item) => {
569 if !current_spans.is_empty() {
572 elements.push(ParsedElement::Block(ParsedBlock {
573 spans: std::mem::take(&mut current_spans),
574 heading_level: None,
575 list_style: current_list_style.clone(),
576 list_indent: current_list_indent,
577 list_prefix: String::new(),
578 list_suffix: String::new(),
579 list_start: current_list_start,
580 marker: None,
581 is_code_block: false,
582 code_language: None,
583 blockquote_depth,
584 blockquote_opens: 0,
585 line_height: None,
586 non_breakable_lines: None,
587 page_break_before: None,
588 front_matter: None,
589 direction: None,
590 background_color: None,
591 alignment: None,
592 top_margin: None,
593 text_indent: None,
594 semantic_role: None,
595 }));
596 }
597 in_block = true;
598 current_list_style = list_stack.last().cloned().flatten();
599 current_list_start = list_start_stack.last().copied().flatten();
600 current_list_indent = if list_stack.is_empty() {
601 0
602 } else {
603 (list_stack.len() - 1) as u32
604 };
605 }
606 Event::End(TagEnd::Item) => {
607 if !current_spans.is_empty() {
610 elements.push(ParsedElement::Block(ParsedBlock {
611 spans: std::mem::take(&mut current_spans),
612 heading_level: None,
613 list_style: current_list_style.clone(),
614 list_indent: current_list_indent,
615 list_prefix: String::new(),
616 list_suffix: String::new(),
617 list_start: current_list_start,
618 marker: None,
619 is_code_block: false,
620 code_language: None,
621 blockquote_depth,
622 blockquote_opens: 0,
623 line_height: None,
624 non_breakable_lines: None,
625 page_break_before: None,
626 front_matter: None,
627 direction: None,
628 background_color: None,
629 alignment: None,
630 top_margin: None,
631 text_indent: None,
632 semantic_role: None,
633 }));
634 }
635 in_block = false;
636 current_list_style = None;
637 current_list_start = None;
638 }
639 Event::Start(Tag::CodeBlock(kind)) => {
640 in_block = true;
641 is_code_block = true;
642 code_language = match &kind {
643 pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
644 Some(lang.to_string())
645 }
646 _ => None,
647 };
648 }
649 Event::End(TagEnd::CodeBlock) => {
650 if let Some(last) = current_spans.last_mut()
652 && last.text.ends_with('\n')
653 {
654 last.text.truncate(last.text.len() - 1);
655 }
656 elements.push(ParsedElement::Block(ParsedBlock {
657 spans: std::mem::take(&mut current_spans),
658 heading_level: None,
659 list_style: None,
660 list_indent: 0,
661 list_prefix: String::new(),
662 list_suffix: String::new(),
663 list_start: None,
664 marker: None,
665 is_code_block: true,
666 code_language: code_language.take(),
667 blockquote_depth,
668 blockquote_opens: 0,
669 line_height: None,
670 non_breakable_lines: None,
671 page_break_before: None,
672 front_matter: None,
673 direction: None,
674 background_color: None,
675 alignment: None,
676 top_margin: None,
677 text_indent: None,
678 semantic_role: None,
679 }));
680 in_block = false;
681 is_code_block = false;
682 }
683 Event::Start(Tag::Table(_)) => {
685 in_table = true;
686 in_table_head = false;
687 table_rows.clear();
688 current_row_cells.clear();
689 current_cell_spans.clear();
690 table_header_rows = 0;
691 }
692 Event::End(TagEnd::Table) => {
693 let own_cells = cells_in_rows(&table_rows);
694 push_table(
695 &mut elements,
696 ParsedTable {
697 header_rows: table_header_rows,
698 rows: std::mem::take(&mut table_rows),
699 blockquote_depth,
700 blockquote_opens: 0,
701 },
702 own_cells,
703 );
704 in_table = false;
705 }
706 Event::Start(Tag::TableHead) => {
707 in_table_head = true;
708 current_row_cells.clear();
709 }
710 Event::End(TagEnd::TableHead) => {
711 table_rows.push(std::mem::take(&mut current_row_cells));
713 table_header_rows += 1;
714 in_table_head = false;
715 }
716 Event::Start(Tag::TableRow) => {
717 current_row_cells.clear();
718 }
719 Event::End(TagEnd::TableRow) if !in_table_head => {
720 table_rows.push(std::mem::take(&mut current_row_cells));
722 }
723 Event::Start(Tag::TableCell) => {
724 current_cell_spans.clear();
725 }
726 Event::End(TagEnd::TableCell) => {
727 current_row_cells.push(ParsedTableCell {
728 spans: std::mem::take(&mut current_cell_spans),
729 });
730 }
731 Event::Start(Tag::Emphasis) => {
733 italic = true;
734 }
735 Event::End(TagEnd::Emphasis) => {
736 italic = false;
737 }
738 Event::Start(Tag::Strong) => {
739 bold = true;
740 }
741 Event::End(TagEnd::Strong) => {
742 bold = false;
743 }
744 Event::Start(Tag::Strikethrough) => {
745 strikeout = true;
746 }
747 Event::End(TagEnd::Strikethrough) => {
748 strikeout = false;
749 }
750 Event::Start(Tag::Link { dest_url, .. }) => {
751 link_href = Some(dest_url.to_string());
752 }
753 Event::End(TagEnd::Link) => {
754 link_href = None;
755 }
756 Event::Start(Tag::Image { dest_url, .. }) => {
759 pending_image = Some(ParsedImage {
760 src: dest_url.to_string(),
761 alt: String::new(),
762 width: 0,
763 height: 0,
764 });
765 }
766 Event::End(TagEnd::Image) => {
767 if let Some(image) = pending_image.take() {
768 let span = ParsedSpan {
769 text: String::new(),
770 bold,
771 italic,
772 underline: false,
773 strikeout,
774 code: false,
775 superscript: false,
776 subscript: false,
777 link_href: link_href.clone(),
778 image: Some(image),
779 footnote_ref: None,
780 };
781 if in_table {
782 current_cell_spans.push(span);
783 } else {
784 if !in_block {
785 in_block = true;
786 }
787 current_spans.push(span);
788 }
789 }
790 }
791 Event::Text(text) => {
792 if let Some(img) = pending_image.as_mut() {
797 img.alt.push_str(&text);
798 continue;
799 }
800 let span = ParsedSpan {
801 text: text.to_string(),
802 bold,
803 italic,
804 underline: false,
805 strikeout,
806 code: is_code_block,
807 superscript: false,
808 subscript: false,
809 link_href: link_href.clone(),
810 image: None,
811 footnote_ref: None,
812 };
813 if in_table {
814 current_cell_spans.push(span);
815 } else {
816 if !in_block {
817 in_block = true;
818 }
819 current_spans.push(span);
820 }
821 }
822 Event::Code(text) => {
823 let span = ParsedSpan {
824 text: text.to_string(),
825 bold,
826 italic,
827 underline: false,
828 strikeout,
829 code: true,
830 superscript: false,
831 subscript: false,
832 link_href: link_href.clone(),
833 image: None,
834 footnote_ref: None,
835 };
836 if in_table {
837 current_cell_spans.push(span);
838 } else {
839 if !in_block {
840 in_block = true;
841 }
842 current_spans.push(span);
843 }
844 }
845 Event::SoftBreak => {
846 let span = ParsedSpan {
847 text: " ".to_string(),
848 bold,
849 italic,
850 underline: false,
851 strikeout,
852 code: false,
853 superscript: false,
854 subscript: false,
855 link_href: link_href.clone(),
856 image: None,
857 footnote_ref: None,
858 };
859 if in_table {
860 current_cell_spans.push(span);
861 } else {
862 current_spans.push(span);
863 }
864 }
865 Event::HardBreak if !current_spans.is_empty() || in_block => {
866 elements.push(ParsedElement::Block(ParsedBlock {
868 spans: std::mem::take(&mut current_spans),
869 heading_level: current_heading.take(),
870 list_style: current_list_style.clone(),
871 list_indent: current_list_indent,
872 list_prefix: String::new(),
873 list_suffix: String::new(),
874 list_start: current_list_start,
875 marker: None,
876 is_code_block,
877 code_language: code_language.clone(),
878 blockquote_depth,
879 blockquote_opens: 0,
880 line_height: None,
881 non_breakable_lines: None,
882 page_break_before: None,
883 front_matter: None,
884 direction: None,
885 background_color: None,
886 alignment: None,
887 top_margin: None,
888 text_indent: None,
889 semantic_role: None,
890 }));
891 }
892 Event::Start(Tag::BlockQuote(_)) => {
893 blockquote_depth += 1;
894 quotations_open.push(elements.len());
895 }
896 Event::End(TagEnd::BlockQuote(_)) => {
897 blockquote_depth = blockquote_depth.saturating_sub(1);
898 if let Some(first) = quotations_open.pop() {
899 mark_quotation_opened(&mut elements, first);
900 }
901 }
902 Event::Start(Tag::FootnoteDefinition(label)) => {
913 if !current_spans.is_empty() {
914 elements.push(ParsedElement::Block(ParsedBlock {
915 spans: std::mem::take(&mut current_spans),
916 heading_level: current_heading.take(),
917 list_style: current_list_style.clone(),
918 list_indent: current_list_indent,
919 list_prefix: String::new(),
920 list_suffix: String::new(),
921 list_start: current_list_start,
922 marker: None,
923 is_code_block: false,
924 code_language: None,
925 blockquote_depth,
926 blockquote_opens: 0,
927 line_height: None,
928 non_breakable_lines: None,
929 page_break_before: None,
930 front_matter: None,
931 direction: None,
932 background_color: None,
933 alignment: None,
934 top_margin: None,
935 text_indent: None,
936 semantic_role: None,
937 }));
938 }
939 footnote_open = Some((label.to_string(), elements.len()));
940 }
941 Event::End(TagEnd::FootnoteDefinition) => {
942 if !current_spans.is_empty() {
943 elements.push(ParsedElement::Block(ParsedBlock {
944 spans: std::mem::take(&mut current_spans),
945 heading_level: current_heading.take(),
946 list_style: current_list_style.clone(),
947 list_indent: current_list_indent,
948 list_prefix: String::new(),
949 list_suffix: String::new(),
950 list_start: current_list_start,
951 marker: None,
952 is_code_block: false,
953 code_language: None,
954 blockquote_depth,
955 blockquote_opens: 0,
956 line_height: None,
957 non_breakable_lines: None,
958 page_break_before: None,
959 front_matter: None,
960 direction: None,
961 background_color: None,
962 alignment: None,
963 top_margin: None,
964 text_indent: None,
965 semantic_role: None,
966 }));
967 }
968 if let Some((label, start)) = footnote_open.take() {
969 let blocks: Vec<ParsedBlock> = elements
970 .drain(start..)
971 .filter_map(|e| match e {
972 ParsedElement::Block(b) => Some(b),
973 _ => None,
976 })
977 .collect();
978 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
979 }
980 }
981 Event::FootnoteReference(label) => {
987 let span = ParsedSpan {
988 text: String::new(),
989 bold,
990 italic,
991 underline: false,
992 strikeout,
993 code: false,
994 superscript: false,
995 subscript: false,
996 link_href: link_href.clone(),
997 image: None,
998 footnote_ref: Some(label.to_string()),
999 };
1000 if in_table {
1001 current_cell_spans.push(span);
1002 } else {
1003 if !in_block {
1004 in_block = true;
1005 }
1006 current_spans.push(span);
1007 }
1008 }
1009 _ => {}
1010 }
1011 }
1012
1013 if !current_spans.is_empty() {
1015 elements.push(ParsedElement::Block(ParsedBlock {
1016 spans: std::mem::take(&mut current_spans),
1017 heading_level: current_heading,
1018 list_style: current_list_style,
1019 list_indent: current_list_indent,
1020 list_prefix: String::new(),
1021 list_suffix: String::new(),
1022 list_start: current_list_start,
1023 marker: None,
1024 is_code_block,
1025 code_language: code_language.take(),
1026 blockquote_depth,
1027 blockquote_opens: 0,
1028 line_height: None,
1029 non_breakable_lines: None,
1030 page_break_before: None,
1031 front_matter: None,
1032 direction: None,
1033 background_color: None,
1034 alignment: None,
1035 top_margin: None,
1036 text_indent: None,
1037 semantic_role: None,
1038 }));
1039 }
1040
1041 if !dangling.is_empty() {
1048 elements.retain(
1049 |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
1050 );
1051 }
1052
1053 if elements.is_empty() {
1055 elements.push(ParsedElement::Block(ParsedBlock {
1056 spans: vec![ParsedSpan {
1057 text: String::new(),
1058 ..Default::default()
1059 }],
1060 heading_level: None,
1061 list_style: None,
1062 list_indent: 0,
1063 list_prefix: String::new(),
1064 list_suffix: String::new(),
1065 list_start: None,
1066 marker: None,
1067 is_code_block: false,
1068 code_language: None,
1069 blockquote_depth: 0,
1070 blockquote_opens: 0,
1071 line_height: None,
1072 non_breakable_lines: None,
1073 page_break_before: None,
1074 front_matter: None,
1075 direction: None,
1076 background_color: None,
1077 alignment: None,
1078 top_margin: None,
1079 text_indent: None,
1080 semantic_role: None,
1081 }));
1082 }
1083
1084 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1087 elements
1088}
1089
1090fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
1091 use pulldown_cmark::HeadingLevel;
1092 match level {
1093 HeadingLevel::H1 => 1,
1094 HeadingLevel::H2 => 2,
1095 HeadingLevel::H3 => 3,
1096 HeadingLevel::H4 => 4,
1097 HeadingLevel::H5 => 5,
1098 HeadingLevel::H6 => 6,
1099 }
1100}
1101
1102use scraper::Node;
1105
1106#[derive(Debug, Clone, Default)]
1108struct BlockStyles {
1109 line_height: Option<i64>,
1110 non_breakable_lines: Option<bool>,
1111 page_break_before: Option<bool>,
1112 direction: Option<TextDirection>,
1113 background_color: Option<String>,
1114 preserve_whitespace: Option<bool>,
1119}
1120
1121fn parse_block_styles(style: &str) -> BlockStyles {
1125 let mut result = BlockStyles::default();
1126 for part in style.split(';') {
1127 let part = part.trim();
1128 if let Some((prop, val)) = part.split_once(':') {
1129 let prop = prop.trim().to_ascii_lowercase();
1130 let val = val.trim();
1131 match prop.as_str() {
1132 "line-height" => {
1133 if let Ok(v) = val.parse::<f64>() {
1135 result.line_height = Some((v * 1000.0) as i64);
1136 }
1137 }
1138 "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1139 result.non_breakable_lines = Some(true);
1140 if val != "nowrap" {
1141 result.preserve_whitespace = Some(true);
1142 }
1143 }
1144 "break-before" | "page-break-before" => {
1148 result.page_break_before = match val.to_ascii_lowercase().as_str() {
1149 "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1150 "avoid" | "auto" => Some(false),
1151 _ => None,
1152 };
1153 }
1154 "direction" => {
1155 if val.eq_ignore_ascii_case("rtl") {
1156 result.direction = Some(TextDirection::RightToLeft);
1157 } else if val.eq_ignore_ascii_case("ltr") {
1158 result.direction = Some(TextDirection::LeftToRight);
1159 }
1160 }
1161 "background-color" | "background" => {
1162 result.background_color = Some(val.to_string());
1163 }
1164 _ => {}
1165 }
1166 }
1167 }
1168 result
1169}
1170
1171fn is_html_space(ch: char) -> bool {
1178 matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1179}
1180
1181fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1183 spans
1184 .iter()
1185 .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1186}
1187
1188fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1202 fn is_replaced(span: &ParsedSpan) -> bool {
1203 span.image.is_some() || span.footnote_ref.is_some()
1204 }
1205
1206 for span in spans.iter_mut() {
1210 if is_replaced(span) {
1211 continue;
1212 }
1213 let mut out = String::with_capacity(span.text.len());
1214 let mut in_run = false;
1215 for ch in span.text.chars() {
1216 if is_html_space(ch) {
1217 if !in_run {
1218 out.push(' ');
1219 in_run = true;
1220 }
1221 } else {
1222 out.push(ch);
1223 in_run = false;
1224 }
1225 }
1226 span.text = out;
1227 }
1228
1229 let mut prev_ends_with_space = true;
1232 for span in spans.iter_mut() {
1233 if is_replaced(span) {
1234 prev_ends_with_space = false;
1235 continue;
1236 }
1237 if span.text.is_empty() {
1238 prev_ends_with_space = false;
1242 continue;
1243 }
1244 if prev_ends_with_space && span.text.starts_with(' ') {
1245 span.text.remove(0);
1246 }
1247 if !span.text.is_empty() {
1250 prev_ends_with_space = span.text.ends_with(' ');
1251 }
1252 }
1253
1254 for span in spans.iter_mut().rev() {
1258 if is_replaced(span) {
1259 break;
1260 }
1261 if span.text.is_empty() {
1262 continue;
1263 }
1264 if span.text.ends_with(' ') {
1265 span.text.pop();
1266 }
1267 break;
1268 }
1269}
1270
1271pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
1292 let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
1293 if !block.spans.iter().any(breaks) {
1294 return vec![block];
1295 }
1296 let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
1297 for mut span in std::mem::take(&mut block.spans) {
1298 if !breaks(&span) {
1299 push_to_last_line(&mut lines, span);
1300 continue;
1301 }
1302 let text = std::mem::take(&mut span.text);
1303 for (i, piece) in text.split('\n').enumerate() {
1304 if i > 0 {
1305 lines.push(Vec::new());
1306 }
1307 if !piece.is_empty() {
1308 push_to_last_line(
1309 &mut lines,
1310 ParsedSpan {
1311 text: piece.to_string(),
1312 ..span.clone()
1313 },
1314 );
1315 }
1316 }
1317 }
1318 if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
1319 lines.pop();
1320 }
1321 let later_line = ParsedBlock {
1322 page_break_before: None,
1323 blockquote_opens: 0,
1324 ..block.clone()
1325 };
1326 let mut first_line = Some(block);
1327 lines
1328 .into_iter()
1329 .map(|spans| {
1330 let spans = if spans.is_empty() {
1331 vec![ParsedSpan::default()]
1332 } else {
1333 spans
1334 };
1335 match first_line.take() {
1336 Some(first) => ParsedBlock { spans, ..first },
1337 None => ParsedBlock {
1338 spans,
1339 ..later_line.clone()
1340 },
1341 }
1342 })
1343 .collect()
1344}
1345
1346fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
1349 match lines.last_mut() {
1350 Some(line) => line.push(span),
1351 None => lines.push(vec![span]),
1352 }
1353}
1354
1355pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
1356 ParsedElement::flatten_to_blocks(parse_html_elements(html))
1357}
1358
1359pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
1373
1374fn html_footnote_span(
1377 el: &scraper::node::Element,
1378 link_href: Option<String>,
1379) -> Option<ParsedSpan> {
1380 let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
1381 if label.is_empty() {
1382 return None;
1383 }
1384 Some(ParsedSpan {
1385 text: String::new(),
1386 link_href,
1387 footnote_ref: Some(label.to_string()),
1388 ..Default::default()
1389 })
1390}
1391
1392fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
1400 let src = el.attr("src")?;
1401 if src.is_empty() {
1402 return None;
1403 }
1404 let dim = |name: &str| -> i64 {
1405 el.attr(name)
1406 .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
1407 .filter(|n| *n > 0)
1408 .unwrap_or(0)
1409 };
1410 Some(ParsedSpan {
1411 text: String::new(),
1412 link_href,
1413 image: Some(ParsedImage {
1414 src: src.to_string(),
1415 alt: el.attr("alt").unwrap_or_default().to_string(),
1416 width: dim("width"),
1417 height: dim("height"),
1418 }),
1419 ..Default::default()
1420 })
1421}
1422
1423fn is_html_list(node: &Node) -> bool {
1425 node.as_element()
1426 .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
1427}
1428
1429fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
1442 use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
1443 use ego_tree::iter::Edge;
1444
1445 let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
1447 let mut depth = 0usize;
1448 for edge in tree.root().traverse() {
1449 match edge {
1450 Edge::Open(node) if is_html_list(node.value()) => {
1451 if depth == MAX_LIST_LEVELS {
1452 too_deep.push(node.id());
1453 }
1454 depth += 1;
1455 }
1456 Edge::Close(node) if is_html_list(node.value()) => {
1457 depth = depth.saturating_sub(1);
1458 }
1459 _ => {}
1460 }
1461 }
1462
1463 for list in too_deep {
1464 let Some(list_ref) = tree.get(list) else {
1468 continue;
1469 };
1470 let mut holder = None;
1471 let mut current = list_ref;
1472 while let Some(parent) = current.parent() {
1473 if is_html_list(parent.value()) {
1474 holder = Some(current.id());
1475 break;
1476 }
1477 current = parent;
1478 }
1479 let Some(holder) = holder else {
1480 continue;
1481 };
1482 let mut after = holder;
1483 let items: Vec<ego_tree::NodeId> = list_ref
1484 .descendants()
1485 .filter(|node| {
1486 node.value()
1487 .as_element()
1488 .is_some_and(|el| el.name() == "li")
1489 })
1490 .map(|node| node.id())
1491 .collect();
1492 for item in items {
1493 if tree.get(after).and_then(|node| node.parent()).is_none() {
1496 break;
1497 }
1498 if let Some(mut node) = tree.get_mut(after) {
1499 node.insert_id_after(item);
1500 after = item;
1501 }
1502 }
1503 let holds_text = tree.get(list).is_some_and(|node| {
1508 node.descendants()
1509 .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
1510 });
1511 if !holds_text {
1512 if let Some(mut node) = tree.get_mut(list) {
1513 node.detach();
1514 }
1515 } else if holder != list
1516 && tree.get(after).and_then(|node| node.parent()).is_some()
1517 && let Some(mut node) = tree.get_mut(after)
1518 {
1519 node.insert_id_after(list);
1520 }
1521 }
1522}
1523
1524const HTML_DEPTH_ON_THE_CALLERS_STACK: usize = 32;
1528
1529const MAX_HTML_READ_DEPTH: usize = 256;
1532
1533const HTML_READER_STACK: usize = 2 << 20;
1536
1537const HTML_STACK_PER_LEVEL: usize = 16 << 10;
1540
1541fn html_integer(value: &str) -> Option<i64> {
1547 let value = value.trim_start_matches(|c: char| c.is_ascii_whitespace());
1548 let (negative, digits) = match value.as_bytes().first() {
1549 Some(b'-') => (true, &value[1..]),
1550 Some(b'+') => (false, &value[1..]),
1551 _ => (false, value),
1552 };
1553 let digits: &str = digits
1554 .split(|c: char| !c.is_ascii_digit())
1555 .next()
1556 .unwrap_or("");
1557 if digits.is_empty() {
1558 return None;
1559 }
1560 let magnitude = digits.bytes().fold(0i64, |number, digit| {
1561 number
1562 .saturating_mul(10)
1563 .saturating_add(i64::from(digit - b'0'))
1564 });
1565 Some(if negative { -magnitude } else { magnitude })
1566}
1567
1568fn html_tree_depth(tree: &ego_tree::Tree<Node>) -> usize {
1571 use ego_tree::iter::Edge;
1572 let mut depth = 0usize;
1573 let mut deepest = 0usize;
1574 for edge in tree.root().traverse() {
1575 match edge {
1576 Edge::Open(node) if node.value().is_element() => {
1577 depth += 1;
1578 deepest = deepest.max(depth);
1579 }
1580 Edge::Close(node) if node.value().is_element() => depth = depth.saturating_sub(1),
1581 _ => {}
1582 }
1583 }
1584 deepest
1585}
1586
1587pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
1599 use scraper::Html;
1600
1601 let read = |depth_limit: usize, html: &str| {
1602 let mut fragment = Html::parse_fragment(html);
1603 flatten_deep_html_lists(&mut fragment.tree);
1604 let mut elements = read_html_tree(&fragment, depth_limit);
1605 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1606 elements
1607 };
1608 let mut fragment = Html::parse_fragment(html);
1609 flatten_deep_html_lists(&mut fragment.tree);
1610 let depth = html_tree_depth(&fragment.tree);
1611 if depth <= HTML_DEPTH_ON_THE_CALLERS_STACK {
1612 let mut elements = read_html_tree(&fragment, MAX_HTML_READ_DEPTH);
1613 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1614 return elements;
1615 }
1616 drop(fragment);
1617 let stack = HTML_READER_STACK + depth.min(MAX_HTML_READ_DEPTH + 1) * HTML_STACK_PER_LEVEL;
1618 let owned = html.to_owned();
1619 let on_its_own_thread = std::thread::Builder::new()
1620 .stack_size(stack)
1621 .spawn(move || read(MAX_HTML_READ_DEPTH, &owned))
1622 .ok()
1623 .and_then(|reader| reader.join().ok());
1624 on_its_own_thread.unwrap_or_else(|| read(HTML_DEPTH_ON_THE_CALLERS_STACK, html))
1625}
1626
1627fn read_html_tree(fragment: &scraper::Html, depth_limit: usize) -> Vec<ParsedElement> {
1631 let mut elements: Vec<ParsedElement> = Vec::new();
1632
1633 let root = fragment.root_element();
1635
1636 #[derive(Clone, Default)]
1637 struct FmtState {
1638 bold: bool,
1639 italic: bool,
1640 underline: bool,
1641 strikeout: bool,
1642 code: bool,
1643 superscript: bool,
1644 subscript: bool,
1645 link_href: Option<String>,
1646 }
1647
1648 type HtmlList = (Option<ListStyle>, Option<i64>);
1651
1652 fn is_metadata_tag(tag: &str) -> bool {
1662 matches!(
1663 tag,
1664 "head"
1665 | "style"
1666 | "script"
1667 | "title"
1668 | "meta"
1669 | "link"
1670 | "base"
1671 | "noscript"
1672 | "template"
1673 )
1674 }
1675
1676 fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
1685 use ego_tree::iter::Edge;
1686 let mut text = String::new();
1687 let mut hidden = 0usize;
1689 for edge in node.traverse() {
1690 match edge {
1691 Edge::Open(open) => match open.value() {
1692 Node::Element(el) => {
1693 if hidden > 0 || is_metadata_tag(el.name()) {
1694 hidden += 1;
1695 } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
1696 text.push(' ');
1697 }
1698 }
1699 Node::Text(t) if hidden == 0 => text.push_str(t),
1700 _ => {}
1701 },
1702 Edge::Close(close) => {
1703 if hidden > 0 && close.value().is_element() {
1704 hidden -= 1;
1705 }
1706 }
1707 }
1708 }
1709 text
1710 }
1711
1712 fn text_span(text: String, state: &FmtState) -> ParsedSpan {
1714 ParsedSpan {
1715 text,
1716 bold: state.bold,
1717 italic: state.italic,
1718 underline: state.underline,
1719 strikeout: state.strikeout,
1720 code: state.code,
1721 superscript: state.superscript,
1722 subscript: state.subscript,
1723 link_href: state.link_href.clone(),
1724 image: None,
1725 footnote_ref: None,
1726 }
1727 }
1728
1729 fn collect_cell_spans(
1731 node: ego_tree::NodeRef<Node>,
1732 state: &FmtState,
1733 spans: &mut Vec<ParsedSpan>,
1734 depth: usize,
1735 limit: usize,
1736 ) {
1737 if depth > limit {
1738 let text = subtree_text(node);
1739 if !text.is_empty() {
1740 spans.push(text_span(text, state));
1741 }
1742 return;
1743 }
1744 for child in node.children() {
1745 match child.value() {
1746 Node::Text(text) => {
1747 let t = text.text.to_string();
1748 if !t.is_empty() {
1749 spans.push(ParsedSpan {
1750 text: t,
1751 bold: state.bold,
1752 italic: state.italic,
1753 underline: state.underline,
1754 strikeout: state.strikeout,
1755 code: state.code,
1756 superscript: state.superscript,
1757 subscript: state.subscript,
1758 link_href: state.link_href.clone(),
1759 image: None,
1760 footnote_ref: None,
1761 });
1762 }
1763 }
1764 Node::Element(el) => {
1765 let tag = el.name();
1766 if is_metadata_tag(tag) {
1767 continue;
1768 }
1769 let mut new_state = state.clone();
1770 match tag {
1771 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
1775 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
1776 {
1777 spans.push(span);
1778 }
1779 continue;
1780 }
1781 "b" | "strong" => new_state.bold = true,
1782 "i" | "em" => new_state.italic = true,
1783 "u" | "ins" => new_state.underline = true,
1784 "s" | "del" | "strike" => new_state.strikeout = true,
1785 "code" => new_state.code = true,
1786 "sup" => new_state.superscript = true,
1787 "sub" => new_state.subscript = true,
1788 "a" => {
1789 if let Some(href) = el.attr("href") {
1790 new_state.link_href = Some(href.to_string());
1791 }
1792 }
1793 "img" => {
1794 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
1795 spans.push(span);
1796 }
1797 continue;
1798 }
1799 "br" => {
1805 spans.push(text_span(" ".to_string(), state));
1806 continue;
1807 }
1808 "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
1809 | "h3" | "h4" | "h5" | "h6" => {
1810 spans.push(text_span(" ".to_string(), state));
1811 }
1812 _ => {}
1813 }
1814 collect_cell_spans(child, &new_state, spans, depth + 1, limit);
1815 }
1816 _ => {}
1817 }
1818 }
1819 }
1820
1821 const MAX_HTML_COLSPAN: usize = 1000;
1824
1825 fn parse_table_element(
1829 table_node: ego_tree::NodeRef<Node>,
1830 depth: usize,
1831 limit: usize,
1832 ) -> (ParsedTable, usize) {
1833 let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
1834 let mut header_rows: usize = 0;
1835 let mut own_cells: usize = 0;
1836
1837 fn collect_rows(
1839 node: ego_tree::NodeRef<Node>,
1840 rows: &mut Vec<Vec<ParsedTableCell>>,
1841 header_rows: &mut usize,
1842 own_cells: &mut usize,
1843 in_thead: bool,
1844 walk: (usize, usize),
1845 ) {
1846 let (depth, limit) = walk;
1847 for child in node.children() {
1848 if let Node::Element(el) = child.value() {
1849 match el.name() {
1850 "thead" => collect_rows(
1851 child,
1852 rows,
1853 header_rows,
1854 own_cells,
1855 true,
1856 (depth + 1, limit),
1857 ),
1858 "tbody" | "tfoot" => collect_rows(
1859 child,
1860 rows,
1861 header_rows,
1862 own_cells,
1863 false,
1864 (depth + 1, limit),
1865 ),
1866 "tr" => {
1867 let mut cells: Vec<ParsedTableCell> = Vec::new();
1868 let mut own_end = 0;
1871 for td in child.children() {
1872 if let Node::Element(td_el) = td.value()
1873 && matches!(td_el.name(), "td" | "th")
1874 {
1875 let mut spans = Vec::new();
1876 let state = FmtState::default();
1877 collect_cell_spans(td, &state, &mut spans, depth + 2, limit);
1878 collapse_inline_whitespace(&mut spans);
1879 if spans.is_empty() {
1880 spans.push(ParsedSpan::default());
1881 }
1882 cells.push(ParsedTableCell { spans });
1883 *own_cells += 1;
1884 own_end = cells.len();
1885 let span = td_el
1890 .attr("colspan")
1891 .and_then(|value| value.trim().parse::<usize>().ok())
1892 .unwrap_or(1)
1893 .clamp(1, MAX_HTML_COLSPAN);
1894 for _ in 1..span {
1895 if cells.len() >= MAX_HTML_COLSPAN {
1896 break;
1897 }
1898 cells.push(ParsedTableCell {
1899 spans: vec![ParsedSpan::default()],
1900 });
1901 }
1902 }
1903 }
1904 cells.truncate(own_end);
1908 if !cells.is_empty() {
1909 rows.push(cells);
1910 if in_thead {
1911 *header_rows += 1;
1912 }
1913 }
1914 }
1915 _ => {}
1916 }
1917 }
1918 }
1919 }
1920
1921 collect_rows(
1922 table_node,
1923 &mut rows,
1924 &mut header_rows,
1925 &mut own_cells,
1926 false,
1927 (depth, limit),
1928 );
1929
1930 if header_rows == 0 && !rows.is_empty() {
1932 header_rows = 1;
1933 }
1934
1935 (
1936 ParsedTable {
1937 header_rows,
1938 rows,
1939 blockquote_depth: 0,
1942 blockquote_opens: 0,
1943 },
1944 own_cells,
1945 )
1946 }
1947
1948 #[allow(clippy::too_many_arguments)]
1949 fn walk_node(
1950 node: ego_tree::NodeRef<Node>,
1951 state: &FmtState,
1952 elements: &mut Vec<ParsedElement>,
1953 current_list: &HtmlList,
1954 blockquote_depth: u32,
1955 list_depth: u32,
1956 depth: usize,
1957 limit: usize,
1958 ) {
1959 if depth > limit {
1960 let text = subtree_text(node);
1961 if !text.trim().is_empty() {
1962 let mut spans = vec![text_span(text, state)];
1963 collapse_inline_whitespace(&mut spans);
1964 elements.push(ParsedElement::Block(ParsedBlock {
1965 spans,
1966 blockquote_depth,
1967 blockquote_opens: 0,
1968 ..ParsedBlock::default()
1969 }));
1970 }
1971 return;
1972 }
1973 match node.value() {
1974 Node::Element(el) => {
1975 let tag = el.name();
1976 if is_metadata_tag(tag) {
1977 return;
1978 }
1979 let mut new_state = state.clone();
1980 let mut new_list = current_list.clone();
1981 let mut bq_depth = blockquote_depth;
1982 let mut new_list_depth = list_depth;
1983
1984 let is_block_tag = matches!(
1986 tag,
1987 "p" | "div"
1988 | "h1"
1989 | "h2"
1990 | "h3"
1991 | "h4"
1992 | "h5"
1993 | "h6"
1994 | "li"
1995 | "pre"
1996 | "br"
1997 | "blockquote"
1998 | "body"
1999 | "html"
2000 );
2001
2002 match tag {
2004 "b" | "strong" => new_state.bold = true,
2005 "i" | "em" => new_state.italic = true,
2006 "u" | "ins" => new_state.underline = true,
2007 "s" | "del" | "strike" => new_state.strikeout = true,
2008 "code" => new_state.code = true,
2009 "sup" => new_state.superscript = true,
2010 "sub" => new_state.subscript = true,
2011 "a" => {
2012 if let Some(href) = el.attr("href") {
2013 new_state.link_href = Some(href.to_string());
2014 }
2015 }
2016 "ul" => {
2017 new_list = (Some(ListStyle::Disc), None);
2018 new_list_depth = list_depth + 1;
2019 }
2020 "ol" => {
2021 let start = el
2025 .attr("start")
2026 .and_then(html_integer)
2027 .and_then(crate::parser_tools::list_start::list_start);
2028 new_list = (Some(ListStyle::Decimal), start);
2029 new_list_depth = list_depth + 1;
2030 }
2031 "blockquote" => {
2032 bq_depth += 1;
2033 }
2034 _ => {}
2035 }
2036
2037 let heading_level = match tag {
2039 "h1" => Some(1),
2040 "h2" => Some(2),
2041 "h3" => Some(3),
2042 "h4" => Some(4),
2043 "h5" => Some(5),
2044 "h6" => Some(6),
2045 _ => None,
2046 };
2047
2048 let is_code_block = tag == "pre";
2049
2050 let code_language = if is_code_block {
2052 node.children().find_map(|child| {
2053 if let Node::Element(cel) = child.value()
2054 && cel.name() == "code"
2055 && let Some(cls) = cel.attr("class")
2056 {
2057 return cls
2058 .split_whitespace()
2059 .find_map(|c| c.strip_prefix("language-"))
2060 .map(|l| l.to_string());
2061 }
2062 None
2063 })
2064 } else {
2065 None
2066 };
2067
2068 let css = if is_block_tag {
2070 el.attr("style").map(parse_block_styles).unwrap_or_default()
2071 } else {
2072 BlockStyles::default()
2073 };
2074
2075 if tag == "table" {
2076 let (mut parsed_table, own_cells) = parse_table_element(node, depth, limit);
2078 if !parsed_table.rows.is_empty() {
2079 parsed_table.blockquote_depth = bq_depth;
2080 push_table(elements, parsed_table, own_cells);
2081 }
2082 return;
2083 }
2084
2085 if tag == "br" {
2086 elements.push(ParsedElement::Block(ParsedBlock {
2088 spans: vec![ParsedSpan {
2089 text: String::new(),
2090 ..Default::default()
2091 }],
2092 heading_level: None,
2093 list_style: None,
2094 list_indent: 0,
2095 list_prefix: String::new(),
2096 list_suffix: String::new(),
2097 list_start: None,
2098 marker: None,
2099 is_code_block: false,
2100 code_language: None,
2101 blockquote_depth: bq_depth,
2102 blockquote_opens: 0,
2103 line_height: None,
2104 non_breakable_lines: None,
2105 page_break_before: None,
2106 front_matter: None,
2107 direction: None,
2108 background_color: None,
2109 alignment: None,
2110 top_margin: None,
2111 text_indent: None,
2112 semantic_role: None,
2113 }));
2114 return;
2115 }
2116
2117 if tag == "blockquote" {
2118 let first = elements.len();
2120 for child in node.children() {
2121 walk_node(
2122 child,
2123 &new_state,
2124 elements,
2125 &new_list,
2126 bq_depth,
2127 new_list_depth,
2128 depth + 1,
2129 limit,
2130 );
2131 }
2132 mark_quotation_opened(elements, first);
2133 } else if is_block_tag && tag != "br" {
2134 let mut spans: Vec<ParsedSpan> = Vec::new();
2139 let mut nested_elements: Vec<ParsedElement> = Vec::new();
2140 collect_inline_spans(
2141 node,
2142 &new_state,
2143 &mut spans,
2144 &new_list,
2145 &mut nested_elements,
2146 bq_depth,
2147 new_list_depth,
2148 depth + 1,
2149 limit,
2150 );
2151
2152 let (list_style_for_block, list_start_for_block) = if tag == "li" {
2153 new_list.clone()
2154 } else {
2155 (None, None)
2156 };
2157
2158 let list_indent_for_block = if tag == "li" {
2159 new_list_depth.saturating_sub(1)
2160 } else {
2161 0
2162 };
2163
2164 if !(is_code_block || css.preserve_whitespace == Some(true)) {
2165 collapse_inline_whitespace(&mut spans);
2166 }
2167
2168 let own_run_is_content =
2176 spans_carry_content(&spans) || nested_elements.is_empty();
2177
2178 if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
2179 let is_code_block = is_code_block
2183 && !spans
2184 .iter()
2185 .any(|span| span.image.is_some() || span.footnote_ref.is_some());
2186 let block = ParsedBlock {
2187 spans,
2188 heading_level,
2189 list_style: list_style_for_block,
2190 list_indent: list_indent_for_block,
2191 list_prefix: String::new(),
2192 list_suffix: String::new(),
2193 list_start: list_start_for_block,
2194 marker: None,
2195 is_code_block,
2196 code_language: code_language.filter(|_| is_code_block),
2197 blockquote_depth: bq_depth,
2198 blockquote_opens: 0,
2199 line_height: css.line_height,
2200 non_breakable_lines: css.non_breakable_lines,
2201 page_break_before: css.page_break_before,
2202 front_matter: None,
2203 direction: css.direction,
2204 background_color: css.background_color,
2205 alignment: None,
2206 top_margin: None,
2207 text_indent: None,
2208 semantic_role: None,
2209 };
2210 if is_code_block {
2214 elements.push(ParsedElement::Block(block));
2215 } else {
2216 elements.extend(
2217 split_block_at_line_breaks(block)
2218 .into_iter()
2219 .map(ParsedElement::Block),
2220 );
2221 }
2222 }
2223 elements.append(&mut nested_elements);
2225 } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
2226 for child in node.children() {
2228 walk_node(
2229 child,
2230 &new_state,
2231 elements,
2232 &new_list,
2233 bq_depth,
2234 new_list_depth,
2235 depth + 1,
2236 limit,
2237 );
2238 }
2239 } else {
2240 for child in node.children() {
2242 walk_node(
2243 child,
2244 &new_state,
2245 elements,
2246 current_list,
2247 bq_depth,
2248 list_depth,
2249 depth + 1,
2250 limit,
2251 );
2252 }
2253 }
2254 }
2255 Node::Text(text) => {
2256 let t = text.text.to_string();
2257 let trimmed = t.trim();
2258 if !trimmed.is_empty() {
2259 elements.push(ParsedElement::Block(ParsedBlock {
2261 spans: vec![ParsedSpan {
2262 text: trimmed.to_string(),
2263 bold: state.bold,
2264 italic: state.italic,
2265 underline: state.underline,
2266 strikeout: state.strikeout,
2267 code: state.code,
2268 superscript: state.superscript,
2269 subscript: state.subscript,
2270 link_href: state.link_href.clone(),
2271 image: None,
2272 footnote_ref: None,
2273 }],
2274 heading_level: None,
2275 list_style: None,
2276 list_indent: 0,
2277 list_prefix: String::new(),
2278 list_suffix: String::new(),
2279 list_start: None,
2280 marker: None,
2281 is_code_block: false,
2282 code_language: None,
2283 blockquote_depth,
2284 blockquote_opens: 0,
2285 line_height: None,
2286 non_breakable_lines: None,
2287 page_break_before: None,
2288 front_matter: None,
2289 direction: None,
2290 background_color: None,
2291 alignment: None,
2292 top_margin: None,
2293 text_indent: None,
2294 semantic_role: None,
2295 }));
2296 }
2297 }
2298 _ => {
2299 for child in node.children() {
2301 walk_node(
2302 child,
2303 state,
2304 elements,
2305 current_list,
2306 blockquote_depth,
2307 list_depth,
2308 depth + 1,
2309 limit,
2310 );
2311 }
2312 }
2313 }
2314 }
2315
2316 #[allow(clippy::too_many_arguments)]
2320 fn collect_inline_spans(
2321 node: ego_tree::NodeRef<Node>,
2322 state: &FmtState,
2323 spans: &mut Vec<ParsedSpan>,
2324 current_list: &HtmlList,
2325 elements: &mut Vec<ParsedElement>,
2326 blockquote_depth: u32,
2327 list_depth: u32,
2328 depth: usize,
2329 limit: usize,
2330 ) {
2331 if depth > limit {
2332 let text = subtree_text(node);
2333 if !text.is_empty() {
2334 spans.push(text_span(text, state));
2335 }
2336 return;
2337 }
2338 for child in node.children() {
2339 match child.value() {
2340 Node::Text(text) => {
2341 let t = text.text.to_string();
2342 if !t.is_empty() {
2343 spans.push(ParsedSpan {
2344 text: t,
2345 bold: state.bold,
2346 italic: state.italic,
2347 underline: state.underline,
2348 strikeout: state.strikeout,
2349 code: state.code,
2350 superscript: state.superscript,
2351 subscript: state.subscript,
2352 link_href: state.link_href.clone(),
2353 image: None,
2354 footnote_ref: None,
2355 });
2356 }
2357 }
2358 Node::Element(el) => {
2359 let tag = el.name();
2360 if is_metadata_tag(tag) {
2361 continue;
2362 }
2363 let mut new_state = state.clone();
2364
2365 match tag {
2366 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2370 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2371 {
2372 spans.push(span);
2373 }
2374 continue;
2375 }
2376 "b" | "strong" => new_state.bold = true,
2377 "i" | "em" => new_state.italic = true,
2378 "u" | "ins" => new_state.underline = true,
2379 "s" | "del" | "strike" => new_state.strikeout = true,
2380 "code" => new_state.code = true,
2381 "sup" => new_state.superscript = true,
2382 "sub" => new_state.subscript = true,
2383 "a" => {
2384 if let Some(href) = el.attr("href") {
2385 new_state.link_href = Some(href.to_string());
2386 }
2387 }
2388 "img" => {
2389 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2390 spans.push(span);
2391 }
2392 continue;
2393 }
2394 _ => {}
2395 }
2396
2397 let nested_block = matches!(
2399 tag,
2400 "p" | "div"
2401 | "h1"
2402 | "h2"
2403 | "h3"
2404 | "h4"
2405 | "h5"
2406 | "h6"
2407 | "li"
2408 | "pre"
2409 | "blockquote"
2410 | "ul"
2411 | "ol"
2412 );
2413
2414 if tag == "br" {
2415 spans.push(ParsedSpan {
2418 text: String::new(),
2419 ..Default::default()
2420 });
2421 } else if nested_block || tag == "table" {
2422 walk_node(
2424 child,
2425 &new_state,
2426 elements,
2427 current_list,
2428 blockquote_depth,
2429 list_depth,
2430 depth + 1,
2431 limit,
2432 );
2433 } else {
2434 collect_inline_spans(
2436 child,
2437 &new_state,
2438 spans,
2439 current_list,
2440 elements,
2441 blockquote_depth,
2442 list_depth,
2443 depth + 1,
2444 limit,
2445 );
2446 }
2447 }
2448 _ => {}
2449 }
2450 }
2451 }
2452
2453 let initial_state = FmtState::default();
2454 let mut root_spans: Vec<ParsedSpan> = Vec::new();
2458 collect_inline_spans(
2459 *root,
2460 &initial_state,
2461 &mut root_spans,
2462 &(None, None),
2463 &mut elements,
2464 0,
2465 0,
2466 0,
2467 depth_limit,
2468 );
2469 collapse_inline_whitespace(&mut root_spans);
2470 if spans_carry_content(&root_spans) {
2473 elements.push(ParsedElement::Block(ParsedBlock {
2474 spans: root_spans,
2475 heading_level: None,
2476 list_style: None,
2477 list_indent: 0,
2478 list_prefix: String::new(),
2479 list_suffix: String::new(),
2480 list_start: None,
2481 marker: None,
2482 is_code_block: false,
2483 code_language: None,
2484 blockquote_depth: 0,
2485 blockquote_opens: 0,
2486 line_height: None,
2487 non_breakable_lines: None,
2488 page_break_before: None,
2489 front_matter: None,
2490 direction: None,
2491 background_color: None,
2492 alignment: None,
2493 top_margin: None,
2494 text_indent: None,
2495 semantic_role: None,
2496 }));
2497 }
2498
2499 if elements.is_empty() {
2501 elements.push(ParsedElement::Block(ParsedBlock {
2502 spans: vec![ParsedSpan {
2503 text: String::new(),
2504 ..Default::default()
2505 }],
2506 heading_level: None,
2507 list_style: None,
2508 list_indent: 0,
2509 list_prefix: String::new(),
2510 list_suffix: String::new(),
2511 list_start: None,
2512 marker: None,
2513 is_code_block: false,
2514 code_language: None,
2515 blockquote_depth: 0,
2516 blockquote_opens: 0,
2517 line_height: None,
2518 non_breakable_lines: None,
2519 page_break_before: None,
2520 front_matter: None,
2521 direction: None,
2522 background_color: None,
2523 alignment: None,
2524 top_margin: None,
2525 text_indent: None,
2526 semantic_role: None,
2527 }));
2528 }
2529
2530 elements
2531}
2532
2533pub fn character_format_from_span(
2537 span: &ParsedSpan,
2538 is_code_block: bool,
2539) -> crate::format_runs::CharacterFormat {
2540 use crate::entities::CharVerticalAlignment;
2541 crate::format_runs::CharacterFormat {
2542 font_bold: if span.bold { Some(true) } else { None },
2543 font_italic: if span.italic { Some(true) } else { None },
2544 font_underline: if span.underline { Some(true) } else { None },
2545 font_strikeout: if span.strikeout { Some(true) } else { None },
2546 font_family: if span.code || is_code_block {
2547 Some("monospace".to_string())
2548 } else {
2549 None
2550 },
2551 anchor_href: span.link_href.clone(),
2552 is_anchor: if span.link_href.is_some() {
2553 Some(true)
2554 } else {
2555 None
2556 },
2557 vertical_alignment: if span.superscript {
2558 Some(CharVerticalAlignment::SuperScript)
2559 } else if span.subscript {
2560 Some(CharVerticalAlignment::SubScript)
2561 } else {
2562 None
2563 },
2564 ..Default::default()
2565 }
2566}
2567
2568pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
2580 use crate::format_runs::{
2581 CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
2582 };
2583
2584 let mut plain_text = String::new();
2585 let mut runs: Vec<FormatRun> = Vec::new();
2586 let mut images: Vec<ImageAnchor> = Vec::new();
2587 let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
2588 let default = CharacterFormat::default();
2589
2590 for span in spans {
2591 let byte_start = plain_text.len() as u32;
2592
2593 if let Some(label) = &span.footnote_ref {
2594 plain_text.push('\u{FFFC}');
2597 let mut format = character_format_from_span(span, is_code_block);
2612 format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
2613 footnote_refs.push(FootnoteRefAnchor {
2614 byte_offset: byte_start,
2615 label: label.clone(),
2616 format,
2617 });
2618 continue;
2619 }
2620
2621 if let Some(image) = &span.image {
2622 plain_text.push('\u{FFFC}');
2626 images.push(ImageAnchor {
2627 byte_offset: byte_start,
2628 name: image.src.clone(),
2629 alt: image.alt.clone(),
2630 width: image.width,
2631 height: image.height,
2632 quality: 100,
2633 format: character_format_from_span(span, is_code_block),
2634 });
2635 continue;
2636 }
2637
2638 plain_text.push_str(&span.text);
2639 let byte_end = plain_text.len() as u32;
2640 if byte_start == byte_end {
2641 continue;
2642 }
2643 let format = character_format_from_span(span, is_code_block);
2644 if format == default {
2645 continue;
2646 }
2647 runs.push(FormatRun {
2648 byte_start,
2649 byte_end,
2650 format,
2651 });
2652 }
2653 coalesce_in_place(&mut runs);
2654 ParsedInline {
2655 plain_text,
2656 runs,
2657 images,
2658 footnote_refs,
2659 }
2660}
2661
2662#[derive(Debug, Clone, Default)]
2669pub struct ParsedInline {
2670 pub plain_text: String,
2671 pub runs: Vec<crate::format_runs::FormatRun>,
2672 pub images: Vec<crate::format_runs::ImageAnchor>,
2673 pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
2674}
2675
2676fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
2684 use jotdown::ListBulletType as B;
2685 match b {
2686 B::Dash => ListStyle::Disc,
2687 B::Star => ListStyle::Circle,
2688 B::Plus => ListStyle::Square,
2689 }
2690}
2691
2692fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
2694 use jotdown::OrderedListNumbering as N;
2695 match n {
2696 N::Decimal => ListStyle::Decimal,
2697 N::AlphaLower => ListStyle::LowerAlpha,
2698 N::AlphaUpper => ListStyle::UpperAlpha,
2699 N::RomanLower => ListStyle::LowerRoman,
2700 N::RomanUpper => ListStyle::UpperRoman,
2701 }
2702}
2703
2704fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
2708 use jotdown::OrderedListStyle as S;
2709 match style {
2710 S::Period => (String::new(), ".".to_string()),
2711 S::Paren => (String::new(), ")".to_string()),
2712 S::ParenParen => ("(".to_string(), ")".to_string()),
2713 }
2714}
2715
2716#[derive(Debug, Clone, Default)]
2720struct DjotBlockStyle {
2721 alignment: Option<Alignment>,
2722 line_height: Option<i64>,
2723 non_breakable_lines: Option<bool>,
2724 page_break_before: Option<bool>,
2725 front_matter: Option<bool>,
2726 direction: Option<TextDirection>,
2727 background_color: Option<String>,
2728 top_margin: Option<i64>,
2729 text_indent: Option<i64>,
2730 semantic_role: Option<SemanticRole>,
2731}
2732
2733impl DjotBlockStyle {
2734 fn merge_from(&mut self, other: DjotBlockStyle) {
2738 if other.alignment.is_some() {
2739 self.alignment = other.alignment;
2740 }
2741 if other.line_height.is_some() {
2742 self.line_height = other.line_height;
2743 }
2744 if other.non_breakable_lines.is_some() {
2745 self.non_breakable_lines = other.non_breakable_lines;
2746 }
2747 if other.page_break_before.is_some() {
2748 self.page_break_before = other.page_break_before;
2749 }
2750 if other.front_matter.is_some() {
2751 self.front_matter = other.front_matter;
2752 }
2753 if other.direction.is_some() {
2754 self.direction = other.direction;
2755 }
2756 if other.background_color.is_some() {
2757 self.background_color = other.background_color;
2758 }
2759 if other.top_margin.is_some() {
2760 self.top_margin = other.top_margin;
2761 }
2762 if other.text_indent.is_some() {
2763 self.text_indent = other.text_indent;
2764 }
2765 if other.semantic_role.is_some() {
2766 self.semantic_role = other.semantic_role.clone();
2767 }
2768 }
2769}
2770
2771fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
2777 let mut style = DjotBlockStyle::default();
2778
2779 if opts.alignment
2780 && let Some(v) = attrs.get_value("alignment")
2781 {
2782 style.alignment = match v.to_string().as_str() {
2783 "left" => Some(Alignment::Left),
2784 "right" => Some(Alignment::Right),
2785 "center" => Some(Alignment::Center),
2786 "justify" => Some(Alignment::Justify),
2787 _ => None,
2788 };
2789 }
2790 if opts.line_height
2791 && let Some(v) = attrs.get_value("line_height")
2792 {
2793 style.line_height = v.to_string().parse::<i64>().ok();
2794 }
2795 if opts.direction
2796 && let Some(v) = attrs.get_value("direction")
2797 {
2798 style.direction = match v.to_string().as_str() {
2799 "ltr" => Some(TextDirection::LeftToRight),
2800 "rtl" => Some(TextDirection::RightToLeft),
2801 _ => None,
2802 };
2803 }
2804 if opts.non_breakable_lines
2805 && let Some(v) = attrs.get_value("non_breakable_lines")
2806 {
2807 style.non_breakable_lines = match v.to_string().as_str() {
2808 "true" => Some(true),
2809 "false" => Some(false),
2810 _ => None,
2811 };
2812 }
2813 if opts.page_break_before
2814 && let Some(v) = attrs.get_value("page_break_before")
2815 {
2816 style.page_break_before = match v.to_string().as_str() {
2817 "true" => Some(true),
2818 "false" => Some(false),
2819 _ => None,
2820 };
2821 }
2822 if opts.front_matter
2823 && let Some(v) = attrs.get_value("front_matter")
2824 {
2825 style.front_matter = match v.to_string().as_str() {
2826 "true" => Some(true),
2827 "false" => Some(false),
2828 _ => None,
2829 };
2830 }
2831 if opts.background_color
2832 && let Some(v) = attrs.get_value("background_color")
2833 {
2834 style.background_color = Some(v.to_string());
2835 }
2836 if opts.top_margin
2837 && let Some(v) = attrs.get_value("top_margin")
2838 {
2839 style.top_margin = v.to_string().parse::<i64>().ok();
2840 }
2841 if opts.text_indent
2842 && let Some(v) = attrs.get_value("text_indent")
2843 {
2844 style.text_indent = v.to_string().parse::<i64>().ok();
2845 }
2846 if opts.semantic_role
2847 && let Some(v) = attrs.get_value("semantic_role")
2848 {
2849 style.semantic_role = match v.to_string().as_str() {
2850 "epigraph" => Some(SemanticRole::Epigraph),
2851 _ => None,
2855 };
2856 }
2857
2858 style
2859}
2860
2861fn list_start_of(start: u64) -> Option<i64> {
2865 crate::parser_tools::list_start::list_start_from_unsigned(start)
2866}
2867
2868#[allow(clippy::too_many_arguments)]
2871fn djot_push_block(
2872 elements: &mut Vec<ParsedElement>,
2873 spans: Vec<ParsedSpan>,
2874 heading_level: Option<i64>,
2875 list_style: Option<ListStyle>,
2876 list_indent: u32,
2877 list_prefix: String,
2878 list_suffix: String,
2879 list_start: Option<i64>,
2880 marker: Option<MarkerType>,
2881 is_code_block: bool,
2882 code_language: Option<String>,
2883 blockquote_depth: u32,
2884 style: DjotBlockStyle,
2885) {
2886 elements.push(ParsedElement::Block(ParsedBlock {
2887 spans,
2888 heading_level,
2889 list_style,
2890 list_indent,
2891 list_prefix,
2892 list_suffix,
2893 list_start,
2894 marker,
2895 is_code_block,
2896 code_language,
2897 blockquote_depth,
2898 blockquote_opens: 0,
2899 line_height: style.line_height,
2900 non_breakable_lines: style.non_breakable_lines,
2901 page_break_before: style.page_break_before,
2902 front_matter: style.front_matter,
2903 direction: style.direction,
2904 background_color: style.background_color,
2905 alignment: style.alignment,
2906 top_margin: style.top_margin,
2907 text_indent: style.text_indent,
2908 semantic_role: style.semantic_role.clone(),
2909 }));
2910}
2911
2912fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
2925 djot.lines()
2926 .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
2927 .filter(|line| !line.is_empty())
2928 .map(|line| {
2929 ParsedElement::Block(ParsedBlock {
2930 spans: vec![ParsedSpan {
2931 text: line.to_string(),
2932 ..Default::default()
2933 }],
2934 ..Default::default()
2935 })
2936 })
2937 .collect()
2938}
2939
2940pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
2961 use jotdown::{Container as C, Event as E, ListKind};
2962
2963 let parsable = crate::parser_tools::djot_depth::parsable(djot);
2989 let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
2990 return djot_source_lines(djot);
2991 };
2992
2993 let mut elements: Vec<ParsedElement> = Vec::new();
2994 let mut current_spans: Vec<ParsedSpan> = Vec::new();
2995 let mut current_heading: Option<i64> = None;
2996 let mut is_code_block = false;
2997 let mut code_language: Option<String> = None;
2998 let mut blockquote_depth: u32 = 0;
2999 let mut quotations_open: Vec<usize> = Vec::new();
3001 let mut pending_style = DjotBlockStyle::default();
3004
3005 let mut bold = false;
3007 let mut italic = false;
3008 let mut underline = false;
3009 let mut strikeout = false;
3010 let mut code = false;
3011 let mut superscript = false;
3012 let mut subscript = false;
3013 let mut link_href: Option<String> = None;
3014 let mut pending_image: Option<ParsedImage> = None;
3017
3018 let mut list_stack: Vec<(ListStyle, String, String, Option<i64>)> = Vec::new();
3020 let mut cur_list_style: Option<ListStyle> = None;
3022 let mut cur_list_prefix = String::new();
3023 let mut cur_list_suffix = String::new();
3024 let mut cur_list_start: Option<i64> = None;
3025 let mut cur_list_indent: u32 = 0;
3026 let mut cur_marker: Option<MarkerType> = None;
3027
3028 let mut in_table_cell = false;
3030 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
3031 let mut current_row: Vec<ParsedTableCell> = Vec::new();
3032 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
3033 let mut table_header_rows: usize = 0;
3034 let mut row_is_head = false;
3035
3036 let mut skip_depth: u32 = 0;
3040
3041 let mut footnote_open: Option<(String, usize)> = None;
3045
3046 macro_rules! push_text {
3051 ($t:expr) => {{
3052 if let Some(img) = pending_image.as_mut() {
3057 img.alt.push_str(($t).as_ref());
3058 } else {
3059 let sp = ParsedSpan {
3060 text: ($t).to_string(),
3061 bold,
3062 italic,
3063 underline,
3064 strikeout,
3065 code,
3066 superscript,
3067 subscript,
3068 link_href: link_href.clone(),
3069 image: None,
3070 footnote_ref: None,
3071 };
3072 if in_table_cell {
3073 current_cell_spans.push(sp);
3074 } else {
3075 current_spans.push(sp);
3076 }
3077 }
3078 }};
3079 }
3080
3081 macro_rules! push_image {
3083 ($img:expr) => {{
3084 let sp = ParsedSpan {
3085 text: String::new(),
3086 bold,
3087 italic,
3088 underline,
3089 strikeout,
3090 code,
3091 superscript,
3092 subscript,
3093 link_href: link_href.clone(),
3094 image: Some($img),
3095 footnote_ref: None,
3096 };
3097 if in_table_cell {
3098 current_cell_spans.push(sp);
3099 } else {
3100 current_spans.push(sp);
3101 }
3102 }};
3103 }
3104
3105 macro_rules! enter_item {
3108 ($marker:expr) => {{
3109 if !current_spans.is_empty() {
3110 djot_push_block(
3111 &mut elements,
3112 std::mem::take(&mut current_spans),
3113 None,
3114 cur_list_style.clone(),
3115 cur_list_indent,
3116 cur_list_prefix.clone(),
3117 cur_list_suffix.clone(),
3118 cur_list_start,
3119 cur_marker.clone(),
3120 false,
3121 None,
3122 blockquote_depth,
3123 DjotBlockStyle::default(),
3124 );
3125 }
3126 let (style, prefix, suffix, start) = list_stack.last().cloned().unwrap_or((
3127 ListStyle::Disc,
3128 String::new(),
3129 String::new(),
3130 None,
3131 ));
3132 cur_list_style = Some(style);
3133 cur_list_prefix = prefix;
3134 cur_list_suffix = suffix;
3135 cur_list_start = start;
3136 cur_list_indent = list_stack.len().saturating_sub(1) as u32;
3137 cur_marker = $marker;
3138 }};
3139 }
3140
3141 macro_rules! line_break {
3148 () => {{
3149 if in_table_cell {
3150 push_text!(" ");
3151 } else if !current_spans.is_empty() && current_heading.is_some() {
3152 djot_push_block(
3157 &mut elements,
3158 std::mem::take(&mut current_spans),
3159 current_heading,
3160 None,
3161 0,
3162 String::new(),
3163 String::new(),
3164 None,
3165 None,
3166 false,
3167 None,
3168 blockquote_depth,
3169 pending_style.clone(),
3170 );
3171 pending_style.page_break_before = None;
3172 } else if !current_spans.is_empty() {
3173 djot_push_block(
3176 &mut elements,
3177 std::mem::take(&mut current_spans),
3178 None,
3179 cur_list_style.clone(),
3180 cur_list_indent,
3181 cur_list_prefix.clone(),
3182 cur_list_suffix.clone(),
3183 cur_list_start,
3184 cur_marker.clone(),
3185 is_code_block,
3186 code_language.clone(),
3187 blockquote_depth,
3188 pending_style.clone(),
3189 );
3190 pending_style.page_break_before = None;
3191 }
3192 }};
3193 }
3194
3195 for event in events {
3196 if skip_depth > 0 {
3197 match event {
3198 E::Start(..) => skip_depth += 1,
3199 E::End(_) => skip_depth -= 1,
3200 _ => {}
3201 }
3202 continue;
3203 }
3204
3205 match event {
3206 E::Start(C::Document, _) | E::End(C::Document) => {}
3208 E::Start(C::Section { .. }, attrs) => {
3209 if list_stack.is_empty() {
3212 pending_style.merge_from(block_attrs_to_style(&attrs, options));
3213 }
3214 }
3215 E::End(C::Section { .. }) => {}
3216 E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
3217
3218 E::Start(C::Blockquote, _) => {
3220 blockquote_depth += 1;
3221 quotations_open.push(elements.len());
3222 }
3223 E::End(C::Blockquote) => {
3224 blockquote_depth = blockquote_depth.saturating_sub(1);
3225 if let Some(first) = quotations_open.pop() {
3226 mark_quotation_opened(&mut elements, first);
3227 }
3228 }
3229
3230 E::Start(C::List { kind, .. }, _) => {
3232 let (style, prefix, suffix, start) = match kind {
3233 ListKind::Unordered(b) | ListKind::Task(b) => {
3234 (djot_bullet_style(b), String::new(), String::new(), None)
3235 }
3236 ListKind::Ordered {
3237 numbering,
3238 style,
3239 start,
3240 } => {
3241 let (p, s) = djot_ordered_affixes(style);
3242 (djot_ordered_style(numbering), p, s, list_start_of(start))
3243 }
3244 };
3245 list_stack.push((style, prefix, suffix, start));
3246 }
3247 E::End(C::List { .. }) => {
3248 list_stack.pop();
3249 cur_list_style = None;
3250 cur_list_start = None;
3251 cur_marker = None;
3252 }
3253 E::Start(C::ListItem, _) => enter_item!(None),
3254 E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
3255 MarkerType::Checked
3256 } else {
3257 MarkerType::Unchecked
3258 })),
3259 E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
3260 if !current_spans.is_empty() {
3262 djot_push_block(
3263 &mut elements,
3264 std::mem::take(&mut current_spans),
3265 None,
3266 cur_list_style.clone(),
3267 cur_list_indent,
3268 cur_list_prefix.clone(),
3269 cur_list_suffix.clone(),
3270 cur_list_start,
3271 cur_marker.clone(),
3272 false,
3273 None,
3274 blockquote_depth,
3275 DjotBlockStyle::default(),
3276 );
3277 }
3278 cur_list_style = None;
3279 cur_list_start = None;
3280 cur_marker = None;
3281 }
3282
3283 E::Start(C::Heading { level, .. }, attrs) => {
3285 current_heading = Some(level as i64);
3286 pending_style.merge_from(block_attrs_to_style(&attrs, options));
3289 }
3290 E::End(C::Heading { .. }) => {
3291 djot_push_block(
3292 &mut elements,
3293 std::mem::take(&mut current_spans),
3294 current_heading.take(),
3295 None,
3296 0,
3297 String::new(),
3298 String::new(),
3299 None,
3300 None,
3301 false,
3302 None,
3303 blockquote_depth,
3304 std::mem::take(&mut pending_style),
3305 );
3306 }
3307 E::Start(C::Paragraph, attrs) => {
3308 current_heading = None;
3309 pending_style = if list_stack.is_empty() {
3313 block_attrs_to_style(&attrs, options)
3314 } else {
3315 DjotBlockStyle::default()
3316 };
3317 }
3318 E::End(C::Paragraph) => {
3319 if !current_spans.is_empty() {
3320 djot_push_block(
3321 &mut elements,
3322 std::mem::take(&mut current_spans),
3323 None,
3324 cur_list_style.clone(),
3325 cur_list_indent,
3326 cur_list_prefix.clone(),
3327 cur_list_suffix.clone(),
3328 cur_list_start,
3329 cur_marker.clone(),
3330 false,
3331 None,
3332 blockquote_depth,
3333 std::mem::take(&mut pending_style),
3334 );
3335 }
3336 cur_list_style = None;
3337 cur_list_start = None;
3338 cur_marker = None;
3339 }
3340 E::Start(C::CodeBlock { language }, _) => {
3341 is_code_block = true;
3342 code_language = if language.is_empty() {
3343 None
3344 } else {
3345 Some(language.to_string())
3346 };
3347 }
3348 E::End(C::CodeBlock { .. }) => {
3349 if let Some(last) = current_spans.last_mut()
3351 && last.text.ends_with('\n')
3352 {
3353 last.text.pop();
3354 }
3355 djot_push_block(
3356 &mut elements,
3357 std::mem::take(&mut current_spans),
3358 None,
3359 None,
3360 0,
3361 String::new(),
3362 String::new(),
3363 None,
3364 None,
3365 true,
3366 code_language.take(),
3367 blockquote_depth,
3368 DjotBlockStyle::default(),
3369 );
3370 is_code_block = false;
3371 }
3372
3373 E::Start(C::Table, _) => {
3375 table_rows.clear();
3376 current_row.clear();
3377 current_cell_spans.clear();
3378 table_header_rows = 0;
3379 }
3380 E::End(C::Table) => {
3381 let own_cells = cells_in_rows(&table_rows);
3382 push_table(
3383 &mut elements,
3384 ParsedTable {
3385 header_rows: table_header_rows,
3386 rows: std::mem::take(&mut table_rows),
3387 blockquote_depth,
3388 blockquote_opens: 0,
3389 },
3390 own_cells,
3391 );
3392 }
3393 E::Start(C::TableRow { head }, _) => {
3394 row_is_head = head;
3395 current_row.clear();
3396 }
3397 E::End(C::TableRow { .. }) => {
3398 if row_is_head {
3399 table_header_rows += 1;
3400 }
3401 table_rows.push(std::mem::take(&mut current_row));
3402 }
3403 E::Start(C::TableCell { .. }, _) => {
3404 in_table_cell = true;
3405 current_cell_spans.clear();
3406 }
3407 E::End(C::TableCell { .. }) => {
3408 in_table_cell = false;
3409 current_row.push(ParsedTableCell {
3410 spans: std::mem::take(&mut current_cell_spans),
3411 });
3412 }
3413
3414 E::Start(C::Strong, _) => bold = true,
3416 E::End(C::Strong) => bold = false,
3417 E::Start(C::Emphasis, _) => italic = true,
3418 E::End(C::Emphasis) => italic = false,
3419 E::Start(C::Verbatim, _) => code = true,
3420 E::End(C::Verbatim) => code = false,
3421 E::Start(C::Superscript, _) => superscript = true,
3422 E::End(C::Superscript) => superscript = false,
3423 E::Start(C::Subscript, _) => subscript = true,
3424 E::End(C::Subscript) => subscript = false,
3425 E::Start(C::Insert, _) => underline = true,
3426 E::End(C::Insert) => underline = false,
3427 E::Start(C::Delete, _) => strikeout = true,
3428 E::End(C::Delete) => strikeout = false,
3429 E::Start(C::Mark, _) | E::End(C::Mark) => {}
3431 E::Start(C::Span, _) | E::End(C::Span) => {}
3432 E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
3433 E::End(C::Link(..)) => link_href = None,
3434 E::Start(C::Image(src, _), attrs) => {
3439 let attr_num = |key: &str| -> i64 {
3440 attrs
3441 .get_value(key)
3442 .map(|v| v.to_string())
3443 .and_then(|v| v.trim().parse::<i64>().ok())
3444 .filter(|n| *n > 0)
3445 .unwrap_or(0)
3446 };
3447 pending_image = Some(ParsedImage {
3448 src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
3451 alt: String::new(),
3452 width: attr_num("width"),
3453 height: attr_num("height"),
3454 });
3455 }
3456 E::End(C::Image(..)) => {
3457 if let Some(img) = pending_image.take() {
3458 push_image!(img);
3459 }
3460 }
3461
3462 E::Start(C::Footnote { label }, _) => {
3471 if !current_spans.is_empty() {
3472 djot_push_block(
3473 &mut elements,
3474 std::mem::take(&mut current_spans),
3475 None,
3476 cur_list_style.clone(),
3477 cur_list_indent,
3478 cur_list_prefix.clone(),
3479 cur_list_suffix.clone(),
3480 cur_list_start,
3481 cur_marker.clone(),
3482 false,
3483 None,
3484 blockquote_depth,
3485 DjotBlockStyle::default(),
3486 );
3487 }
3488 footnote_open = Some((label.to_string(), elements.len()));
3489 }
3490 E::End(C::Footnote { .. }) => {
3491 if !current_spans.is_empty() {
3492 djot_push_block(
3493 &mut elements,
3494 std::mem::take(&mut current_spans),
3495 None,
3496 cur_list_style.clone(),
3497 cur_list_indent,
3498 cur_list_prefix.clone(),
3499 cur_list_suffix.clone(),
3500 cur_list_start,
3501 cur_marker.clone(),
3502 false,
3503 None,
3504 blockquote_depth,
3505 DjotBlockStyle::default(),
3506 );
3507 }
3508 if let Some((label, start)) = footnote_open.take() {
3509 let blocks: Vec<ParsedBlock> = elements
3510 .drain(start..)
3511 .filter_map(|e| match e {
3512 ParsedElement::Block(b) => Some(b),
3513 _ => None,
3517 })
3518 .collect();
3519 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
3520 }
3521 }
3522
3523 E::Start(
3525 C::Math { .. }
3526 | C::RawBlock { .. }
3527 | C::RawInline { .. }
3528 | C::DescriptionList
3529 | C::DescriptionDetails
3530 | C::DescriptionTerm
3531 | C::Caption
3532 | C::LinkDefinition { .. },
3533 _,
3534 ) => skip_depth = 1,
3535
3536 E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
3548 for (i, piece) in s.split('\n').enumerate() {
3549 if i > 0 {
3550 line_break!();
3551 }
3552 if !piece.is_empty() {
3553 push_text!(piece);
3554 }
3555 }
3556 }
3557 E::Str(s) => push_text!(s.as_ref()),
3558 E::Softbreak => push_text!(" "),
3559 E::LeftSingleQuote => push_text!("\u{2018}"),
3560 E::RightSingleQuote => push_text!("\u{2019}"),
3561 E::LeftDoubleQuote => push_text!("\u{201C}"),
3562 E::RightDoubleQuote => push_text!("\u{201D}"),
3563 E::Ellipsis => push_text!("\u{2026}"),
3564 E::EnDash => push_text!("\u{2013}"),
3565 E::EmDash => push_text!("\u{2014}"),
3566 E::NonBreakingSpace => push_text!("\u{00A0}"),
3567 E::Hardbreak => line_break!(),
3568 E::FootnoteReference(label) => {
3574 let sp = ParsedSpan {
3575 text: String::new(),
3576 bold,
3577 italic,
3578 underline,
3579 strikeout,
3580 code,
3581 superscript,
3582 subscript,
3583 link_href: link_href.clone(),
3584 image: None,
3585 footnote_ref: Some(label.to_string()),
3586 };
3587 if in_table_cell {
3588 current_cell_spans.push(sp);
3589 } else {
3590 current_spans.push(sp);
3591 }
3592 }
3593 E::Symbol(_) => {}
3596 E::Escape | E::Blankline => {}
3597 E::ThematicBreak(_) | E::Attributes(_) => {}
3598
3599 _ => {}
3602 }
3603 }
3604
3605 if !current_spans.is_empty() {
3607 djot_push_block(
3608 &mut elements,
3609 std::mem::take(&mut current_spans),
3610 current_heading.take(),
3611 cur_list_style.clone(),
3612 cur_list_indent,
3613 cur_list_prefix.clone(),
3614 cur_list_suffix.clone(),
3615 cur_list_start,
3616 cur_marker.clone(),
3617 is_code_block,
3618 code_language.take(),
3619 blockquote_depth,
3620 std::mem::take(&mut pending_style),
3621 );
3622 }
3623
3624 if elements.is_empty() {
3627 djot_push_block(
3628 &mut elements,
3629 vec![ParsedSpan {
3630 text: String::new(),
3631 ..Default::default()
3632 }],
3633 None,
3634 None,
3635 0,
3636 String::new(),
3637 String::new(),
3638 None,
3639 None,
3640 false,
3641 None,
3642 0,
3643 DjotBlockStyle::default(),
3644 );
3645 }
3646
3647 elements
3648}
3649
3650#[cfg(test)]
3651mod tests {
3652 use super::*;
3653
3654 fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
3656 ParsedElement::flatten_to_blocks(parse_markdown(md))
3657 }
3658
3659 #[test]
3660 fn test_parse_markdown_simple_paragraph() {
3661 let blocks = parse_markdown_blocks("Hello **world**");
3662 assert_eq!(blocks.len(), 1);
3663 assert!(blocks[0].spans.len() >= 2);
3664 let plain_span = blocks[0]
3666 .spans
3667 .iter()
3668 .find(|s| s.text.contains("Hello"))
3669 .unwrap();
3670 assert!(!plain_span.bold);
3671 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3672 assert!(bold_span.bold);
3673 }
3674
3675 #[test]
3676 fn test_parse_markdown_heading() {
3677 let blocks = parse_markdown_blocks("# Title");
3678 assert_eq!(blocks.len(), 1);
3679 assert_eq!(blocks[0].heading_level, Some(1));
3680 assert_eq!(blocks[0].spans[0].text, "Title");
3681 }
3682
3683 #[test]
3684 fn test_parse_markdown_list() {
3685 let blocks = parse_markdown_blocks("- item1\n- item2");
3686 assert!(blocks.len() >= 2);
3687 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3688 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3689 }
3690
3691 fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
3693 elements
3694 .iter()
3695 .map(|e| match e {
3696 ParsedElement::Block(b) => (false, b.blockquote_depth),
3697 ParsedElement::Table(t) => (true, t.blockquote_depth),
3698 ParsedElement::FootnoteDefinition { .. } => (false, 0),
3701 })
3702 .collect()
3703 }
3704
3705 #[test]
3706 fn test_parse_markdown_table_in_blockquote_records_depth() {
3707 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3708 assert_eq!(element_depths(&elements), vec![(true, 1)]);
3709 }
3710
3711 #[test]
3712 fn test_parse_markdown_text_then_table_in_blockquote() {
3713 let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
3714 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
3715 }
3716
3717 #[test]
3718 fn test_parse_markdown_table_after_blockquote_closes() {
3719 let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
3720 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
3721 }
3722
3723 #[test]
3724 fn test_parse_markdown_table_in_nested_blockquote() {
3725 let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
3726 assert_eq!(element_depths(&elements), vec![(true, 2)]);
3727 }
3728
3729 #[test]
3730 fn test_parse_markdown_list_in_blockquote_records_depth() {
3731 let elements = parse_markdown("> - item1\n> - item2");
3732 let depths = element_depths(&elements);
3733 assert_eq!(depths, vec![(false, 1), (false, 1)]);
3734 for e in &elements {
3735 if let ParsedElement::Block(b) = e {
3736 assert_eq!(b.list_style, Some(ListStyle::Disc));
3737 }
3738 }
3739 }
3740
3741 #[test]
3742 fn test_parse_html_table_in_blockquote_records_depth() {
3743 let elements = parse_html_elements(
3744 "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
3745 );
3746 assert_eq!(element_depths(&elements), vec![(true, 1)]);
3747 }
3748
3749 #[test]
3750 fn test_parse_html_table_after_blockquote() {
3751 let elements = parse_html_elements(
3752 "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
3753 );
3754 let depths = element_depths(&elements);
3755 assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
3757 assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
3758 }
3759
3760 #[test]
3761 fn test_flatten_to_blocks_propagates_blockquote_depth() {
3762 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3763 let blocks = ParsedElement::flatten_to_blocks(elements);
3764 assert!(!blocks.is_empty());
3765 for b in &blocks {
3766 assert_eq!(b.blockquote_depth, 1);
3767 }
3768 }
3769
3770 #[test]
3771 fn test_parse_html_simple() {
3772 let blocks = parse_html("<p>Hello <b>world</b></p>");
3773 assert_eq!(blocks.len(), 1);
3774 assert!(blocks[0].spans.len() >= 2);
3775 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3776 assert!(bold_span.bold);
3777 }
3778
3779 #[test]
3780 fn test_parse_html_multiple_paragraphs() {
3781 let blocks = parse_html("<p>A</p><p>B</p>");
3782 assert_eq!(blocks.len(), 2);
3783 }
3784
3785 #[test]
3786 fn test_parse_html_heading() {
3787 let blocks = parse_html("<h2>Subtitle</h2>");
3788 assert_eq!(blocks.len(), 1);
3789 assert_eq!(blocks[0].heading_level, Some(2));
3790 }
3791
3792 #[test]
3793 fn test_parse_html_list() {
3794 let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
3795 assert!(blocks.len() >= 2);
3796 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3797 }
3798
3799 #[test]
3800 fn test_parse_markdown_code_block() {
3801 let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
3802 assert_eq!(blocks.len(), 1);
3803 assert!(blocks[0].is_code_block);
3804 assert!(blocks[0].spans[0].code);
3805 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3807 assert_eq!(
3808 text, "fn main() {}",
3809 "code block text should not have trailing newline"
3810 );
3811 }
3812
3813 #[test]
3814 fn test_parse_markdown_nested_formatting() {
3815 let blocks = parse_markdown_blocks("***bold italic***");
3816 assert_eq!(blocks.len(), 1);
3817 let span = &blocks[0].spans[0];
3818 assert!(span.bold);
3819 assert!(span.italic);
3820 }
3821
3822 #[test]
3823 fn test_parse_markdown_link() {
3824 let blocks = parse_markdown_blocks("[click](http://example.com)");
3825 assert_eq!(blocks.len(), 1);
3826 let span = &blocks[0].spans[0];
3827 assert_eq!(span.text, "click");
3828 assert_eq!(span.link_href, Some("http://example.com".to_string()));
3829 }
3830
3831 #[test]
3832 fn test_parse_markdown_empty() {
3833 let blocks = parse_markdown_blocks("");
3834 assert_eq!(blocks.len(), 1);
3835 assert!(blocks[0].spans[0].text.is_empty());
3836 }
3837
3838 #[test]
3839 fn test_parse_html_empty() {
3840 let blocks = parse_html("");
3841 assert_eq!(blocks.len(), 1);
3842 assert!(blocks[0].spans[0].text.is_empty());
3843 }
3844
3845 #[test]
3846 fn test_parse_html_nested_formatting() {
3847 let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
3848 assert_eq!(blocks.len(), 1);
3849 let span = &blocks[0].spans[0];
3850 assert!(span.bold);
3851 assert!(span.italic);
3852 }
3853
3854 #[test]
3855 fn test_parse_html_link() {
3856 let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
3857 assert_eq!(blocks.len(), 1);
3858 let span = &blocks[0].spans[0];
3859 assert_eq!(span.text, "click");
3860 assert_eq!(span.link_href, Some("http://example.com".to_string()));
3861 }
3862
3863 #[test]
3864 fn test_parse_html_ordered_list() {
3865 let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
3866 assert!(blocks.len() >= 2);
3867 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3868 }
3869
3870 #[test]
3871 fn test_parse_markdown_ordered_list() {
3872 let blocks = parse_markdown_blocks("1. first\n2. second");
3873 assert!(blocks.len() >= 2);
3874 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3875 }
3876
3877 #[test]
3878 fn test_parse_html_blockquote_nested() {
3879 let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
3880 assert!(blocks.len() >= 3);
3881 }
3882
3883 #[test]
3884 fn test_parse_block_styles_line_height() {
3885 let styles = parse_block_styles("line-height: 1.5");
3886 assert_eq!(styles.line_height, Some(1500));
3887 }
3888
3889 #[test]
3890 fn test_parse_block_styles_direction_rtl() {
3891 let styles = parse_block_styles("direction: rtl");
3892 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3893 }
3894
3895 #[test]
3896 fn test_parse_block_styles_background_color() {
3897 let styles = parse_block_styles("background-color: #ff0000");
3898 assert_eq!(styles.background_color, Some("#ff0000".to_string()));
3899 }
3900
3901 #[test]
3902 fn test_parse_block_styles_white_space_pre() {
3903 let styles = parse_block_styles("white-space: pre");
3904 assert_eq!(styles.non_breakable_lines, Some(true));
3905 }
3906
3907 #[test]
3908 fn test_parse_block_styles_multiple() {
3909 let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
3910 assert_eq!(styles.line_height, Some(2000));
3911 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3912 assert_eq!(styles.background_color, Some("blue".to_string()));
3913 }
3914
3915 #[test]
3916 fn test_parse_html_block_styles_extracted() {
3917 let blocks = parse_html(
3918 r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
3919 );
3920 assert_eq!(blocks.len(), 1);
3921 assert_eq!(blocks[0].line_height, Some(1500));
3922 assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
3923 assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
3924 }
3925
3926 #[test]
3930 fn test_parse_html_collapses_exporter_line_wrapping() {
3931 let blocks = parse_html(
3932 "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
3933 <p>\nsecond para</p>\n</body>",
3934 );
3935 let texts: Vec<String> = blocks
3936 .iter()
3937 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
3938 .collect();
3939 assert_eq!(texts, vec!["first line wrapped here", "second para"]);
3940 }
3941
3942 #[test]
3946 fn test_parse_html_drops_whitespace_between_blocks() {
3947 let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
3948 assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
3949 assert!(
3950 blocks
3951 .iter()
3952 .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
3953 "no block keeps a literal newline"
3954 );
3955 }
3956
3957 #[test]
3960 fn test_parse_html_collapses_across_span_boundaries() {
3961 let blocks = parse_html("<p> <b>bold</b>\n <i>italic</i> </p>");
3962 assert_eq!(blocks.len(), 1);
3963 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3964 assert_eq!(text, "bold italic");
3965 }
3966
3967 #[test]
3970 fn test_parse_html_keeps_no_break_space() {
3971 let blocks = parse_html("<p>\nAttention : ici</p>");
3972 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3973 assert_eq!(text, "Attention\u{a0}: ici");
3974 }
3975
3976 #[test]
3979 fn test_parse_html_keeps_empty_paragraph() {
3980 let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
3981 assert_eq!(blocks.len(), 3);
3982 let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
3983 assert_eq!(text, "");
3984 }
3985
3986 #[test]
3990 fn test_parse_html_preserves_whitespace_in_pre() {
3991 let blocks = parse_html("<pre>\nfn main() {\n let x = 1;\n}</pre>");
3994 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3995 assert_eq!(text, "fn main() {\n let x = 1;\n}");
3996
3997 let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n b</p>");
4000 let texts: Vec<String> = styled
4001 .iter()
4002 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4003 .collect();
4004 assert_eq!(texts, ["a", " b"]);
4005
4006 let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n b</p>");
4007 let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
4008 assert_eq!(text, "a b");
4009 }
4010
4011 #[test]
4014 fn test_parse_html_collapses_table_cell_whitespace() {
4015 let elements = parse_html_elements(
4016 "<table>\n<tr>\n<td>\n one\n</td>\n<td>\n two\n</td>\n</tr>\n</table>",
4017 );
4018 let table = elements
4019 .iter()
4020 .find_map(|e| match e {
4021 ParsedElement::Table(t) => Some(t),
4022 _ => None,
4023 })
4024 .expect("a table");
4025 let cells: Vec<String> = table.rows[0]
4026 .iter()
4027 .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
4028 .collect();
4029 assert_eq!(cells, vec!["one", "two"]);
4030 }
4031
4032 #[test]
4033 fn test_parse_html_white_space_pre() {
4034 let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
4035 assert_eq!(blocks.len(), 1);
4036 assert_eq!(blocks[0].non_breakable_lines, Some(true));
4037 }
4038
4039 #[test]
4040 fn test_parse_html_no_styles_returns_none() {
4041 let blocks = parse_html("<p>plain</p>");
4042 assert_eq!(blocks.len(), 1);
4043 assert_eq!(blocks[0].line_height, None);
4044 assert_eq!(blocks[0].direction, None);
4045 assert_eq!(blocks[0].background_color, None);
4046 assert_eq!(blocks[0].non_breakable_lines, None);
4047 }
4048
4049 #[test]
4050 fn test_parse_markdown_nested_list_indent() {
4051 let md = "- top\n - nested\n - deep";
4052 let blocks = parse_markdown_blocks(md);
4053 assert_eq!(blocks.len(), 3);
4054 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4055 assert_eq!(blocks[0].list_indent, 0);
4056 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4057 assert_eq!(blocks[1].list_indent, 1);
4058 assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
4059 assert_eq!(blocks[2].list_indent, 2);
4060 }
4061
4062 #[test]
4063 fn test_parse_markdown_nested_ordered_list_indent() {
4064 let md = "1. first\n 1. nested\n 2. nested2";
4065 let blocks = parse_markdown_blocks(md);
4066 assert_eq!(blocks.len(), 3);
4067 assert_eq!(blocks[0].list_indent, 0);
4068 assert_eq!(blocks[1].list_indent, 1);
4069 assert_eq!(blocks[2].list_indent, 1);
4070 }
4071
4072 #[test]
4073 fn test_parse_html_nested_list_indent() {
4074 let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
4075 let blocks = parse_html(html);
4076 assert!(blocks.len() >= 2);
4077 assert_eq!(blocks[0].list_indent, 0);
4078 assert_eq!(blocks[1].list_indent, 1);
4079 }
4080
4081 #[test]
4082 fn test_parse_markdown_table() {
4083 let md = "| A | B |\n|---|---|\n| 1 | 2 |";
4084 let elements = parse_markdown(md);
4085 assert_eq!(elements.len(), 1);
4086 match &elements[0] {
4087 ParsedElement::Table(table) => {
4088 assert_eq!(table.header_rows, 1);
4089 assert_eq!(table.rows.len(), 2); assert_eq!(table.rows[0].len(), 2);
4092 assert_eq!(table.rows[0][0].spans[0].text, "A");
4093 assert_eq!(table.rows[0][1].spans[0].text, "B");
4094 assert_eq!(table.rows[1].len(), 2);
4096 assert_eq!(table.rows[1][0].spans[0].text, "1");
4097 assert_eq!(table.rows[1][1].spans[0].text, "2");
4098 }
4099 _ => panic!("Expected ParsedElement::Table"),
4100 }
4101 }
4102
4103 #[test]
4104 fn test_parse_markdown_table_with_formatting() {
4105 let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
4106 let elements = parse_markdown(md);
4107 assert_eq!(elements.len(), 1);
4108 match &elements[0] {
4109 ParsedElement::Table(table) => {
4110 assert_eq!(table.rows.len(), 2);
4111 assert!(table.rows[0][0].spans[0].bold);
4113 assert!(table.rows[0][1].spans[0].code);
4115 assert!(table.rows[0][2].spans[0].italic);
4117 assert!(table.rows[1][0].spans[0].strikeout);
4119 assert_eq!(
4121 table.rows[1][2].spans[0].link_href,
4122 Some("http://x.com".to_string())
4123 );
4124 }
4125 _ => panic!("Expected ParsedElement::Table"),
4126 }
4127 }
4128
4129 #[test]
4130 fn test_parse_markdown_mixed_content_with_table() {
4131 let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
4132 let elements = parse_markdown(md);
4133 assert_eq!(elements.len(), 3);
4134 assert!(matches!(&elements[0], ParsedElement::Block(_)));
4135 assert!(matches!(&elements[1], ParsedElement::Table(_)));
4136 assert!(matches!(&elements[2], ParsedElement::Block(_)));
4137 }
4138}
4139
4140#[cfg(test)]
4141mod djot_tests {
4142 use super::*;
4143 use crate::entities::MarkerType;
4144
4145 fn blocks(d: &str) -> Vec<ParsedBlock> {
4146 ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
4147 }
4148
4149 fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
4150 b.spans.iter().find(|s| pred(s)).expect("span not found")
4151 }
4152
4153 #[test]
4154 fn paragraph_bold_italic() {
4155 let b = blocks("normal *bold* _italic_");
4156 assert_eq!(b.len(), 1);
4157 assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
4158 assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
4159 }
4160
4161 #[test]
4162 fn heading_levels() {
4163 assert_eq!(blocks("# H1")[0].heading_level, Some(1));
4164 assert_eq!(blocks("### H3")[0].heading_level, Some(3));
4165 assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
4166 }
4167
4168 #[test]
4169 fn unordered_bullet_styles_are_distinct() {
4170 assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
4171 assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
4172 assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
4173 }
4174
4175 #[test]
4176 fn ordered_delimiters() {
4177 let period = blocks("1. a");
4178 assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
4179 assert_eq!(period[0].list_prefix, "");
4180 assert_eq!(period[0].list_suffix, ".");
4181
4182 let paren = blocks("1) a");
4183 assert_eq!(paren[0].list_suffix, ")");
4184 assert_eq!(paren[0].list_prefix, "");
4185
4186 let paren_paren = blocks("(1) a");
4187 assert_eq!(paren_paren[0].list_prefix, "(");
4188 assert_eq!(paren_paren[0].list_suffix, ")");
4189 }
4190
4191 #[test]
4192 fn task_list_markers() {
4193 let b = blocks("- [ ] a\n- [x] b");
4194 assert_eq!(b.len(), 2);
4195 assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
4196 assert_eq!(b[1].marker, Some(MarkerType::Checked));
4197 }
4198
4199 #[test]
4200 fn code_block_with_language() {
4201 let b = blocks("```rust\nfn main() {}\n```");
4202 assert_eq!(b.len(), 1);
4203 assert!(b[0].is_code_block);
4204 assert_eq!(b[0].code_language.as_deref(), Some("rust"));
4205 let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
4206 assert_eq!(text, "fn main() {}");
4207 }
4208
4209 #[test]
4210 fn link_href() {
4211 let b = blocks("[text](http://example.com)");
4212 let s = first_span_with(&b[0], |s| s.text == "text");
4213 assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
4214 }
4215
4216 #[test]
4217 fn superscript_subscript() {
4218 assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
4219 assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
4220 }
4221
4222 #[test]
4223 fn delete_insert_verbatim() {
4224 assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
4225 assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
4226 assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
4227 }
4228
4229 #[test]
4230 fn blockquote_depth() {
4231 let els = parse_djot("> quoted", &DjotImportOptions::default());
4232 match &els[0] {
4233 ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
4234 _ => panic!("expected block"),
4235 }
4236 }
4237
4238 #[test]
4239 fn nested_list_indent() {
4240 let b = blocks("- a\n\n - b\n\n - c");
4245 assert_eq!(b.len(), 3);
4246 assert_eq!(b[0].list_indent, 0);
4247 assert_eq!(b[1].list_indent, 1);
4248 assert_eq!(b[2].list_indent, 2);
4249 }
4250
4251 #[test]
4252 fn table_parsed_as_table() {
4253 let els = parse_djot(
4254 "| a | b |\n|---|---|\n| c | d |",
4255 &DjotImportOptions::default(),
4256 );
4257 assert_eq!(els.len(), 1);
4258 match &els[0] {
4259 ParsedElement::Table(t) => {
4260 assert_eq!(t.header_rows, 1);
4261 assert_eq!(t.rows.len(), 2);
4262 assert_eq!(t.rows[0][0].spans[0].text, "a");
4263 assert_eq!(t.rows[1][1].spans[0].text, "d");
4264 }
4265 _ => panic!("expected table"),
4266 }
4267 }
4268
4269 #[test]
4270 fn smart_punctuation_normalised_to_unicode() {
4271 let text: String = blocks("a... b---c")[0]
4272 .spans
4273 .iter()
4274 .map(|s| s.text.as_str())
4275 .collect();
4276 assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
4277 assert!(text.contains('\u{2014}'), "em dash: {text:?}");
4278 }
4279
4280 #[test]
4281 fn unrepresentable_constructs_dropped_without_leaking_text() {
4282 let b = blocks("para1\n\n---\n\npara2");
4284 assert_eq!(b.len(), 2);
4285 assert_eq!(
4286 b[0].spans
4287 .iter()
4288 .map(|s| s.text.as_str())
4289 .collect::<String>(),
4290 "para1"
4291 );
4292 assert_eq!(
4293 b[1].spans
4294 .iter()
4295 .map(|s| s.text.as_str())
4296 .collect::<String>(),
4297 "para2"
4298 );
4299
4300 let d = blocks(":::\ninside\n:::");
4302 let joined: String = d
4303 .iter()
4304 .flat_map(|b| b.spans.iter())
4305 .map(|s| s.text.as_str())
4306 .collect();
4307 assert_eq!(joined, "inside");
4308
4309 let m = blocks("before $`E=mc^2` after");
4311 let joined: String = m
4312 .iter()
4313 .flat_map(|b| b.spans.iter())
4314 .map(|s| s.text.as_str())
4315 .collect();
4316 assert!(joined.contains("before"), "{joined:?}");
4317 assert!(joined.contains("after"), "{joined:?}");
4318 assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
4319 }
4320
4321 #[test]
4322 fn empty_document_yields_one_empty_block() {
4323 let b = blocks("");
4324 assert_eq!(b.len(), 1);
4325 assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
4326 }
4327
4328 #[test]
4329 fn block_attributes_parse_into_block() {
4330 let b = blocks(
4331 "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
4332 );
4333 assert_eq!(b.len(), 1);
4334 assert_eq!(b[0].alignment, Some(Alignment::Center));
4335 assert_eq!(b[0].line_height, Some(1500));
4336 assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
4337 assert_eq!(b[0].non_breakable_lines, Some(true));
4338 assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
4339 }
4340
4341 #[test]
4342 fn spacing_block_attributes_parse_into_block() {
4343 let b = blocks("{top_margin=24 text_indent=0}\nhello");
4347 assert_eq!(b.len(), 1);
4348 assert_eq!(b[0].top_margin, Some(24));
4349 assert_eq!(b[0].text_indent, Some(0));
4350 }
4351
4352 #[test]
4353 fn a_zero_text_indent_is_distinct_from_an_absent_one() {
4354 let explicit = blocks("{text_indent=0}\nhello");
4358 let absent = blocks("hello");
4359 assert_eq!(explicit[0].text_indent, Some(0));
4360 assert_eq!(absent[0].text_indent, None);
4361 assert!(!explicit[0].is_inline_only());
4362 assert!(absent[0].is_inline_only());
4363 }
4364
4365 #[test]
4366 fn spacing_block_attributes_respect_import_options() {
4367 let src = "{top_margin=24 text_indent=0}\nhello";
4368 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4369 assert_eq!(b[0].top_margin, None);
4370 assert_eq!(b[0].text_indent, None);
4371 }
4372
4373 #[test]
4374 fn block_attributes_on_heading() {
4375 let b = blocks("{alignment=right}\n# Title");
4376 assert_eq!(b[0].heading_level, Some(1));
4377 assert_eq!(b[0].alignment, Some(Alignment::Right));
4378 }
4379
4380 #[test]
4381 fn block_attributes_respect_import_options() {
4382 let src = "{alignment=center line_height=1500}\nhello";
4385 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4386 assert_eq!(b[0].alignment, None);
4387 assert_eq!(b[0].line_height, None);
4388 assert_eq!(
4389 b[0].spans
4390 .iter()
4391 .map(|s| s.text.as_str())
4392 .collect::<String>(),
4393 "hello"
4394 );
4395 }
4396
4397 #[test]
4398 fn list_item_block_attributes_are_dropped() {
4399 let b = blocks("{alignment=center}\n- item");
4402 assert!(b.iter().all(|blk| blk.alignment.is_none()));
4403 }
4404
4405 #[test]
4406 fn unknown_alignment_value_is_ignored() {
4407 let b = blocks("{alignment=sideways}\nhello");
4408 assert_eq!(b[0].alignment, None);
4409 }
4410}
4411
4412pub const TABLE_ANCHOR: &str = "\u{FFFC}";
4424
4425fn span_prose(span: &ParsedSpan, out: &mut String) {
4433 if span.image.is_some() || span.footnote_ref.is_some() {
4434 out.push('\u{FFFC}');
4435 return;
4436 }
4437 out.push_str(&span.text);
4438}
4439
4440fn block_prose(block: &ParsedBlock) -> String {
4441 let mut prose = String::new();
4442 for span in &block.spans {
4443 span_prose(span, &mut prose);
4444 }
4445 prose
4446}
4447
4448fn cell_prose(cell: &ParsedTableCell) -> String {
4449 let mut prose = String::new();
4450 for span in &cell.spans {
4451 span_prose(span, &mut prose);
4452 }
4453 prose
4454}
4455
4456pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
4507 let elements = parse_djot(djot, options);
4511
4512 let mut out = String::with_capacity(djot.len());
4515
4516 let mut first = true;
4521 let push = |text: &str, out: &mut String, first: &mut bool| {
4522 if *first {
4523 *first = false;
4524 } else {
4525 out.push('\n');
4526 }
4527 out.push_str(text);
4528 };
4529
4530 for element in &elements {
4531 match element {
4532 ParsedElement::Block(block) => {
4533 push(&block_prose(block), &mut out, &mut first);
4534 }
4535 ParsedElement::FootnoteDefinition { .. } => {}
4545 ParsedElement::Table(table) => {
4546 push(TABLE_ANCHOR, &mut out, &mut first);
4553 let columns = table.column_count();
4556 for row in &table.rows {
4557 for column in 0..columns {
4558 let prose = row.get(column).map(cell_prose).unwrap_or_default();
4559 push(&prose, &mut out, &mut first);
4560 }
4561 }
4562 }
4563 }
4564 }
4565 out
4566}
4567
4568#[cfg(test)]
4569mod html_footnote_tests {
4570 use super::*;
4571
4572 #[test]
4576 fn a_literal_reference_in_text_would_be_escaped_into_prose() {
4577 let blocks = parse_html("<p>The ferry.[^1]</p>");
4578 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4579 assert!(
4580 text.contains("[^1]"),
4581 "the parser keeps it as characters: {text:?}"
4582 );
4583 assert!(
4584 blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
4585 "text is text — it must not become a reference by accident"
4586 );
4587 }
4588
4589 #[test]
4590 fn an_attributed_element_becomes_a_real_footnote_reference() {
4591 let blocks = parse_html(
4592 r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
4593 );
4594 let refs: Vec<&str> = blocks
4595 .iter()
4596 .flat_map(|b| b.spans.iter())
4597 .filter_map(|s| s.footnote_ref.as_deref())
4598 .collect();
4599 assert_eq!(refs, vec!["3"]);
4600
4601 let marker = blocks[0]
4604 .spans
4605 .iter()
4606 .find(|s| s.footnote_ref.is_some())
4607 .expect("the reference span");
4608 assert!(marker.text.is_empty(), "{marker:?}");
4609 }
4610
4611 #[test]
4614 fn any_element_carrying_the_attribute_works() {
4615 for html in [
4616 r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
4617 r#"<p>a<span data-footnote-ref="1"></span></p>"#,
4618 r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
4619 ] {
4620 let blocks = parse_html(html);
4621 assert_eq!(
4622 blocks
4623 .iter()
4624 .flat_map(|b| b.spans.iter())
4625 .filter_map(|s| s.footnote_ref.as_deref())
4626 .collect::<Vec<_>>(),
4627 vec!["1"],
4628 "failed for {html}"
4629 );
4630 }
4631 }
4632
4633 #[test]
4634 fn an_empty_or_absent_label_is_not_a_reference() {
4635 for html in [
4636 r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
4637 r#"<p>a<sup></sup></p>"#,
4638 ] {
4639 let blocks = parse_html(html);
4640 assert!(
4641 blocks
4642 .iter()
4643 .flat_map(|b| b.spans.iter())
4644 .all(|s| s.footnote_ref.is_none()),
4645 "failed for {html}"
4646 );
4647 }
4648 }
4649}
4650
4651#[cfg(test)]
4652mod html_depth_tests {
4653 use super::*;
4656
4657 fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
4659 blocks
4660 .iter()
4661 .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
4662 .collect()
4663 }
4664
4665 #[test]
4670 fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
4671 use crate::parser_tools::list_depth::MAX_LIST_INDENT;
4672 let depth = 300;
4673 let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
4674 html.push_str(&"</li></ol>".repeat(depth));
4675 let blocks = parse_html(&html);
4676 let items: Vec<(String, u32)> = blocks
4677 .iter()
4678 .filter(|block| block.list_style.is_some())
4679 .map(|block| {
4680 let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
4681 (text, block.list_indent)
4682 })
4683 .collect();
4684 let expected: Vec<(String, u32)> = (0..depth)
4685 .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
4686 .collect();
4687 assert_eq!(items, expected);
4688 }
4689
4690 #[test]
4692 fn text_in_a_deep_html_list_outside_its_items_is_kept() {
4693 let depth = 40;
4694 let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
4695 html.push_str("<ul>loose words<li>last</li></ul>");
4696 html.push_str(&"</li></ul>".repeat(depth));
4697 let texts = block_texts(&parse_html(&html)).join(" ");
4698 assert!(texts.contains("loose words"), "{texts:?}");
4699 assert!(texts.contains("last"), "{texts:?}");
4700 assert!(texts.contains("item 39"), "{texts:?}");
4701 }
4702
4703 #[test]
4706 fn html_nested_past_the_reader_depth_keeps_its_words() {
4707 for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
4708 let html = format!(
4709 "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
4710 open.repeat(300),
4711 close.repeat(300)
4712 );
4713 let texts = block_texts(&parse_html(&html)).join(" ");
4714 let words: Vec<&str> = texts.split_whitespace().collect();
4715 assert_eq!(
4716 words,
4717 ["Top.", "Deep", "words", "here.", "End."],
4718 "for {open}"
4719 );
4720 }
4721 }
4722
4723 #[test]
4725 fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
4726 let html = format!(
4727 "<table><tr><td>{}deep words{}</td></tr></table>",
4728 "<span>".repeat(300),
4729 "</span>".repeat(300)
4730 );
4731 let elements = parse_html_elements(&html);
4732 let Some(ParsedElement::Table(table)) = elements.first() else {
4733 panic!("a table: {elements:?}");
4734 };
4735 let text: String = table.rows[0][0]
4736 .spans
4737 .iter()
4738 .map(|span| span.text.as_str())
4739 .collect();
4740 assert_eq!(text, "deep words");
4741 }
4742
4743 #[test]
4747 fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
4748 for cell in [
4749 "One<br/>Two",
4750 "<p>One</p><p>Two</p>",
4751 "<ul><li>One</li><li>Two</li></ul>",
4752 ] {
4753 let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
4754 let Some(ParsedElement::Table(table)) = elements.first() else {
4755 panic!("a table: {elements:?}");
4756 };
4757 let text: String = table.rows[0][0]
4758 .spans
4759 .iter()
4760 .map(|span| span.text.as_str())
4761 .collect();
4762 assert_eq!(text, "One Two", "for {cell}");
4763 }
4764 }
4765}