1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3
4#[derive(Debug, Clone, Default, PartialEq, Eq)]
10pub struct ParsedImage {
11 pub src: String,
12 pub alt: String,
13 pub width: i64,
19 pub height: i64,
20}
21
22#[derive(Debug, Clone, Default)]
29pub struct ParsedSpan {
30 pub text: String,
31 pub bold: bool,
32 pub italic: bool,
33 pub underline: bool,
34 pub strikeout: bool,
35 pub code: bool,
36 pub superscript: bool,
38 pub subscript: bool,
40 pub link_href: Option<String>,
41 pub image: Option<ParsedImage>,
43 pub footnote_ref: Option<String>,
46}
47
48#[derive(Debug, Clone)]
50pub struct ParsedTableCell {
51 pub spans: Vec<ParsedSpan>,
52}
53
54#[derive(Debug, Clone)]
56pub struct ParsedTable {
57 pub header_rows: usize,
59 pub rows: Vec<Vec<ParsedTableCell>>,
61 pub blockquote_depth: u32,
64 pub blockquote_opens: u32,
67}
68
69impl ParsedTable {
70 pub fn column_count(&self) -> usize {
80 self.rows.iter().map(Vec::len).max().unwrap_or(0)
81 }
82}
83
84const SQUARED_CELLS_ALWAYS: usize = 4096;
88
89const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
99
100fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
111 let squared = table.rows.len().saturating_mul(table.column_count());
112 let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
113 if squared <= allowed {
114 elements.push(ParsedElement::Table(table));
115 return;
116 }
117 elements.extend(
118 ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
119 .into_iter()
120 .filter(|block| spans_carry_content(&block.spans))
121 .map(ParsedElement::Block),
122 );
123}
124
125fn mark_quotation_opened(elements: &mut [ParsedElement], first: usize) {
129 let Some(rest) = elements.get_mut(first..) else {
130 return;
131 };
132 for element in rest {
133 match element {
134 ParsedElement::Block(block) => {
135 block.blockquote_opens += 1;
136 return;
137 }
138 ParsedElement::Table(table) => {
139 table.blockquote_opens += 1;
140 return;
141 }
142 ParsedElement::FootnoteDefinition { .. } => {}
143 }
144 }
145}
146
147fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
149 rows.iter().map(Vec::len).sum()
150}
151
152#[derive(Debug, Clone)]
154pub enum ParsedElement {
155 Block(ParsedBlock),
156 Table(ParsedTable),
157 FootnoteDefinition {
164 label: String,
165 blocks: Vec<ParsedBlock>,
166 },
167}
168
169impl ParsedElement {
170 pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
173 let mut blocks = Vec::new();
174 for elem in elements {
175 match elem {
176 ParsedElement::Block(b) => blocks.push(b),
177 ParsedElement::FootnoteDefinition { .. } => {}
182 ParsedElement::Table(t) => {
183 for row in t.rows {
184 for cell in row {
185 blocks.push(ParsedBlock {
186 spans: cell.spans,
187 heading_level: None,
188 list_style: None,
189 list_indent: 0,
190 list_prefix: String::new(),
191 list_suffix: String::new(),
192 list_start: None,
193 marker: None,
194 is_code_block: false,
195 code_language: None,
196 blockquote_depth: t.blockquote_depth,
197 blockquote_opens: 0,
198 line_height: None,
199 non_breakable_lines: None,
200 page_break_before: None,
201 front_matter: None,
202 direction: None,
203 background_color: None,
204 alignment: None,
205 top_margin: None,
206 text_indent: None,
207 semantic_role: None,
208 });
209 }
210 }
211 }
212 }
213 }
214 if blocks.is_empty() {
215 blocks.push(ParsedBlock {
216 spans: vec![ParsedSpan {
217 text: String::new(),
218 ..Default::default()
219 }],
220 heading_level: None,
221 list_style: None,
222 list_indent: 0,
223 list_prefix: String::new(),
224 list_suffix: String::new(),
225 list_start: None,
226 marker: None,
227 is_code_block: false,
228 code_language: None,
229 blockquote_depth: 0,
230 blockquote_opens: 0,
231 line_height: None,
232 non_breakable_lines: None,
233 page_break_before: None,
234 front_matter: None,
235 direction: None,
236 background_color: None,
237 alignment: None,
238 top_margin: None,
239 text_indent: None,
240 semantic_role: None,
241 });
242 }
243 blocks
244 }
245}
246
247#[derive(Debug, Clone, Default)]
254pub struct ParsedBlock {
255 pub spans: Vec<ParsedSpan>,
256 pub heading_level: Option<i64>,
257 pub list_style: Option<ListStyle>,
258 pub list_indent: u32,
259 pub list_prefix: String,
262 pub list_suffix: String,
265 pub list_start: Option<i64>,
269 pub marker: Option<MarkerType>,
272 pub is_code_block: bool,
273 pub code_language: Option<String>,
274 pub blockquote_depth: u32,
275 pub blockquote_opens: u32,
282 pub line_height: Option<i64>,
283 pub non_breakable_lines: Option<bool>,
284 pub page_break_before: Option<bool>,
287 pub front_matter: Option<bool>,
290 pub direction: Option<TextDirection>,
291 pub background_color: Option<String>,
292 pub alignment: Option<Alignment>,
295 pub top_margin: Option<i64>,
299 pub text_indent: Option<i64>,
303 pub semantic_role: Option<SemanticRole>,
307}
308
309impl ParsedBlock {
310 pub fn is_inline_only(&self) -> bool {
313 self.heading_level.is_none()
314 && self.list_style.is_none()
315 && !self.is_code_block
316 && self.blockquote_depth == 0
317 && self.line_height.is_none()
318 && self.non_breakable_lines.is_none()
319 && self.page_break_before.is_none()
320 && self.front_matter.is_none()
321 && self.direction.is_none()
322 && self.background_color.is_none()
323 && self.alignment.is_none()
324 && self.top_margin.is_none()
325 && self.text_indent.is_none()
326 }
327}
328
329fn dangling_footnote_labels(
358 markdown: &str,
359 options: pulldown_cmark::Options,
360) -> std::collections::BTreeSet<String> {
361 use pulldown_cmark::{Event, Parser, Tag};
362
363 let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
364 for event in Parser::new_ext(markdown, options) {
365 if let Event::Start(Tag::FootnoteDefinition(label)) = event {
366 defined.insert(label.to_string());
367 }
368 }
369
370 let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
371 let bytes = markdown.as_bytes();
372 let mut search_from = 0usize;
373 while let Some(rel) = markdown[search_from..].find("[^") {
374 let open = search_from + rel;
375 let label_start = open + 2;
376 let Some(close_rel) = markdown[label_start..].find(']') else {
377 break;
378 };
379 let close = label_start + close_rel;
380 let label = &markdown[label_start..close];
381 let looks_like_definition = bytes.get(close + 1) == Some(&b':');
386 if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
387 referenced.insert(label.to_string());
388 }
389 search_from = close + 1;
390 }
391
392 referenced.difference(&defined).cloned().collect()
393}
394
395pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
403 use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
404
405 let options = Options::ENABLE_STRIKETHROUGH
416 | Options::ENABLE_TABLES
417 | Options::ENABLE_TASKLISTS
418 | Options::ENABLE_FOOTNOTES;
419
420 let dangling = dangling_footnote_labels(markdown, options);
427 let augmented_owner;
428 let source: &str = if dangling.is_empty() {
429 markdown
430 } else {
431 augmented_owner = dangling
432 .iter()
433 .fold(markdown.to_string(), |mut acc, label| {
434 acc.push_str("\n\n[^");
435 acc.push_str(label);
436 acc.push_str("]:\n");
437 acc
438 });
439 &augmented_owner
440 };
441 let parser = Parser::new_ext(source, options);
442
443 let mut elements: Vec<ParsedElement> = Vec::new();
444 let mut current_spans: Vec<ParsedSpan> = Vec::new();
445 let mut current_heading: Option<i64> = None;
446 let mut current_list_style: Option<ListStyle> = None;
447 let mut current_list_start: Option<i64> = None;
448 let mut is_code_block = false;
449 let mut code_language: Option<String> = None;
450 let mut blockquote_depth: u32 = 0;
451 let mut quotations_open: Vec<usize> = Vec::new();
453 let mut in_block = false;
454
455 let mut bold = false;
457 let mut italic = false;
458 let mut strikeout = false;
459 let mut link_href: Option<String> = None;
460 let mut pending_image: Option<ParsedImage> = None;
462
463 let mut footnote_open: Option<(String, usize)> = None;
470
471 let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
474 let mut list_start_stack: Vec<Option<i64>> = Vec::new();
475 let mut current_list_indent: u32 = 0;
476
477 let mut in_table = false;
479 let mut in_table_head = false;
480 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
481 let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
482 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
483 let mut table_header_rows: usize = 0;
484
485 for event in parser {
486 match event {
487 Event::Start(Tag::Paragraph) => {
488 in_block = true;
489 current_heading = None;
490 is_code_block = false;
491 }
492 Event::End(TagEnd::Paragraph) => {
493 if !current_spans.is_empty() || in_block {
494 elements.push(ParsedElement::Block(ParsedBlock {
495 spans: std::mem::take(&mut current_spans),
496 heading_level: current_heading.take(),
497 list_style: current_list_style.clone(),
498 list_indent: current_list_indent,
499 list_prefix: String::new(),
500 list_suffix: String::new(),
501 list_start: current_list_start,
502 marker: None,
503 is_code_block: false,
504 code_language: None,
505 blockquote_depth,
506 blockquote_opens: 0,
507 line_height: None,
508 non_breakable_lines: None,
509 page_break_before: None,
510 front_matter: None,
511 direction: None,
512 background_color: None,
513 alignment: None,
514 top_margin: None,
515 text_indent: None,
516 semantic_role: None,
517 }));
518 }
519 in_block = false;
520 current_list_style = None;
521 current_list_start = None;
522 }
523 Event::Start(Tag::Heading { level, .. }) => {
524 in_block = true;
525 current_heading = Some(heading_level_to_i64(level));
526 is_code_block = false;
527 }
528 Event::End(TagEnd::Heading(_)) => {
529 elements.push(ParsedElement::Block(ParsedBlock {
530 spans: std::mem::take(&mut current_spans),
531 heading_level: current_heading.take(),
532 list_style: None,
533 list_indent: 0,
534 list_prefix: String::new(),
535 list_suffix: String::new(),
536 list_start: None,
537 marker: None,
538 is_code_block: false,
539 code_language: None,
540 blockquote_depth,
541 blockquote_opens: 0,
542 line_height: None,
543 non_breakable_lines: None,
544 page_break_before: None,
545 front_matter: None,
546 direction: None,
547 background_color: None,
548 alignment: None,
549 top_margin: None,
550 text_indent: None,
551 semantic_role: None,
552 }));
553 in_block = false;
554 }
555 Event::Start(Tag::List(ordered)) => {
556 let style = if ordered.is_some() {
557 Some(ListStyle::Decimal)
558 } else {
559 Some(ListStyle::Disc)
560 };
561 list_stack.push(style);
562 list_start_stack.push(ordered.and_then(list_start_of));
563 }
564 Event::End(TagEnd::List(_)) => {
565 list_stack.pop();
566 list_start_stack.pop();
567 }
568 Event::Start(Tag::Item) => {
569 if !current_spans.is_empty() {
572 elements.push(ParsedElement::Block(ParsedBlock {
573 spans: std::mem::take(&mut current_spans),
574 heading_level: None,
575 list_style: current_list_style.clone(),
576 list_indent: current_list_indent,
577 list_prefix: String::new(),
578 list_suffix: String::new(),
579 list_start: current_list_start,
580 marker: None,
581 is_code_block: false,
582 code_language: None,
583 blockquote_depth,
584 blockquote_opens: 0,
585 line_height: None,
586 non_breakable_lines: None,
587 page_break_before: None,
588 front_matter: None,
589 direction: None,
590 background_color: None,
591 alignment: None,
592 top_margin: None,
593 text_indent: None,
594 semantic_role: None,
595 }));
596 }
597 in_block = true;
598 current_list_style = list_stack.last().cloned().flatten();
599 current_list_start = list_start_stack.last().copied().flatten();
600 current_list_indent = if list_stack.is_empty() {
601 0
602 } else {
603 (list_stack.len() - 1) as u32
604 };
605 }
606 Event::End(TagEnd::Item) => {
607 if !current_spans.is_empty() {
610 elements.push(ParsedElement::Block(ParsedBlock {
611 spans: std::mem::take(&mut current_spans),
612 heading_level: None,
613 list_style: current_list_style.clone(),
614 list_indent: current_list_indent,
615 list_prefix: String::new(),
616 list_suffix: String::new(),
617 list_start: current_list_start,
618 marker: None,
619 is_code_block: false,
620 code_language: None,
621 blockquote_depth,
622 blockquote_opens: 0,
623 line_height: None,
624 non_breakable_lines: None,
625 page_break_before: None,
626 front_matter: None,
627 direction: None,
628 background_color: None,
629 alignment: None,
630 top_margin: None,
631 text_indent: None,
632 semantic_role: None,
633 }));
634 }
635 in_block = false;
636 current_list_style = None;
637 current_list_start = None;
638 }
639 Event::Start(Tag::CodeBlock(kind)) => {
640 in_block = true;
641 is_code_block = true;
642 code_language = match &kind {
643 pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
644 Some(lang.to_string())
645 }
646 _ => None,
647 };
648 }
649 Event::End(TagEnd::CodeBlock) => {
650 if let Some(last) = current_spans.last_mut()
652 && last.text.ends_with('\n')
653 {
654 last.text.truncate(last.text.len() - 1);
655 }
656 elements.push(ParsedElement::Block(ParsedBlock {
657 spans: std::mem::take(&mut current_spans),
658 heading_level: None,
659 list_style: None,
660 list_indent: 0,
661 list_prefix: String::new(),
662 list_suffix: String::new(),
663 list_start: None,
664 marker: None,
665 is_code_block: true,
666 code_language: code_language.take(),
667 blockquote_depth,
668 blockquote_opens: 0,
669 line_height: None,
670 non_breakable_lines: None,
671 page_break_before: None,
672 front_matter: None,
673 direction: None,
674 background_color: None,
675 alignment: None,
676 top_margin: None,
677 text_indent: None,
678 semantic_role: None,
679 }));
680 in_block = false;
681 is_code_block = false;
682 }
683 Event::Start(Tag::Table(_)) => {
685 in_table = true;
686 in_table_head = false;
687 table_rows.clear();
688 current_row_cells.clear();
689 current_cell_spans.clear();
690 table_header_rows = 0;
691 }
692 Event::End(TagEnd::Table) => {
693 let own_cells = cells_in_rows(&table_rows);
694 push_table(
695 &mut elements,
696 ParsedTable {
697 header_rows: table_header_rows,
698 rows: std::mem::take(&mut table_rows),
699 blockquote_depth,
700 blockquote_opens: 0,
701 },
702 own_cells,
703 );
704 in_table = false;
705 }
706 Event::Start(Tag::TableHead) => {
707 in_table_head = true;
708 current_row_cells.clear();
709 }
710 Event::End(TagEnd::TableHead) => {
711 table_rows.push(std::mem::take(&mut current_row_cells));
713 table_header_rows += 1;
714 in_table_head = false;
715 }
716 Event::Start(Tag::TableRow) => {
717 current_row_cells.clear();
718 }
719 Event::End(TagEnd::TableRow) if !in_table_head => {
720 table_rows.push(std::mem::take(&mut current_row_cells));
722 }
723 Event::Start(Tag::TableCell) => {
724 current_cell_spans.clear();
725 }
726 Event::End(TagEnd::TableCell) => {
727 current_row_cells.push(ParsedTableCell {
728 spans: std::mem::take(&mut current_cell_spans),
729 });
730 }
731 Event::Start(Tag::Emphasis) => {
733 italic = true;
734 }
735 Event::End(TagEnd::Emphasis) => {
736 italic = false;
737 }
738 Event::Start(Tag::Strong) => {
739 bold = true;
740 }
741 Event::End(TagEnd::Strong) => {
742 bold = false;
743 }
744 Event::Start(Tag::Strikethrough) => {
745 strikeout = true;
746 }
747 Event::End(TagEnd::Strikethrough) => {
748 strikeout = false;
749 }
750 Event::Start(Tag::Link { dest_url, .. }) => {
751 link_href = Some(dest_url.to_string());
752 }
753 Event::End(TagEnd::Link) => {
754 link_href = None;
755 }
756 Event::Start(Tag::Image { dest_url, .. }) => {
759 pending_image = Some(ParsedImage {
760 src: dest_url.to_string(),
761 alt: String::new(),
762 width: 0,
763 height: 0,
764 });
765 }
766 Event::End(TagEnd::Image) => {
767 if let Some(image) = pending_image.take() {
768 let span = ParsedSpan {
769 text: String::new(),
770 bold,
771 italic,
772 underline: false,
773 strikeout,
774 code: false,
775 superscript: false,
776 subscript: false,
777 link_href: link_href.clone(),
778 image: Some(image),
779 footnote_ref: None,
780 };
781 if in_table {
782 current_cell_spans.push(span);
783 } else {
784 if !in_block {
785 in_block = true;
786 }
787 current_spans.push(span);
788 }
789 }
790 }
791 Event::Text(text) => {
792 if let Some(img) = pending_image.as_mut() {
797 img.alt.push_str(&text);
798 continue;
799 }
800 let span = ParsedSpan {
801 text: text.to_string(),
802 bold,
803 italic,
804 underline: false,
805 strikeout,
806 code: is_code_block,
807 superscript: false,
808 subscript: false,
809 link_href: link_href.clone(),
810 image: None,
811 footnote_ref: None,
812 };
813 if in_table {
814 current_cell_spans.push(span);
815 } else {
816 if !in_block {
817 in_block = true;
818 }
819 current_spans.push(span);
820 }
821 }
822 Event::Code(text) => {
823 let span = ParsedSpan {
824 text: text.to_string(),
825 bold,
826 italic,
827 underline: false,
828 strikeout,
829 code: true,
830 superscript: false,
831 subscript: false,
832 link_href: link_href.clone(),
833 image: None,
834 footnote_ref: None,
835 };
836 if in_table {
837 current_cell_spans.push(span);
838 } else {
839 if !in_block {
840 in_block = true;
841 }
842 current_spans.push(span);
843 }
844 }
845 Event::SoftBreak => {
846 let span = ParsedSpan {
847 text: " ".to_string(),
848 bold,
849 italic,
850 underline: false,
851 strikeout,
852 code: false,
853 superscript: false,
854 subscript: false,
855 link_href: link_href.clone(),
856 image: None,
857 footnote_ref: None,
858 };
859 if in_table {
860 current_cell_spans.push(span);
861 } else {
862 current_spans.push(span);
863 }
864 }
865 Event::HardBreak if pending_image.is_some() => {
869 if let Some(img) = pending_image.as_mut() {
870 img.alt.push(' ');
871 }
872 }
873 Event::HardBreak => {
874 let span = ParsedSpan {
875 text: if in_table { " " } else { LINE_SEPARATOR_STR }.to_string(),
876 bold,
877 italic,
878 underline: false,
879 strikeout,
880 code: false,
881 superscript: false,
882 subscript: false,
883 link_href: link_href.clone(),
884 image: None,
885 footnote_ref: None,
886 };
887 if in_table {
888 current_cell_spans.push(span);
889 } else {
890 current_spans.push(span);
891 }
892 }
893 Event::Start(Tag::BlockQuote(_)) => {
894 blockquote_depth += 1;
895 quotations_open.push(elements.len());
896 }
897 Event::End(TagEnd::BlockQuote(_)) => {
898 blockquote_depth = blockquote_depth.saturating_sub(1);
899 if let Some(first) = quotations_open.pop() {
900 mark_quotation_opened(&mut elements, first);
901 }
902 }
903 Event::Start(Tag::FootnoteDefinition(label)) => {
914 if !current_spans.is_empty() {
915 elements.push(ParsedElement::Block(ParsedBlock {
916 spans: std::mem::take(&mut current_spans),
917 heading_level: current_heading.take(),
918 list_style: current_list_style.clone(),
919 list_indent: current_list_indent,
920 list_prefix: String::new(),
921 list_suffix: String::new(),
922 list_start: current_list_start,
923 marker: None,
924 is_code_block: false,
925 code_language: None,
926 blockquote_depth,
927 blockquote_opens: 0,
928 line_height: None,
929 non_breakable_lines: None,
930 page_break_before: None,
931 front_matter: None,
932 direction: None,
933 background_color: None,
934 alignment: None,
935 top_margin: None,
936 text_indent: None,
937 semantic_role: None,
938 }));
939 }
940 footnote_open = Some((label.to_string(), elements.len()));
941 }
942 Event::End(TagEnd::FootnoteDefinition) => {
943 if !current_spans.is_empty() {
944 elements.push(ParsedElement::Block(ParsedBlock {
945 spans: std::mem::take(&mut current_spans),
946 heading_level: current_heading.take(),
947 list_style: current_list_style.clone(),
948 list_indent: current_list_indent,
949 list_prefix: String::new(),
950 list_suffix: String::new(),
951 list_start: current_list_start,
952 marker: None,
953 is_code_block: false,
954 code_language: None,
955 blockquote_depth,
956 blockquote_opens: 0,
957 line_height: None,
958 non_breakable_lines: None,
959 page_break_before: None,
960 front_matter: None,
961 direction: None,
962 background_color: None,
963 alignment: None,
964 top_margin: None,
965 text_indent: None,
966 semantic_role: None,
967 }));
968 }
969 if let Some((label, start)) = footnote_open.take() {
970 let blocks: Vec<ParsedBlock> = elements
971 .drain(start..)
972 .filter_map(|e| match e {
973 ParsedElement::Block(b) => Some(b),
974 _ => None,
977 })
978 .collect();
979 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
980 }
981 }
982 Event::FootnoteReference(label) => {
988 let span = ParsedSpan {
989 text: String::new(),
990 bold,
991 italic,
992 underline: false,
993 strikeout,
994 code: false,
995 superscript: false,
996 subscript: false,
997 link_href: link_href.clone(),
998 image: None,
999 footnote_ref: Some(label.to_string()),
1000 };
1001 if in_table {
1002 current_cell_spans.push(span);
1003 } else {
1004 if !in_block {
1005 in_block = true;
1006 }
1007 current_spans.push(span);
1008 }
1009 }
1010 _ => {}
1011 }
1012 }
1013
1014 if !current_spans.is_empty() {
1016 elements.push(ParsedElement::Block(ParsedBlock {
1017 spans: std::mem::take(&mut current_spans),
1018 heading_level: current_heading,
1019 list_style: current_list_style,
1020 list_indent: current_list_indent,
1021 list_prefix: String::new(),
1022 list_suffix: String::new(),
1023 list_start: current_list_start,
1024 marker: None,
1025 is_code_block,
1026 code_language: code_language.take(),
1027 blockquote_depth,
1028 blockquote_opens: 0,
1029 line_height: None,
1030 non_breakable_lines: None,
1031 page_break_before: None,
1032 front_matter: None,
1033 direction: None,
1034 background_color: None,
1035 alignment: None,
1036 top_margin: None,
1037 text_indent: None,
1038 semantic_role: None,
1039 }));
1040 }
1041
1042 if !dangling.is_empty() {
1049 elements.retain(
1050 |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
1051 );
1052 }
1053
1054 if elements.is_empty() {
1056 elements.push(ParsedElement::Block(ParsedBlock {
1057 spans: vec![ParsedSpan {
1058 text: String::new(),
1059 ..Default::default()
1060 }],
1061 heading_level: None,
1062 list_style: None,
1063 list_indent: 0,
1064 list_prefix: String::new(),
1065 list_suffix: String::new(),
1066 list_start: None,
1067 marker: None,
1068 is_code_block: false,
1069 code_language: None,
1070 blockquote_depth: 0,
1071 blockquote_opens: 0,
1072 line_height: None,
1073 non_breakable_lines: None,
1074 page_break_before: None,
1075 front_matter: None,
1076 direction: None,
1077 background_color: None,
1078 alignment: None,
1079 top_margin: None,
1080 text_indent: None,
1081 semantic_role: None,
1082 }));
1083 }
1084
1085 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1088 elements
1089}
1090
1091fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
1092 use pulldown_cmark::HeadingLevel;
1093 match level {
1094 HeadingLevel::H1 => 1,
1095 HeadingLevel::H2 => 2,
1096 HeadingLevel::H3 => 3,
1097 HeadingLevel::H4 => 4,
1098 HeadingLevel::H5 => 5,
1099 HeadingLevel::H6 => 6,
1100 }
1101}
1102
1103use scraper::Node;
1106
1107#[derive(Debug, Clone, Default)]
1109struct BlockStyles {
1110 line_height: Option<i64>,
1111 non_breakable_lines: Option<bool>,
1112 page_break_before: Option<bool>,
1113 direction: Option<TextDirection>,
1114 background_color: Option<String>,
1115 preserve_whitespace: Option<bool>,
1120}
1121
1122fn parse_block_styles(style: &str) -> BlockStyles {
1126 let mut result = BlockStyles::default();
1127 for part in style.split(';') {
1128 let part = part.trim();
1129 if let Some((prop, val)) = part.split_once(':') {
1130 let prop = prop.trim().to_ascii_lowercase();
1131 let val = val.trim();
1132 match prop.as_str() {
1133 "line-height" => {
1134 if let Ok(v) = val.parse::<f64>() {
1136 result.line_height = Some((v * 1000.0) as i64);
1137 }
1138 }
1139 "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1140 result.non_breakable_lines = Some(true);
1141 if val != "nowrap" {
1142 result.preserve_whitespace = Some(true);
1143 }
1144 }
1145 "break-before" | "page-break-before" => {
1149 result.page_break_before = match val.to_ascii_lowercase().as_str() {
1150 "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1151 "avoid" | "auto" => Some(false),
1152 _ => None,
1153 };
1154 }
1155 "direction" => {
1156 if val.eq_ignore_ascii_case("rtl") {
1157 result.direction = Some(TextDirection::RightToLeft);
1158 } else if val.eq_ignore_ascii_case("ltr") {
1159 result.direction = Some(TextDirection::LeftToRight);
1160 }
1161 }
1162 "background-color" | "background" => {
1163 result.background_color = Some(val.to_string());
1164 }
1165 _ => {}
1166 }
1167 }
1168 }
1169 result
1170}
1171
1172fn is_html_space(ch: char) -> bool {
1179 matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1180}
1181
1182fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1184 spans
1185 .iter()
1186 .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1187}
1188
1189fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1203 fn is_replaced(span: &ParsedSpan) -> bool {
1204 span.image.is_some() || span.footnote_ref.is_some()
1205 }
1206
1207 for span in spans.iter_mut() {
1211 if is_replaced(span) {
1212 continue;
1213 }
1214 let mut out = String::with_capacity(span.text.len());
1215 let mut in_run = false;
1216 for ch in span.text.chars() {
1217 if is_html_space(ch) {
1218 if !in_run {
1219 out.push(' ');
1220 in_run = true;
1221 }
1222 } else {
1223 out.push(ch);
1224 in_run = false;
1225 }
1226 }
1227 span.text = out;
1228 }
1229
1230 let mut prev_ends_with_space = true;
1235 let mut line_end: Option<usize> = None;
1238 for i in 0..spans.len() {
1239 let span = &mut spans[i];
1240 if is_replaced(span) {
1241 prev_ends_with_space = false;
1242 line_end = None;
1243 continue;
1244 }
1245 if span.text.is_empty() {
1246 prev_ends_with_space = false;
1249 continue;
1250 }
1251 if span.text == LINE_SEPARATOR_STR {
1252 if let Some(end) = line_end.take()
1253 && spans[end].text.ends_with(' ')
1254 {
1255 spans[end].text.pop();
1256 }
1257 prev_ends_with_space = true;
1258 continue;
1259 }
1260 if prev_ends_with_space && span.text.starts_with(' ') {
1261 span.text.remove(0);
1262 }
1263 if !span.text.is_empty() {
1266 prev_ends_with_space = span.text.ends_with(' ');
1267 line_end = Some(i);
1268 }
1269 }
1270
1271 let mut trailing_break_dropped = false;
1277 for span in spans.iter_mut().rev() {
1278 if is_replaced(span) {
1279 break;
1280 }
1281 if span.text.is_empty() {
1282 continue;
1283 }
1284 if span.text == LINE_SEPARATOR_STR {
1285 if trailing_break_dropped {
1286 break;
1287 }
1288 span.text.clear();
1289 trailing_break_dropped = true;
1290 continue;
1291 }
1292 if span.text.ends_with(' ') {
1293 span.text.pop();
1294 }
1295 break;
1296 }
1297}
1298
1299pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
1320 let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
1321 if !block.spans.iter().any(breaks) {
1322 return vec![block];
1323 }
1324 let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
1325 for mut span in std::mem::take(&mut block.spans) {
1326 if !breaks(&span) {
1327 push_to_last_line(&mut lines, span);
1328 continue;
1329 }
1330 let text = std::mem::take(&mut span.text);
1331 for (i, piece) in text.split('\n').enumerate() {
1332 if i > 0 {
1333 lines.push(Vec::new());
1334 }
1335 if !piece.is_empty() {
1336 push_to_last_line(
1337 &mut lines,
1338 ParsedSpan {
1339 text: piece.to_string(),
1340 ..span.clone()
1341 },
1342 );
1343 }
1344 }
1345 }
1346 if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
1347 lines.pop();
1348 }
1349 let later_line = ParsedBlock {
1350 page_break_before: None,
1351 blockquote_opens: 0,
1352 ..block.clone()
1353 };
1354 let mut first_line = Some(block);
1355 lines
1356 .into_iter()
1357 .map(|spans| {
1358 let spans = if spans.is_empty() {
1359 vec![ParsedSpan::default()]
1360 } else {
1361 spans
1362 };
1363 match first_line.take() {
1364 Some(first) => ParsedBlock { spans, ..first },
1365 None => ParsedBlock {
1366 spans,
1367 ..later_line.clone()
1368 },
1369 }
1370 })
1371 .collect()
1372}
1373
1374fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
1377 match lines.last_mut() {
1378 Some(line) => line.push(span),
1379 None => lines.push(vec![span]),
1380 }
1381}
1382
1383pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
1384 ParsedElement::flatten_to_blocks(parse_html_elements(html))
1385}
1386
1387pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
1401
1402fn html_footnote_span(
1405 el: &scraper::node::Element,
1406 link_href: Option<String>,
1407) -> Option<ParsedSpan> {
1408 let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
1409 if label.is_empty() {
1410 return None;
1411 }
1412 Some(ParsedSpan {
1413 text: String::new(),
1414 link_href,
1415 footnote_ref: Some(label.to_string()),
1416 ..Default::default()
1417 })
1418}
1419
1420fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
1428 let src = el.attr("src")?;
1429 if src.is_empty() {
1430 return None;
1431 }
1432 let dim = |name: &str| -> i64 {
1433 el.attr(name)
1434 .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
1435 .filter(|n| *n > 0)
1436 .unwrap_or(0)
1437 };
1438 Some(ParsedSpan {
1439 text: String::new(),
1440 link_href,
1441 image: Some(ParsedImage {
1442 src: src.to_string(),
1443 alt: el.attr("alt").unwrap_or_default().to_string(),
1444 width: dim("width"),
1445 height: dim("height"),
1446 }),
1447 ..Default::default()
1448 })
1449}
1450
1451fn is_html_list(node: &Node) -> bool {
1453 node.as_element()
1454 .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
1455}
1456
1457fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
1470 use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
1471 use ego_tree::iter::Edge;
1472
1473 let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
1475 let mut depth = 0usize;
1476 for edge in tree.root().traverse() {
1477 match edge {
1478 Edge::Open(node) if is_html_list(node.value()) => {
1479 if depth == MAX_LIST_LEVELS {
1480 too_deep.push(node.id());
1481 }
1482 depth += 1;
1483 }
1484 Edge::Close(node) if is_html_list(node.value()) => {
1485 depth = depth.saturating_sub(1);
1486 }
1487 _ => {}
1488 }
1489 }
1490
1491 for list in too_deep {
1492 let Some(list_ref) = tree.get(list) else {
1496 continue;
1497 };
1498 let mut holder = None;
1499 let mut current = list_ref;
1500 while let Some(parent) = current.parent() {
1501 if is_html_list(parent.value()) {
1502 holder = Some(current.id());
1503 break;
1504 }
1505 current = parent;
1506 }
1507 let Some(holder) = holder else {
1508 continue;
1509 };
1510 let mut after = holder;
1511 let items: Vec<ego_tree::NodeId> = list_ref
1512 .descendants()
1513 .filter(|node| {
1514 node.value()
1515 .as_element()
1516 .is_some_and(|el| el.name() == "li")
1517 })
1518 .map(|node| node.id())
1519 .collect();
1520 for item in items {
1521 if tree.get(after).and_then(|node| node.parent()).is_none() {
1524 break;
1525 }
1526 if let Some(mut node) = tree.get_mut(after) {
1527 node.insert_id_after(item);
1528 after = item;
1529 }
1530 }
1531 let holds_text = tree.get(list).is_some_and(|node| {
1536 node.descendants()
1537 .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
1538 });
1539 if !holds_text {
1540 if let Some(mut node) = tree.get_mut(list) {
1541 node.detach();
1542 }
1543 } else if holder != list
1544 && tree.get(after).and_then(|node| node.parent()).is_some()
1545 && let Some(mut node) = tree.get_mut(after)
1546 {
1547 node.insert_id_after(list);
1548 }
1549 }
1550}
1551
1552const HTML_DEPTH_ON_THE_CALLERS_STACK: usize = 32;
1556
1557const MAX_HTML_READ_DEPTH: usize = 256;
1560
1561const HTML_READER_STACK: usize = 2 << 20;
1564
1565const HTML_STACK_PER_LEVEL: usize = 16 << 10;
1568
1569fn html_integer(value: &str) -> Option<i64> {
1575 let value = value.trim_start_matches(|c: char| c.is_ascii_whitespace());
1576 let (negative, digits) = match value.as_bytes().first() {
1577 Some(b'-') => (true, &value[1..]),
1578 Some(b'+') => (false, &value[1..]),
1579 _ => (false, value),
1580 };
1581 let digits: &str = digits
1582 .split(|c: char| !c.is_ascii_digit())
1583 .next()
1584 .unwrap_or("");
1585 if digits.is_empty() {
1586 return None;
1587 }
1588 let magnitude = digits.bytes().fold(0i64, |number, digit| {
1589 number
1590 .saturating_mul(10)
1591 .saturating_add(i64::from(digit - b'0'))
1592 });
1593 Some(if negative { -magnitude } else { magnitude })
1594}
1595
1596fn html_tree_depth(tree: &ego_tree::Tree<Node>) -> usize {
1599 use ego_tree::iter::Edge;
1600 let mut depth = 0usize;
1601 let mut deepest = 0usize;
1602 for edge in tree.root().traverse() {
1603 match edge {
1604 Edge::Open(node) if node.value().is_element() => {
1605 depth += 1;
1606 deepest = deepest.max(depth);
1607 }
1608 Edge::Close(node) if node.value().is_element() => depth = depth.saturating_sub(1),
1609 _ => {}
1610 }
1611 }
1612 deepest
1613}
1614
1615pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
1627 use scraper::Html;
1628
1629 let read = |depth_limit: usize, html: &str| {
1630 let mut fragment = Html::parse_fragment(html);
1631 flatten_deep_html_lists(&mut fragment.tree);
1632 let mut elements = read_html_tree(&fragment, depth_limit);
1633 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1634 elements
1635 };
1636 let mut fragment = Html::parse_fragment(html);
1637 flatten_deep_html_lists(&mut fragment.tree);
1638 let depth = html_tree_depth(&fragment.tree);
1639 if depth <= HTML_DEPTH_ON_THE_CALLERS_STACK {
1640 let mut elements = read_html_tree(&fragment, MAX_HTML_READ_DEPTH);
1641 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1642 return elements;
1643 }
1644 drop(fragment);
1645 let stack = HTML_READER_STACK + depth.min(MAX_HTML_READ_DEPTH + 1) * HTML_STACK_PER_LEVEL;
1646 let owned = html.to_owned();
1647 let on_its_own_thread = std::thread::Builder::new()
1648 .stack_size(stack)
1649 .spawn(move || read(MAX_HTML_READ_DEPTH, &owned))
1650 .ok()
1651 .and_then(|reader| reader.join().ok());
1652 on_its_own_thread.unwrap_or_else(|| read(HTML_DEPTH_ON_THE_CALLERS_STACK, html))
1653}
1654
1655fn read_html_tree(fragment: &scraper::Html, depth_limit: usize) -> Vec<ParsedElement> {
1659 let mut elements: Vec<ParsedElement> = Vec::new();
1660
1661 let root = fragment.root_element();
1663
1664 #[derive(Clone, Default)]
1665 struct FmtState {
1666 bold: bool,
1667 italic: bool,
1668 underline: bool,
1669 strikeout: bool,
1670 code: bool,
1671 superscript: bool,
1672 subscript: bool,
1673 link_href: Option<String>,
1674 }
1675
1676 type HtmlList = (Option<ListStyle>, Option<i64>);
1679
1680 fn is_metadata_tag(tag: &str) -> bool {
1690 matches!(
1691 tag,
1692 "head"
1693 | "style"
1694 | "script"
1695 | "title"
1696 | "meta"
1697 | "link"
1698 | "base"
1699 | "noscript"
1700 | "template"
1701 )
1702 }
1703
1704 fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
1713 use ego_tree::iter::Edge;
1714 let mut text = String::new();
1715 let mut hidden = 0usize;
1717 for edge in node.traverse() {
1718 match edge {
1719 Edge::Open(open) => match open.value() {
1720 Node::Element(el) => {
1721 if hidden > 0 || is_metadata_tag(el.name()) {
1722 hidden += 1;
1723 } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
1724 text.push(' ');
1725 }
1726 }
1727 Node::Text(t) if hidden == 0 => text.push_str(t),
1728 _ => {}
1729 },
1730 Edge::Close(close) => {
1731 if hidden > 0 && close.value().is_element() {
1732 hidden -= 1;
1733 }
1734 }
1735 }
1736 }
1737 text
1738 }
1739
1740 fn text_span(text: String, state: &FmtState) -> ParsedSpan {
1742 ParsedSpan {
1743 text,
1744 bold: state.bold,
1745 italic: state.italic,
1746 underline: state.underline,
1747 strikeout: state.strikeout,
1748 code: state.code,
1749 superscript: state.superscript,
1750 subscript: state.subscript,
1751 link_href: state.link_href.clone(),
1752 image: None,
1753 footnote_ref: None,
1754 }
1755 }
1756
1757 fn collect_cell_spans(
1759 node: ego_tree::NodeRef<Node>,
1760 state: &FmtState,
1761 spans: &mut Vec<ParsedSpan>,
1762 depth: usize,
1763 limit: usize,
1764 ) {
1765 if depth > limit {
1766 let text = subtree_text(node);
1767 if !text.is_empty() {
1768 spans.push(text_span(text, state));
1769 }
1770 return;
1771 }
1772 for child in node.children() {
1773 match child.value() {
1774 Node::Text(text) => {
1775 let t = text.text.to_string();
1776 if !t.is_empty() {
1777 spans.push(ParsedSpan {
1778 text: t,
1779 bold: state.bold,
1780 italic: state.italic,
1781 underline: state.underline,
1782 strikeout: state.strikeout,
1783 code: state.code,
1784 superscript: state.superscript,
1785 subscript: state.subscript,
1786 link_href: state.link_href.clone(),
1787 image: None,
1788 footnote_ref: None,
1789 });
1790 }
1791 }
1792 Node::Element(el) => {
1793 let tag = el.name();
1794 if is_metadata_tag(tag) {
1795 continue;
1796 }
1797 let mut new_state = state.clone();
1798 match tag {
1799 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
1803 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
1804 {
1805 spans.push(span);
1806 }
1807 continue;
1808 }
1809 "b" | "strong" => new_state.bold = true,
1810 "i" | "em" => new_state.italic = true,
1811 "u" | "ins" => new_state.underline = true,
1812 "s" | "del" | "strike" => new_state.strikeout = true,
1813 "code" => new_state.code = true,
1814 "sup" => new_state.superscript = true,
1815 "sub" => new_state.subscript = true,
1816 "a" => {
1817 if let Some(href) = el.attr("href") {
1818 new_state.link_href = Some(href.to_string());
1819 }
1820 }
1821 "img" => {
1822 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
1823 spans.push(span);
1824 }
1825 continue;
1826 }
1827 "br" => {
1833 spans.push(text_span(" ".to_string(), state));
1834 continue;
1835 }
1836 "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
1837 | "h3" | "h4" | "h5" | "h6" => {
1838 spans.push(text_span(" ".to_string(), state));
1839 }
1840 _ => {}
1841 }
1842 collect_cell_spans(child, &new_state, spans, depth + 1, limit);
1843 }
1844 _ => {}
1845 }
1846 }
1847 }
1848
1849 const MAX_HTML_COLSPAN: usize = 1000;
1852
1853 fn parse_table_element(
1857 table_node: ego_tree::NodeRef<Node>,
1858 depth: usize,
1859 limit: usize,
1860 ) -> (ParsedTable, usize) {
1861 let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
1862 let mut header_rows: usize = 0;
1863 let mut own_cells: usize = 0;
1864
1865 fn collect_rows(
1867 node: ego_tree::NodeRef<Node>,
1868 rows: &mut Vec<Vec<ParsedTableCell>>,
1869 header_rows: &mut usize,
1870 own_cells: &mut usize,
1871 in_thead: bool,
1872 walk: (usize, usize),
1873 ) {
1874 let (depth, limit) = walk;
1875 for child in node.children() {
1876 if let Node::Element(el) = child.value() {
1877 match el.name() {
1878 "thead" => collect_rows(
1879 child,
1880 rows,
1881 header_rows,
1882 own_cells,
1883 true,
1884 (depth + 1, limit),
1885 ),
1886 "tbody" | "tfoot" => collect_rows(
1887 child,
1888 rows,
1889 header_rows,
1890 own_cells,
1891 false,
1892 (depth + 1, limit),
1893 ),
1894 "tr" => {
1895 let mut cells: Vec<ParsedTableCell> = Vec::new();
1896 let mut own_end = 0;
1899 for td in child.children() {
1900 if let Node::Element(td_el) = td.value()
1901 && matches!(td_el.name(), "td" | "th")
1902 {
1903 let mut spans = Vec::new();
1904 let state = FmtState::default();
1905 collect_cell_spans(td, &state, &mut spans, depth + 2, limit);
1906 collapse_inline_whitespace(&mut spans);
1907 if spans.is_empty() {
1908 spans.push(ParsedSpan::default());
1909 }
1910 cells.push(ParsedTableCell { spans });
1911 *own_cells += 1;
1912 own_end = cells.len();
1913 let span = td_el
1918 .attr("colspan")
1919 .and_then(|value| value.trim().parse::<usize>().ok())
1920 .unwrap_or(1)
1921 .clamp(1, MAX_HTML_COLSPAN);
1922 for _ in 1..span {
1923 if cells.len() >= MAX_HTML_COLSPAN {
1924 break;
1925 }
1926 cells.push(ParsedTableCell {
1927 spans: vec![ParsedSpan::default()],
1928 });
1929 }
1930 }
1931 }
1932 cells.truncate(own_end);
1936 if !cells.is_empty() {
1937 rows.push(cells);
1938 if in_thead {
1939 *header_rows += 1;
1940 }
1941 }
1942 }
1943 _ => {}
1944 }
1945 }
1946 }
1947 }
1948
1949 collect_rows(
1950 table_node,
1951 &mut rows,
1952 &mut header_rows,
1953 &mut own_cells,
1954 false,
1955 (depth, limit),
1956 );
1957
1958 if header_rows == 0 && !rows.is_empty() {
1960 header_rows = 1;
1961 }
1962
1963 (
1964 ParsedTable {
1965 header_rows,
1966 rows,
1967 blockquote_depth: 0,
1970 blockquote_opens: 0,
1971 },
1972 own_cells,
1973 )
1974 }
1975
1976 #[allow(clippy::too_many_arguments)]
1977 fn walk_node(
1978 node: ego_tree::NodeRef<Node>,
1979 state: &FmtState,
1980 elements: &mut Vec<ParsedElement>,
1981 current_list: &HtmlList,
1982 blockquote_depth: u32,
1983 list_depth: u32,
1984 depth: usize,
1985 limit: usize,
1986 ) {
1987 if depth > limit {
1988 let text = subtree_text(node);
1989 if !text.trim().is_empty() {
1990 let mut spans = vec![text_span(text, state)];
1991 collapse_inline_whitespace(&mut spans);
1992 elements.push(ParsedElement::Block(ParsedBlock {
1993 spans,
1994 blockquote_depth,
1995 blockquote_opens: 0,
1996 ..ParsedBlock::default()
1997 }));
1998 }
1999 return;
2000 }
2001 match node.value() {
2002 Node::Element(el) => {
2003 let tag = el.name();
2004 if is_metadata_tag(tag) {
2005 return;
2006 }
2007 let mut new_state = state.clone();
2008 let mut new_list = current_list.clone();
2009 let mut bq_depth = blockquote_depth;
2010 let mut new_list_depth = list_depth;
2011
2012 let is_block_tag = matches!(
2014 tag,
2015 "p" | "div"
2016 | "h1"
2017 | "h2"
2018 | "h3"
2019 | "h4"
2020 | "h5"
2021 | "h6"
2022 | "li"
2023 | "pre"
2024 | "br"
2025 | "blockquote"
2026 | "body"
2027 | "html"
2028 );
2029
2030 match tag {
2032 "b" | "strong" => new_state.bold = true,
2033 "i" | "em" => new_state.italic = true,
2034 "u" | "ins" => new_state.underline = true,
2035 "s" | "del" | "strike" => new_state.strikeout = true,
2036 "code" => new_state.code = true,
2037 "sup" => new_state.superscript = true,
2038 "sub" => new_state.subscript = true,
2039 "a" => {
2040 if let Some(href) = el.attr("href") {
2041 new_state.link_href = Some(href.to_string());
2042 }
2043 }
2044 "ul" => {
2045 new_list = (Some(ListStyle::Disc), None);
2046 new_list_depth = list_depth + 1;
2047 }
2048 "ol" => {
2049 let start = el
2053 .attr("start")
2054 .and_then(html_integer)
2055 .and_then(crate::parser_tools::list_start::list_start);
2056 new_list = (Some(ListStyle::Decimal), start);
2057 new_list_depth = list_depth + 1;
2058 }
2059 "blockquote" => {
2060 bq_depth += 1;
2061 }
2062 _ => {}
2063 }
2064
2065 let heading_level = match tag {
2067 "h1" => Some(1),
2068 "h2" => Some(2),
2069 "h3" => Some(3),
2070 "h4" => Some(4),
2071 "h5" => Some(5),
2072 "h6" => Some(6),
2073 _ => None,
2074 };
2075
2076 let is_code_block = tag == "pre";
2077
2078 let code_language = if is_code_block {
2080 node.children().find_map(|child| {
2081 if let Node::Element(cel) = child.value()
2082 && cel.name() == "code"
2083 && let Some(cls) = cel.attr("class")
2084 {
2085 return cls
2086 .split_whitespace()
2087 .find_map(|c| c.strip_prefix("language-"))
2088 .map(|l| l.to_string());
2089 }
2090 None
2091 })
2092 } else {
2093 None
2094 };
2095
2096 let css = if is_block_tag {
2098 el.attr("style").map(parse_block_styles).unwrap_or_default()
2099 } else {
2100 BlockStyles::default()
2101 };
2102
2103 if tag == "table" {
2104 let (mut parsed_table, own_cells) = parse_table_element(node, depth, limit);
2106 if !parsed_table.rows.is_empty() {
2107 parsed_table.blockquote_depth = bq_depth;
2108 push_table(elements, parsed_table, own_cells);
2109 }
2110 return;
2111 }
2112
2113 if tag == "br" {
2114 elements.push(ParsedElement::Block(ParsedBlock {
2116 spans: vec![ParsedSpan {
2117 text: String::new(),
2118 ..Default::default()
2119 }],
2120 heading_level: None,
2121 list_style: None,
2122 list_indent: 0,
2123 list_prefix: String::new(),
2124 list_suffix: String::new(),
2125 list_start: None,
2126 marker: None,
2127 is_code_block: false,
2128 code_language: None,
2129 blockquote_depth: bq_depth,
2130 blockquote_opens: 0,
2131 line_height: None,
2132 non_breakable_lines: None,
2133 page_break_before: None,
2134 front_matter: None,
2135 direction: None,
2136 background_color: None,
2137 alignment: None,
2138 top_margin: None,
2139 text_indent: None,
2140 semantic_role: None,
2141 }));
2142 return;
2143 }
2144
2145 if tag == "blockquote" {
2146 let first = elements.len();
2147 let inline_only = node.children().all(|child| match child.value() {
2152 Node::Element(el) => !matches!(
2153 el.name(),
2154 "p" | "div"
2155 | "h1"
2156 | "h2"
2157 | "h3"
2158 | "h4"
2159 | "h5"
2160 | "h6"
2161 | "li"
2162 | "pre"
2163 | "blockquote"
2164 | "ul"
2165 | "ol"
2166 | "table"
2167 ),
2168 _ => true,
2169 });
2170 if inline_only {
2171 let mut spans: Vec<ParsedSpan> = Vec::new();
2172 let mut nested_elements: Vec<ParsedElement> = Vec::new();
2173 collect_inline_spans(
2174 node,
2175 &new_state,
2176 &mut spans,
2177 &new_list,
2178 &mut nested_elements,
2179 bq_depth,
2180 new_list_depth,
2181 depth + 1,
2182 limit,
2183 );
2184 collapse_inline_whitespace(&mut spans);
2185 if spans_carry_content(&spans) {
2186 elements.push(ParsedElement::Block(ParsedBlock {
2187 spans,
2188 blockquote_depth: bq_depth,
2189 blockquote_opens: 0,
2190 ..ParsedBlock::default()
2191 }));
2192 }
2193 elements.append(&mut nested_elements);
2194 } else {
2195 for child in node.children() {
2197 walk_node(
2198 child,
2199 &new_state,
2200 elements,
2201 &new_list,
2202 bq_depth,
2203 new_list_depth,
2204 depth + 1,
2205 limit,
2206 );
2207 }
2208 }
2209 mark_quotation_opened(elements, first);
2210 } else if is_block_tag && tag != "br" {
2211 let mut spans: Vec<ParsedSpan> = Vec::new();
2216 let mut nested_elements: Vec<ParsedElement> = Vec::new();
2217 collect_inline_spans(
2218 node,
2219 &new_state,
2220 &mut spans,
2221 &new_list,
2222 &mut nested_elements,
2223 bq_depth,
2224 new_list_depth,
2225 depth + 1,
2226 limit,
2227 );
2228
2229 let (list_style_for_block, list_start_for_block) = if tag == "li" {
2230 new_list.clone()
2231 } else {
2232 (None, None)
2233 };
2234
2235 let list_indent_for_block = if tag == "li" {
2236 new_list_depth.saturating_sub(1)
2237 } else {
2238 0
2239 };
2240
2241 if !(is_code_block || css.preserve_whitespace == Some(true)) {
2242 collapse_inline_whitespace(&mut spans);
2243 }
2244
2245 let own_run_is_content =
2253 spans_carry_content(&spans) || nested_elements.is_empty();
2254
2255 if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
2256 let is_code_block = is_code_block
2260 && !spans
2261 .iter()
2262 .any(|span| span.image.is_some() || span.footnote_ref.is_some());
2263 let block = ParsedBlock {
2264 spans,
2265 heading_level,
2266 list_style: list_style_for_block,
2267 list_indent: list_indent_for_block,
2268 list_prefix: String::new(),
2269 list_suffix: String::new(),
2270 list_start: list_start_for_block,
2271 marker: None,
2272 is_code_block,
2273 code_language: code_language.filter(|_| is_code_block),
2274 blockquote_depth: bq_depth,
2275 blockquote_opens: 0,
2276 line_height: css.line_height,
2277 non_breakable_lines: css.non_breakable_lines,
2278 page_break_before: css.page_break_before,
2279 front_matter: None,
2280 direction: css.direction,
2281 background_color: css.background_color,
2282 alignment: None,
2283 top_margin: None,
2284 text_indent: None,
2285 semantic_role: None,
2286 };
2287 if is_code_block {
2291 elements.push(ParsedElement::Block(block));
2292 } else {
2293 elements.extend(
2294 split_block_at_line_breaks(block)
2295 .into_iter()
2296 .map(ParsedElement::Block),
2297 );
2298 }
2299 }
2300 elements.append(&mut nested_elements);
2302 } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
2303 for child in node.children() {
2305 walk_node(
2306 child,
2307 &new_state,
2308 elements,
2309 &new_list,
2310 bq_depth,
2311 new_list_depth,
2312 depth + 1,
2313 limit,
2314 );
2315 }
2316 } else {
2317 for child in node.children() {
2319 walk_node(
2320 child,
2321 &new_state,
2322 elements,
2323 current_list,
2324 bq_depth,
2325 list_depth,
2326 depth + 1,
2327 limit,
2328 );
2329 }
2330 }
2331 }
2332 Node::Text(text) => {
2333 let t = text.text.to_string();
2334 let trimmed = t.trim();
2335 if !trimmed.is_empty() {
2336 elements.push(ParsedElement::Block(ParsedBlock {
2338 spans: vec![ParsedSpan {
2339 text: trimmed.to_string(),
2340 bold: state.bold,
2341 italic: state.italic,
2342 underline: state.underline,
2343 strikeout: state.strikeout,
2344 code: state.code,
2345 superscript: state.superscript,
2346 subscript: state.subscript,
2347 link_href: state.link_href.clone(),
2348 image: None,
2349 footnote_ref: None,
2350 }],
2351 heading_level: None,
2352 list_style: None,
2353 list_indent: 0,
2354 list_prefix: String::new(),
2355 list_suffix: String::new(),
2356 list_start: None,
2357 marker: None,
2358 is_code_block: false,
2359 code_language: None,
2360 blockquote_depth,
2361 blockquote_opens: 0,
2362 line_height: None,
2363 non_breakable_lines: None,
2364 page_break_before: None,
2365 front_matter: None,
2366 direction: None,
2367 background_color: None,
2368 alignment: None,
2369 top_margin: None,
2370 text_indent: None,
2371 semantic_role: None,
2372 }));
2373 }
2374 }
2375 _ => {
2376 for child in node.children() {
2378 walk_node(
2379 child,
2380 state,
2381 elements,
2382 current_list,
2383 blockquote_depth,
2384 list_depth,
2385 depth + 1,
2386 limit,
2387 );
2388 }
2389 }
2390 }
2391 }
2392
2393 #[allow(clippy::too_many_arguments)]
2397 fn collect_inline_spans(
2398 node: ego_tree::NodeRef<Node>,
2399 state: &FmtState,
2400 spans: &mut Vec<ParsedSpan>,
2401 current_list: &HtmlList,
2402 elements: &mut Vec<ParsedElement>,
2403 blockquote_depth: u32,
2404 list_depth: u32,
2405 depth: usize,
2406 limit: usize,
2407 ) {
2408 if depth > limit {
2409 let text = subtree_text(node);
2410 if !text.is_empty() {
2411 spans.push(text_span(text, state));
2412 }
2413 return;
2414 }
2415 for child in node.children() {
2416 match child.value() {
2417 Node::Text(text) => {
2418 let t = text.text.to_string();
2419 if !t.is_empty() {
2420 spans.push(ParsedSpan {
2421 text: t,
2422 bold: state.bold,
2423 italic: state.italic,
2424 underline: state.underline,
2425 strikeout: state.strikeout,
2426 code: state.code,
2427 superscript: state.superscript,
2428 subscript: state.subscript,
2429 link_href: state.link_href.clone(),
2430 image: None,
2431 footnote_ref: None,
2432 });
2433 }
2434 }
2435 Node::Element(el) => {
2436 let tag = el.name();
2437 if is_metadata_tag(tag) {
2438 continue;
2439 }
2440 let mut new_state = state.clone();
2441
2442 match tag {
2443 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2447 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2448 {
2449 spans.push(span);
2450 }
2451 continue;
2452 }
2453 "b" | "strong" => new_state.bold = true,
2454 "i" | "em" => new_state.italic = true,
2455 "u" | "ins" => new_state.underline = true,
2456 "s" | "del" | "strike" => new_state.strikeout = true,
2457 "code" => new_state.code = true,
2458 "sup" => new_state.superscript = true,
2459 "sub" => new_state.subscript = true,
2460 "a" => {
2461 if let Some(href) = el.attr("href") {
2462 new_state.link_href = Some(href.to_string());
2463 }
2464 }
2465 "img" => {
2466 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2467 spans.push(span);
2468 }
2469 continue;
2470 }
2471 _ => {}
2472 }
2473
2474 let nested_block = matches!(
2476 tag,
2477 "p" | "div"
2478 | "h1"
2479 | "h2"
2480 | "h3"
2481 | "h4"
2482 | "h5"
2483 | "h6"
2484 | "li"
2485 | "pre"
2486 | "blockquote"
2487 | "ul"
2488 | "ol"
2489 );
2490
2491 if tag == "br" {
2492 spans.push(text_span(LINE_SEPARATOR_STR.to_string(), state));
2496 } else if nested_block || tag == "table" {
2497 walk_node(
2499 child,
2500 &new_state,
2501 elements,
2502 current_list,
2503 blockquote_depth,
2504 list_depth,
2505 depth + 1,
2506 limit,
2507 );
2508 } else {
2509 collect_inline_spans(
2511 child,
2512 &new_state,
2513 spans,
2514 current_list,
2515 elements,
2516 blockquote_depth,
2517 list_depth,
2518 depth + 1,
2519 limit,
2520 );
2521 }
2522 }
2523 _ => {}
2524 }
2525 }
2526 }
2527
2528 let initial_state = FmtState::default();
2529 let mut root_spans: Vec<ParsedSpan> = Vec::new();
2533 collect_inline_spans(
2534 *root,
2535 &initial_state,
2536 &mut root_spans,
2537 &(None, None),
2538 &mut elements,
2539 0,
2540 0,
2541 0,
2542 depth_limit,
2543 );
2544 collapse_inline_whitespace(&mut root_spans);
2545 if spans_carry_content(&root_spans) {
2548 elements.push(ParsedElement::Block(ParsedBlock {
2549 spans: root_spans,
2550 heading_level: None,
2551 list_style: None,
2552 list_indent: 0,
2553 list_prefix: String::new(),
2554 list_suffix: String::new(),
2555 list_start: None,
2556 marker: None,
2557 is_code_block: false,
2558 code_language: None,
2559 blockquote_depth: 0,
2560 blockquote_opens: 0,
2561 line_height: None,
2562 non_breakable_lines: None,
2563 page_break_before: None,
2564 front_matter: None,
2565 direction: None,
2566 background_color: None,
2567 alignment: None,
2568 top_margin: None,
2569 text_indent: None,
2570 semantic_role: None,
2571 }));
2572 }
2573
2574 if elements.is_empty() {
2576 elements.push(ParsedElement::Block(ParsedBlock {
2577 spans: vec![ParsedSpan {
2578 text: String::new(),
2579 ..Default::default()
2580 }],
2581 heading_level: None,
2582 list_style: None,
2583 list_indent: 0,
2584 list_prefix: String::new(),
2585 list_suffix: String::new(),
2586 list_start: None,
2587 marker: None,
2588 is_code_block: false,
2589 code_language: None,
2590 blockquote_depth: 0,
2591 blockquote_opens: 0,
2592 line_height: None,
2593 non_breakable_lines: None,
2594 page_break_before: None,
2595 front_matter: None,
2596 direction: None,
2597 background_color: None,
2598 alignment: None,
2599 top_margin: None,
2600 text_indent: None,
2601 semantic_role: None,
2602 }));
2603 }
2604
2605 elements
2606}
2607
2608pub fn character_format_from_span(
2612 span: &ParsedSpan,
2613 is_code_block: bool,
2614) -> crate::format_runs::CharacterFormat {
2615 use crate::entities::CharVerticalAlignment;
2616 crate::format_runs::CharacterFormat {
2617 font_bold: if span.bold { Some(true) } else { None },
2618 font_italic: if span.italic { Some(true) } else { None },
2619 font_underline: if span.underline { Some(true) } else { None },
2620 font_strikeout: if span.strikeout { Some(true) } else { None },
2621 font_family: if span.code || is_code_block {
2622 Some("monospace".to_string())
2623 } else {
2624 None
2625 },
2626 anchor_href: span.link_href.clone(),
2627 is_anchor: if span.link_href.is_some() {
2628 Some(true)
2629 } else {
2630 None
2631 },
2632 vertical_alignment: if span.superscript {
2633 Some(CharVerticalAlignment::SuperScript)
2634 } else if span.subscript {
2635 Some(CharVerticalAlignment::SubScript)
2636 } else {
2637 None
2638 },
2639 ..Default::default()
2640 }
2641}
2642
2643pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
2655 use crate::format_runs::{
2656 CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
2657 };
2658
2659 let mut plain_text = String::new();
2660 let mut runs: Vec<FormatRun> = Vec::new();
2661 let mut images: Vec<ImageAnchor> = Vec::new();
2662 let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
2663 let default = CharacterFormat::default();
2664
2665 for span in spans {
2666 let byte_start = plain_text.len() as u32;
2667
2668 if let Some(label) = &span.footnote_ref {
2669 plain_text.push('\u{FFFC}');
2672 let mut format = character_format_from_span(span, is_code_block);
2687 format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
2688 footnote_refs.push(FootnoteRefAnchor {
2689 byte_offset: byte_start,
2690 label: label.clone(),
2691 format,
2692 });
2693 continue;
2694 }
2695
2696 if let Some(image) = &span.image {
2697 plain_text.push('\u{FFFC}');
2701 images.push(ImageAnchor {
2702 byte_offset: byte_start,
2703 name: image.src.clone(),
2704 alt: image.alt.clone(),
2705 width: image.width,
2706 height: image.height,
2707 quality: 100,
2708 format: character_format_from_span(span, is_code_block),
2709 });
2710 continue;
2711 }
2712
2713 plain_text.push_str(&span.text);
2714 let byte_end = plain_text.len() as u32;
2715 if byte_start == byte_end {
2716 continue;
2717 }
2718 let format = character_format_from_span(span, is_code_block);
2719 if format == default {
2720 continue;
2721 }
2722 runs.push(FormatRun {
2723 byte_start,
2724 byte_end,
2725 format,
2726 });
2727 }
2728 coalesce_in_place(&mut runs);
2729 ParsedInline {
2730 plain_text,
2731 runs,
2732 images,
2733 footnote_refs,
2734 }
2735}
2736
2737#[derive(Debug, Clone, Default)]
2744pub struct ParsedInline {
2745 pub plain_text: String,
2746 pub runs: Vec<crate::format_runs::FormatRun>,
2747 pub images: Vec<crate::format_runs::ImageAnchor>,
2748 pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
2749}
2750
2751fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
2759 use jotdown::ListBulletType as B;
2760 match b {
2761 B::Dash => ListStyle::Disc,
2762 B::Star => ListStyle::Circle,
2763 B::Plus => ListStyle::Square,
2764 }
2765}
2766
2767fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
2769 use jotdown::OrderedListNumbering as N;
2770 match n {
2771 N::Decimal => ListStyle::Decimal,
2772 N::AlphaLower => ListStyle::LowerAlpha,
2773 N::AlphaUpper => ListStyle::UpperAlpha,
2774 N::RomanLower => ListStyle::LowerRoman,
2775 N::RomanUpper => ListStyle::UpperRoman,
2776 }
2777}
2778
2779fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
2783 use jotdown::OrderedListStyle as S;
2784 match style {
2785 S::Period => (String::new(), ".".to_string()),
2786 S::Paren => (String::new(), ")".to_string()),
2787 S::ParenParen => ("(".to_string(), ")".to_string()),
2788 }
2789}
2790
2791#[derive(Debug, Clone, Default)]
2795struct DjotBlockStyle {
2796 alignment: Option<Alignment>,
2797 line_height: Option<i64>,
2798 non_breakable_lines: Option<bool>,
2799 page_break_before: Option<bool>,
2800 front_matter: Option<bool>,
2801 direction: Option<TextDirection>,
2802 background_color: Option<String>,
2803 top_margin: Option<i64>,
2804 text_indent: Option<i64>,
2805 semantic_role: Option<SemanticRole>,
2806}
2807
2808impl DjotBlockStyle {
2809 fn merge_from(&mut self, other: DjotBlockStyle) {
2813 if other.alignment.is_some() {
2814 self.alignment = other.alignment;
2815 }
2816 if other.line_height.is_some() {
2817 self.line_height = other.line_height;
2818 }
2819 if other.non_breakable_lines.is_some() {
2820 self.non_breakable_lines = other.non_breakable_lines;
2821 }
2822 if other.page_break_before.is_some() {
2823 self.page_break_before = other.page_break_before;
2824 }
2825 if other.front_matter.is_some() {
2826 self.front_matter = other.front_matter;
2827 }
2828 if other.direction.is_some() {
2829 self.direction = other.direction;
2830 }
2831 if other.background_color.is_some() {
2832 self.background_color = other.background_color;
2833 }
2834 if other.top_margin.is_some() {
2835 self.top_margin = other.top_margin;
2836 }
2837 if other.text_indent.is_some() {
2838 self.text_indent = other.text_indent;
2839 }
2840 if other.semantic_role.is_some() {
2841 self.semantic_role = other.semantic_role.clone();
2842 }
2843 }
2844}
2845
2846fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
2852 let mut style = DjotBlockStyle::default();
2853
2854 if opts.alignment
2855 && let Some(v) = attrs.get_value("alignment")
2856 {
2857 style.alignment = match v.to_string().as_str() {
2858 "left" => Some(Alignment::Left),
2859 "right" => Some(Alignment::Right),
2860 "center" => Some(Alignment::Center),
2861 "justify" => Some(Alignment::Justify),
2862 _ => None,
2863 };
2864 }
2865 if opts.line_height
2866 && let Some(v) = attrs.get_value("line_height")
2867 {
2868 style.line_height = v.to_string().parse::<i64>().ok();
2869 }
2870 if opts.direction
2871 && let Some(v) = attrs.get_value("direction")
2872 {
2873 style.direction = match v.to_string().as_str() {
2874 "ltr" => Some(TextDirection::LeftToRight),
2875 "rtl" => Some(TextDirection::RightToLeft),
2876 _ => None,
2877 };
2878 }
2879 if opts.non_breakable_lines
2880 && let Some(v) = attrs.get_value("non_breakable_lines")
2881 {
2882 style.non_breakable_lines = match v.to_string().as_str() {
2883 "true" => Some(true),
2884 "false" => Some(false),
2885 _ => None,
2886 };
2887 }
2888 if opts.page_break_before
2889 && let Some(v) = attrs.get_value("page_break_before")
2890 {
2891 style.page_break_before = match v.to_string().as_str() {
2892 "true" => Some(true),
2893 "false" => Some(false),
2894 _ => None,
2895 };
2896 }
2897 if opts.front_matter
2898 && let Some(v) = attrs.get_value("front_matter")
2899 {
2900 style.front_matter = match v.to_string().as_str() {
2901 "true" => Some(true),
2902 "false" => Some(false),
2903 _ => None,
2904 };
2905 }
2906 if opts.background_color
2907 && let Some(v) = attrs.get_value("background_color")
2908 {
2909 style.background_color = Some(v.to_string());
2910 }
2911 if opts.top_margin
2912 && let Some(v) = attrs.get_value("top_margin")
2913 {
2914 style.top_margin = v.to_string().parse::<i64>().ok();
2915 }
2916 if opts.text_indent
2917 && let Some(v) = attrs.get_value("text_indent")
2918 {
2919 style.text_indent = v.to_string().parse::<i64>().ok();
2920 }
2921 if opts.semantic_role
2922 && let Some(v) = attrs.get_value("semantic_role")
2923 {
2924 style.semantic_role = match v.to_string().as_str() {
2925 "epigraph" => Some(SemanticRole::Epigraph),
2926 _ => None,
2930 };
2931 }
2932
2933 style
2934}
2935
2936fn list_start_of(start: u64) -> Option<i64> {
2940 crate::parser_tools::list_start::list_start_from_unsigned(start)
2941}
2942
2943#[allow(clippy::too_many_arguments)]
2946fn djot_push_block(
2947 elements: &mut Vec<ParsedElement>,
2948 spans: Vec<ParsedSpan>,
2949 heading_level: Option<i64>,
2950 list_style: Option<ListStyle>,
2951 list_indent: u32,
2952 list_prefix: String,
2953 list_suffix: String,
2954 list_start: Option<i64>,
2955 marker: Option<MarkerType>,
2956 is_code_block: bool,
2957 code_language: Option<String>,
2958 blockquote_depth: u32,
2959 style: DjotBlockStyle,
2960) {
2961 elements.push(ParsedElement::Block(ParsedBlock {
2962 spans,
2963 heading_level,
2964 list_style,
2965 list_indent,
2966 list_prefix,
2967 list_suffix,
2968 list_start,
2969 marker,
2970 is_code_block,
2971 code_language,
2972 blockquote_depth,
2973 blockquote_opens: 0,
2974 line_height: style.line_height,
2975 non_breakable_lines: style.non_breakable_lines,
2976 page_break_before: style.page_break_before,
2977 front_matter: style.front_matter,
2978 direction: style.direction,
2979 background_color: style.background_color,
2980 alignment: style.alignment,
2981 top_margin: style.top_margin,
2982 text_indent: style.text_indent,
2983 semantic_role: style.semantic_role.clone(),
2984 }));
2985}
2986
2987fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
3000 djot.lines()
3001 .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
3002 .filter(|line| !line.is_empty())
3003 .map(|line| {
3004 ParsedElement::Block(ParsedBlock {
3005 spans: vec![ParsedSpan {
3006 text: line.to_string(),
3007 ..Default::default()
3008 }],
3009 ..Default::default()
3010 })
3011 })
3012 .collect()
3013}
3014
3015pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
3036 use jotdown::{Container as C, Event as E, ListKind};
3037
3038 let parsable = crate::parser_tools::djot_depth::parsable(djot);
3064 let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
3065 return djot_source_lines(djot);
3066 };
3067
3068 let mut elements: Vec<ParsedElement> = Vec::new();
3069 let mut current_spans: Vec<ParsedSpan> = Vec::new();
3070 let mut current_heading: Option<i64> = None;
3071 let mut is_code_block = false;
3072 let mut code_language: Option<String> = None;
3073 let mut blockquote_depth: u32 = 0;
3074 let mut quotations_open: Vec<usize> = Vec::new();
3076 let mut pending_style = DjotBlockStyle::default();
3079
3080 let mut bold = false;
3082 let mut italic = false;
3083 let mut underline = false;
3084 let mut strikeout = false;
3085 let mut code = false;
3086 let mut superscript = false;
3087 let mut subscript = false;
3088 let mut link_href: Option<String> = None;
3089 let mut pending_image: Option<ParsedImage> = None;
3092
3093 let mut list_stack: Vec<(ListStyle, String, String, Option<i64>)> = Vec::new();
3095 let mut cur_list_style: Option<ListStyle> = None;
3097 let mut cur_list_prefix = String::new();
3098 let mut cur_list_suffix = String::new();
3099 let mut cur_list_start: Option<i64> = None;
3100 let mut cur_list_indent: u32 = 0;
3101 let mut cur_marker: Option<MarkerType> = None;
3102
3103 let mut in_table_cell = false;
3105 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
3106 let mut current_row: Vec<ParsedTableCell> = Vec::new();
3107 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
3108 let mut table_header_rows: usize = 0;
3109 let mut row_is_head = false;
3110
3111 let mut skip_depth: u32 = 0;
3115
3116 let mut footnote_open: Option<(String, usize)> = None;
3120
3121 macro_rules! push_text {
3126 ($t:expr) => {{
3127 if let Some(img) = pending_image.as_mut() {
3132 img.alt.push_str(($t).as_ref());
3133 } else {
3134 let sp = ParsedSpan {
3135 text: ($t).to_string(),
3136 bold,
3137 italic,
3138 underline,
3139 strikeout,
3140 code,
3141 superscript,
3142 subscript,
3143 link_href: link_href.clone(),
3144 image: None,
3145 footnote_ref: None,
3146 };
3147 if in_table_cell {
3148 current_cell_spans.push(sp);
3149 } else {
3150 current_spans.push(sp);
3151 }
3152 }
3153 }};
3154 }
3155
3156 macro_rules! push_image {
3158 ($img:expr) => {{
3159 let sp = ParsedSpan {
3160 text: String::new(),
3161 bold,
3162 italic,
3163 underline,
3164 strikeout,
3165 code,
3166 superscript,
3167 subscript,
3168 link_href: link_href.clone(),
3169 image: Some($img),
3170 footnote_ref: None,
3171 };
3172 if in_table_cell {
3173 current_cell_spans.push(sp);
3174 } else {
3175 current_spans.push(sp);
3176 }
3177 }};
3178 }
3179
3180 macro_rules! enter_item {
3183 ($marker:expr) => {{
3184 if !current_spans.is_empty() {
3185 djot_push_block(
3186 &mut elements,
3187 std::mem::take(&mut current_spans),
3188 None,
3189 cur_list_style.clone(),
3190 cur_list_indent,
3191 cur_list_prefix.clone(),
3192 cur_list_suffix.clone(),
3193 cur_list_start,
3194 cur_marker.clone(),
3195 false,
3196 None,
3197 blockquote_depth,
3198 DjotBlockStyle::default(),
3199 );
3200 }
3201 let (style, prefix, suffix, start) = list_stack.last().cloned().unwrap_or((
3202 ListStyle::Disc,
3203 String::new(),
3204 String::new(),
3205 None,
3206 ));
3207 cur_list_style = Some(style);
3208 cur_list_prefix = prefix;
3209 cur_list_suffix = suffix;
3210 cur_list_start = start;
3211 cur_list_indent = list_stack.len().saturating_sub(1) as u32;
3212 cur_marker = $marker;
3213 }};
3214 }
3215
3216 macro_rules! line_break {
3220 () => {{
3221 if in_table_cell || pending_image.is_some() {
3222 push_text!(" ");
3223 } else {
3224 push_text!(LINE_SEPARATOR_STR);
3225 }
3226 }};
3227 }
3228
3229 macro_rules! split_line {
3236 () => {{
3237 if in_table_cell {
3238 push_text!(" ");
3239 } else if !current_spans.is_empty() && current_heading.is_some() {
3240 djot_push_block(
3245 &mut elements,
3246 std::mem::take(&mut current_spans),
3247 current_heading,
3248 None,
3249 0,
3250 String::new(),
3251 String::new(),
3252 None,
3253 None,
3254 false,
3255 None,
3256 blockquote_depth,
3257 pending_style.clone(),
3258 );
3259 pending_style.page_break_before = None;
3260 } else if !current_spans.is_empty() {
3261 djot_push_block(
3262 &mut elements,
3263 std::mem::take(&mut current_spans),
3264 None,
3265 cur_list_style.clone(),
3266 cur_list_indent,
3267 cur_list_prefix.clone(),
3268 cur_list_suffix.clone(),
3269 cur_list_start,
3270 cur_marker.clone(),
3271 is_code_block,
3272 code_language.clone(),
3273 blockquote_depth,
3274 pending_style.clone(),
3275 );
3276 pending_style.page_break_before = None;
3277 }
3278 }};
3279 }
3280
3281 for event in events {
3282 if skip_depth > 0 {
3283 match event {
3284 E::Start(..) => skip_depth += 1,
3285 E::End(_) => skip_depth -= 1,
3286 _ => {}
3287 }
3288 continue;
3289 }
3290
3291 match event {
3292 E::Start(C::Document, _) | E::End(C::Document) => {}
3294 E::Start(C::Section { .. }, attrs) => {
3295 if list_stack.is_empty() {
3298 pending_style.merge_from(block_attrs_to_style(&attrs, options));
3299 }
3300 }
3301 E::End(C::Section { .. }) => {}
3302 E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
3303
3304 E::Start(C::Blockquote, _) => {
3306 blockquote_depth += 1;
3307 quotations_open.push(elements.len());
3308 }
3309 E::End(C::Blockquote) => {
3310 blockquote_depth = blockquote_depth.saturating_sub(1);
3311 if let Some(first) = quotations_open.pop() {
3312 mark_quotation_opened(&mut elements, first);
3313 }
3314 }
3315
3316 E::Start(C::List { kind, .. }, _) => {
3318 let (style, prefix, suffix, start) = match kind {
3319 ListKind::Unordered(b) | ListKind::Task(b) => {
3320 (djot_bullet_style(b), String::new(), String::new(), None)
3321 }
3322 ListKind::Ordered {
3323 numbering,
3324 style,
3325 start,
3326 } => {
3327 let (p, s) = djot_ordered_affixes(style);
3328 (djot_ordered_style(numbering), p, s, list_start_of(start))
3329 }
3330 };
3331 list_stack.push((style, prefix, suffix, start));
3332 }
3333 E::End(C::List { .. }) => {
3334 list_stack.pop();
3335 cur_list_style = None;
3336 cur_list_start = None;
3337 cur_marker = None;
3338 }
3339 E::Start(C::ListItem, _) => enter_item!(None),
3340 E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
3341 MarkerType::Checked
3342 } else {
3343 MarkerType::Unchecked
3344 })),
3345 E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
3346 if !current_spans.is_empty() {
3348 djot_push_block(
3349 &mut elements,
3350 std::mem::take(&mut current_spans),
3351 None,
3352 cur_list_style.clone(),
3353 cur_list_indent,
3354 cur_list_prefix.clone(),
3355 cur_list_suffix.clone(),
3356 cur_list_start,
3357 cur_marker.clone(),
3358 false,
3359 None,
3360 blockquote_depth,
3361 DjotBlockStyle::default(),
3362 );
3363 }
3364 cur_list_style = None;
3365 cur_list_start = None;
3366 cur_marker = None;
3367 }
3368
3369 E::Start(C::Heading { level, .. }, attrs) => {
3371 current_heading = Some(level as i64);
3372 pending_style.merge_from(block_attrs_to_style(&attrs, options));
3375 }
3376 E::End(C::Heading { .. }) => {
3377 djot_push_block(
3378 &mut elements,
3379 std::mem::take(&mut current_spans),
3380 current_heading.take(),
3381 None,
3382 0,
3383 String::new(),
3384 String::new(),
3385 None,
3386 None,
3387 false,
3388 None,
3389 blockquote_depth,
3390 std::mem::take(&mut pending_style),
3391 );
3392 }
3393 E::Start(C::Paragraph, attrs) => {
3394 current_heading = None;
3395 pending_style = if list_stack.is_empty() {
3399 block_attrs_to_style(&attrs, options)
3400 } else {
3401 DjotBlockStyle::default()
3402 };
3403 }
3404 E::End(C::Paragraph) => {
3405 if !current_spans.is_empty() {
3406 djot_push_block(
3407 &mut elements,
3408 std::mem::take(&mut current_spans),
3409 None,
3410 cur_list_style.clone(),
3411 cur_list_indent,
3412 cur_list_prefix.clone(),
3413 cur_list_suffix.clone(),
3414 cur_list_start,
3415 cur_marker.clone(),
3416 false,
3417 None,
3418 blockquote_depth,
3419 std::mem::take(&mut pending_style),
3420 );
3421 }
3422 cur_list_style = None;
3423 cur_list_start = None;
3424 cur_marker = None;
3425 }
3426 E::Start(C::CodeBlock { language }, _) => {
3427 is_code_block = true;
3428 code_language = if language.is_empty() {
3429 None
3430 } else {
3431 Some(language.to_string())
3432 };
3433 }
3434 E::End(C::CodeBlock { .. }) => {
3435 if let Some(last) = current_spans.last_mut()
3437 && last.text.ends_with('\n')
3438 {
3439 last.text.pop();
3440 }
3441 djot_push_block(
3442 &mut elements,
3443 std::mem::take(&mut current_spans),
3444 None,
3445 None,
3446 0,
3447 String::new(),
3448 String::new(),
3449 None,
3450 None,
3451 true,
3452 code_language.take(),
3453 blockquote_depth,
3454 DjotBlockStyle::default(),
3455 );
3456 is_code_block = false;
3457 }
3458
3459 E::Start(C::Table, _) => {
3461 table_rows.clear();
3462 current_row.clear();
3463 current_cell_spans.clear();
3464 table_header_rows = 0;
3465 }
3466 E::End(C::Table) => {
3467 let own_cells = cells_in_rows(&table_rows);
3468 push_table(
3469 &mut elements,
3470 ParsedTable {
3471 header_rows: table_header_rows,
3472 rows: std::mem::take(&mut table_rows),
3473 blockquote_depth,
3474 blockquote_opens: 0,
3475 },
3476 own_cells,
3477 );
3478 }
3479 E::Start(C::TableRow { head }, _) => {
3480 row_is_head = head;
3481 current_row.clear();
3482 }
3483 E::End(C::TableRow { .. }) => {
3484 if row_is_head {
3485 table_header_rows += 1;
3486 }
3487 table_rows.push(std::mem::take(&mut current_row));
3488 }
3489 E::Start(C::TableCell { .. }, _) => {
3490 in_table_cell = true;
3491 current_cell_spans.clear();
3492 }
3493 E::End(C::TableCell { .. }) => {
3494 in_table_cell = false;
3495 current_row.push(ParsedTableCell {
3496 spans: std::mem::take(&mut current_cell_spans),
3497 });
3498 }
3499
3500 E::Start(C::Strong, _) => bold = true,
3502 E::End(C::Strong) => bold = false,
3503 E::Start(C::Emphasis, _) => italic = true,
3504 E::End(C::Emphasis) => italic = false,
3505 E::Start(C::Verbatim, _) => code = true,
3506 E::End(C::Verbatim) => code = false,
3507 E::Start(C::Superscript, _) => superscript = true,
3508 E::End(C::Superscript) => superscript = false,
3509 E::Start(C::Subscript, _) => subscript = true,
3510 E::End(C::Subscript) => subscript = false,
3511 E::Start(C::Insert, _) => underline = true,
3512 E::End(C::Insert) => underline = false,
3513 E::Start(C::Delete, _) => strikeout = true,
3514 E::End(C::Delete) => strikeout = false,
3515 E::Start(C::Mark, _) | E::End(C::Mark) => {}
3517 E::Start(C::Span, _) | E::End(C::Span) => {}
3518 E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
3519 E::End(C::Link(..)) => link_href = None,
3520 E::Start(C::Image(src, _), attrs) => {
3525 let attr_num = |key: &str| -> i64 {
3526 attrs
3527 .get_value(key)
3528 .map(|v| v.to_string())
3529 .and_then(|v| v.trim().parse::<i64>().ok())
3530 .filter(|n| *n > 0)
3531 .unwrap_or(0)
3532 };
3533 pending_image = Some(ParsedImage {
3534 src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
3537 alt: String::new(),
3538 width: attr_num("width"),
3539 height: attr_num("height"),
3540 });
3541 }
3542 E::End(C::Image(..)) => {
3543 if let Some(img) = pending_image.take() {
3544 push_image!(img);
3545 }
3546 }
3547
3548 E::Start(C::Footnote { label }, _) => {
3557 if !current_spans.is_empty() {
3558 djot_push_block(
3559 &mut elements,
3560 std::mem::take(&mut current_spans),
3561 None,
3562 cur_list_style.clone(),
3563 cur_list_indent,
3564 cur_list_prefix.clone(),
3565 cur_list_suffix.clone(),
3566 cur_list_start,
3567 cur_marker.clone(),
3568 false,
3569 None,
3570 blockquote_depth,
3571 DjotBlockStyle::default(),
3572 );
3573 }
3574 footnote_open = Some((label.to_string(), elements.len()));
3575 }
3576 E::End(C::Footnote { .. }) => {
3577 if !current_spans.is_empty() {
3578 djot_push_block(
3579 &mut elements,
3580 std::mem::take(&mut current_spans),
3581 None,
3582 cur_list_style.clone(),
3583 cur_list_indent,
3584 cur_list_prefix.clone(),
3585 cur_list_suffix.clone(),
3586 cur_list_start,
3587 cur_marker.clone(),
3588 false,
3589 None,
3590 blockquote_depth,
3591 DjotBlockStyle::default(),
3592 );
3593 }
3594 if let Some((label, start)) = footnote_open.take() {
3595 let blocks: Vec<ParsedBlock> = elements
3596 .drain(start..)
3597 .filter_map(|e| match e {
3598 ParsedElement::Block(b) => Some(b),
3599 _ => None,
3603 })
3604 .collect();
3605 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
3606 }
3607 }
3608
3609 E::Start(
3611 C::Math { .. }
3612 | C::RawBlock { .. }
3613 | C::RawInline { .. }
3614 | C::DescriptionList
3615 | C::DescriptionDetails
3616 | C::DescriptionTerm
3617 | C::Caption
3618 | C::LinkDefinition { .. },
3619 _,
3620 ) => skip_depth = 1,
3621
3622 E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
3634 for (i, piece) in s.split('\n').enumerate() {
3635 if i > 0 {
3636 split_line!();
3637 }
3638 if !piece.is_empty() {
3639 push_text!(piece);
3640 }
3641 }
3642 }
3643 E::Str(s) => push_text!(s.as_ref()),
3644 E::Softbreak => push_text!(" "),
3645 E::LeftSingleQuote => push_text!("\u{2018}"),
3646 E::RightSingleQuote => push_text!("\u{2019}"),
3647 E::LeftDoubleQuote => push_text!("\u{201C}"),
3648 E::RightDoubleQuote => push_text!("\u{201D}"),
3649 E::Ellipsis => push_text!("\u{2026}"),
3650 E::EnDash => push_text!("\u{2013}"),
3651 E::EmDash => push_text!("\u{2014}"),
3652 E::NonBreakingSpace => push_text!("\u{00A0}"),
3653 E::Hardbreak => line_break!(),
3654 E::FootnoteReference(label) => {
3660 let sp = ParsedSpan {
3661 text: String::new(),
3662 bold,
3663 italic,
3664 underline,
3665 strikeout,
3666 code,
3667 superscript,
3668 subscript,
3669 link_href: link_href.clone(),
3670 image: None,
3671 footnote_ref: Some(label.to_string()),
3672 };
3673 if in_table_cell {
3674 current_cell_spans.push(sp);
3675 } else {
3676 current_spans.push(sp);
3677 }
3678 }
3679 E::Symbol(_) => {}
3682 E::Escape | E::Blankline => {}
3683 E::ThematicBreak(_) | E::Attributes(_) => {}
3684
3685 _ => {}
3688 }
3689 }
3690
3691 if !current_spans.is_empty() {
3693 djot_push_block(
3694 &mut elements,
3695 std::mem::take(&mut current_spans),
3696 current_heading.take(),
3697 cur_list_style.clone(),
3698 cur_list_indent,
3699 cur_list_prefix.clone(),
3700 cur_list_suffix.clone(),
3701 cur_list_start,
3702 cur_marker.clone(),
3703 is_code_block,
3704 code_language.take(),
3705 blockquote_depth,
3706 std::mem::take(&mut pending_style),
3707 );
3708 }
3709
3710 if elements.is_empty() {
3713 djot_push_block(
3714 &mut elements,
3715 vec![ParsedSpan {
3716 text: String::new(),
3717 ..Default::default()
3718 }],
3719 None,
3720 None,
3721 0,
3722 String::new(),
3723 String::new(),
3724 None,
3725 None,
3726 false,
3727 None,
3728 0,
3729 DjotBlockStyle::default(),
3730 );
3731 }
3732
3733 elements
3734}
3735
3736#[cfg(test)]
3737mod tests {
3738 use super::*;
3739
3740 fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
3742 ParsedElement::flatten_to_blocks(parse_markdown(md))
3743 }
3744
3745 #[test]
3746 fn test_parse_markdown_simple_paragraph() {
3747 let blocks = parse_markdown_blocks("Hello **world**");
3748 assert_eq!(blocks.len(), 1);
3749 assert!(blocks[0].spans.len() >= 2);
3750 let plain_span = blocks[0]
3752 .spans
3753 .iter()
3754 .find(|s| s.text.contains("Hello"))
3755 .unwrap();
3756 assert!(!plain_span.bold);
3757 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3758 assert!(bold_span.bold);
3759 }
3760
3761 #[test]
3762 fn test_parse_markdown_heading() {
3763 let blocks = parse_markdown_blocks("# Title");
3764 assert_eq!(blocks.len(), 1);
3765 assert_eq!(blocks[0].heading_level, Some(1));
3766 assert_eq!(blocks[0].spans[0].text, "Title");
3767 }
3768
3769 #[test]
3770 fn test_parse_markdown_list() {
3771 let blocks = parse_markdown_blocks("- item1\n- item2");
3772 assert!(blocks.len() >= 2);
3773 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3774 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3775 }
3776
3777 fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
3779 elements
3780 .iter()
3781 .map(|e| match e {
3782 ParsedElement::Block(b) => (false, b.blockquote_depth),
3783 ParsedElement::Table(t) => (true, t.blockquote_depth),
3784 ParsedElement::FootnoteDefinition { .. } => (false, 0),
3787 })
3788 .collect()
3789 }
3790
3791 #[test]
3792 fn test_parse_markdown_table_in_blockquote_records_depth() {
3793 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3794 assert_eq!(element_depths(&elements), vec![(true, 1)]);
3795 }
3796
3797 #[test]
3798 fn test_parse_markdown_text_then_table_in_blockquote() {
3799 let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
3800 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
3801 }
3802
3803 #[test]
3804 fn test_parse_markdown_table_after_blockquote_closes() {
3805 let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
3806 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
3807 }
3808
3809 #[test]
3810 fn test_parse_markdown_table_in_nested_blockquote() {
3811 let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
3812 assert_eq!(element_depths(&elements), vec![(true, 2)]);
3813 }
3814
3815 #[test]
3816 fn test_parse_markdown_list_in_blockquote_records_depth() {
3817 let elements = parse_markdown("> - item1\n> - item2");
3818 let depths = element_depths(&elements);
3819 assert_eq!(depths, vec![(false, 1), (false, 1)]);
3820 for e in &elements {
3821 if let ParsedElement::Block(b) = e {
3822 assert_eq!(b.list_style, Some(ListStyle::Disc));
3823 }
3824 }
3825 }
3826
3827 #[test]
3828 fn test_parse_html_table_in_blockquote_records_depth() {
3829 let elements = parse_html_elements(
3830 "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
3831 );
3832 assert_eq!(element_depths(&elements), vec![(true, 1)]);
3833 }
3834
3835 #[test]
3836 fn test_parse_html_table_after_blockquote() {
3837 let elements = parse_html_elements(
3838 "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
3839 );
3840 let depths = element_depths(&elements);
3841 assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
3843 assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
3844 }
3845
3846 #[test]
3847 fn test_flatten_to_blocks_propagates_blockquote_depth() {
3848 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3849 let blocks = ParsedElement::flatten_to_blocks(elements);
3850 assert!(!blocks.is_empty());
3851 for b in &blocks {
3852 assert_eq!(b.blockquote_depth, 1);
3853 }
3854 }
3855
3856 #[test]
3857 fn test_parse_html_simple() {
3858 let blocks = parse_html("<p>Hello <b>world</b></p>");
3859 assert_eq!(blocks.len(), 1);
3860 assert!(blocks[0].spans.len() >= 2);
3861 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3862 assert!(bold_span.bold);
3863 }
3864
3865 #[test]
3866 fn test_parse_html_multiple_paragraphs() {
3867 let blocks = parse_html("<p>A</p><p>B</p>");
3868 assert_eq!(blocks.len(), 2);
3869 }
3870
3871 #[test]
3872 fn test_parse_html_heading() {
3873 let blocks = parse_html("<h2>Subtitle</h2>");
3874 assert_eq!(blocks.len(), 1);
3875 assert_eq!(blocks[0].heading_level, Some(2));
3876 }
3877
3878 #[test]
3879 fn test_parse_html_list() {
3880 let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
3881 assert!(blocks.len() >= 2);
3882 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3883 }
3884
3885 #[test]
3886 fn test_parse_markdown_code_block() {
3887 let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
3888 assert_eq!(blocks.len(), 1);
3889 assert!(blocks[0].is_code_block);
3890 assert!(blocks[0].spans[0].code);
3891 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3893 assert_eq!(
3894 text, "fn main() {}",
3895 "code block text should not have trailing newline"
3896 );
3897 }
3898
3899 #[test]
3900 fn test_parse_markdown_nested_formatting() {
3901 let blocks = parse_markdown_blocks("***bold italic***");
3902 assert_eq!(blocks.len(), 1);
3903 let span = &blocks[0].spans[0];
3904 assert!(span.bold);
3905 assert!(span.italic);
3906 }
3907
3908 #[test]
3909 fn test_parse_markdown_link() {
3910 let blocks = parse_markdown_blocks("[click](http://example.com)");
3911 assert_eq!(blocks.len(), 1);
3912 let span = &blocks[0].spans[0];
3913 assert_eq!(span.text, "click");
3914 assert_eq!(span.link_href, Some("http://example.com".to_string()));
3915 }
3916
3917 #[test]
3918 fn test_parse_markdown_empty() {
3919 let blocks = parse_markdown_blocks("");
3920 assert_eq!(blocks.len(), 1);
3921 assert!(blocks[0].spans[0].text.is_empty());
3922 }
3923
3924 #[test]
3925 fn test_parse_html_empty() {
3926 let blocks = parse_html("");
3927 assert_eq!(blocks.len(), 1);
3928 assert!(blocks[0].spans[0].text.is_empty());
3929 }
3930
3931 #[test]
3932 fn test_parse_html_nested_formatting() {
3933 let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
3934 assert_eq!(blocks.len(), 1);
3935 let span = &blocks[0].spans[0];
3936 assert!(span.bold);
3937 assert!(span.italic);
3938 }
3939
3940 #[test]
3941 fn test_parse_html_link() {
3942 let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
3943 assert_eq!(blocks.len(), 1);
3944 let span = &blocks[0].spans[0];
3945 assert_eq!(span.text, "click");
3946 assert_eq!(span.link_href, Some("http://example.com".to_string()));
3947 }
3948
3949 #[test]
3950 fn test_parse_html_ordered_list() {
3951 let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
3952 assert!(blocks.len() >= 2);
3953 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3954 }
3955
3956 #[test]
3957 fn test_parse_markdown_ordered_list() {
3958 let blocks = parse_markdown_blocks("1. first\n2. second");
3959 assert!(blocks.len() >= 2);
3960 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3961 }
3962
3963 #[test]
3964 fn test_parse_html_blockquote_nested() {
3965 let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
3966 assert!(blocks.len() >= 3);
3967 }
3968
3969 #[test]
3970 fn test_parse_block_styles_line_height() {
3971 let styles = parse_block_styles("line-height: 1.5");
3972 assert_eq!(styles.line_height, Some(1500));
3973 }
3974
3975 #[test]
3976 fn test_parse_block_styles_direction_rtl() {
3977 let styles = parse_block_styles("direction: rtl");
3978 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3979 }
3980
3981 #[test]
3982 fn test_parse_block_styles_background_color() {
3983 let styles = parse_block_styles("background-color: #ff0000");
3984 assert_eq!(styles.background_color, Some("#ff0000".to_string()));
3985 }
3986
3987 #[test]
3988 fn test_parse_block_styles_white_space_pre() {
3989 let styles = parse_block_styles("white-space: pre");
3990 assert_eq!(styles.non_breakable_lines, Some(true));
3991 }
3992
3993 #[test]
3994 fn test_parse_block_styles_multiple() {
3995 let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
3996 assert_eq!(styles.line_height, Some(2000));
3997 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3998 assert_eq!(styles.background_color, Some("blue".to_string()));
3999 }
4000
4001 #[test]
4002 fn test_parse_html_block_styles_extracted() {
4003 let blocks = parse_html(
4004 r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
4005 );
4006 assert_eq!(blocks.len(), 1);
4007 assert_eq!(blocks[0].line_height, Some(1500));
4008 assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
4009 assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
4010 }
4011
4012 #[test]
4016 fn test_parse_html_collapses_exporter_line_wrapping() {
4017 let blocks = parse_html(
4018 "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
4019 <p>\nsecond para</p>\n</body>",
4020 );
4021 let texts: Vec<String> = blocks
4022 .iter()
4023 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4024 .collect();
4025 assert_eq!(texts, vec!["first line wrapped here", "second para"]);
4026 }
4027
4028 #[test]
4032 fn test_parse_html_drops_whitespace_between_blocks() {
4033 let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
4034 assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
4035 assert!(
4036 blocks
4037 .iter()
4038 .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
4039 "no block keeps a literal newline"
4040 );
4041 }
4042
4043 #[test]
4046 fn test_parse_html_collapses_across_span_boundaries() {
4047 let blocks = parse_html("<p> <b>bold</b>\n <i>italic</i> </p>");
4048 assert_eq!(blocks.len(), 1);
4049 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4050 assert_eq!(text, "bold italic");
4051 }
4052
4053 #[test]
4056 fn test_parse_html_keeps_no_break_space() {
4057 let blocks = parse_html("<p>\nAttention : ici</p>");
4058 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4059 assert_eq!(text, "Attention\u{a0}: ici");
4060 }
4061
4062 #[test]
4065 fn test_parse_html_keeps_empty_paragraph() {
4066 let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
4067 assert_eq!(blocks.len(), 3);
4068 let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
4069 assert_eq!(text, "");
4070 }
4071
4072 #[test]
4076 fn test_parse_html_preserves_whitespace_in_pre() {
4077 let blocks = parse_html("<pre>\nfn main() {\n let x = 1;\n}</pre>");
4080 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4081 assert_eq!(text, "fn main() {\n let x = 1;\n}");
4082
4083 let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n b</p>");
4086 let texts: Vec<String> = styled
4087 .iter()
4088 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4089 .collect();
4090 assert_eq!(texts, ["a", " b"]);
4091
4092 let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n b</p>");
4093 let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
4094 assert_eq!(text, "a b");
4095 }
4096
4097 #[test]
4100 fn test_parse_html_collapses_table_cell_whitespace() {
4101 let elements = parse_html_elements(
4102 "<table>\n<tr>\n<td>\n one\n</td>\n<td>\n two\n</td>\n</tr>\n</table>",
4103 );
4104 let table = elements
4105 .iter()
4106 .find_map(|e| match e {
4107 ParsedElement::Table(t) => Some(t),
4108 _ => None,
4109 })
4110 .expect("a table");
4111 let cells: Vec<String> = table.rows[0]
4112 .iter()
4113 .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
4114 .collect();
4115 assert_eq!(cells, vec!["one", "two"]);
4116 }
4117
4118 #[test]
4119 fn test_parse_html_white_space_pre() {
4120 let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
4121 assert_eq!(blocks.len(), 1);
4122 assert_eq!(blocks[0].non_breakable_lines, Some(true));
4123 }
4124
4125 #[test]
4126 fn test_parse_html_no_styles_returns_none() {
4127 let blocks = parse_html("<p>plain</p>");
4128 assert_eq!(blocks.len(), 1);
4129 assert_eq!(blocks[0].line_height, None);
4130 assert_eq!(blocks[0].direction, None);
4131 assert_eq!(blocks[0].background_color, None);
4132 assert_eq!(blocks[0].non_breakable_lines, None);
4133 }
4134
4135 #[test]
4136 fn test_parse_markdown_nested_list_indent() {
4137 let md = "- top\n - nested\n - deep";
4138 let blocks = parse_markdown_blocks(md);
4139 assert_eq!(blocks.len(), 3);
4140 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4141 assert_eq!(blocks[0].list_indent, 0);
4142 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4143 assert_eq!(blocks[1].list_indent, 1);
4144 assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
4145 assert_eq!(blocks[2].list_indent, 2);
4146 }
4147
4148 #[test]
4149 fn test_parse_markdown_nested_ordered_list_indent() {
4150 let md = "1. first\n 1. nested\n 2. nested2";
4151 let blocks = parse_markdown_blocks(md);
4152 assert_eq!(blocks.len(), 3);
4153 assert_eq!(blocks[0].list_indent, 0);
4154 assert_eq!(blocks[1].list_indent, 1);
4155 assert_eq!(blocks[2].list_indent, 1);
4156 }
4157
4158 #[test]
4159 fn test_parse_html_nested_list_indent() {
4160 let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
4161 let blocks = parse_html(html);
4162 assert!(blocks.len() >= 2);
4163 assert_eq!(blocks[0].list_indent, 0);
4164 assert_eq!(blocks[1].list_indent, 1);
4165 }
4166
4167 #[test]
4168 fn test_parse_markdown_table() {
4169 let md = "| A | B |\n|---|---|\n| 1 | 2 |";
4170 let elements = parse_markdown(md);
4171 assert_eq!(elements.len(), 1);
4172 match &elements[0] {
4173 ParsedElement::Table(table) => {
4174 assert_eq!(table.header_rows, 1);
4175 assert_eq!(table.rows.len(), 2); assert_eq!(table.rows[0].len(), 2);
4178 assert_eq!(table.rows[0][0].spans[0].text, "A");
4179 assert_eq!(table.rows[0][1].spans[0].text, "B");
4180 assert_eq!(table.rows[1].len(), 2);
4182 assert_eq!(table.rows[1][0].spans[0].text, "1");
4183 assert_eq!(table.rows[1][1].spans[0].text, "2");
4184 }
4185 _ => panic!("Expected ParsedElement::Table"),
4186 }
4187 }
4188
4189 #[test]
4190 fn test_parse_markdown_table_with_formatting() {
4191 let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
4192 let elements = parse_markdown(md);
4193 assert_eq!(elements.len(), 1);
4194 match &elements[0] {
4195 ParsedElement::Table(table) => {
4196 assert_eq!(table.rows.len(), 2);
4197 assert!(table.rows[0][0].spans[0].bold);
4199 assert!(table.rows[0][1].spans[0].code);
4201 assert!(table.rows[0][2].spans[0].italic);
4203 assert!(table.rows[1][0].spans[0].strikeout);
4205 assert_eq!(
4207 table.rows[1][2].spans[0].link_href,
4208 Some("http://x.com".to_string())
4209 );
4210 }
4211 _ => panic!("Expected ParsedElement::Table"),
4212 }
4213 }
4214
4215 #[test]
4216 fn test_parse_markdown_mixed_content_with_table() {
4217 let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
4218 let elements = parse_markdown(md);
4219 assert_eq!(elements.len(), 3);
4220 assert!(matches!(&elements[0], ParsedElement::Block(_)));
4221 assert!(matches!(&elements[1], ParsedElement::Table(_)));
4222 assert!(matches!(&elements[2], ParsedElement::Block(_)));
4223 }
4224}
4225
4226#[cfg(test)]
4227mod djot_tests {
4228 use super::*;
4229 use crate::entities::MarkerType;
4230
4231 fn blocks(d: &str) -> Vec<ParsedBlock> {
4232 ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
4233 }
4234
4235 fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
4236 b.spans.iter().find(|s| pred(s)).expect("span not found")
4237 }
4238
4239 #[test]
4240 fn paragraph_bold_italic() {
4241 let b = blocks("normal *bold* _italic_");
4242 assert_eq!(b.len(), 1);
4243 assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
4244 assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
4245 }
4246
4247 #[test]
4248 fn heading_levels() {
4249 assert_eq!(blocks("# H1")[0].heading_level, Some(1));
4250 assert_eq!(blocks("### H3")[0].heading_level, Some(3));
4251 assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
4252 }
4253
4254 #[test]
4255 fn unordered_bullet_styles_are_distinct() {
4256 assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
4257 assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
4258 assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
4259 }
4260
4261 #[test]
4262 fn ordered_delimiters() {
4263 let period = blocks("1. a");
4264 assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
4265 assert_eq!(period[0].list_prefix, "");
4266 assert_eq!(period[0].list_suffix, ".");
4267
4268 let paren = blocks("1) a");
4269 assert_eq!(paren[0].list_suffix, ")");
4270 assert_eq!(paren[0].list_prefix, "");
4271
4272 let paren_paren = blocks("(1) a");
4273 assert_eq!(paren_paren[0].list_prefix, "(");
4274 assert_eq!(paren_paren[0].list_suffix, ")");
4275 }
4276
4277 #[test]
4278 fn task_list_markers() {
4279 let b = blocks("- [ ] a\n- [x] b");
4280 assert_eq!(b.len(), 2);
4281 assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
4282 assert_eq!(b[1].marker, Some(MarkerType::Checked));
4283 }
4284
4285 #[test]
4286 fn code_block_with_language() {
4287 let b = blocks("```rust\nfn main() {}\n```");
4288 assert_eq!(b.len(), 1);
4289 assert!(b[0].is_code_block);
4290 assert_eq!(b[0].code_language.as_deref(), Some("rust"));
4291 let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
4292 assert_eq!(text, "fn main() {}");
4293 }
4294
4295 #[test]
4296 fn link_href() {
4297 let b = blocks("[text](http://example.com)");
4298 let s = first_span_with(&b[0], |s| s.text == "text");
4299 assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
4300 }
4301
4302 #[test]
4303 fn superscript_subscript() {
4304 assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
4305 assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
4306 }
4307
4308 #[test]
4309 fn delete_insert_verbatim() {
4310 assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
4311 assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
4312 assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
4313 }
4314
4315 #[test]
4316 fn blockquote_depth() {
4317 let els = parse_djot("> quoted", &DjotImportOptions::default());
4318 match &els[0] {
4319 ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
4320 _ => panic!("expected block"),
4321 }
4322 }
4323
4324 #[test]
4325 fn nested_list_indent() {
4326 let b = blocks("- a\n\n - b\n\n - c");
4331 assert_eq!(b.len(), 3);
4332 assert_eq!(b[0].list_indent, 0);
4333 assert_eq!(b[1].list_indent, 1);
4334 assert_eq!(b[2].list_indent, 2);
4335 }
4336
4337 #[test]
4338 fn table_parsed_as_table() {
4339 let els = parse_djot(
4340 "| a | b |\n|---|---|\n| c | d |",
4341 &DjotImportOptions::default(),
4342 );
4343 assert_eq!(els.len(), 1);
4344 match &els[0] {
4345 ParsedElement::Table(t) => {
4346 assert_eq!(t.header_rows, 1);
4347 assert_eq!(t.rows.len(), 2);
4348 assert_eq!(t.rows[0][0].spans[0].text, "a");
4349 assert_eq!(t.rows[1][1].spans[0].text, "d");
4350 }
4351 _ => panic!("expected table"),
4352 }
4353 }
4354
4355 #[test]
4356 fn smart_punctuation_normalised_to_unicode() {
4357 let text: String = blocks("a... b---c")[0]
4358 .spans
4359 .iter()
4360 .map(|s| s.text.as_str())
4361 .collect();
4362 assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
4363 assert!(text.contains('\u{2014}'), "em dash: {text:?}");
4364 }
4365
4366 #[test]
4367 fn unrepresentable_constructs_dropped_without_leaking_text() {
4368 let b = blocks("para1\n\n---\n\npara2");
4370 assert_eq!(b.len(), 2);
4371 assert_eq!(
4372 b[0].spans
4373 .iter()
4374 .map(|s| s.text.as_str())
4375 .collect::<String>(),
4376 "para1"
4377 );
4378 assert_eq!(
4379 b[1].spans
4380 .iter()
4381 .map(|s| s.text.as_str())
4382 .collect::<String>(),
4383 "para2"
4384 );
4385
4386 let d = blocks(":::\ninside\n:::");
4388 let joined: String = d
4389 .iter()
4390 .flat_map(|b| b.spans.iter())
4391 .map(|s| s.text.as_str())
4392 .collect();
4393 assert_eq!(joined, "inside");
4394
4395 let m = blocks("before $`E=mc^2` after");
4397 let joined: String = m
4398 .iter()
4399 .flat_map(|b| b.spans.iter())
4400 .map(|s| s.text.as_str())
4401 .collect();
4402 assert!(joined.contains("before"), "{joined:?}");
4403 assert!(joined.contains("after"), "{joined:?}");
4404 assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
4405 }
4406
4407 #[test]
4408 fn empty_document_yields_one_empty_block() {
4409 let b = blocks("");
4410 assert_eq!(b.len(), 1);
4411 assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
4412 }
4413
4414 #[test]
4415 fn block_attributes_parse_into_block() {
4416 let b = blocks(
4417 "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
4418 );
4419 assert_eq!(b.len(), 1);
4420 assert_eq!(b[0].alignment, Some(Alignment::Center));
4421 assert_eq!(b[0].line_height, Some(1500));
4422 assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
4423 assert_eq!(b[0].non_breakable_lines, Some(true));
4424 assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
4425 }
4426
4427 #[test]
4428 fn spacing_block_attributes_parse_into_block() {
4429 let b = blocks("{top_margin=24 text_indent=0}\nhello");
4433 assert_eq!(b.len(), 1);
4434 assert_eq!(b[0].top_margin, Some(24));
4435 assert_eq!(b[0].text_indent, Some(0));
4436 }
4437
4438 #[test]
4439 fn a_zero_text_indent_is_distinct_from_an_absent_one() {
4440 let explicit = blocks("{text_indent=0}\nhello");
4444 let absent = blocks("hello");
4445 assert_eq!(explicit[0].text_indent, Some(0));
4446 assert_eq!(absent[0].text_indent, None);
4447 assert!(!explicit[0].is_inline_only());
4448 assert!(absent[0].is_inline_only());
4449 }
4450
4451 #[test]
4452 fn spacing_block_attributes_respect_import_options() {
4453 let src = "{top_margin=24 text_indent=0}\nhello";
4454 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4455 assert_eq!(b[0].top_margin, None);
4456 assert_eq!(b[0].text_indent, None);
4457 }
4458
4459 #[test]
4460 fn block_attributes_on_heading() {
4461 let b = blocks("{alignment=right}\n# Title");
4462 assert_eq!(b[0].heading_level, Some(1));
4463 assert_eq!(b[0].alignment, Some(Alignment::Right));
4464 }
4465
4466 #[test]
4467 fn block_attributes_respect_import_options() {
4468 let src = "{alignment=center line_height=1500}\nhello";
4471 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4472 assert_eq!(b[0].alignment, None);
4473 assert_eq!(b[0].line_height, None);
4474 assert_eq!(
4475 b[0].spans
4476 .iter()
4477 .map(|s| s.text.as_str())
4478 .collect::<String>(),
4479 "hello"
4480 );
4481 }
4482
4483 #[test]
4484 fn list_item_block_attributes_are_dropped() {
4485 let b = blocks("{alignment=center}\n- item");
4488 assert!(b.iter().all(|blk| blk.alignment.is_none()));
4489 }
4490
4491 #[test]
4492 fn unknown_alignment_value_is_ignored() {
4493 let b = blocks("{alignment=sideways}\nhello");
4494 assert_eq!(b[0].alignment, None);
4495 }
4496}
4497
4498pub const TABLE_ANCHOR: &str = "\u{FFFC}";
4510
4511pub const LINE_SEPARATOR: char = '\u{2028}';
4525
4526const LINE_SEPARATOR_STR: &str = "\u{2028}";
4528
4529fn span_prose(span: &ParsedSpan, out: &mut String) {
4537 if span.image.is_some() || span.footnote_ref.is_some() {
4538 out.push('\u{FFFC}');
4539 return;
4540 }
4541 out.push_str(&span.text);
4542}
4543
4544fn block_prose(block: &ParsedBlock) -> String {
4545 let mut prose = String::new();
4546 for span in &block.spans {
4547 span_prose(span, &mut prose);
4548 }
4549 prose
4550}
4551
4552fn cell_prose(cell: &ParsedTableCell) -> String {
4553 let mut prose = String::new();
4554 for span in &cell.spans {
4555 span_prose(span, &mut prose);
4556 }
4557 prose
4558}
4559
4560pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
4611 let elements = parse_djot(djot, options);
4615
4616 let mut out = String::with_capacity(djot.len());
4619
4620 let mut first = true;
4625 let push = |text: &str, out: &mut String, first: &mut bool| {
4626 if *first {
4627 *first = false;
4628 } else {
4629 out.push('\n');
4630 }
4631 out.push_str(text);
4632 };
4633
4634 for element in &elements {
4635 match element {
4636 ParsedElement::Block(block) => {
4637 push(&block_prose(block), &mut out, &mut first);
4638 }
4639 ParsedElement::FootnoteDefinition { .. } => {}
4649 ParsedElement::Table(table) => {
4650 push(TABLE_ANCHOR, &mut out, &mut first);
4657 let columns = table.column_count();
4660 for row in &table.rows {
4661 for column in 0..columns {
4662 let prose = row.get(column).map(cell_prose).unwrap_or_default();
4663 push(&prose, &mut out, &mut first);
4664 }
4665 }
4666 }
4667 }
4668 }
4669 out
4670}
4671
4672#[cfg(test)]
4673mod html_footnote_tests {
4674 use super::*;
4675
4676 #[test]
4680 fn a_literal_reference_in_text_would_be_escaped_into_prose() {
4681 let blocks = parse_html("<p>The ferry.[^1]</p>");
4682 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4683 assert!(
4684 text.contains("[^1]"),
4685 "the parser keeps it as characters: {text:?}"
4686 );
4687 assert!(
4688 blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
4689 "text is text — it must not become a reference by accident"
4690 );
4691 }
4692
4693 #[test]
4694 fn an_attributed_element_becomes_a_real_footnote_reference() {
4695 let blocks = parse_html(
4696 r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
4697 );
4698 let refs: Vec<&str> = blocks
4699 .iter()
4700 .flat_map(|b| b.spans.iter())
4701 .filter_map(|s| s.footnote_ref.as_deref())
4702 .collect();
4703 assert_eq!(refs, vec!["3"]);
4704
4705 let marker = blocks[0]
4708 .spans
4709 .iter()
4710 .find(|s| s.footnote_ref.is_some())
4711 .expect("the reference span");
4712 assert!(marker.text.is_empty(), "{marker:?}");
4713 }
4714
4715 #[test]
4718 fn any_element_carrying_the_attribute_works() {
4719 for html in [
4720 r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
4721 r#"<p>a<span data-footnote-ref="1"></span></p>"#,
4722 r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
4723 ] {
4724 let blocks = parse_html(html);
4725 assert_eq!(
4726 blocks
4727 .iter()
4728 .flat_map(|b| b.spans.iter())
4729 .filter_map(|s| s.footnote_ref.as_deref())
4730 .collect::<Vec<_>>(),
4731 vec!["1"],
4732 "failed for {html}"
4733 );
4734 }
4735 }
4736
4737 #[test]
4738 fn an_empty_or_absent_label_is_not_a_reference() {
4739 for html in [
4740 r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
4741 r#"<p>a<sup></sup></p>"#,
4742 ] {
4743 let blocks = parse_html(html);
4744 assert!(
4745 blocks
4746 .iter()
4747 .flat_map(|b| b.spans.iter())
4748 .all(|s| s.footnote_ref.is_none()),
4749 "failed for {html}"
4750 );
4751 }
4752 }
4753}
4754
4755#[cfg(test)]
4756mod html_depth_tests {
4757 use super::*;
4760
4761 fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
4763 blocks
4764 .iter()
4765 .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
4766 .collect()
4767 }
4768
4769 #[test]
4774 fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
4775 use crate::parser_tools::list_depth::MAX_LIST_INDENT;
4776 let depth = 300;
4777 let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
4778 html.push_str(&"</li></ol>".repeat(depth));
4779 let blocks = parse_html(&html);
4780 let items: Vec<(String, u32)> = blocks
4781 .iter()
4782 .filter(|block| block.list_style.is_some())
4783 .map(|block| {
4784 let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
4785 (text, block.list_indent)
4786 })
4787 .collect();
4788 let expected: Vec<(String, u32)> = (0..depth)
4789 .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
4790 .collect();
4791 assert_eq!(items, expected);
4792 }
4793
4794 #[test]
4796 fn text_in_a_deep_html_list_outside_its_items_is_kept() {
4797 let depth = 40;
4798 let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
4799 html.push_str("<ul>loose words<li>last</li></ul>");
4800 html.push_str(&"</li></ul>".repeat(depth));
4801 let texts = block_texts(&parse_html(&html)).join(" ");
4802 assert!(texts.contains("loose words"), "{texts:?}");
4803 assert!(texts.contains("last"), "{texts:?}");
4804 assert!(texts.contains("item 39"), "{texts:?}");
4805 }
4806
4807 #[test]
4810 fn html_nested_past_the_reader_depth_keeps_its_words() {
4811 for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
4812 let html = format!(
4813 "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
4814 open.repeat(300),
4815 close.repeat(300)
4816 );
4817 let texts = block_texts(&parse_html(&html)).join(" ");
4818 let words: Vec<&str> = texts.split_whitespace().collect();
4819 assert_eq!(
4820 words,
4821 ["Top.", "Deep", "words", "here.", "End."],
4822 "for {open}"
4823 );
4824 }
4825 }
4826
4827 #[test]
4829 fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
4830 let html = format!(
4831 "<table><tr><td>{}deep words{}</td></tr></table>",
4832 "<span>".repeat(300),
4833 "</span>".repeat(300)
4834 );
4835 let elements = parse_html_elements(&html);
4836 let Some(ParsedElement::Table(table)) = elements.first() else {
4837 panic!("a table: {elements:?}");
4838 };
4839 let text: String = table.rows[0][0]
4840 .spans
4841 .iter()
4842 .map(|span| span.text.as_str())
4843 .collect();
4844 assert_eq!(text, "deep words");
4845 }
4846
4847 #[test]
4851 fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
4852 for cell in [
4853 "One<br/>Two",
4854 "<p>One</p><p>Two</p>",
4855 "<ul><li>One</li><li>Two</li></ul>",
4856 ] {
4857 let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
4858 let Some(ParsedElement::Table(table)) = elements.first() else {
4859 panic!("a table: {elements:?}");
4860 };
4861 let text: String = table.rows[0][0]
4862 .spans
4863 .iter()
4864 .map(|span| span.text.as_str())
4865 .collect();
4866 assert_eq!(text, "One Two", "for {cell}");
4867 }
4868 }
4869}