1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3
4#[derive(Debug, Clone, Default, PartialEq, Eq)]
10pub struct ParsedImage {
11 pub src: String,
12 pub alt: String,
13 pub width: i64,
19 pub height: i64,
20}
21
22#[derive(Debug, Clone, Default)]
29pub struct ParsedSpan {
30 pub text: String,
31 pub bold: bool,
32 pub italic: bool,
33 pub underline: bool,
34 pub strikeout: bool,
35 pub code: bool,
36 pub superscript: bool,
38 pub subscript: bool,
40 pub link_href: Option<String>,
41 pub image: Option<ParsedImage>,
43 pub footnote_ref: Option<String>,
46 pub preserve_whitespace: bool,
50}
51
52#[derive(Debug, Clone)]
54pub struct ParsedTableCell {
55 pub spans: Vec<ParsedSpan>,
56}
57
58#[derive(Debug, Clone)]
60pub struct ParsedTable {
61 pub header_rows: usize,
63 pub rows: Vec<Vec<ParsedTableCell>>,
65 pub blockquote_depth: u32,
68 pub blockquote_opens: u32,
71}
72
73impl ParsedTable {
74 pub fn column_count(&self) -> usize {
84 self.rows.iter().map(Vec::len).max().unwrap_or(0)
85 }
86}
87
88const SQUARED_CELLS_ALWAYS: usize = 4096;
92
93const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
103
104fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
115 let squared = table.rows.len().saturating_mul(table.column_count());
116 let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
117 if squared <= allowed {
118 elements.push(ParsedElement::Table(table));
119 return;
120 }
121 elements.extend(
122 ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
123 .into_iter()
124 .filter(|block| spans_carry_content(&block.spans))
125 .map(ParsedElement::Block),
126 );
127}
128
129fn mark_quotation_opened(elements: &mut [ParsedElement], first: usize) {
133 let Some(rest) = elements.get_mut(first..) else {
134 return;
135 };
136 for element in rest {
137 match element {
138 ParsedElement::Block(block) => {
139 block.blockquote_opens += 1;
140 return;
141 }
142 ParsedElement::Table(table) => {
143 table.blockquote_opens += 1;
144 return;
145 }
146 ParsedElement::FootnoteDefinition { .. } => {}
147 }
148 }
149}
150
151fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
153 rows.iter().map(Vec::len).sum()
154}
155
156#[derive(Debug, Clone)]
158pub enum ParsedElement {
159 Block(ParsedBlock),
160 Table(ParsedTable),
161 FootnoteDefinition {
168 label: String,
169 blocks: Vec<ParsedBlock>,
170 },
171}
172
173impl ParsedElement {
174 pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
177 let mut blocks = Vec::new();
178 for elem in elements {
179 match elem {
180 ParsedElement::Block(b) => blocks.push(b),
181 ParsedElement::FootnoteDefinition { .. } => {}
186 ParsedElement::Table(t) => {
187 for row in t.rows {
188 for cell in row {
189 blocks.push(ParsedBlock {
190 spans: cell.spans,
191 heading_level: None,
192 list_style: None,
193 list_indent: 0,
194 list_prefix: String::new(),
195 list_suffix: String::new(),
196 list_start: None,
197 marker: None,
198 is_code_block: false,
199 code_language: None,
200 blockquote_depth: t.blockquote_depth,
201 blockquote_opens: 0,
202 line_height: None,
203 non_breakable_lines: None,
204 page_break_before: None,
205 front_matter: None,
206 direction: None,
207 background_color: None,
208 alignment: None,
209 top_margin: None,
210 text_indent: None,
211 semantic_role: None,
212 });
213 }
214 }
215 }
216 }
217 }
218 if blocks.is_empty() {
219 blocks.push(ParsedBlock {
220 spans: vec![ParsedSpan {
221 text: String::new(),
222 ..Default::default()
223 }],
224 heading_level: None,
225 list_style: None,
226 list_indent: 0,
227 list_prefix: String::new(),
228 list_suffix: String::new(),
229 list_start: None,
230 marker: None,
231 is_code_block: false,
232 code_language: None,
233 blockquote_depth: 0,
234 blockquote_opens: 0,
235 line_height: None,
236 non_breakable_lines: None,
237 page_break_before: None,
238 front_matter: None,
239 direction: None,
240 background_color: None,
241 alignment: None,
242 top_margin: None,
243 text_indent: None,
244 semantic_role: None,
245 });
246 }
247 blocks
248 }
249}
250
251#[derive(Debug, Clone, Default)]
258pub struct ParsedBlock {
259 pub spans: Vec<ParsedSpan>,
260 pub heading_level: Option<i64>,
261 pub list_style: Option<ListStyle>,
262 pub list_indent: u32,
263 pub list_prefix: String,
266 pub list_suffix: String,
269 pub list_start: Option<i64>,
273 pub marker: Option<MarkerType>,
276 pub is_code_block: bool,
277 pub code_language: Option<String>,
278 pub blockquote_depth: u32,
279 pub blockquote_opens: u32,
286 pub line_height: Option<i64>,
287 pub non_breakable_lines: Option<bool>,
288 pub page_break_before: Option<bool>,
291 pub front_matter: Option<bool>,
294 pub direction: Option<TextDirection>,
295 pub background_color: Option<String>,
296 pub alignment: Option<Alignment>,
299 pub top_margin: Option<i64>,
303 pub text_indent: Option<i64>,
307 pub semantic_role: Option<SemanticRole>,
311}
312
313impl ParsedBlock {
314 pub fn is_inline_only(&self) -> bool {
317 self.heading_level.is_none()
318 && self.list_style.is_none()
319 && !self.is_code_block
320 && self.blockquote_depth == 0
321 && self.line_height.is_none()
322 && self.non_breakable_lines.is_none()
323 && self.page_break_before.is_none()
324 && self.front_matter.is_none()
325 && self.direction.is_none()
326 && self.background_color.is_none()
327 && self.alignment.is_none()
328 && self.top_margin.is_none()
329 && self.text_indent.is_none()
330 }
331}
332
333fn dangling_footnote_labels(
362 markdown: &str,
363 options: pulldown_cmark::Options,
364) -> std::collections::BTreeSet<String> {
365 use pulldown_cmark::{Event, Parser, Tag};
366
367 let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
368 for event in Parser::new_ext(markdown, options) {
369 if let Event::Start(Tag::FootnoteDefinition(label)) = event {
370 defined.insert(label.to_string());
371 }
372 }
373
374 let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
375 let bytes = markdown.as_bytes();
376 let mut search_from = 0usize;
377 while let Some(rel) = markdown[search_from..].find("[^") {
378 let open = search_from + rel;
379 let label_start = open + 2;
380 let Some(close_rel) = markdown[label_start..].find(']') else {
381 break;
382 };
383 let close = label_start + close_rel;
384 let label = &markdown[label_start..close];
385 let looks_like_definition = bytes.get(close + 1) == Some(&b':');
390 if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
391 referenced.insert(label.to_string());
392 }
393 search_from = close + 1;
394 }
395
396 referenced.difference(&defined).cloned().collect()
397}
398
399pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
407 use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
408
409 let options = Options::ENABLE_STRIKETHROUGH
420 | Options::ENABLE_TABLES
421 | Options::ENABLE_TASKLISTS
422 | Options::ENABLE_FOOTNOTES;
423
424 let dangling = dangling_footnote_labels(markdown, options);
431 let augmented_owner;
432 let source: &str = if dangling.is_empty() {
433 markdown
434 } else {
435 augmented_owner = dangling
436 .iter()
437 .fold(markdown.to_string(), |mut acc, label| {
438 acc.push_str("\n\n[^");
439 acc.push_str(label);
440 acc.push_str("]:\n");
441 acc
442 });
443 &augmented_owner
444 };
445 let parser = Parser::new_ext(source, options);
446
447 let mut elements: Vec<ParsedElement> = Vec::new();
448 let mut current_spans: Vec<ParsedSpan> = Vec::new();
449 let mut current_heading: Option<i64> = None;
450 let mut current_list_style: Option<ListStyle> = None;
451 let mut current_list_start: Option<i64> = None;
452 let mut is_code_block = false;
453 let mut code_language: Option<String> = None;
454 let mut blockquote_depth: u32 = 0;
455 let mut quotations_open: Vec<usize> = Vec::new();
457 let mut in_block = false;
458
459 let mut bold = false;
461 let mut italic = false;
462 let mut strikeout = false;
463 let mut link_href: Option<String> = None;
464 let mut pending_image: Option<ParsedImage> = None;
466
467 let mut footnote_open: Option<(String, usize)> = None;
474
475 let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
478 let mut list_start_stack: Vec<Option<i64>> = Vec::new();
479 let mut current_list_indent: u32 = 0;
480
481 let mut in_table = false;
483 let mut in_table_head = false;
484 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
485 let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
486 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
487 let mut table_header_rows: usize = 0;
488
489 for event in parser {
490 match event {
491 Event::Start(Tag::Paragraph) => {
492 in_block = true;
493 current_heading = None;
494 is_code_block = false;
495 }
496 Event::End(TagEnd::Paragraph) => {
497 if !current_spans.is_empty() || in_block {
498 elements.push(ParsedElement::Block(ParsedBlock {
499 spans: std::mem::take(&mut current_spans),
500 heading_level: current_heading.take(),
501 list_style: current_list_style.clone(),
502 list_indent: current_list_indent,
503 list_prefix: String::new(),
504 list_suffix: String::new(),
505 list_start: current_list_start,
506 marker: None,
507 is_code_block: false,
508 code_language: None,
509 blockquote_depth,
510 blockquote_opens: 0,
511 line_height: None,
512 non_breakable_lines: None,
513 page_break_before: None,
514 front_matter: None,
515 direction: None,
516 background_color: None,
517 alignment: None,
518 top_margin: None,
519 text_indent: None,
520 semantic_role: None,
521 }));
522 }
523 in_block = false;
524 current_list_style = None;
525 current_list_start = None;
526 }
527 Event::Start(Tag::Heading { level, .. }) => {
528 in_block = true;
529 current_heading = Some(heading_level_to_i64(level));
530 is_code_block = false;
531 }
532 Event::End(TagEnd::Heading(_)) => {
533 elements.push(ParsedElement::Block(ParsedBlock {
534 spans: std::mem::take(&mut current_spans),
535 heading_level: current_heading.take(),
536 list_style: None,
537 list_indent: 0,
538 list_prefix: String::new(),
539 list_suffix: String::new(),
540 list_start: None,
541 marker: None,
542 is_code_block: false,
543 code_language: None,
544 blockquote_depth,
545 blockquote_opens: 0,
546 line_height: None,
547 non_breakable_lines: None,
548 page_break_before: None,
549 front_matter: None,
550 direction: None,
551 background_color: None,
552 alignment: None,
553 top_margin: None,
554 text_indent: None,
555 semantic_role: None,
556 }));
557 in_block = false;
558 }
559 Event::Start(Tag::List(ordered)) => {
560 let style = if ordered.is_some() {
561 Some(ListStyle::Decimal)
562 } else {
563 Some(ListStyle::Disc)
564 };
565 list_stack.push(style);
566 list_start_stack.push(ordered.and_then(list_start_of));
567 }
568 Event::End(TagEnd::List(_)) => {
569 list_stack.pop();
570 list_start_stack.pop();
571 }
572 Event::Start(Tag::Item) => {
573 if !current_spans.is_empty() {
576 elements.push(ParsedElement::Block(ParsedBlock {
577 spans: std::mem::take(&mut current_spans),
578 heading_level: None,
579 list_style: current_list_style.clone(),
580 list_indent: current_list_indent,
581 list_prefix: String::new(),
582 list_suffix: String::new(),
583 list_start: current_list_start,
584 marker: None,
585 is_code_block: false,
586 code_language: None,
587 blockquote_depth,
588 blockquote_opens: 0,
589 line_height: None,
590 non_breakable_lines: None,
591 page_break_before: None,
592 front_matter: None,
593 direction: None,
594 background_color: None,
595 alignment: None,
596 top_margin: None,
597 text_indent: None,
598 semantic_role: None,
599 }));
600 }
601 in_block = true;
602 current_list_style = list_stack.last().cloned().flatten();
603 current_list_start = list_start_stack.last().copied().flatten();
604 current_list_indent = if list_stack.is_empty() {
605 0
606 } else {
607 (list_stack.len() - 1) as u32
608 };
609 }
610 Event::End(TagEnd::Item) => {
611 if !current_spans.is_empty() {
614 elements.push(ParsedElement::Block(ParsedBlock {
615 spans: std::mem::take(&mut current_spans),
616 heading_level: None,
617 list_style: current_list_style.clone(),
618 list_indent: current_list_indent,
619 list_prefix: String::new(),
620 list_suffix: String::new(),
621 list_start: current_list_start,
622 marker: None,
623 is_code_block: false,
624 code_language: None,
625 blockquote_depth,
626 blockquote_opens: 0,
627 line_height: None,
628 non_breakable_lines: None,
629 page_break_before: None,
630 front_matter: None,
631 direction: None,
632 background_color: None,
633 alignment: None,
634 top_margin: None,
635 text_indent: None,
636 semantic_role: None,
637 }));
638 }
639 in_block = false;
640 current_list_style = None;
641 current_list_start = None;
642 }
643 Event::Start(Tag::CodeBlock(kind)) => {
644 in_block = true;
645 is_code_block = true;
646 code_language = match &kind {
647 pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
648 Some(lang.to_string())
649 }
650 _ => None,
651 };
652 }
653 Event::End(TagEnd::CodeBlock) => {
654 if let Some(last) = current_spans.last_mut()
656 && last.text.ends_with('\n')
657 {
658 last.text.truncate(last.text.len() - 1);
659 }
660 elements.push(ParsedElement::Block(ParsedBlock {
661 spans: std::mem::take(&mut current_spans),
662 heading_level: None,
663 list_style: None,
664 list_indent: 0,
665 list_prefix: String::new(),
666 list_suffix: String::new(),
667 list_start: None,
668 marker: None,
669 is_code_block: true,
670 code_language: code_language.take(),
671 blockquote_depth,
672 blockquote_opens: 0,
673 line_height: None,
674 non_breakable_lines: None,
675 page_break_before: None,
676 front_matter: None,
677 direction: None,
678 background_color: None,
679 alignment: None,
680 top_margin: None,
681 text_indent: None,
682 semantic_role: None,
683 }));
684 in_block = false;
685 is_code_block = false;
686 }
687 Event::Start(Tag::Table(_)) => {
689 in_table = true;
690 in_table_head = false;
691 table_rows.clear();
692 current_row_cells.clear();
693 current_cell_spans.clear();
694 table_header_rows = 0;
695 }
696 Event::End(TagEnd::Table) => {
697 let own_cells = cells_in_rows(&table_rows);
698 push_table(
699 &mut elements,
700 ParsedTable {
701 header_rows: table_header_rows,
702 rows: std::mem::take(&mut table_rows),
703 blockquote_depth,
704 blockquote_opens: 0,
705 },
706 own_cells,
707 );
708 in_table = false;
709 }
710 Event::Start(Tag::TableHead) => {
711 in_table_head = true;
712 current_row_cells.clear();
713 }
714 Event::End(TagEnd::TableHead) => {
715 table_rows.push(std::mem::take(&mut current_row_cells));
717 table_header_rows += 1;
718 in_table_head = false;
719 }
720 Event::Start(Tag::TableRow) => {
721 current_row_cells.clear();
722 }
723 Event::End(TagEnd::TableRow) if !in_table_head => {
724 table_rows.push(std::mem::take(&mut current_row_cells));
726 }
727 Event::Start(Tag::TableCell) => {
728 current_cell_spans.clear();
729 }
730 Event::End(TagEnd::TableCell) => {
731 current_row_cells.push(ParsedTableCell {
732 spans: std::mem::take(&mut current_cell_spans),
733 });
734 }
735 Event::Start(Tag::Emphasis) => {
737 italic = true;
738 }
739 Event::End(TagEnd::Emphasis) => {
740 italic = false;
741 }
742 Event::Start(Tag::Strong) => {
743 bold = true;
744 }
745 Event::End(TagEnd::Strong) => {
746 bold = false;
747 }
748 Event::Start(Tag::Strikethrough) => {
749 strikeout = true;
750 }
751 Event::End(TagEnd::Strikethrough) => {
752 strikeout = false;
753 }
754 Event::Start(Tag::Link { dest_url, .. }) => {
755 link_href = Some(dest_url.to_string());
756 }
757 Event::End(TagEnd::Link) => {
758 link_href = None;
759 }
760 Event::Start(Tag::Image { dest_url, .. }) => {
763 pending_image = Some(ParsedImage {
764 src: dest_url.to_string(),
765 alt: String::new(),
766 width: 0,
767 height: 0,
768 });
769 }
770 Event::End(TagEnd::Image) => {
771 if let Some(image) = pending_image.take() {
772 let span = ParsedSpan {
773 text: String::new(),
774 bold,
775 italic,
776 underline: false,
777 strikeout,
778 code: false,
779 superscript: false,
780 subscript: false,
781 link_href: link_href.clone(),
782 image: Some(image),
783 footnote_ref: None,
784 preserve_whitespace: false,
785 };
786 if in_table {
787 current_cell_spans.push(span);
788 } else {
789 if !in_block {
790 in_block = true;
791 }
792 current_spans.push(span);
793 }
794 }
795 }
796 Event::Text(text) => {
797 if let Some(img) = pending_image.as_mut() {
802 img.alt.push_str(&text);
803 continue;
804 }
805 let span = ParsedSpan {
806 text: text.to_string(),
807 bold,
808 italic,
809 underline: false,
810 strikeout,
811 code: is_code_block,
812 superscript: false,
813 subscript: false,
814 link_href: link_href.clone(),
815 image: None,
816 footnote_ref: None,
817 preserve_whitespace: false,
818 };
819 if in_table {
820 current_cell_spans.push(span);
821 } else {
822 if !in_block {
823 in_block = true;
824 }
825 current_spans.push(span);
826 }
827 }
828 Event::Code(text) => {
829 let span = ParsedSpan {
830 text: text.to_string(),
831 bold,
832 italic,
833 underline: false,
834 strikeout,
835 code: true,
836 superscript: false,
837 subscript: false,
838 link_href: link_href.clone(),
839 image: None,
840 footnote_ref: None,
841 preserve_whitespace: false,
842 };
843 if in_table {
844 current_cell_spans.push(span);
845 } else {
846 if !in_block {
847 in_block = true;
848 }
849 current_spans.push(span);
850 }
851 }
852 Event::SoftBreak => {
853 let span = ParsedSpan {
854 text: " ".to_string(),
855 bold,
856 italic,
857 underline: false,
858 strikeout,
859 code: false,
860 superscript: false,
861 subscript: false,
862 link_href: link_href.clone(),
863 image: None,
864 footnote_ref: None,
865 preserve_whitespace: false,
866 };
867 if in_table {
868 current_cell_spans.push(span);
869 } else {
870 current_spans.push(span);
871 }
872 }
873 Event::HardBreak if pending_image.is_some() => {
877 if let Some(img) = pending_image.as_mut() {
878 img.alt.push(' ');
879 }
880 }
881 Event::HardBreak => {
882 let span = ParsedSpan {
883 text: if in_table { " " } else { LINE_SEPARATOR_STR }.to_string(),
884 bold,
885 italic,
886 underline: false,
887 strikeout,
888 code: false,
889 superscript: false,
890 subscript: false,
891 link_href: link_href.clone(),
892 image: None,
893 footnote_ref: None,
894 preserve_whitespace: false,
895 };
896 if in_table {
897 current_cell_spans.push(span);
898 } else {
899 current_spans.push(span);
900 }
901 }
902 Event::Start(Tag::BlockQuote(_)) => {
903 blockquote_depth += 1;
904 quotations_open.push(elements.len());
905 }
906 Event::End(TagEnd::BlockQuote(_)) => {
907 blockquote_depth = blockquote_depth.saturating_sub(1);
908 if let Some(first) = quotations_open.pop() {
909 mark_quotation_opened(&mut elements, first);
910 }
911 }
912 Event::Start(Tag::FootnoteDefinition(label)) => {
923 if !current_spans.is_empty() {
924 elements.push(ParsedElement::Block(ParsedBlock {
925 spans: std::mem::take(&mut current_spans),
926 heading_level: current_heading.take(),
927 list_style: current_list_style.clone(),
928 list_indent: current_list_indent,
929 list_prefix: String::new(),
930 list_suffix: String::new(),
931 list_start: current_list_start,
932 marker: None,
933 is_code_block: false,
934 code_language: None,
935 blockquote_depth,
936 blockquote_opens: 0,
937 line_height: None,
938 non_breakable_lines: None,
939 page_break_before: None,
940 front_matter: None,
941 direction: None,
942 background_color: None,
943 alignment: None,
944 top_margin: None,
945 text_indent: None,
946 semantic_role: None,
947 }));
948 }
949 footnote_open = Some((label.to_string(), elements.len()));
950 }
951 Event::End(TagEnd::FootnoteDefinition) => {
952 if !current_spans.is_empty() {
953 elements.push(ParsedElement::Block(ParsedBlock {
954 spans: std::mem::take(&mut current_spans),
955 heading_level: current_heading.take(),
956 list_style: current_list_style.clone(),
957 list_indent: current_list_indent,
958 list_prefix: String::new(),
959 list_suffix: String::new(),
960 list_start: current_list_start,
961 marker: None,
962 is_code_block: false,
963 code_language: None,
964 blockquote_depth,
965 blockquote_opens: 0,
966 line_height: None,
967 non_breakable_lines: None,
968 page_break_before: None,
969 front_matter: None,
970 direction: None,
971 background_color: None,
972 alignment: None,
973 top_margin: None,
974 text_indent: None,
975 semantic_role: None,
976 }));
977 }
978 if let Some((label, start)) = footnote_open.take() {
979 let blocks: Vec<ParsedBlock> = elements
980 .drain(start..)
981 .filter_map(|e| match e {
982 ParsedElement::Block(b) => Some(b),
983 _ => None,
986 })
987 .collect();
988 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
989 }
990 }
991 Event::FootnoteReference(label) => {
997 let span = ParsedSpan {
998 text: String::new(),
999 bold,
1000 italic,
1001 underline: false,
1002 strikeout,
1003 code: false,
1004 superscript: false,
1005 subscript: false,
1006 link_href: link_href.clone(),
1007 image: None,
1008 footnote_ref: Some(label.to_string()),
1009 preserve_whitespace: false,
1010 };
1011 if in_table {
1012 current_cell_spans.push(span);
1013 } else {
1014 if !in_block {
1015 in_block = true;
1016 }
1017 current_spans.push(span);
1018 }
1019 }
1020 _ => {}
1021 }
1022 }
1023
1024 if !current_spans.is_empty() {
1026 elements.push(ParsedElement::Block(ParsedBlock {
1027 spans: std::mem::take(&mut current_spans),
1028 heading_level: current_heading,
1029 list_style: current_list_style,
1030 list_indent: current_list_indent,
1031 list_prefix: String::new(),
1032 list_suffix: String::new(),
1033 list_start: current_list_start,
1034 marker: None,
1035 is_code_block,
1036 code_language: code_language.take(),
1037 blockquote_depth,
1038 blockquote_opens: 0,
1039 line_height: None,
1040 non_breakable_lines: None,
1041 page_break_before: None,
1042 front_matter: None,
1043 direction: None,
1044 background_color: None,
1045 alignment: None,
1046 top_margin: None,
1047 text_indent: None,
1048 semantic_role: None,
1049 }));
1050 }
1051
1052 if !dangling.is_empty() {
1059 elements.retain(
1060 |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
1061 );
1062 }
1063
1064 if elements.is_empty() {
1066 elements.push(ParsedElement::Block(ParsedBlock {
1067 spans: vec![ParsedSpan {
1068 text: String::new(),
1069 ..Default::default()
1070 }],
1071 heading_level: None,
1072 list_style: None,
1073 list_indent: 0,
1074 list_prefix: String::new(),
1075 list_suffix: String::new(),
1076 list_start: None,
1077 marker: None,
1078 is_code_block: false,
1079 code_language: None,
1080 blockquote_depth: 0,
1081 blockquote_opens: 0,
1082 line_height: None,
1083 non_breakable_lines: None,
1084 page_break_before: None,
1085 front_matter: None,
1086 direction: None,
1087 background_color: None,
1088 alignment: None,
1089 top_margin: None,
1090 text_indent: None,
1091 semantic_role: None,
1092 }));
1093 }
1094
1095 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1098 elements
1099}
1100
1101fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
1102 use pulldown_cmark::HeadingLevel;
1103 match level {
1104 HeadingLevel::H1 => 1,
1105 HeadingLevel::H2 => 2,
1106 HeadingLevel::H3 => 3,
1107 HeadingLevel::H4 => 4,
1108 HeadingLevel::H5 => 5,
1109 HeadingLevel::H6 => 6,
1110 }
1111}
1112
1113use scraper::Node;
1116
1117#[derive(Debug, Clone, Default)]
1119struct BlockStyles {
1120 line_height: Option<i64>,
1121 non_breakable_lines: Option<bool>,
1122 page_break_before: Option<bool>,
1123 direction: Option<TextDirection>,
1124 background_color: Option<String>,
1125 preserve_whitespace: Option<bool>,
1130}
1131
1132fn parse_block_styles(style: &str) -> BlockStyles {
1136 let mut result = BlockStyles::default();
1137 for part in style.split(';') {
1138 let part = part.trim();
1139 if let Some((prop, val)) = part.split_once(':') {
1140 let prop = prop.trim().to_ascii_lowercase();
1141 let val = val.trim();
1142 match prop.as_str() {
1143 "line-height" => {
1144 if let Ok(v) = val.parse::<f64>() {
1146 result.line_height = Some((v * 1000.0) as i64);
1147 }
1148 }
1149 "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1150 result.non_breakable_lines = Some(true);
1151 if val != "nowrap" {
1152 result.preserve_whitespace = Some(true);
1153 }
1154 }
1155 "break-before" | "page-break-before" => {
1159 result.page_break_before = match val.to_ascii_lowercase().as_str() {
1160 "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1161 "avoid" | "auto" => Some(false),
1162 _ => None,
1163 };
1164 }
1165 "direction" => {
1166 if val.eq_ignore_ascii_case("rtl") {
1167 result.direction = Some(TextDirection::RightToLeft);
1168 } else if val.eq_ignore_ascii_case("ltr") {
1169 result.direction = Some(TextDirection::LeftToRight);
1170 }
1171 }
1172 "background-color" | "background" => {
1173 result.background_color = Some(val.to_string());
1174 }
1175 _ => {}
1176 }
1177 }
1178 }
1179 result
1180}
1181
1182fn style_keeps_whitespace(style: &str) -> Option<bool> {
1188 let mut kept = None;
1189 let mut important_kept = false;
1190 for part in style.split(';') {
1191 let Some((property, value)) = part.split_once(':') else {
1192 continue;
1193 };
1194 if !property.trim().eq_ignore_ascii_case("white-space") {
1195 continue;
1196 }
1197 let value = value.trim().to_ascii_lowercase();
1198 let (value, important) = value
1199 .strip_suffix("important")
1200 .and_then(|rest| rest.trim_end().strip_suffix('!'))
1201 .map_or((value.as_str(), false), |rest| (rest.trim_end(), true));
1202 let says = match value {
1203 "pre" | "pre-wrap" | "break-spaces" => true,
1204 "normal" | "nowrap" | "pre-line" => false,
1205 _ => continue,
1206 };
1207 if important || !important_kept {
1208 kept = Some(says);
1209 important_kept |= important;
1210 }
1211 }
1212 kept
1213}
1214
1215pub fn push_html_text(
1230 out: &mut String,
1231 text: &str,
1232 after_space: bool,
1233 ends_line: bool,
1234 escape: impl Fn(&str) -> String,
1235 line_break: &str,
1236) {
1237 let mut lines = text.split(LINE_SEPARATOR).peekable();
1238 let mut first = true;
1239 while let Some(line) = lines.next() {
1240 if !first {
1241 out.push_str(line_break);
1242 }
1243 let opens_on_space = if first { after_space } else { true };
1244 let ends = ends_line || lines.peek().is_some();
1245 first = false;
1246 let collapses = line.chars().any(|ch| ch != ' ' && is_html_space(ch))
1247 || line.contains(" ")
1248 || (opens_on_space && line.starts_with(' '))
1249 || (ends && line.ends_with(' '));
1250 if collapses {
1251 out.push_str("<span style=\"white-space: pre-wrap\">");
1252 out.push_str(&escape(line));
1253 out.push_str("</span>");
1254 } else {
1255 out.push_str(&escape(line));
1256 }
1257 }
1258}
1259
1260pub fn html_line_ends_before<'a>(
1264 rest: impl IntoIterator<Item = &'a crate::format_runs::InlineContent>,
1265) -> bool {
1266 use crate::format_runs::InlineContent;
1267 for content in rest {
1268 match content {
1269 InlineContent::Empty => {}
1270 InlineContent::Text(text) if text.is_empty() => {}
1271 InlineContent::Text(text) => return text.starts_with(LINE_SEPARATOR),
1272 _ => return false,
1273 }
1274 }
1275 true
1276}
1277
1278pub fn html_text_ends_open(text: &str, after_space: bool) -> bool {
1283 match text.chars().next_back() {
1284 None => after_space,
1285 Some(ch) => ch == LINE_SEPARATOR || is_html_space(ch),
1286 }
1287}
1288
1289fn is_html_space(ch: char) -> bool {
1296 matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1297}
1298
1299fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1301 spans
1302 .iter()
1303 .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1304}
1305
1306fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1320 fn is_replaced(span: &ParsedSpan) -> bool {
1321 span.image.is_some() || span.footnote_ref.is_some()
1322 }
1323
1324 for span in spans.iter_mut() {
1328 if is_replaced(span) || span.preserve_whitespace {
1329 continue;
1330 }
1331 let mut out = String::with_capacity(span.text.len());
1332 let mut in_run = false;
1333 for ch in span.text.chars() {
1334 if is_html_space(ch) {
1335 if !in_run {
1336 out.push(' ');
1337 in_run = true;
1338 }
1339 } else {
1340 out.push(ch);
1341 in_run = false;
1342 }
1343 }
1344 span.text = out;
1345 }
1346
1347 let mut prev_ends_with_space = true;
1352 let mut line_end: Option<usize> = None;
1355 for i in 0..spans.len() {
1356 let span = &mut spans[i];
1357 if is_replaced(span) {
1358 prev_ends_with_space = false;
1359 line_end = None;
1360 continue;
1361 }
1362 if span.text.is_empty() {
1363 prev_ends_with_space = false;
1366 continue;
1367 }
1368 if span.text == LINE_SEPARATOR_STR {
1369 if let Some(end) = line_end.take()
1370 && spans[end].text.ends_with(' ')
1371 {
1372 spans[end].text.pop();
1373 }
1374 prev_ends_with_space = true;
1375 continue;
1376 }
1377 if span.preserve_whitespace {
1380 prev_ends_with_space = false;
1381 line_end = None;
1382 continue;
1383 }
1384 if prev_ends_with_space && span.text.starts_with(' ') {
1385 span.text.remove(0);
1386 }
1387 if !span.text.is_empty() {
1390 prev_ends_with_space = span.text.ends_with(' ');
1391 line_end = Some(i);
1392 }
1393 }
1394
1395 let mut trailing_break_dropped = false;
1401 for span in spans.iter_mut().rev() {
1402 if is_replaced(span) {
1403 break;
1404 }
1405 if span.text.is_empty() {
1406 continue;
1407 }
1408 if span.text == LINE_SEPARATOR_STR {
1409 if trailing_break_dropped {
1410 break;
1411 }
1412 span.text.clear();
1413 trailing_break_dropped = true;
1414 continue;
1415 }
1416 if span.preserve_whitespace {
1417 break;
1418 }
1419 if span.text.ends_with(' ') {
1420 span.text.pop();
1421 }
1422 break;
1423 }
1424}
1425
1426pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
1447 let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
1448 if !block.spans.iter().any(breaks) {
1449 return vec![block];
1450 }
1451 let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
1452 for mut span in std::mem::take(&mut block.spans) {
1453 if !breaks(&span) {
1454 push_to_last_line(&mut lines, span);
1455 continue;
1456 }
1457 let text = std::mem::take(&mut span.text);
1458 for (i, piece) in text.split('\n').enumerate() {
1459 if i > 0 {
1460 lines.push(Vec::new());
1461 }
1462 if !piece.is_empty() {
1463 push_to_last_line(
1464 &mut lines,
1465 ParsedSpan {
1466 text: piece.to_string(),
1467 ..span.clone()
1468 },
1469 );
1470 }
1471 }
1472 }
1473 if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
1474 lines.pop();
1475 }
1476 let later_line = ParsedBlock {
1477 page_break_before: None,
1478 blockquote_opens: 0,
1479 ..block.clone()
1480 };
1481 let mut first_line = Some(block);
1482 lines
1483 .into_iter()
1484 .map(|spans| {
1485 let spans = if spans.is_empty() {
1486 vec![ParsedSpan::default()]
1487 } else {
1488 spans
1489 };
1490 match first_line.take() {
1491 Some(first) => ParsedBlock { spans, ..first },
1492 None => ParsedBlock {
1493 spans,
1494 ..later_line.clone()
1495 },
1496 }
1497 })
1498 .collect()
1499}
1500
1501fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
1504 match lines.last_mut() {
1505 Some(line) => line.push(span),
1506 None => lines.push(vec![span]),
1507 }
1508}
1509
1510pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
1511 ParsedElement::flatten_to_blocks(parse_html_elements(html))
1512}
1513
1514pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
1528
1529fn html_footnote_span(
1532 el: &scraper::node::Element,
1533 link_href: Option<String>,
1534) -> Option<ParsedSpan> {
1535 let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
1536 if label.is_empty() {
1537 return None;
1538 }
1539 Some(ParsedSpan {
1540 text: String::new(),
1541 link_href,
1542 footnote_ref: Some(label.to_string()),
1543 ..Default::default()
1544 })
1545}
1546
1547fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
1555 let src = el.attr("src")?;
1556 if src.is_empty() {
1557 return None;
1558 }
1559 let dim = |name: &str| -> i64 {
1560 el.attr(name)
1561 .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
1562 .filter(|n| *n > 0)
1563 .unwrap_or(0)
1564 };
1565 Some(ParsedSpan {
1566 text: String::new(),
1567 link_href,
1568 image: Some(ParsedImage {
1569 src: src.to_string(),
1570 alt: el.attr("alt").unwrap_or_default().to_string(),
1571 width: dim("width"),
1572 height: dim("height"),
1573 }),
1574 ..Default::default()
1575 })
1576}
1577
1578fn is_html_list(node: &Node) -> bool {
1580 node.as_element()
1581 .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
1582}
1583
1584fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
1597 use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
1598 use ego_tree::iter::Edge;
1599
1600 let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
1602 let mut depth = 0usize;
1603 for edge in tree.root().traverse() {
1604 match edge {
1605 Edge::Open(node) if is_html_list(node.value()) => {
1606 if depth == MAX_LIST_LEVELS {
1607 too_deep.push(node.id());
1608 }
1609 depth += 1;
1610 }
1611 Edge::Close(node) if is_html_list(node.value()) => {
1612 depth = depth.saturating_sub(1);
1613 }
1614 _ => {}
1615 }
1616 }
1617
1618 for list in too_deep {
1619 let Some(list_ref) = tree.get(list) else {
1623 continue;
1624 };
1625 let mut holder = None;
1626 let mut current = list_ref;
1627 while let Some(parent) = current.parent() {
1628 if is_html_list(parent.value()) {
1629 holder = Some(current.id());
1630 break;
1631 }
1632 current = parent;
1633 }
1634 let Some(holder) = holder else {
1635 continue;
1636 };
1637 let mut after = holder;
1638 let items: Vec<ego_tree::NodeId> = list_ref
1639 .descendants()
1640 .filter(|node| {
1641 node.value()
1642 .as_element()
1643 .is_some_and(|el| el.name() == "li")
1644 })
1645 .map(|node| node.id())
1646 .collect();
1647 for item in items {
1648 if tree.get(after).and_then(|node| node.parent()).is_none() {
1651 break;
1652 }
1653 if let Some(mut node) = tree.get_mut(after) {
1654 node.insert_id_after(item);
1655 after = item;
1656 }
1657 }
1658 let holds_text = tree.get(list).is_some_and(|node| {
1663 node.descendants()
1664 .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
1665 });
1666 if !holds_text {
1667 if let Some(mut node) = tree.get_mut(list) {
1668 node.detach();
1669 }
1670 } else if holder != list
1671 && tree.get(after).and_then(|node| node.parent()).is_some()
1672 && let Some(mut node) = tree.get_mut(after)
1673 {
1674 node.insert_id_after(list);
1675 }
1676 }
1677}
1678
1679const HTML_DEPTH_ON_THE_CALLERS_STACK: usize = 32;
1683
1684const MAX_HTML_READ_DEPTH: usize = 256;
1687
1688const HTML_READER_STACK: usize = 2 << 20;
1691
1692const HTML_STACK_PER_LEVEL: usize = 16 << 10;
1695
1696fn html_integer(value: &str) -> Option<i64> {
1702 let value = value.trim_start_matches(|c: char| c.is_ascii_whitespace());
1703 let (negative, digits) = match value.as_bytes().first() {
1704 Some(b'-') => (true, &value[1..]),
1705 Some(b'+') => (false, &value[1..]),
1706 _ => (false, value),
1707 };
1708 let digits: &str = digits
1709 .split(|c: char| !c.is_ascii_digit())
1710 .next()
1711 .unwrap_or("");
1712 if digits.is_empty() {
1713 return None;
1714 }
1715 let magnitude = digits.bytes().fold(0i64, |number, digit| {
1716 number
1717 .saturating_mul(10)
1718 .saturating_add(i64::from(digit - b'0'))
1719 });
1720 Some(if negative { -magnitude } else { magnitude })
1721}
1722
1723fn html_tree_depth(tree: &ego_tree::Tree<Node>) -> usize {
1726 use ego_tree::iter::Edge;
1727 let mut depth = 0usize;
1728 let mut deepest = 0usize;
1729 for edge in tree.root().traverse() {
1730 match edge {
1731 Edge::Open(node) if node.value().is_element() => {
1732 depth += 1;
1733 deepest = deepest.max(depth);
1734 }
1735 Edge::Close(node) if node.value().is_element() => depth = depth.saturating_sub(1),
1736 _ => {}
1737 }
1738 }
1739 deepest
1740}
1741
1742pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
1754 use scraper::Html;
1755
1756 let read = |depth_limit: usize, html: &str| {
1757 let mut fragment = Html::parse_fragment(html);
1758 flatten_deep_html_lists(&mut fragment.tree);
1759 let mut elements = read_html_tree(&fragment, depth_limit);
1760 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1761 elements
1762 };
1763 let mut fragment = Html::parse_fragment(html);
1764 flatten_deep_html_lists(&mut fragment.tree);
1765 let depth = html_tree_depth(&fragment.tree);
1766 if depth <= HTML_DEPTH_ON_THE_CALLERS_STACK {
1767 let mut elements = read_html_tree(&fragment, MAX_HTML_READ_DEPTH);
1768 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1769 return elements;
1770 }
1771 drop(fragment);
1772 let stack = HTML_READER_STACK + depth.min(MAX_HTML_READ_DEPTH + 1) * HTML_STACK_PER_LEVEL;
1773 let owned = html.to_owned();
1774 let on_its_own_thread = std::thread::Builder::new()
1775 .stack_size(stack)
1776 .spawn(move || read(MAX_HTML_READ_DEPTH, &owned))
1777 .ok()
1778 .and_then(|reader| reader.join().ok());
1779 on_its_own_thread.unwrap_or_else(|| read(HTML_DEPTH_ON_THE_CALLERS_STACK, html))
1780}
1781
1782fn read_html_tree(fragment: &scraper::Html, depth_limit: usize) -> Vec<ParsedElement> {
1786 let mut elements: Vec<ParsedElement> = Vec::new();
1787
1788 let root = fragment.root_element();
1790
1791 #[derive(Clone, Default)]
1792 struct FmtState {
1793 bold: bool,
1794 italic: bool,
1795 underline: bool,
1796 strikeout: bool,
1797 code: bool,
1798 superscript: bool,
1799 subscript: bool,
1800 link_href: Option<String>,
1801 preserve_whitespace: bool,
1803 }
1804
1805 type HtmlList = (Option<ListStyle>, Option<i64>);
1808
1809 fn is_metadata_tag(tag: &str) -> bool {
1819 matches!(
1820 tag,
1821 "head"
1822 | "style"
1823 | "script"
1824 | "title"
1825 | "meta"
1826 | "link"
1827 | "base"
1828 | "noscript"
1829 | "template"
1830 )
1831 }
1832
1833 fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
1842 use ego_tree::iter::Edge;
1843 let mut text = String::new();
1844 let mut hidden = 0usize;
1846 for edge in node.traverse() {
1847 match edge {
1848 Edge::Open(open) => match open.value() {
1849 Node::Element(el) => {
1850 if hidden > 0 || is_metadata_tag(el.name()) {
1851 hidden += 1;
1852 } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
1853 text.push(' ');
1854 }
1855 }
1856 Node::Text(t) if hidden == 0 => text.push_str(t),
1857 _ => {}
1858 },
1859 Edge::Close(close) => {
1860 if hidden > 0 && close.value().is_element() {
1861 hidden -= 1;
1862 }
1863 }
1864 }
1865 }
1866 text
1867 }
1868
1869 fn text_span(text: String, state: &FmtState) -> ParsedSpan {
1871 ParsedSpan {
1872 text,
1873 bold: state.bold,
1874 italic: state.italic,
1875 underline: state.underline,
1876 strikeout: state.strikeout,
1877 code: state.code,
1878 superscript: state.superscript,
1879 subscript: state.subscript,
1880 link_href: state.link_href.clone(),
1881 image: None,
1882 footnote_ref: None,
1883 preserve_whitespace: state.preserve_whitespace,
1884 }
1885 }
1886
1887 fn collect_cell_spans(
1889 node: ego_tree::NodeRef<Node>,
1890 state: &FmtState,
1891 spans: &mut Vec<ParsedSpan>,
1892 depth: usize,
1893 limit: usize,
1894 ) {
1895 if depth > limit {
1896 let text = subtree_text(node);
1897 if !text.is_empty() {
1898 spans.push(text_span(text, state));
1899 }
1900 return;
1901 }
1902 for child in node.children() {
1903 match child.value() {
1904 Node::Text(text) => {
1905 let t = text.text.to_string();
1906 if !t.is_empty() {
1907 spans.push(ParsedSpan {
1908 text: t,
1909 bold: state.bold,
1910 italic: state.italic,
1911 underline: state.underline,
1912 strikeout: state.strikeout,
1913 code: state.code,
1914 superscript: state.superscript,
1915 subscript: state.subscript,
1916 link_href: state.link_href.clone(),
1917 image: None,
1918 footnote_ref: None,
1919 preserve_whitespace: state.preserve_whitespace,
1920 });
1921 }
1922 }
1923 Node::Element(el) => {
1924 let tag = el.name();
1925 if is_metadata_tag(tag) {
1926 continue;
1927 }
1928 let mut new_state = state.clone();
1929 if let Some(keep) = el.attr("style").and_then(style_keeps_whitespace) {
1930 new_state.preserve_whitespace = keep;
1931 }
1932 match tag {
1933 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
1937 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
1938 {
1939 spans.push(span);
1940 }
1941 continue;
1942 }
1943 "b" | "strong" => new_state.bold = true,
1944 "i" | "em" => new_state.italic = true,
1945 "u" | "ins" => new_state.underline = true,
1946 "s" | "del" | "strike" => new_state.strikeout = true,
1947 "code" => new_state.code = true,
1948 "sup" => new_state.superscript = true,
1949 "sub" => new_state.subscript = true,
1950 "a" => {
1951 if let Some(href) = el.attr("href") {
1952 new_state.link_href = Some(href.to_string());
1953 }
1954 }
1955 "img" => {
1956 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
1957 spans.push(span);
1958 }
1959 continue;
1960 }
1961 "br" => {
1967 spans.push(text_span(" ".to_string(), state));
1968 continue;
1969 }
1970 "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
1971 | "h3" | "h4" | "h5" | "h6" => {
1972 spans.push(text_span(" ".to_string(), state));
1973 }
1974 _ => {}
1975 }
1976 collect_cell_spans(child, &new_state, spans, depth + 1, limit);
1977 }
1978 _ => {}
1979 }
1980 }
1981 }
1982
1983 const MAX_HTML_COLSPAN: usize = 1000;
1986
1987 fn parse_table_element(
1991 table_node: ego_tree::NodeRef<Node>,
1992 depth: usize,
1993 limit: usize,
1994 ) -> (ParsedTable, usize) {
1995 let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
1996 let mut header_rows: usize = 0;
1997 let mut own_cells: usize = 0;
1998
1999 fn collect_rows(
2001 node: ego_tree::NodeRef<Node>,
2002 rows: &mut Vec<Vec<ParsedTableCell>>,
2003 header_rows: &mut usize,
2004 own_cells: &mut usize,
2005 in_thead: bool,
2006 walk: (usize, usize),
2007 ) {
2008 let (depth, limit) = walk;
2009 for child in node.children() {
2010 if let Node::Element(el) = child.value() {
2011 match el.name() {
2012 "thead" => collect_rows(
2013 child,
2014 rows,
2015 header_rows,
2016 own_cells,
2017 true,
2018 (depth + 1, limit),
2019 ),
2020 "tbody" | "tfoot" => collect_rows(
2021 child,
2022 rows,
2023 header_rows,
2024 own_cells,
2025 false,
2026 (depth + 1, limit),
2027 ),
2028 "tr" => {
2029 let mut cells: Vec<ParsedTableCell> = Vec::new();
2030 let mut own_end = 0;
2033 for td in child.children() {
2034 if let Node::Element(td_el) = td.value()
2035 && matches!(td_el.name(), "td" | "th")
2036 {
2037 let mut spans = Vec::new();
2038 let state = FmtState::default();
2039 collect_cell_spans(td, &state, &mut spans, depth + 2, limit);
2040 collapse_inline_whitespace(&mut spans);
2041 if spans.is_empty() {
2042 spans.push(ParsedSpan::default());
2043 }
2044 cells.push(ParsedTableCell { spans });
2045 *own_cells += 1;
2046 own_end = cells.len();
2047 let span = td_el
2052 .attr("colspan")
2053 .and_then(|value| value.trim().parse::<usize>().ok())
2054 .unwrap_or(1)
2055 .clamp(1, MAX_HTML_COLSPAN);
2056 for _ in 1..span {
2057 if cells.len() >= MAX_HTML_COLSPAN {
2058 break;
2059 }
2060 cells.push(ParsedTableCell {
2061 spans: vec![ParsedSpan::default()],
2062 });
2063 }
2064 }
2065 }
2066 cells.truncate(own_end);
2070 if !cells.is_empty() {
2071 rows.push(cells);
2072 if in_thead {
2073 *header_rows += 1;
2074 }
2075 }
2076 }
2077 _ => {}
2078 }
2079 }
2080 }
2081 }
2082
2083 collect_rows(
2084 table_node,
2085 &mut rows,
2086 &mut header_rows,
2087 &mut own_cells,
2088 false,
2089 (depth, limit),
2090 );
2091
2092 if header_rows == 0 && !rows.is_empty() {
2094 header_rows = 1;
2095 }
2096
2097 (
2098 ParsedTable {
2099 header_rows,
2100 rows,
2101 blockquote_depth: 0,
2104 blockquote_opens: 0,
2105 },
2106 own_cells,
2107 )
2108 }
2109
2110 #[allow(clippy::too_many_arguments)]
2111 fn walk_node(
2112 node: ego_tree::NodeRef<Node>,
2113 state: &FmtState,
2114 elements: &mut Vec<ParsedElement>,
2115 current_list: &HtmlList,
2116 blockquote_depth: u32,
2117 list_depth: u32,
2118 depth: usize,
2119 limit: usize,
2120 ) {
2121 if depth > limit {
2122 let text = subtree_text(node);
2123 if !text.trim().is_empty() {
2124 let mut spans = vec![text_span(text, state)];
2125 collapse_inline_whitespace(&mut spans);
2126 elements.push(ParsedElement::Block(ParsedBlock {
2127 spans,
2128 blockquote_depth,
2129 blockquote_opens: 0,
2130 ..ParsedBlock::default()
2131 }));
2132 }
2133 return;
2134 }
2135 match node.value() {
2136 Node::Element(el) => {
2137 let tag = el.name();
2138 if is_metadata_tag(tag) {
2139 return;
2140 }
2141 let mut new_state = state.clone();
2142 if let Some(keep) = el.attr("style").and_then(style_keeps_whitespace) {
2143 new_state.preserve_whitespace = keep;
2144 }
2145 let mut new_list = current_list.clone();
2146 let mut bq_depth = blockquote_depth;
2147 let mut new_list_depth = list_depth;
2148
2149 let is_block_tag = matches!(
2151 tag,
2152 "p" | "div"
2153 | "h1"
2154 | "h2"
2155 | "h3"
2156 | "h4"
2157 | "h5"
2158 | "h6"
2159 | "li"
2160 | "pre"
2161 | "br"
2162 | "blockquote"
2163 | "body"
2164 | "html"
2165 );
2166
2167 match tag {
2169 "b" | "strong" => new_state.bold = true,
2170 "i" | "em" => new_state.italic = true,
2171 "u" | "ins" => new_state.underline = true,
2172 "s" | "del" | "strike" => new_state.strikeout = true,
2173 "code" => new_state.code = true,
2174 "sup" => new_state.superscript = true,
2175 "sub" => new_state.subscript = true,
2176 "a" => {
2177 if let Some(href) = el.attr("href") {
2178 new_state.link_href = Some(href.to_string());
2179 }
2180 }
2181 "ul" => {
2182 new_list = (Some(ListStyle::Disc), None);
2183 new_list_depth = list_depth + 1;
2184 }
2185 "ol" => {
2186 let start = el
2190 .attr("start")
2191 .and_then(html_integer)
2192 .and_then(crate::parser_tools::list_start::list_start);
2193 new_list = (Some(ListStyle::Decimal), start);
2194 new_list_depth = list_depth + 1;
2195 }
2196 "blockquote" => {
2197 bq_depth += 1;
2198 }
2199 _ => {}
2200 }
2201
2202 let heading_level = match tag {
2204 "h1" => Some(1),
2205 "h2" => Some(2),
2206 "h3" => Some(3),
2207 "h4" => Some(4),
2208 "h5" => Some(5),
2209 "h6" => Some(6),
2210 _ => None,
2211 };
2212
2213 let is_code_block = tag == "pre";
2214
2215 let code_language = if is_code_block {
2217 node.children().find_map(|child| {
2218 if let Node::Element(cel) = child.value()
2219 && cel.name() == "code"
2220 && let Some(cls) = cel.attr("class")
2221 {
2222 return cls
2223 .split_whitespace()
2224 .find_map(|c| c.strip_prefix("language-"))
2225 .map(|l| l.to_string());
2226 }
2227 None
2228 })
2229 } else {
2230 None
2231 };
2232
2233 let css = if is_block_tag {
2235 el.attr("style").map(parse_block_styles).unwrap_or_default()
2236 } else {
2237 BlockStyles::default()
2238 };
2239
2240 if tag == "table" {
2241 let (mut parsed_table, own_cells) = parse_table_element(node, depth, limit);
2243 if !parsed_table.rows.is_empty() {
2244 parsed_table.blockquote_depth = bq_depth;
2245 push_table(elements, parsed_table, own_cells);
2246 }
2247 return;
2248 }
2249
2250 if tag == "br" {
2251 elements.push(ParsedElement::Block(ParsedBlock {
2253 spans: vec![ParsedSpan {
2254 text: String::new(),
2255 ..Default::default()
2256 }],
2257 heading_level: None,
2258 list_style: None,
2259 list_indent: 0,
2260 list_prefix: String::new(),
2261 list_suffix: String::new(),
2262 list_start: None,
2263 marker: None,
2264 is_code_block: false,
2265 code_language: None,
2266 blockquote_depth: bq_depth,
2267 blockquote_opens: 0,
2268 line_height: None,
2269 non_breakable_lines: None,
2270 page_break_before: None,
2271 front_matter: None,
2272 direction: None,
2273 background_color: None,
2274 alignment: None,
2275 top_margin: None,
2276 text_indent: None,
2277 semantic_role: None,
2278 }));
2279 return;
2280 }
2281
2282 if tag == "blockquote" {
2283 let first = elements.len();
2284 let inline_only = node.children().all(|child| match child.value() {
2289 Node::Element(el) => !matches!(
2290 el.name(),
2291 "p" | "div"
2292 | "h1"
2293 | "h2"
2294 | "h3"
2295 | "h4"
2296 | "h5"
2297 | "h6"
2298 | "li"
2299 | "pre"
2300 | "blockquote"
2301 | "ul"
2302 | "ol"
2303 | "table"
2304 ),
2305 _ => true,
2306 });
2307 if inline_only {
2308 let mut spans: Vec<ParsedSpan> = Vec::new();
2309 let mut nested_elements: Vec<ParsedElement> = Vec::new();
2310 collect_inline_spans(
2311 node,
2312 &new_state,
2313 &mut spans,
2314 &new_list,
2315 &mut nested_elements,
2316 bq_depth,
2317 new_list_depth,
2318 depth + 1,
2319 limit,
2320 );
2321 collapse_inline_whitespace(&mut spans);
2322 if spans_carry_content(&spans) {
2323 elements.push(ParsedElement::Block(ParsedBlock {
2324 spans,
2325 blockquote_depth: bq_depth,
2326 blockquote_opens: 0,
2327 ..ParsedBlock::default()
2328 }));
2329 }
2330 elements.append(&mut nested_elements);
2331 } else {
2332 for child in node.children() {
2334 walk_node(
2335 child,
2336 &new_state,
2337 elements,
2338 &new_list,
2339 bq_depth,
2340 new_list_depth,
2341 depth + 1,
2342 limit,
2343 );
2344 }
2345 }
2346 mark_quotation_opened(elements, first);
2347 } else if is_block_tag && tag != "br" {
2348 let mut spans: Vec<ParsedSpan> = Vec::new();
2353 let mut nested_elements: Vec<ParsedElement> = Vec::new();
2354 collect_inline_spans(
2355 node,
2356 &new_state,
2357 &mut spans,
2358 &new_list,
2359 &mut nested_elements,
2360 bq_depth,
2361 new_list_depth,
2362 depth + 1,
2363 limit,
2364 );
2365
2366 let (list_style_for_block, list_start_for_block) = if tag == "li" {
2367 new_list.clone()
2368 } else {
2369 (None, None)
2370 };
2371
2372 let list_indent_for_block = if tag == "li" {
2373 new_list_depth.saturating_sub(1)
2374 } else {
2375 0
2376 };
2377
2378 if !(is_code_block || css.preserve_whitespace == Some(true)) {
2379 collapse_inline_whitespace(&mut spans);
2380 }
2381
2382 let own_run_is_content =
2390 spans_carry_content(&spans) || nested_elements.is_empty();
2391
2392 if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
2393 let is_code_block = is_code_block
2397 && !spans
2398 .iter()
2399 .any(|span| span.image.is_some() || span.footnote_ref.is_some());
2400 let block = ParsedBlock {
2401 spans,
2402 heading_level,
2403 list_style: list_style_for_block,
2404 list_indent: list_indent_for_block,
2405 list_prefix: String::new(),
2406 list_suffix: String::new(),
2407 list_start: list_start_for_block,
2408 marker: None,
2409 is_code_block,
2410 code_language: code_language.filter(|_| is_code_block),
2411 blockquote_depth: bq_depth,
2412 blockquote_opens: 0,
2413 line_height: css.line_height,
2414 non_breakable_lines: css.non_breakable_lines,
2415 page_break_before: css.page_break_before,
2416 front_matter: None,
2417 direction: css.direction,
2418 background_color: css.background_color,
2419 alignment: None,
2420 top_margin: None,
2421 text_indent: None,
2422 semantic_role: None,
2423 };
2424 if is_code_block {
2428 elements.push(ParsedElement::Block(block));
2429 } else {
2430 elements.extend(
2431 split_block_at_line_breaks(block)
2432 .into_iter()
2433 .map(ParsedElement::Block),
2434 );
2435 }
2436 }
2437 elements.append(&mut nested_elements);
2439 } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
2440 for child in node.children() {
2442 walk_node(
2443 child,
2444 &new_state,
2445 elements,
2446 &new_list,
2447 bq_depth,
2448 new_list_depth,
2449 depth + 1,
2450 limit,
2451 );
2452 }
2453 } else {
2454 for child in node.children() {
2456 walk_node(
2457 child,
2458 &new_state,
2459 elements,
2460 current_list,
2461 bq_depth,
2462 list_depth,
2463 depth + 1,
2464 limit,
2465 );
2466 }
2467 }
2468 }
2469 Node::Text(text) => {
2470 let mut spans = vec![text_span(text.text.to_string(), state)];
2475 collapse_inline_whitespace(&mut spans);
2476 if spans_carry_content(&spans) {
2477 let block = ParsedBlock {
2478 spans,
2479 blockquote_depth,
2480 blockquote_opens: 0,
2481 ..ParsedBlock::default()
2482 };
2483 elements.extend(
2484 split_block_at_line_breaks(block)
2485 .into_iter()
2486 .map(ParsedElement::Block),
2487 );
2488 }
2489 }
2490 _ => {
2491 for child in node.children() {
2493 walk_node(
2494 child,
2495 state,
2496 elements,
2497 current_list,
2498 blockquote_depth,
2499 list_depth,
2500 depth + 1,
2501 limit,
2502 );
2503 }
2504 }
2505 }
2506 }
2507
2508 #[allow(clippy::too_many_arguments)]
2512 fn collect_inline_spans(
2513 node: ego_tree::NodeRef<Node>,
2514 state: &FmtState,
2515 spans: &mut Vec<ParsedSpan>,
2516 current_list: &HtmlList,
2517 elements: &mut Vec<ParsedElement>,
2518 blockquote_depth: u32,
2519 list_depth: u32,
2520 depth: usize,
2521 limit: usize,
2522 ) {
2523 if depth > limit {
2524 let text = subtree_text(node);
2525 if !text.is_empty() {
2526 spans.push(text_span(text, state));
2527 }
2528 return;
2529 }
2530 for child in node.children() {
2531 match child.value() {
2532 Node::Text(text) => {
2533 let t = text.text.to_string();
2534 if !t.is_empty() {
2535 spans.push(ParsedSpan {
2536 text: t,
2537 bold: state.bold,
2538 italic: state.italic,
2539 underline: state.underline,
2540 strikeout: state.strikeout,
2541 code: state.code,
2542 superscript: state.superscript,
2543 subscript: state.subscript,
2544 link_href: state.link_href.clone(),
2545 image: None,
2546 footnote_ref: None,
2547 preserve_whitespace: state.preserve_whitespace,
2548 });
2549 }
2550 }
2551 Node::Element(el) => {
2552 let tag = el.name();
2553 if is_metadata_tag(tag) {
2554 continue;
2555 }
2556 let mut new_state = state.clone();
2557 if let Some(keep) = el.attr("style").and_then(style_keeps_whitespace) {
2558 new_state.preserve_whitespace = keep;
2559 }
2560
2561 match tag {
2562 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2566 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2567 {
2568 spans.push(span);
2569 }
2570 continue;
2571 }
2572 "b" | "strong" => new_state.bold = true,
2573 "i" | "em" => new_state.italic = true,
2574 "u" | "ins" => new_state.underline = true,
2575 "s" | "del" | "strike" => new_state.strikeout = true,
2576 "code" => new_state.code = true,
2577 "sup" => new_state.superscript = true,
2578 "sub" => new_state.subscript = true,
2579 "a" => {
2580 if let Some(href) = el.attr("href") {
2581 new_state.link_href = Some(href.to_string());
2582 }
2583 }
2584 "img" => {
2585 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2586 spans.push(span);
2587 }
2588 continue;
2589 }
2590 _ => {}
2591 }
2592
2593 let nested_block = matches!(
2595 tag,
2596 "p" | "div"
2597 | "h1"
2598 | "h2"
2599 | "h3"
2600 | "h4"
2601 | "h5"
2602 | "h6"
2603 | "li"
2604 | "pre"
2605 | "blockquote"
2606 | "ul"
2607 | "ol"
2608 );
2609
2610 if tag == "br" {
2611 spans.push(text_span(LINE_SEPARATOR_STR.to_string(), state));
2615 } else if nested_block || tag == "table" {
2616 walk_node(
2618 child,
2619 &new_state,
2620 elements,
2621 current_list,
2622 blockquote_depth,
2623 list_depth,
2624 depth + 1,
2625 limit,
2626 );
2627 } else {
2628 collect_inline_spans(
2630 child,
2631 &new_state,
2632 spans,
2633 current_list,
2634 elements,
2635 blockquote_depth,
2636 list_depth,
2637 depth + 1,
2638 limit,
2639 );
2640 }
2641 }
2642 _ => {}
2643 }
2644 }
2645 }
2646
2647 let initial_state = FmtState::default();
2648 let mut root_spans: Vec<ParsedSpan> = Vec::new();
2652 collect_inline_spans(
2653 *root,
2654 &initial_state,
2655 &mut root_spans,
2656 &(None, None),
2657 &mut elements,
2658 0,
2659 0,
2660 0,
2661 depth_limit,
2662 );
2663 collapse_inline_whitespace(&mut root_spans);
2664 if spans_carry_content(&root_spans) {
2667 elements.push(ParsedElement::Block(ParsedBlock {
2668 spans: root_spans,
2669 heading_level: None,
2670 list_style: None,
2671 list_indent: 0,
2672 list_prefix: String::new(),
2673 list_suffix: String::new(),
2674 list_start: None,
2675 marker: None,
2676 is_code_block: false,
2677 code_language: None,
2678 blockquote_depth: 0,
2679 blockquote_opens: 0,
2680 line_height: None,
2681 non_breakable_lines: None,
2682 page_break_before: None,
2683 front_matter: None,
2684 direction: None,
2685 background_color: None,
2686 alignment: None,
2687 top_margin: None,
2688 text_indent: None,
2689 semantic_role: None,
2690 }));
2691 }
2692
2693 if elements.is_empty() {
2695 elements.push(ParsedElement::Block(ParsedBlock {
2696 spans: vec![ParsedSpan {
2697 text: String::new(),
2698 ..Default::default()
2699 }],
2700 heading_level: None,
2701 list_style: None,
2702 list_indent: 0,
2703 list_prefix: String::new(),
2704 list_suffix: String::new(),
2705 list_start: None,
2706 marker: None,
2707 is_code_block: false,
2708 code_language: None,
2709 blockquote_depth: 0,
2710 blockquote_opens: 0,
2711 line_height: None,
2712 non_breakable_lines: None,
2713 page_break_before: None,
2714 front_matter: None,
2715 direction: None,
2716 background_color: None,
2717 alignment: None,
2718 top_margin: None,
2719 text_indent: None,
2720 semantic_role: None,
2721 }));
2722 }
2723
2724 elements
2725}
2726
2727pub fn character_format_from_span(
2731 span: &ParsedSpan,
2732 is_code_block: bool,
2733) -> crate::format_runs::CharacterFormat {
2734 use crate::entities::CharVerticalAlignment;
2735 crate::format_runs::CharacterFormat {
2736 font_bold: if span.bold { Some(true) } else { None },
2737 font_italic: if span.italic { Some(true) } else { None },
2738 font_underline: if span.underline { Some(true) } else { None },
2739 font_strikeout: if span.strikeout { Some(true) } else { None },
2740 font_family: if span.code || is_code_block {
2741 Some("monospace".to_string())
2742 } else {
2743 None
2744 },
2745 anchor_href: span.link_href.clone(),
2746 is_anchor: if span.link_href.is_some() {
2747 Some(true)
2748 } else {
2749 None
2750 },
2751 vertical_alignment: if span.superscript {
2752 Some(CharVerticalAlignment::SuperScript)
2753 } else if span.subscript {
2754 Some(CharVerticalAlignment::SubScript)
2755 } else {
2756 None
2757 },
2758 ..Default::default()
2759 }
2760}
2761
2762pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
2774 use crate::format_runs::{
2775 CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
2776 };
2777
2778 let mut plain_text = String::new();
2779 let mut runs: Vec<FormatRun> = Vec::new();
2780 let mut images: Vec<ImageAnchor> = Vec::new();
2781 let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
2782 let default = CharacterFormat::default();
2783
2784 for span in spans {
2785 let byte_start = plain_text.len() as u32;
2786
2787 if let Some(label) = &span.footnote_ref {
2788 plain_text.push('\u{FFFC}');
2791 let mut format = character_format_from_span(span, is_code_block);
2806 format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
2807 footnote_refs.push(FootnoteRefAnchor {
2808 byte_offset: byte_start,
2809 label: label.clone(),
2810 format,
2811 });
2812 continue;
2813 }
2814
2815 if let Some(image) = &span.image {
2816 plain_text.push('\u{FFFC}');
2820 images.push(ImageAnchor {
2821 byte_offset: byte_start,
2822 name: image.src.clone(),
2823 alt: image.alt.clone(),
2824 width: image.width,
2825 height: image.height,
2826 quality: 100,
2827 format: character_format_from_span(span, is_code_block),
2828 });
2829 continue;
2830 }
2831
2832 plain_text.push_str(&span.text);
2833 let byte_end = plain_text.len() as u32;
2834 if byte_start == byte_end {
2835 continue;
2836 }
2837 let format = character_format_from_span(span, is_code_block);
2838 if format == default {
2839 continue;
2840 }
2841 runs.push(FormatRun {
2842 byte_start,
2843 byte_end,
2844 format,
2845 });
2846 }
2847 coalesce_in_place(&mut runs);
2848 ParsedInline {
2849 plain_text,
2850 runs,
2851 images,
2852 footnote_refs,
2853 }
2854}
2855
2856#[derive(Debug, Clone, Default)]
2863pub struct ParsedInline {
2864 pub plain_text: String,
2865 pub runs: Vec<crate::format_runs::FormatRun>,
2866 pub images: Vec<crate::format_runs::ImageAnchor>,
2867 pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
2868}
2869
2870fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
2878 use jotdown::ListBulletType as B;
2879 match b {
2880 B::Dash => ListStyle::Disc,
2881 B::Star => ListStyle::Circle,
2882 B::Plus => ListStyle::Square,
2883 }
2884}
2885
2886fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
2888 use jotdown::OrderedListNumbering as N;
2889 match n {
2890 N::Decimal => ListStyle::Decimal,
2891 N::AlphaLower => ListStyle::LowerAlpha,
2892 N::AlphaUpper => ListStyle::UpperAlpha,
2893 N::RomanLower => ListStyle::LowerRoman,
2894 N::RomanUpper => ListStyle::UpperRoman,
2895 }
2896}
2897
2898fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
2902 use jotdown::OrderedListStyle as S;
2903 match style {
2904 S::Period => (String::new(), ".".to_string()),
2905 S::Paren => (String::new(), ")".to_string()),
2906 S::ParenParen => ("(".to_string(), ")".to_string()),
2907 }
2908}
2909
2910#[derive(Debug, Clone, Default)]
2914struct DjotBlockStyle {
2915 alignment: Option<Alignment>,
2916 line_height: Option<i64>,
2917 non_breakable_lines: Option<bool>,
2918 page_break_before: Option<bool>,
2919 front_matter: Option<bool>,
2920 direction: Option<TextDirection>,
2921 background_color: Option<String>,
2922 top_margin: Option<i64>,
2923 text_indent: Option<i64>,
2924 semantic_role: Option<SemanticRole>,
2925}
2926
2927impl DjotBlockStyle {
2928 fn merge_from(&mut self, other: DjotBlockStyle) {
2932 if other.alignment.is_some() {
2933 self.alignment = other.alignment;
2934 }
2935 if other.line_height.is_some() {
2936 self.line_height = other.line_height;
2937 }
2938 if other.non_breakable_lines.is_some() {
2939 self.non_breakable_lines = other.non_breakable_lines;
2940 }
2941 if other.page_break_before.is_some() {
2942 self.page_break_before = other.page_break_before;
2943 }
2944 if other.front_matter.is_some() {
2945 self.front_matter = other.front_matter;
2946 }
2947 if other.direction.is_some() {
2948 self.direction = other.direction;
2949 }
2950 if other.background_color.is_some() {
2951 self.background_color = other.background_color;
2952 }
2953 if other.top_margin.is_some() {
2954 self.top_margin = other.top_margin;
2955 }
2956 if other.text_indent.is_some() {
2957 self.text_indent = other.text_indent;
2958 }
2959 if other.semantic_role.is_some() {
2960 self.semantic_role = other.semantic_role.clone();
2961 }
2962 }
2963}
2964
2965fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
2971 let mut style = DjotBlockStyle::default();
2972
2973 if opts.alignment
2974 && let Some(v) = attrs.get_value("alignment")
2975 {
2976 style.alignment = match v.to_string().as_str() {
2977 "left" => Some(Alignment::Left),
2978 "right" => Some(Alignment::Right),
2979 "center" => Some(Alignment::Center),
2980 "justify" => Some(Alignment::Justify),
2981 _ => None,
2982 };
2983 }
2984 if opts.line_height
2985 && let Some(v) = attrs.get_value("line_height")
2986 {
2987 style.line_height = v.to_string().parse::<i64>().ok();
2988 }
2989 if opts.direction
2990 && let Some(v) = attrs.get_value("direction")
2991 {
2992 style.direction = match v.to_string().as_str() {
2993 "ltr" => Some(TextDirection::LeftToRight),
2994 "rtl" => Some(TextDirection::RightToLeft),
2995 _ => None,
2996 };
2997 }
2998 if opts.non_breakable_lines
2999 && let Some(v) = attrs.get_value("non_breakable_lines")
3000 {
3001 style.non_breakable_lines = match v.to_string().as_str() {
3002 "true" => Some(true),
3003 "false" => Some(false),
3004 _ => None,
3005 };
3006 }
3007 if opts.page_break_before
3008 && let Some(v) = attrs.get_value("page_break_before")
3009 {
3010 style.page_break_before = match v.to_string().as_str() {
3011 "true" => Some(true),
3012 "false" => Some(false),
3013 _ => None,
3014 };
3015 }
3016 if opts.front_matter
3017 && let Some(v) = attrs.get_value("front_matter")
3018 {
3019 style.front_matter = match v.to_string().as_str() {
3020 "true" => Some(true),
3021 "false" => Some(false),
3022 _ => None,
3023 };
3024 }
3025 if opts.background_color
3026 && let Some(v) = attrs.get_value("background_color")
3027 {
3028 style.background_color = Some(v.to_string());
3029 }
3030 if opts.top_margin
3031 && let Some(v) = attrs.get_value("top_margin")
3032 {
3033 style.top_margin = v.to_string().parse::<i64>().ok();
3034 }
3035 if opts.text_indent
3036 && let Some(v) = attrs.get_value("text_indent")
3037 {
3038 style.text_indent = v.to_string().parse::<i64>().ok();
3039 }
3040 if opts.semantic_role
3041 && let Some(v) = attrs.get_value("semantic_role")
3042 {
3043 style.semantic_role = match v.to_string().as_str() {
3044 "epigraph" => Some(SemanticRole::Epigraph),
3045 _ => None,
3049 };
3050 }
3051
3052 style
3053}
3054
3055fn list_start_of(start: u64) -> Option<i64> {
3059 crate::parser_tools::list_start::list_start_from_unsigned(start)
3060}
3061
3062#[allow(clippy::too_many_arguments)]
3065fn djot_push_block(
3066 elements: &mut Vec<ParsedElement>,
3067 spans: Vec<ParsedSpan>,
3068 heading_level: Option<i64>,
3069 list_style: Option<ListStyle>,
3070 list_indent: u32,
3071 list_prefix: String,
3072 list_suffix: String,
3073 list_start: Option<i64>,
3074 marker: Option<MarkerType>,
3075 is_code_block: bool,
3076 code_language: Option<String>,
3077 blockquote_depth: u32,
3078 style: DjotBlockStyle,
3079) {
3080 elements.push(ParsedElement::Block(ParsedBlock {
3081 spans,
3082 heading_level,
3083 list_style,
3084 list_indent,
3085 list_prefix,
3086 list_suffix,
3087 list_start,
3088 marker,
3089 is_code_block,
3090 code_language,
3091 blockquote_depth,
3092 blockquote_opens: 0,
3093 line_height: style.line_height,
3094 non_breakable_lines: style.non_breakable_lines,
3095 page_break_before: style.page_break_before,
3096 front_matter: style.front_matter,
3097 direction: style.direction,
3098 background_color: style.background_color,
3099 alignment: style.alignment,
3100 top_margin: style.top_margin,
3101 text_indent: style.text_indent,
3102 semantic_role: style.semantic_role.clone(),
3103 }));
3104}
3105
3106fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
3119 djot.lines()
3120 .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
3121 .filter(|line| !line.is_empty())
3122 .map(|line| {
3123 ParsedElement::Block(ParsedBlock {
3124 spans: vec![ParsedSpan {
3125 text: line.to_string(),
3126 ..Default::default()
3127 }],
3128 ..Default::default()
3129 })
3130 })
3131 .collect()
3132}
3133
3134pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
3155 use jotdown::{Container as C, Event as E, ListKind};
3156
3157 let parsable = crate::parser_tools::djot_depth::parsable(djot);
3183 let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
3184 return djot_source_lines(djot);
3185 };
3186
3187 let mut elements: Vec<ParsedElement> = Vec::new();
3188 let mut current_spans: Vec<ParsedSpan> = Vec::new();
3189 let mut current_heading: Option<i64> = None;
3190 let mut is_code_block = false;
3191 let mut code_language: Option<String> = None;
3192 let mut blockquote_depth: u32 = 0;
3193 let mut quotations_open: Vec<usize> = Vec::new();
3195 let mut pending_style = DjotBlockStyle::default();
3198
3199 let mut bold = false;
3201 let mut italic = false;
3202 let mut underline = false;
3203 let mut strikeout = false;
3204 let mut code = false;
3205 let mut superscript = false;
3206 let mut subscript = false;
3207 let mut link_href: Option<String> = None;
3208 let mut pending_image: Option<ParsedImage> = None;
3211
3212 let mut list_stack: Vec<(ListStyle, String, String, Option<i64>)> = Vec::new();
3214 let mut cur_list_style: Option<ListStyle> = None;
3216 let mut cur_list_prefix = String::new();
3217 let mut cur_list_suffix = String::new();
3218 let mut cur_list_start: Option<i64> = None;
3219 let mut cur_list_indent: u32 = 0;
3220 let mut cur_marker: Option<MarkerType> = None;
3221
3222 let mut in_table_cell = false;
3224 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
3225 let mut current_row: Vec<ParsedTableCell> = Vec::new();
3226 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
3227 let mut table_header_rows: usize = 0;
3228 let mut row_is_head = false;
3229
3230 let mut skip_depth: u32 = 0;
3234
3235 let mut footnote_open: Option<(String, usize)> = None;
3239
3240 macro_rules! push_text {
3245 ($t:expr) => {{
3246 if let Some(img) = pending_image.as_mut() {
3251 img.alt.push_str(($t).as_ref());
3252 } else {
3253 let sp = ParsedSpan {
3254 text: ($t).to_string(),
3255 bold,
3256 italic,
3257 underline,
3258 strikeout,
3259 code,
3260 superscript,
3261 subscript,
3262 link_href: link_href.clone(),
3263 image: None,
3264 footnote_ref: None,
3265 preserve_whitespace: false,
3266 };
3267 if in_table_cell {
3268 current_cell_spans.push(sp);
3269 } else {
3270 current_spans.push(sp);
3271 }
3272 }
3273 }};
3274 }
3275
3276 macro_rules! push_image {
3278 ($img:expr) => {{
3279 let sp = ParsedSpan {
3280 text: String::new(),
3281 bold,
3282 italic,
3283 underline,
3284 strikeout,
3285 code,
3286 superscript,
3287 subscript,
3288 link_href: link_href.clone(),
3289 image: Some($img),
3290 footnote_ref: None,
3291 preserve_whitespace: false,
3292 };
3293 if in_table_cell {
3294 current_cell_spans.push(sp);
3295 } else {
3296 current_spans.push(sp);
3297 }
3298 }};
3299 }
3300
3301 macro_rules! enter_item {
3304 ($marker:expr) => {{
3305 if !current_spans.is_empty() {
3306 djot_push_block(
3307 &mut elements,
3308 std::mem::take(&mut current_spans),
3309 None,
3310 cur_list_style.clone(),
3311 cur_list_indent,
3312 cur_list_prefix.clone(),
3313 cur_list_suffix.clone(),
3314 cur_list_start,
3315 cur_marker.clone(),
3316 false,
3317 None,
3318 blockquote_depth,
3319 DjotBlockStyle::default(),
3320 );
3321 }
3322 let (style, prefix, suffix, start) = list_stack.last().cloned().unwrap_or((
3323 ListStyle::Disc,
3324 String::new(),
3325 String::new(),
3326 None,
3327 ));
3328 cur_list_style = Some(style);
3329 cur_list_prefix = prefix;
3330 cur_list_suffix = suffix;
3331 cur_list_start = start;
3332 cur_list_indent = list_stack.len().saturating_sub(1) as u32;
3333 cur_marker = $marker;
3334 }};
3335 }
3336
3337 macro_rules! line_break {
3341 () => {{
3342 if in_table_cell || pending_image.is_some() {
3343 push_text!(" ");
3344 } else {
3345 push_text!(LINE_SEPARATOR_STR);
3346 }
3347 }};
3348 }
3349
3350 macro_rules! split_line {
3357 () => {{
3358 if in_table_cell {
3359 push_text!(" ");
3360 } else if !current_spans.is_empty() && current_heading.is_some() {
3361 djot_push_block(
3366 &mut elements,
3367 std::mem::take(&mut current_spans),
3368 current_heading,
3369 None,
3370 0,
3371 String::new(),
3372 String::new(),
3373 None,
3374 None,
3375 false,
3376 None,
3377 blockquote_depth,
3378 pending_style.clone(),
3379 );
3380 pending_style.page_break_before = None;
3381 } else if !current_spans.is_empty() {
3382 djot_push_block(
3383 &mut elements,
3384 std::mem::take(&mut current_spans),
3385 None,
3386 cur_list_style.clone(),
3387 cur_list_indent,
3388 cur_list_prefix.clone(),
3389 cur_list_suffix.clone(),
3390 cur_list_start,
3391 cur_marker.clone(),
3392 is_code_block,
3393 code_language.clone(),
3394 blockquote_depth,
3395 pending_style.clone(),
3396 );
3397 pending_style.page_break_before = None;
3398 }
3399 }};
3400 }
3401
3402 for event in events {
3403 if skip_depth > 0 {
3404 match event {
3405 E::Start(..) => skip_depth += 1,
3406 E::End(_) => skip_depth -= 1,
3407 _ => {}
3408 }
3409 continue;
3410 }
3411
3412 match event {
3413 E::Start(C::Document, _) | E::End(C::Document) => {}
3415 E::Start(C::Section { .. }, attrs) => {
3416 if list_stack.is_empty() {
3419 pending_style.merge_from(block_attrs_to_style(&attrs, options));
3420 }
3421 }
3422 E::End(C::Section { .. }) => {}
3423 E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
3424
3425 E::Start(C::Blockquote, _) => {
3427 blockquote_depth += 1;
3428 quotations_open.push(elements.len());
3429 }
3430 E::End(C::Blockquote) => {
3431 blockquote_depth = blockquote_depth.saturating_sub(1);
3432 if let Some(first) = quotations_open.pop() {
3433 mark_quotation_opened(&mut elements, first);
3434 }
3435 }
3436
3437 E::Start(C::List { kind, .. }, _) => {
3439 let (style, prefix, suffix, start) = match kind {
3440 ListKind::Unordered(b) | ListKind::Task(b) => {
3441 (djot_bullet_style(b), String::new(), String::new(), None)
3442 }
3443 ListKind::Ordered {
3444 numbering,
3445 style,
3446 start,
3447 } => {
3448 let (p, s) = djot_ordered_affixes(style);
3449 (djot_ordered_style(numbering), p, s, list_start_of(start))
3450 }
3451 };
3452 list_stack.push((style, prefix, suffix, start));
3453 }
3454 E::End(C::List { .. }) => {
3455 list_stack.pop();
3456 cur_list_style = None;
3457 cur_list_start = None;
3458 cur_marker = None;
3459 }
3460 E::Start(C::ListItem, _) => enter_item!(None),
3461 E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
3462 MarkerType::Checked
3463 } else {
3464 MarkerType::Unchecked
3465 })),
3466 E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
3467 if !current_spans.is_empty() {
3469 djot_push_block(
3470 &mut elements,
3471 std::mem::take(&mut current_spans),
3472 None,
3473 cur_list_style.clone(),
3474 cur_list_indent,
3475 cur_list_prefix.clone(),
3476 cur_list_suffix.clone(),
3477 cur_list_start,
3478 cur_marker.clone(),
3479 false,
3480 None,
3481 blockquote_depth,
3482 DjotBlockStyle::default(),
3483 );
3484 }
3485 cur_list_style = None;
3486 cur_list_start = None;
3487 cur_marker = None;
3488 }
3489
3490 E::Start(C::Heading { level, .. }, attrs) => {
3492 current_heading = Some(level as i64);
3493 pending_style.merge_from(block_attrs_to_style(&attrs, options));
3496 }
3497 E::End(C::Heading { .. }) => {
3498 djot_push_block(
3499 &mut elements,
3500 std::mem::take(&mut current_spans),
3501 current_heading.take(),
3502 None,
3503 0,
3504 String::new(),
3505 String::new(),
3506 None,
3507 None,
3508 false,
3509 None,
3510 blockquote_depth,
3511 std::mem::take(&mut pending_style),
3512 );
3513 }
3514 E::Start(C::Paragraph, attrs) => {
3515 current_heading = None;
3516 pending_style = if list_stack.is_empty() {
3520 block_attrs_to_style(&attrs, options)
3521 } else {
3522 DjotBlockStyle::default()
3523 };
3524 }
3525 E::End(C::Paragraph) => {
3526 if !current_spans.is_empty() {
3527 djot_push_block(
3528 &mut elements,
3529 std::mem::take(&mut current_spans),
3530 None,
3531 cur_list_style.clone(),
3532 cur_list_indent,
3533 cur_list_prefix.clone(),
3534 cur_list_suffix.clone(),
3535 cur_list_start,
3536 cur_marker.clone(),
3537 false,
3538 None,
3539 blockquote_depth,
3540 std::mem::take(&mut pending_style),
3541 );
3542 }
3543 cur_list_style = None;
3544 cur_list_start = None;
3545 cur_marker = None;
3546 }
3547 E::Start(C::CodeBlock { language }, _) => {
3548 is_code_block = true;
3549 code_language = if language.is_empty() {
3550 None
3551 } else {
3552 Some(language.to_string())
3553 };
3554 }
3555 E::End(C::CodeBlock { .. }) => {
3556 if let Some(last) = current_spans.last_mut()
3558 && last.text.ends_with('\n')
3559 {
3560 last.text.pop();
3561 }
3562 djot_push_block(
3563 &mut elements,
3564 std::mem::take(&mut current_spans),
3565 None,
3566 None,
3567 0,
3568 String::new(),
3569 String::new(),
3570 None,
3571 None,
3572 true,
3573 code_language.take(),
3574 blockquote_depth,
3575 DjotBlockStyle::default(),
3576 );
3577 is_code_block = false;
3578 }
3579
3580 E::Start(C::Table, _) => {
3582 table_rows.clear();
3583 current_row.clear();
3584 current_cell_spans.clear();
3585 table_header_rows = 0;
3586 }
3587 E::End(C::Table) => {
3588 let own_cells = cells_in_rows(&table_rows);
3589 push_table(
3590 &mut elements,
3591 ParsedTable {
3592 header_rows: table_header_rows,
3593 rows: std::mem::take(&mut table_rows),
3594 blockquote_depth,
3595 blockquote_opens: 0,
3596 },
3597 own_cells,
3598 );
3599 }
3600 E::Start(C::TableRow { head }, _) => {
3601 row_is_head = head;
3602 current_row.clear();
3603 }
3604 E::End(C::TableRow { .. }) => {
3605 if row_is_head {
3606 table_header_rows += 1;
3607 }
3608 table_rows.push(std::mem::take(&mut current_row));
3609 }
3610 E::Start(C::TableCell { .. }, _) => {
3611 in_table_cell = true;
3612 current_cell_spans.clear();
3613 }
3614 E::End(C::TableCell { .. }) => {
3615 in_table_cell = false;
3616 current_row.push(ParsedTableCell {
3617 spans: std::mem::take(&mut current_cell_spans),
3618 });
3619 }
3620
3621 E::Start(C::Strong, _) => bold = true,
3623 E::End(C::Strong) => bold = false,
3624 E::Start(C::Emphasis, _) => italic = true,
3625 E::End(C::Emphasis) => italic = false,
3626 E::Start(C::Verbatim, _) => code = true,
3627 E::End(C::Verbatim) => code = false,
3628 E::Start(C::Superscript, _) => superscript = true,
3629 E::End(C::Superscript) => superscript = false,
3630 E::Start(C::Subscript, _) => subscript = true,
3631 E::End(C::Subscript) => subscript = false,
3632 E::Start(C::Insert, _) => underline = true,
3633 E::End(C::Insert) => underline = false,
3634 E::Start(C::Delete, _) => strikeout = true,
3635 E::End(C::Delete) => strikeout = false,
3636 E::Start(C::Mark, _) | E::End(C::Mark) => {}
3638 E::Start(C::Span, _) | E::End(C::Span) => {}
3639 E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
3640 E::End(C::Link(..)) => link_href = None,
3641 E::Start(C::Image(src, _), attrs) => {
3646 let attr_num = |key: &str| -> i64 {
3647 attrs
3648 .get_value(key)
3649 .map(|v| v.to_string())
3650 .and_then(|v| v.trim().parse::<i64>().ok())
3651 .filter(|n| *n > 0)
3652 .unwrap_or(0)
3653 };
3654 pending_image = Some(ParsedImage {
3655 src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
3658 alt: String::new(),
3659 width: attr_num("width"),
3660 height: attr_num("height"),
3661 });
3662 }
3663 E::End(C::Image(..)) => {
3664 if let Some(img) = pending_image.take() {
3665 push_image!(img);
3666 }
3667 }
3668
3669 E::Start(C::Footnote { label }, _) => {
3678 if !current_spans.is_empty() {
3679 djot_push_block(
3680 &mut elements,
3681 std::mem::take(&mut current_spans),
3682 None,
3683 cur_list_style.clone(),
3684 cur_list_indent,
3685 cur_list_prefix.clone(),
3686 cur_list_suffix.clone(),
3687 cur_list_start,
3688 cur_marker.clone(),
3689 false,
3690 None,
3691 blockquote_depth,
3692 DjotBlockStyle::default(),
3693 );
3694 }
3695 footnote_open = Some((label.to_string(), elements.len()));
3696 }
3697 E::End(C::Footnote { .. }) => {
3698 if !current_spans.is_empty() {
3699 djot_push_block(
3700 &mut elements,
3701 std::mem::take(&mut current_spans),
3702 None,
3703 cur_list_style.clone(),
3704 cur_list_indent,
3705 cur_list_prefix.clone(),
3706 cur_list_suffix.clone(),
3707 cur_list_start,
3708 cur_marker.clone(),
3709 false,
3710 None,
3711 blockquote_depth,
3712 DjotBlockStyle::default(),
3713 );
3714 }
3715 if let Some((label, start)) = footnote_open.take() {
3716 let blocks: Vec<ParsedBlock> = elements
3717 .drain(start..)
3718 .filter_map(|e| match e {
3719 ParsedElement::Block(b) => Some(b),
3720 _ => None,
3724 })
3725 .collect();
3726 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
3727 }
3728 }
3729
3730 E::Start(
3732 C::Math { .. }
3733 | C::RawBlock { .. }
3734 | C::RawInline { .. }
3735 | C::DescriptionList
3736 | C::DescriptionDetails
3737 | C::DescriptionTerm
3738 | C::Caption
3739 | C::LinkDefinition { .. },
3740 _,
3741 ) => skip_depth = 1,
3742
3743 E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
3755 for (i, piece) in s.split('\n').enumerate() {
3756 if i > 0 {
3757 split_line!();
3758 }
3759 if !piece.is_empty() {
3760 push_text!(piece);
3761 }
3762 }
3763 }
3764 E::Str(s) => push_text!(s.as_ref()),
3765 E::Softbreak => push_text!(" "),
3766 E::LeftSingleQuote => push_text!("\u{2018}"),
3767 E::RightSingleQuote => push_text!("\u{2019}"),
3768 E::LeftDoubleQuote => push_text!("\u{201C}"),
3769 E::RightDoubleQuote => push_text!("\u{201D}"),
3770 E::Ellipsis => push_text!("\u{2026}"),
3771 E::EnDash => push_text!("\u{2013}"),
3772 E::EmDash => push_text!("\u{2014}"),
3773 E::NonBreakingSpace => push_text!("\u{00A0}"),
3774 E::Hardbreak => line_break!(),
3775 E::FootnoteReference(label) => {
3781 let sp = ParsedSpan {
3782 text: String::new(),
3783 bold,
3784 italic,
3785 underline,
3786 strikeout,
3787 code,
3788 superscript,
3789 subscript,
3790 link_href: link_href.clone(),
3791 image: None,
3792 footnote_ref: Some(label.to_string()),
3793 preserve_whitespace: false,
3794 };
3795 if in_table_cell {
3796 current_cell_spans.push(sp);
3797 } else {
3798 current_spans.push(sp);
3799 }
3800 }
3801 E::Symbol(_) => {}
3804 E::Escape | E::Blankline => {}
3805 E::ThematicBreak(_) | E::Attributes(_) => {}
3806
3807 _ => {}
3810 }
3811 }
3812
3813 if !current_spans.is_empty() {
3815 djot_push_block(
3816 &mut elements,
3817 std::mem::take(&mut current_spans),
3818 current_heading.take(),
3819 cur_list_style.clone(),
3820 cur_list_indent,
3821 cur_list_prefix.clone(),
3822 cur_list_suffix.clone(),
3823 cur_list_start,
3824 cur_marker.clone(),
3825 is_code_block,
3826 code_language.take(),
3827 blockquote_depth,
3828 std::mem::take(&mut pending_style),
3829 );
3830 }
3831
3832 if elements.is_empty() {
3835 djot_push_block(
3836 &mut elements,
3837 vec![ParsedSpan {
3838 text: String::new(),
3839 ..Default::default()
3840 }],
3841 None,
3842 None,
3843 0,
3844 String::new(),
3845 String::new(),
3846 None,
3847 None,
3848 false,
3849 None,
3850 0,
3851 DjotBlockStyle::default(),
3852 );
3853 }
3854
3855 elements
3856}
3857
3858#[cfg(test)]
3859mod tests {
3860 use super::*;
3861
3862 #[test]
3863 fn test_style_keeps_whitespace_reads_important() {
3864 assert_eq!(style_keeps_whitespace("white-space: pre-wrap"), Some(true));
3865 assert_eq!(
3866 style_keeps_whitespace("white-space: pre-wrap !important"),
3867 Some(true)
3868 );
3869 assert_eq!(
3870 style_keeps_whitespace("white-space:PRE!IMPORTANT; color: red"),
3871 Some(true)
3872 );
3873 assert_eq!(
3874 style_keeps_whitespace("white-space: pre ! important; white-space: normal"),
3875 Some(true),
3876 "a later declaration that is not important does not win"
3877 );
3878 assert_eq!(
3879 style_keeps_whitespace("white-space: pre; white-space: normal !important"),
3880 Some(false)
3881 );
3882 assert_eq!(
3883 style_keeps_whitespace("white-space: pre; white-space: normal"),
3884 Some(false)
3885 );
3886 assert_eq!(style_keeps_whitespace("white-space: inherit"), None);
3887 }
3888
3889 fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
3891 ParsedElement::flatten_to_blocks(parse_markdown(md))
3892 }
3893
3894 #[test]
3895 fn test_parse_markdown_simple_paragraph() {
3896 let blocks = parse_markdown_blocks("Hello **world**");
3897 assert_eq!(blocks.len(), 1);
3898 assert!(blocks[0].spans.len() >= 2);
3899 let plain_span = blocks[0]
3901 .spans
3902 .iter()
3903 .find(|s| s.text.contains("Hello"))
3904 .unwrap();
3905 assert!(!plain_span.bold);
3906 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3907 assert!(bold_span.bold);
3908 }
3909
3910 #[test]
3911 fn test_parse_markdown_heading() {
3912 let blocks = parse_markdown_blocks("# Title");
3913 assert_eq!(blocks.len(), 1);
3914 assert_eq!(blocks[0].heading_level, Some(1));
3915 assert_eq!(blocks[0].spans[0].text, "Title");
3916 }
3917
3918 #[test]
3919 fn test_parse_markdown_list() {
3920 let blocks = parse_markdown_blocks("- item1\n- item2");
3921 assert!(blocks.len() >= 2);
3922 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3923 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3924 }
3925
3926 fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
3928 elements
3929 .iter()
3930 .map(|e| match e {
3931 ParsedElement::Block(b) => (false, b.blockquote_depth),
3932 ParsedElement::Table(t) => (true, t.blockquote_depth),
3933 ParsedElement::FootnoteDefinition { .. } => (false, 0),
3936 })
3937 .collect()
3938 }
3939
3940 #[test]
3941 fn test_parse_markdown_table_in_blockquote_records_depth() {
3942 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3943 assert_eq!(element_depths(&elements), vec![(true, 1)]);
3944 }
3945
3946 #[test]
3947 fn test_parse_markdown_text_then_table_in_blockquote() {
3948 let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
3949 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
3950 }
3951
3952 #[test]
3953 fn test_parse_markdown_table_after_blockquote_closes() {
3954 let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
3955 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
3956 }
3957
3958 #[test]
3959 fn test_parse_markdown_table_in_nested_blockquote() {
3960 let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
3961 assert_eq!(element_depths(&elements), vec![(true, 2)]);
3962 }
3963
3964 #[test]
3965 fn test_parse_markdown_list_in_blockquote_records_depth() {
3966 let elements = parse_markdown("> - item1\n> - item2");
3967 let depths = element_depths(&elements);
3968 assert_eq!(depths, vec![(false, 1), (false, 1)]);
3969 for e in &elements {
3970 if let ParsedElement::Block(b) = e {
3971 assert_eq!(b.list_style, Some(ListStyle::Disc));
3972 }
3973 }
3974 }
3975
3976 #[test]
3977 fn test_parse_html_table_in_blockquote_records_depth() {
3978 let elements = parse_html_elements(
3979 "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
3980 );
3981 assert_eq!(element_depths(&elements), vec![(true, 1)]);
3982 }
3983
3984 #[test]
3985 fn test_parse_html_table_after_blockquote() {
3986 let elements = parse_html_elements(
3987 "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
3988 );
3989 let depths = element_depths(&elements);
3990 assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
3992 assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
3993 }
3994
3995 #[test]
3996 fn test_flatten_to_blocks_propagates_blockquote_depth() {
3997 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3998 let blocks = ParsedElement::flatten_to_blocks(elements);
3999 assert!(!blocks.is_empty());
4000 for b in &blocks {
4001 assert_eq!(b.blockquote_depth, 1);
4002 }
4003 }
4004
4005 #[test]
4006 fn test_parse_html_simple() {
4007 let blocks = parse_html("<p>Hello <b>world</b></p>");
4008 assert_eq!(blocks.len(), 1);
4009 assert!(blocks[0].spans.len() >= 2);
4010 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
4011 assert!(bold_span.bold);
4012 }
4013
4014 #[test]
4015 fn test_parse_html_multiple_paragraphs() {
4016 let blocks = parse_html("<p>A</p><p>B</p>");
4017 assert_eq!(blocks.len(), 2);
4018 }
4019
4020 #[test]
4021 fn test_parse_html_heading() {
4022 let blocks = parse_html("<h2>Subtitle</h2>");
4023 assert_eq!(blocks.len(), 1);
4024 assert_eq!(blocks[0].heading_level, Some(2));
4025 }
4026
4027 #[test]
4028 fn test_parse_html_list() {
4029 let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
4030 assert!(blocks.len() >= 2);
4031 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4032 }
4033
4034 #[test]
4035 fn test_parse_markdown_code_block() {
4036 let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
4037 assert_eq!(blocks.len(), 1);
4038 assert!(blocks[0].is_code_block);
4039 assert!(blocks[0].spans[0].code);
4040 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4042 assert_eq!(
4043 text, "fn main() {}",
4044 "code block text should not have trailing newline"
4045 );
4046 }
4047
4048 #[test]
4049 fn test_parse_markdown_nested_formatting() {
4050 let blocks = parse_markdown_blocks("***bold italic***");
4051 assert_eq!(blocks.len(), 1);
4052 let span = &blocks[0].spans[0];
4053 assert!(span.bold);
4054 assert!(span.italic);
4055 }
4056
4057 #[test]
4058 fn test_parse_markdown_link() {
4059 let blocks = parse_markdown_blocks("[click](http://example.com)");
4060 assert_eq!(blocks.len(), 1);
4061 let span = &blocks[0].spans[0];
4062 assert_eq!(span.text, "click");
4063 assert_eq!(span.link_href, Some("http://example.com".to_string()));
4064 }
4065
4066 #[test]
4067 fn test_parse_markdown_empty() {
4068 let blocks = parse_markdown_blocks("");
4069 assert_eq!(blocks.len(), 1);
4070 assert!(blocks[0].spans[0].text.is_empty());
4071 }
4072
4073 #[test]
4074 fn test_parse_html_empty() {
4075 let blocks = parse_html("");
4076 assert_eq!(blocks.len(), 1);
4077 assert!(blocks[0].spans[0].text.is_empty());
4078 }
4079
4080 #[test]
4081 fn test_parse_html_nested_formatting() {
4082 let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
4083 assert_eq!(blocks.len(), 1);
4084 let span = &blocks[0].spans[0];
4085 assert!(span.bold);
4086 assert!(span.italic);
4087 }
4088
4089 #[test]
4090 fn test_parse_html_link() {
4091 let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
4092 assert_eq!(blocks.len(), 1);
4093 let span = &blocks[0].spans[0];
4094 assert_eq!(span.text, "click");
4095 assert_eq!(span.link_href, Some("http://example.com".to_string()));
4096 }
4097
4098 #[test]
4099 fn test_parse_html_ordered_list() {
4100 let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
4101 assert!(blocks.len() >= 2);
4102 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
4103 }
4104
4105 #[test]
4106 fn test_parse_markdown_ordered_list() {
4107 let blocks = parse_markdown_blocks("1. first\n2. second");
4108 assert!(blocks.len() >= 2);
4109 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
4110 }
4111
4112 #[test]
4113 fn test_parse_html_blockquote_nested() {
4114 let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
4115 assert!(blocks.len() >= 3);
4116 }
4117
4118 #[test]
4119 fn test_parse_block_styles_line_height() {
4120 let styles = parse_block_styles("line-height: 1.5");
4121 assert_eq!(styles.line_height, Some(1500));
4122 }
4123
4124 #[test]
4125 fn test_parse_block_styles_direction_rtl() {
4126 let styles = parse_block_styles("direction: rtl");
4127 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
4128 }
4129
4130 #[test]
4131 fn test_parse_block_styles_background_color() {
4132 let styles = parse_block_styles("background-color: #ff0000");
4133 assert_eq!(styles.background_color, Some("#ff0000".to_string()));
4134 }
4135
4136 #[test]
4137 fn test_parse_block_styles_white_space_pre() {
4138 let styles = parse_block_styles("white-space: pre");
4139 assert_eq!(styles.non_breakable_lines, Some(true));
4140 }
4141
4142 #[test]
4143 fn test_parse_block_styles_multiple() {
4144 let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
4145 assert_eq!(styles.line_height, Some(2000));
4146 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
4147 assert_eq!(styles.background_color, Some("blue".to_string()));
4148 }
4149
4150 #[test]
4151 fn test_parse_html_block_styles_extracted() {
4152 let blocks = parse_html(
4153 r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
4154 );
4155 assert_eq!(blocks.len(), 1);
4156 assert_eq!(blocks[0].line_height, Some(1500));
4157 assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
4158 assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
4159 }
4160
4161 #[test]
4165 fn test_parse_html_collapses_exporter_line_wrapping() {
4166 let blocks = parse_html(
4167 "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
4168 <p>\nsecond para</p>\n</body>",
4169 );
4170 let texts: Vec<String> = blocks
4171 .iter()
4172 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4173 .collect();
4174 assert_eq!(texts, vec!["first line wrapped here", "second para"]);
4175 }
4176
4177 #[test]
4181 fn test_parse_html_drops_whitespace_between_blocks() {
4182 let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
4183 assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
4184 assert!(
4185 blocks
4186 .iter()
4187 .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
4188 "no block keeps a literal newline"
4189 );
4190 }
4191
4192 #[test]
4195 fn test_parse_html_collapses_across_span_boundaries() {
4196 let blocks = parse_html("<p> <b>bold</b>\n <i>italic</i> </p>");
4197 assert_eq!(blocks.len(), 1);
4198 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4199 assert_eq!(text, "bold italic");
4200 }
4201
4202 #[test]
4205 fn test_parse_html_keeps_no_break_space() {
4206 let blocks = parse_html("<p>\nAttention : ici</p>");
4207 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4208 assert_eq!(text, "Attention\u{a0}: ici");
4209 }
4210
4211 #[test]
4214 fn test_parse_html_keeps_empty_paragraph() {
4215 let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
4216 assert_eq!(blocks.len(), 3);
4217 let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
4218 assert_eq!(text, "");
4219 }
4220
4221 #[test]
4225 fn test_parse_html_preserves_whitespace_in_pre() {
4226 let blocks = parse_html("<pre>\nfn main() {\n let x = 1;\n}</pre>");
4229 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4230 assert_eq!(text, "fn main() {\n let x = 1;\n}");
4231
4232 let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n b</p>");
4235 let texts: Vec<String> = styled
4236 .iter()
4237 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4238 .collect();
4239 assert_eq!(texts, ["a", " b"]);
4240
4241 let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n b</p>");
4242 let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
4243 assert_eq!(text, "a b");
4244 }
4245
4246 #[test]
4249 fn test_parse_html_collapses_table_cell_whitespace() {
4250 let elements = parse_html_elements(
4251 "<table>\n<tr>\n<td>\n one\n</td>\n<td>\n two\n</td>\n</tr>\n</table>",
4252 );
4253 let table = elements
4254 .iter()
4255 .find_map(|e| match e {
4256 ParsedElement::Table(t) => Some(t),
4257 _ => None,
4258 })
4259 .expect("a table");
4260 let cells: Vec<String> = table.rows[0]
4261 .iter()
4262 .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
4263 .collect();
4264 assert_eq!(cells, vec!["one", "two"]);
4265 }
4266
4267 #[test]
4268 fn test_parse_html_white_space_pre() {
4269 let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
4270 assert_eq!(blocks.len(), 1);
4271 assert_eq!(blocks[0].non_breakable_lines, Some(true));
4272 }
4273
4274 #[test]
4275 fn test_parse_html_no_styles_returns_none() {
4276 let blocks = parse_html("<p>plain</p>");
4277 assert_eq!(blocks.len(), 1);
4278 assert_eq!(blocks[0].line_height, None);
4279 assert_eq!(blocks[0].direction, None);
4280 assert_eq!(blocks[0].background_color, None);
4281 assert_eq!(blocks[0].non_breakable_lines, None);
4282 }
4283
4284 #[test]
4285 fn test_parse_markdown_nested_list_indent() {
4286 let md = "- top\n - nested\n - deep";
4287 let blocks = parse_markdown_blocks(md);
4288 assert_eq!(blocks.len(), 3);
4289 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4290 assert_eq!(blocks[0].list_indent, 0);
4291 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4292 assert_eq!(blocks[1].list_indent, 1);
4293 assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
4294 assert_eq!(blocks[2].list_indent, 2);
4295 }
4296
4297 #[test]
4298 fn test_parse_markdown_nested_ordered_list_indent() {
4299 let md = "1. first\n 1. nested\n 2. nested2";
4300 let blocks = parse_markdown_blocks(md);
4301 assert_eq!(blocks.len(), 3);
4302 assert_eq!(blocks[0].list_indent, 0);
4303 assert_eq!(blocks[1].list_indent, 1);
4304 assert_eq!(blocks[2].list_indent, 1);
4305 }
4306
4307 #[test]
4308 fn test_parse_html_nested_list_indent() {
4309 let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
4310 let blocks = parse_html(html);
4311 assert!(blocks.len() >= 2);
4312 assert_eq!(blocks[0].list_indent, 0);
4313 assert_eq!(blocks[1].list_indent, 1);
4314 }
4315
4316 #[test]
4317 fn test_parse_markdown_table() {
4318 let md = "| A | B |\n|---|---|\n| 1 | 2 |";
4319 let elements = parse_markdown(md);
4320 assert_eq!(elements.len(), 1);
4321 match &elements[0] {
4322 ParsedElement::Table(table) => {
4323 assert_eq!(table.header_rows, 1);
4324 assert_eq!(table.rows.len(), 2); assert_eq!(table.rows[0].len(), 2);
4327 assert_eq!(table.rows[0][0].spans[0].text, "A");
4328 assert_eq!(table.rows[0][1].spans[0].text, "B");
4329 assert_eq!(table.rows[1].len(), 2);
4331 assert_eq!(table.rows[1][0].spans[0].text, "1");
4332 assert_eq!(table.rows[1][1].spans[0].text, "2");
4333 }
4334 _ => panic!("Expected ParsedElement::Table"),
4335 }
4336 }
4337
4338 #[test]
4339 fn test_parse_markdown_table_with_formatting() {
4340 let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
4341 let elements = parse_markdown(md);
4342 assert_eq!(elements.len(), 1);
4343 match &elements[0] {
4344 ParsedElement::Table(table) => {
4345 assert_eq!(table.rows.len(), 2);
4346 assert!(table.rows[0][0].spans[0].bold);
4348 assert!(table.rows[0][1].spans[0].code);
4350 assert!(table.rows[0][2].spans[0].italic);
4352 assert!(table.rows[1][0].spans[0].strikeout);
4354 assert_eq!(
4356 table.rows[1][2].spans[0].link_href,
4357 Some("http://x.com".to_string())
4358 );
4359 }
4360 _ => panic!("Expected ParsedElement::Table"),
4361 }
4362 }
4363
4364 #[test]
4365 fn test_parse_markdown_mixed_content_with_table() {
4366 let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
4367 let elements = parse_markdown(md);
4368 assert_eq!(elements.len(), 3);
4369 assert!(matches!(&elements[0], ParsedElement::Block(_)));
4370 assert!(matches!(&elements[1], ParsedElement::Table(_)));
4371 assert!(matches!(&elements[2], ParsedElement::Block(_)));
4372 }
4373}
4374
4375#[cfg(test)]
4376mod djot_tests {
4377 use super::*;
4378 use crate::entities::MarkerType;
4379
4380 fn blocks(d: &str) -> Vec<ParsedBlock> {
4381 ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
4382 }
4383
4384 fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
4385 b.spans.iter().find(|s| pred(s)).expect("span not found")
4386 }
4387
4388 #[test]
4389 fn paragraph_bold_italic() {
4390 let b = blocks("normal *bold* _italic_");
4391 assert_eq!(b.len(), 1);
4392 assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
4393 assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
4394 }
4395
4396 #[test]
4397 fn heading_levels() {
4398 assert_eq!(blocks("# H1")[0].heading_level, Some(1));
4399 assert_eq!(blocks("### H3")[0].heading_level, Some(3));
4400 assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
4401 }
4402
4403 #[test]
4404 fn unordered_bullet_styles_are_distinct() {
4405 assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
4406 assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
4407 assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
4408 }
4409
4410 #[test]
4411 fn ordered_delimiters() {
4412 let period = blocks("1. a");
4413 assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
4414 assert_eq!(period[0].list_prefix, "");
4415 assert_eq!(period[0].list_suffix, ".");
4416
4417 let paren = blocks("1) a");
4418 assert_eq!(paren[0].list_suffix, ")");
4419 assert_eq!(paren[0].list_prefix, "");
4420
4421 let paren_paren = blocks("(1) a");
4422 assert_eq!(paren_paren[0].list_prefix, "(");
4423 assert_eq!(paren_paren[0].list_suffix, ")");
4424 }
4425
4426 #[test]
4427 fn task_list_markers() {
4428 let b = blocks("- [ ] a\n- [x] b");
4429 assert_eq!(b.len(), 2);
4430 assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
4431 assert_eq!(b[1].marker, Some(MarkerType::Checked));
4432 }
4433
4434 #[test]
4435 fn code_block_with_language() {
4436 let b = blocks("```rust\nfn main() {}\n```");
4437 assert_eq!(b.len(), 1);
4438 assert!(b[0].is_code_block);
4439 assert_eq!(b[0].code_language.as_deref(), Some("rust"));
4440 let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
4441 assert_eq!(text, "fn main() {}");
4442 }
4443
4444 #[test]
4445 fn link_href() {
4446 let b = blocks("[text](http://example.com)");
4447 let s = first_span_with(&b[0], |s| s.text == "text");
4448 assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
4449 }
4450
4451 #[test]
4452 fn superscript_subscript() {
4453 assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
4454 assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
4455 }
4456
4457 #[test]
4458 fn delete_insert_verbatim() {
4459 assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
4460 assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
4461 assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
4462 }
4463
4464 #[test]
4465 fn blockquote_depth() {
4466 let els = parse_djot("> quoted", &DjotImportOptions::default());
4467 match &els[0] {
4468 ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
4469 _ => panic!("expected block"),
4470 }
4471 }
4472
4473 #[test]
4474 fn nested_list_indent() {
4475 let b = blocks("- a\n\n - b\n\n - c");
4480 assert_eq!(b.len(), 3);
4481 assert_eq!(b[0].list_indent, 0);
4482 assert_eq!(b[1].list_indent, 1);
4483 assert_eq!(b[2].list_indent, 2);
4484 }
4485
4486 #[test]
4487 fn table_parsed_as_table() {
4488 let els = parse_djot(
4489 "| a | b |\n|---|---|\n| c | d |",
4490 &DjotImportOptions::default(),
4491 );
4492 assert_eq!(els.len(), 1);
4493 match &els[0] {
4494 ParsedElement::Table(t) => {
4495 assert_eq!(t.header_rows, 1);
4496 assert_eq!(t.rows.len(), 2);
4497 assert_eq!(t.rows[0][0].spans[0].text, "a");
4498 assert_eq!(t.rows[1][1].spans[0].text, "d");
4499 }
4500 _ => panic!("expected table"),
4501 }
4502 }
4503
4504 #[test]
4505 fn smart_punctuation_normalised_to_unicode() {
4506 let text: String = blocks("a... b---c")[0]
4507 .spans
4508 .iter()
4509 .map(|s| s.text.as_str())
4510 .collect();
4511 assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
4512 assert!(text.contains('\u{2014}'), "em dash: {text:?}");
4513 }
4514
4515 #[test]
4516 fn unrepresentable_constructs_dropped_without_leaking_text() {
4517 let b = blocks("para1\n\n---\n\npara2");
4519 assert_eq!(b.len(), 2);
4520 assert_eq!(
4521 b[0].spans
4522 .iter()
4523 .map(|s| s.text.as_str())
4524 .collect::<String>(),
4525 "para1"
4526 );
4527 assert_eq!(
4528 b[1].spans
4529 .iter()
4530 .map(|s| s.text.as_str())
4531 .collect::<String>(),
4532 "para2"
4533 );
4534
4535 let d = blocks(":::\ninside\n:::");
4537 let joined: String = d
4538 .iter()
4539 .flat_map(|b| b.spans.iter())
4540 .map(|s| s.text.as_str())
4541 .collect();
4542 assert_eq!(joined, "inside");
4543
4544 let m = blocks("before $`E=mc^2` after");
4546 let joined: String = m
4547 .iter()
4548 .flat_map(|b| b.spans.iter())
4549 .map(|s| s.text.as_str())
4550 .collect();
4551 assert!(joined.contains("before"), "{joined:?}");
4552 assert!(joined.contains("after"), "{joined:?}");
4553 assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
4554 }
4555
4556 #[test]
4557 fn empty_document_yields_one_empty_block() {
4558 let b = blocks("");
4559 assert_eq!(b.len(), 1);
4560 assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
4561 }
4562
4563 #[test]
4564 fn block_attributes_parse_into_block() {
4565 let b = blocks(
4566 "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
4567 );
4568 assert_eq!(b.len(), 1);
4569 assert_eq!(b[0].alignment, Some(Alignment::Center));
4570 assert_eq!(b[0].line_height, Some(1500));
4571 assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
4572 assert_eq!(b[0].non_breakable_lines, Some(true));
4573 assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
4574 }
4575
4576 #[test]
4577 fn spacing_block_attributes_parse_into_block() {
4578 let b = blocks("{top_margin=24 text_indent=0}\nhello");
4582 assert_eq!(b.len(), 1);
4583 assert_eq!(b[0].top_margin, Some(24));
4584 assert_eq!(b[0].text_indent, Some(0));
4585 }
4586
4587 #[test]
4588 fn a_zero_text_indent_is_distinct_from_an_absent_one() {
4589 let explicit = blocks("{text_indent=0}\nhello");
4593 let absent = blocks("hello");
4594 assert_eq!(explicit[0].text_indent, Some(0));
4595 assert_eq!(absent[0].text_indent, None);
4596 assert!(!explicit[0].is_inline_only());
4597 assert!(absent[0].is_inline_only());
4598 }
4599
4600 #[test]
4601 fn spacing_block_attributes_respect_import_options() {
4602 let src = "{top_margin=24 text_indent=0}\nhello";
4603 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4604 assert_eq!(b[0].top_margin, None);
4605 assert_eq!(b[0].text_indent, None);
4606 }
4607
4608 #[test]
4609 fn block_attributes_on_heading() {
4610 let b = blocks("{alignment=right}\n# Title");
4611 assert_eq!(b[0].heading_level, Some(1));
4612 assert_eq!(b[0].alignment, Some(Alignment::Right));
4613 }
4614
4615 #[test]
4616 fn block_attributes_respect_import_options() {
4617 let src = "{alignment=center line_height=1500}\nhello";
4620 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4621 assert_eq!(b[0].alignment, None);
4622 assert_eq!(b[0].line_height, None);
4623 assert_eq!(
4624 b[0].spans
4625 .iter()
4626 .map(|s| s.text.as_str())
4627 .collect::<String>(),
4628 "hello"
4629 );
4630 }
4631
4632 #[test]
4633 fn list_item_block_attributes_are_dropped() {
4634 let b = blocks("{alignment=center}\n- item");
4637 assert!(b.iter().all(|blk| blk.alignment.is_none()));
4638 }
4639
4640 #[test]
4641 fn unknown_alignment_value_is_ignored() {
4642 let b = blocks("{alignment=sideways}\nhello");
4643 assert_eq!(b[0].alignment, None);
4644 }
4645}
4646
4647pub const TABLE_ANCHOR: &str = "\u{FFFC}";
4659
4660pub const LINE_SEPARATOR: char = '\u{2028}';
4674
4675const LINE_SEPARATOR_STR: &str = "\u{2028}";
4677
4678fn span_prose(span: &ParsedSpan, out: &mut String) {
4686 if span.image.is_some() || span.footnote_ref.is_some() {
4687 out.push('\u{FFFC}');
4688 return;
4689 }
4690 out.push_str(&span.text);
4691}
4692
4693fn block_prose(block: &ParsedBlock) -> String {
4694 let mut prose = String::new();
4695 for span in &block.spans {
4696 span_prose(span, &mut prose);
4697 }
4698 prose
4699}
4700
4701fn cell_prose(cell: &ParsedTableCell) -> String {
4702 let mut prose = String::new();
4703 for span in &cell.spans {
4704 span_prose(span, &mut prose);
4705 }
4706 prose
4707}
4708
4709pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
4760 let elements = parse_djot(djot, options);
4764
4765 let mut out = String::with_capacity(djot.len());
4768
4769 let mut first = true;
4774 let push = |text: &str, out: &mut String, first: &mut bool| {
4775 if *first {
4776 *first = false;
4777 } else {
4778 out.push('\n');
4779 }
4780 out.push_str(text);
4781 };
4782
4783 for element in &elements {
4784 match element {
4785 ParsedElement::Block(block) => {
4786 push(&block_prose(block), &mut out, &mut first);
4787 }
4788 ParsedElement::FootnoteDefinition { .. } => {}
4798 ParsedElement::Table(table) => {
4799 push(TABLE_ANCHOR, &mut out, &mut first);
4806 let columns = table.column_count();
4809 for row in &table.rows {
4810 for column in 0..columns {
4811 let prose = row.get(column).map(cell_prose).unwrap_or_default();
4812 push(&prose, &mut out, &mut first);
4813 }
4814 }
4815 }
4816 }
4817 }
4818 out
4819}
4820
4821#[cfg(test)]
4822mod html_footnote_tests {
4823 use super::*;
4824
4825 #[test]
4829 fn a_literal_reference_in_text_would_be_escaped_into_prose() {
4830 let blocks = parse_html("<p>The ferry.[^1]</p>");
4831 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4832 assert!(
4833 text.contains("[^1]"),
4834 "the parser keeps it as characters: {text:?}"
4835 );
4836 assert!(
4837 blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
4838 "text is text — it must not become a reference by accident"
4839 );
4840 }
4841
4842 #[test]
4843 fn an_attributed_element_becomes_a_real_footnote_reference() {
4844 let blocks = parse_html(
4845 r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
4846 );
4847 let refs: Vec<&str> = blocks
4848 .iter()
4849 .flat_map(|b| b.spans.iter())
4850 .filter_map(|s| s.footnote_ref.as_deref())
4851 .collect();
4852 assert_eq!(refs, vec!["3"]);
4853
4854 let marker = blocks[0]
4857 .spans
4858 .iter()
4859 .find(|s| s.footnote_ref.is_some())
4860 .expect("the reference span");
4861 assert!(marker.text.is_empty(), "{marker:?}");
4862 }
4863
4864 #[test]
4867 fn any_element_carrying_the_attribute_works() {
4868 for html in [
4869 r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
4870 r#"<p>a<span data-footnote-ref="1"></span></p>"#,
4871 r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
4872 ] {
4873 let blocks = parse_html(html);
4874 assert_eq!(
4875 blocks
4876 .iter()
4877 .flat_map(|b| b.spans.iter())
4878 .filter_map(|s| s.footnote_ref.as_deref())
4879 .collect::<Vec<_>>(),
4880 vec!["1"],
4881 "failed for {html}"
4882 );
4883 }
4884 }
4885
4886 #[test]
4887 fn an_empty_or_absent_label_is_not_a_reference() {
4888 for html in [
4889 r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
4890 r#"<p>a<sup></sup></p>"#,
4891 ] {
4892 let blocks = parse_html(html);
4893 assert!(
4894 blocks
4895 .iter()
4896 .flat_map(|b| b.spans.iter())
4897 .all(|s| s.footnote_ref.is_none()),
4898 "failed for {html}"
4899 );
4900 }
4901 }
4902}
4903
4904#[cfg(test)]
4905mod html_depth_tests {
4906 use super::*;
4909
4910 fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
4912 blocks
4913 .iter()
4914 .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
4915 .collect()
4916 }
4917
4918 #[test]
4923 fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
4924 use crate::parser_tools::list_depth::MAX_LIST_INDENT;
4925 let depth = 300;
4926 let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
4927 html.push_str(&"</li></ol>".repeat(depth));
4928 let blocks = parse_html(&html);
4929 let items: Vec<(String, u32)> = blocks
4930 .iter()
4931 .filter(|block| block.list_style.is_some())
4932 .map(|block| {
4933 let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
4934 (text, block.list_indent)
4935 })
4936 .collect();
4937 let expected: Vec<(String, u32)> = (0..depth)
4938 .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
4939 .collect();
4940 assert_eq!(items, expected);
4941 }
4942
4943 #[test]
4945 fn text_in_a_deep_html_list_outside_its_items_is_kept() {
4946 let depth = 40;
4947 let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
4948 html.push_str("<ul>loose words<li>last</li></ul>");
4949 html.push_str(&"</li></ul>".repeat(depth));
4950 let texts = block_texts(&parse_html(&html)).join(" ");
4951 assert!(texts.contains("loose words"), "{texts:?}");
4952 assert!(texts.contains("last"), "{texts:?}");
4953 assert!(texts.contains("item 39"), "{texts:?}");
4954 }
4955
4956 #[test]
4959 fn html_nested_past_the_reader_depth_keeps_its_words() {
4960 for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
4961 let html = format!(
4962 "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
4963 open.repeat(300),
4964 close.repeat(300)
4965 );
4966 let texts = block_texts(&parse_html(&html)).join(" ");
4967 let words: Vec<&str> = texts.split_whitespace().collect();
4968 assert_eq!(
4969 words,
4970 ["Top.", "Deep", "words", "here.", "End."],
4971 "for {open}"
4972 );
4973 }
4974 }
4975
4976 #[test]
4978 fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
4979 let html = format!(
4980 "<table><tr><td>{}deep words{}</td></tr></table>",
4981 "<span>".repeat(300),
4982 "</span>".repeat(300)
4983 );
4984 let elements = parse_html_elements(&html);
4985 let Some(ParsedElement::Table(table)) = elements.first() else {
4986 panic!("a table: {elements:?}");
4987 };
4988 let text: String = table.rows[0][0]
4989 .spans
4990 .iter()
4991 .map(|span| span.text.as_str())
4992 .collect();
4993 assert_eq!(text, "deep words");
4994 }
4995
4996 #[test]
5000 fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
5001 for cell in [
5002 "One<br/>Two",
5003 "<p>One</p><p>Two</p>",
5004 "<ul><li>One</li><li>Two</li></ul>",
5005 ] {
5006 let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
5007 let Some(ParsedElement::Table(table)) = elements.first() else {
5008 panic!("a table: {elements:?}");
5009 };
5010 let text: String = table.rows[0][0]
5011 .spans
5012 .iter()
5013 .map(|span| span.text.as_str())
5014 .collect();
5015 assert_eq!(text, "One Two", "for {cell}");
5016 }
5017 }
5018}