1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3
4#[derive(Debug, Clone, Default, PartialEq, Eq)]
10pub struct ParsedImage {
11 pub src: String,
12 pub alt: String,
13 pub width: i64,
19 pub height: i64,
20}
21
22#[derive(Debug, Clone, Default)]
29pub struct ParsedSpan {
30 pub text: String,
31 pub bold: bool,
32 pub italic: bool,
33 pub underline: bool,
34 pub strikeout: bool,
35 pub code: bool,
36 pub superscript: bool,
38 pub subscript: bool,
40 pub link_href: Option<String>,
41 pub image: Option<ParsedImage>,
43 pub footnote_ref: Option<String>,
46}
47
48#[derive(Debug, Clone)]
50pub struct ParsedTableCell {
51 pub spans: Vec<ParsedSpan>,
52}
53
54#[derive(Debug, Clone)]
56pub struct ParsedTable {
57 pub header_rows: usize,
59 pub rows: Vec<Vec<ParsedTableCell>>,
61 pub blockquote_depth: u32,
64}
65
66impl ParsedTable {
67 pub fn column_count(&self) -> usize {
77 self.rows.iter().map(Vec::len).max().unwrap_or(0)
78 }
79}
80
81const SQUARED_CELLS_ALWAYS: usize = 4096;
85
86const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
96
97fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
108 let squared = table.rows.len().saturating_mul(table.column_count());
109 let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
110 if squared <= allowed {
111 elements.push(ParsedElement::Table(table));
112 return;
113 }
114 elements.extend(
115 ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
116 .into_iter()
117 .filter(|block| spans_carry_content(&block.spans))
118 .map(ParsedElement::Block),
119 );
120}
121
122fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
124 rows.iter().map(Vec::len).sum()
125}
126
127#[derive(Debug, Clone)]
129pub enum ParsedElement {
130 Block(ParsedBlock),
131 Table(ParsedTable),
132 FootnoteDefinition {
139 label: String,
140 blocks: Vec<ParsedBlock>,
141 },
142}
143
144impl ParsedElement {
145 pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
148 let mut blocks = Vec::new();
149 for elem in elements {
150 match elem {
151 ParsedElement::Block(b) => blocks.push(b),
152 ParsedElement::FootnoteDefinition { .. } => {}
157 ParsedElement::Table(t) => {
158 for row in t.rows {
159 for cell in row {
160 blocks.push(ParsedBlock {
161 spans: cell.spans,
162 heading_level: None,
163 list_style: None,
164 list_indent: 0,
165 list_prefix: String::new(),
166 list_suffix: String::new(),
167 marker: None,
168 is_code_block: false,
169 code_language: None,
170 blockquote_depth: t.blockquote_depth,
171 line_height: None,
172 non_breakable_lines: None,
173 page_break_before: None,
174 direction: None,
175 background_color: None,
176 alignment: None,
177 top_margin: None,
178 text_indent: None,
179 semantic_role: None,
180 });
181 }
182 }
183 }
184 }
185 }
186 if blocks.is_empty() {
187 blocks.push(ParsedBlock {
188 spans: vec![ParsedSpan {
189 text: String::new(),
190 ..Default::default()
191 }],
192 heading_level: None,
193 list_style: None,
194 list_indent: 0,
195 list_prefix: String::new(),
196 list_suffix: String::new(),
197 marker: None,
198 is_code_block: false,
199 code_language: None,
200 blockquote_depth: 0,
201 line_height: None,
202 non_breakable_lines: None,
203 page_break_before: None,
204 direction: None,
205 background_color: None,
206 alignment: None,
207 top_margin: None,
208 text_indent: None,
209 semantic_role: None,
210 });
211 }
212 blocks
213 }
214}
215
216#[derive(Debug, Clone, Default)]
223pub struct ParsedBlock {
224 pub spans: Vec<ParsedSpan>,
225 pub heading_level: Option<i64>,
226 pub list_style: Option<ListStyle>,
227 pub list_indent: u32,
228 pub list_prefix: String,
231 pub list_suffix: String,
234 pub marker: Option<MarkerType>,
237 pub is_code_block: bool,
238 pub code_language: Option<String>,
239 pub blockquote_depth: u32,
240 pub line_height: Option<i64>,
241 pub non_breakable_lines: Option<bool>,
242 pub page_break_before: Option<bool>,
245 pub direction: Option<TextDirection>,
246 pub background_color: Option<String>,
247 pub alignment: Option<Alignment>,
250 pub top_margin: Option<i64>,
254 pub text_indent: Option<i64>,
258 pub semantic_role: Option<SemanticRole>,
262}
263
264impl ParsedBlock {
265 pub fn is_inline_only(&self) -> bool {
268 self.heading_level.is_none()
269 && self.list_style.is_none()
270 && !self.is_code_block
271 && self.blockquote_depth == 0
272 && self.line_height.is_none()
273 && self.non_breakable_lines.is_none()
274 && self.page_break_before.is_none()
275 && self.direction.is_none()
276 && self.background_color.is_none()
277 && self.alignment.is_none()
278 && self.top_margin.is_none()
279 && self.text_indent.is_none()
280 }
281}
282
283fn dangling_footnote_labels(
312 markdown: &str,
313 options: pulldown_cmark::Options,
314) -> std::collections::BTreeSet<String> {
315 use pulldown_cmark::{Event, Parser, Tag};
316
317 let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
318 for event in Parser::new_ext(markdown, options) {
319 if let Event::Start(Tag::FootnoteDefinition(label)) = event {
320 defined.insert(label.to_string());
321 }
322 }
323
324 let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
325 let bytes = markdown.as_bytes();
326 let mut search_from = 0usize;
327 while let Some(rel) = markdown[search_from..].find("[^") {
328 let open = search_from + rel;
329 let label_start = open + 2;
330 let Some(close_rel) = markdown[label_start..].find(']') else {
331 break;
332 };
333 let close = label_start + close_rel;
334 let label = &markdown[label_start..close];
335 let looks_like_definition = bytes.get(close + 1) == Some(&b':');
340 if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
341 referenced.insert(label.to_string());
342 }
343 search_from = close + 1;
344 }
345
346 referenced.difference(&defined).cloned().collect()
347}
348
349pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
350 use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
351
352 let options = Options::ENABLE_STRIKETHROUGH
363 | Options::ENABLE_TABLES
364 | Options::ENABLE_TASKLISTS
365 | Options::ENABLE_FOOTNOTES;
366
367 let dangling = dangling_footnote_labels(markdown, options);
374 let augmented_owner;
375 let source: &str = if dangling.is_empty() {
376 markdown
377 } else {
378 augmented_owner = dangling
379 .iter()
380 .fold(markdown.to_string(), |mut acc, label| {
381 acc.push_str("\n\n[^");
382 acc.push_str(label);
383 acc.push_str("]:\n");
384 acc
385 });
386 &augmented_owner
387 };
388 let parser = Parser::new_ext(source, options);
389
390 let mut elements: Vec<ParsedElement> = Vec::new();
391 let mut current_spans: Vec<ParsedSpan> = Vec::new();
392 let mut current_heading: Option<i64> = None;
393 let mut current_list_style: Option<ListStyle> = None;
394 let mut is_code_block = false;
395 let mut code_language: Option<String> = None;
396 let mut blockquote_depth: u32 = 0;
397 let mut in_block = false;
398
399 let mut bold = false;
401 let mut italic = false;
402 let mut strikeout = false;
403 let mut link_href: Option<String> = None;
404 let mut pending_image: Option<ParsedImage> = None;
406
407 let mut footnote_open: Option<(String, usize)> = None;
414
415 let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
417 let mut current_list_indent: u32 = 0;
418
419 let mut in_table = false;
421 let mut in_table_head = false;
422 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
423 let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
424 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
425 let mut table_header_rows: usize = 0;
426
427 for event in parser {
428 match event {
429 Event::Start(Tag::Paragraph) => {
430 in_block = true;
431 current_heading = None;
432 is_code_block = false;
433 }
434 Event::End(TagEnd::Paragraph) => {
435 if !current_spans.is_empty() || in_block {
436 elements.push(ParsedElement::Block(ParsedBlock {
437 spans: std::mem::take(&mut current_spans),
438 heading_level: current_heading.take(),
439 list_style: current_list_style.clone(),
440 list_indent: current_list_indent,
441 list_prefix: String::new(),
442 list_suffix: String::new(),
443 marker: None,
444 is_code_block: false,
445 code_language: None,
446 blockquote_depth,
447 line_height: None,
448 non_breakable_lines: None,
449 page_break_before: None,
450 direction: None,
451 background_color: None,
452 alignment: None,
453 top_margin: None,
454 text_indent: None,
455 semantic_role: None,
456 }));
457 }
458 in_block = false;
459 current_list_style = None;
460 }
461 Event::Start(Tag::Heading { level, .. }) => {
462 in_block = true;
463 current_heading = Some(heading_level_to_i64(level));
464 is_code_block = false;
465 }
466 Event::End(TagEnd::Heading(_)) => {
467 elements.push(ParsedElement::Block(ParsedBlock {
468 spans: std::mem::take(&mut current_spans),
469 heading_level: current_heading.take(),
470 list_style: None,
471 list_indent: 0,
472 list_prefix: String::new(),
473 list_suffix: String::new(),
474 marker: None,
475 is_code_block: false,
476 code_language: None,
477 blockquote_depth,
478 line_height: None,
479 non_breakable_lines: None,
480 page_break_before: None,
481 direction: None,
482 background_color: None,
483 alignment: None,
484 top_margin: None,
485 text_indent: None,
486 semantic_role: None,
487 }));
488 in_block = false;
489 }
490 Event::Start(Tag::List(ordered)) => {
491 let style = if ordered.is_some() {
492 Some(ListStyle::Decimal)
493 } else {
494 Some(ListStyle::Disc)
495 };
496 list_stack.push(style);
497 }
498 Event::End(TagEnd::List(_)) => {
499 list_stack.pop();
500 }
501 Event::Start(Tag::Item) => {
502 if !current_spans.is_empty() {
505 elements.push(ParsedElement::Block(ParsedBlock {
506 spans: std::mem::take(&mut current_spans),
507 heading_level: None,
508 list_style: current_list_style.clone(),
509 list_indent: current_list_indent,
510 list_prefix: String::new(),
511 list_suffix: String::new(),
512 marker: None,
513 is_code_block: false,
514 code_language: None,
515 blockquote_depth,
516 line_height: None,
517 non_breakable_lines: None,
518 page_break_before: None,
519 direction: None,
520 background_color: None,
521 alignment: None,
522 top_margin: None,
523 text_indent: None,
524 semantic_role: None,
525 }));
526 }
527 in_block = true;
528 current_list_style = list_stack.last().cloned().flatten();
529 current_list_indent = if list_stack.is_empty() {
530 0
531 } else {
532 (list_stack.len() - 1) as u32
533 };
534 }
535 Event::End(TagEnd::Item) => {
536 if !current_spans.is_empty() {
539 elements.push(ParsedElement::Block(ParsedBlock {
540 spans: std::mem::take(&mut current_spans),
541 heading_level: None,
542 list_style: current_list_style.clone(),
543 list_indent: current_list_indent,
544 list_prefix: String::new(),
545 list_suffix: String::new(),
546 marker: None,
547 is_code_block: false,
548 code_language: None,
549 blockquote_depth,
550 line_height: None,
551 non_breakable_lines: None,
552 page_break_before: None,
553 direction: None,
554 background_color: None,
555 alignment: None,
556 top_margin: None,
557 text_indent: None,
558 semantic_role: None,
559 }));
560 }
561 in_block = false;
562 current_list_style = None;
563 }
564 Event::Start(Tag::CodeBlock(kind)) => {
565 in_block = true;
566 is_code_block = true;
567 code_language = match &kind {
568 pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
569 Some(lang.to_string())
570 }
571 _ => None,
572 };
573 }
574 Event::End(TagEnd::CodeBlock) => {
575 if let Some(last) = current_spans.last_mut()
577 && last.text.ends_with('\n')
578 {
579 last.text.truncate(last.text.len() - 1);
580 }
581 elements.push(ParsedElement::Block(ParsedBlock {
582 spans: std::mem::take(&mut current_spans),
583 heading_level: None,
584 list_style: None,
585 list_indent: 0,
586 list_prefix: String::new(),
587 list_suffix: String::new(),
588 marker: None,
589 is_code_block: true,
590 code_language: code_language.take(),
591 blockquote_depth,
592 line_height: None,
593 non_breakable_lines: None,
594 page_break_before: None,
595 direction: None,
596 background_color: None,
597 alignment: None,
598 top_margin: None,
599 text_indent: None,
600 semantic_role: None,
601 }));
602 in_block = false;
603 is_code_block = false;
604 }
605 Event::Start(Tag::Table(_)) => {
607 in_table = true;
608 in_table_head = false;
609 table_rows.clear();
610 current_row_cells.clear();
611 current_cell_spans.clear();
612 table_header_rows = 0;
613 }
614 Event::End(TagEnd::Table) => {
615 let own_cells = cells_in_rows(&table_rows);
616 push_table(
617 &mut elements,
618 ParsedTable {
619 header_rows: table_header_rows,
620 rows: std::mem::take(&mut table_rows),
621 blockquote_depth,
622 },
623 own_cells,
624 );
625 in_table = false;
626 }
627 Event::Start(Tag::TableHead) => {
628 in_table_head = true;
629 current_row_cells.clear();
630 }
631 Event::End(TagEnd::TableHead) => {
632 table_rows.push(std::mem::take(&mut current_row_cells));
634 table_header_rows += 1;
635 in_table_head = false;
636 }
637 Event::Start(Tag::TableRow) => {
638 current_row_cells.clear();
639 }
640 Event::End(TagEnd::TableRow) if !in_table_head => {
641 table_rows.push(std::mem::take(&mut current_row_cells));
643 }
644 Event::Start(Tag::TableCell) => {
645 current_cell_spans.clear();
646 }
647 Event::End(TagEnd::TableCell) => {
648 current_row_cells.push(ParsedTableCell {
649 spans: std::mem::take(&mut current_cell_spans),
650 });
651 }
652 Event::Start(Tag::Emphasis) => {
654 italic = true;
655 }
656 Event::End(TagEnd::Emphasis) => {
657 italic = false;
658 }
659 Event::Start(Tag::Strong) => {
660 bold = true;
661 }
662 Event::End(TagEnd::Strong) => {
663 bold = false;
664 }
665 Event::Start(Tag::Strikethrough) => {
666 strikeout = true;
667 }
668 Event::End(TagEnd::Strikethrough) => {
669 strikeout = false;
670 }
671 Event::Start(Tag::Link { dest_url, .. }) => {
672 link_href = Some(dest_url.to_string());
673 }
674 Event::End(TagEnd::Link) => {
675 link_href = None;
676 }
677 Event::Start(Tag::Image { dest_url, .. }) => {
680 pending_image = Some(ParsedImage {
681 src: dest_url.to_string(),
682 alt: String::new(),
683 width: 0,
684 height: 0,
685 });
686 }
687 Event::End(TagEnd::Image) => {
688 if let Some(image) = pending_image.take() {
689 let span = ParsedSpan {
690 text: String::new(),
691 bold,
692 italic,
693 underline: false,
694 strikeout,
695 code: false,
696 superscript: false,
697 subscript: false,
698 link_href: link_href.clone(),
699 image: Some(image),
700 footnote_ref: None,
701 };
702 if in_table {
703 current_cell_spans.push(span);
704 } else {
705 if !in_block {
706 in_block = true;
707 }
708 current_spans.push(span);
709 }
710 }
711 }
712 Event::Text(text) => {
713 if let Some(img) = pending_image.as_mut() {
718 img.alt.push_str(&text);
719 continue;
720 }
721 let span = ParsedSpan {
722 text: text.to_string(),
723 bold,
724 italic,
725 underline: false,
726 strikeout,
727 code: is_code_block,
728 superscript: false,
729 subscript: false,
730 link_href: link_href.clone(),
731 image: None,
732 footnote_ref: None,
733 };
734 if in_table {
735 current_cell_spans.push(span);
736 } else {
737 if !in_block {
738 in_block = true;
739 }
740 current_spans.push(span);
741 }
742 }
743 Event::Code(text) => {
744 let span = ParsedSpan {
745 text: text.to_string(),
746 bold,
747 italic,
748 underline: false,
749 strikeout,
750 code: true,
751 superscript: false,
752 subscript: false,
753 link_href: link_href.clone(),
754 image: None,
755 footnote_ref: None,
756 };
757 if in_table {
758 current_cell_spans.push(span);
759 } else {
760 if !in_block {
761 in_block = true;
762 }
763 current_spans.push(span);
764 }
765 }
766 Event::SoftBreak => {
767 let span = ParsedSpan {
768 text: " ".to_string(),
769 bold,
770 italic,
771 underline: false,
772 strikeout,
773 code: false,
774 superscript: false,
775 subscript: false,
776 link_href: link_href.clone(),
777 image: None,
778 footnote_ref: None,
779 };
780 if in_table {
781 current_cell_spans.push(span);
782 } else {
783 current_spans.push(span);
784 }
785 }
786 Event::HardBreak if !current_spans.is_empty() || in_block => {
787 elements.push(ParsedElement::Block(ParsedBlock {
789 spans: std::mem::take(&mut current_spans),
790 heading_level: current_heading.take(),
791 list_style: current_list_style.clone(),
792 list_indent: current_list_indent,
793 list_prefix: String::new(),
794 list_suffix: String::new(),
795 marker: None,
796 is_code_block,
797 code_language: code_language.clone(),
798 blockquote_depth,
799 line_height: None,
800 non_breakable_lines: None,
801 page_break_before: None,
802 direction: None,
803 background_color: None,
804 alignment: None,
805 top_margin: None,
806 text_indent: None,
807 semantic_role: None,
808 }));
809 }
810 Event::Start(Tag::BlockQuote(_)) => {
811 blockquote_depth += 1;
812 }
813 Event::End(TagEnd::BlockQuote(_)) => {
814 blockquote_depth = blockquote_depth.saturating_sub(1);
815 }
816 Event::Start(Tag::FootnoteDefinition(label)) => {
827 if !current_spans.is_empty() {
828 elements.push(ParsedElement::Block(ParsedBlock {
829 spans: std::mem::take(&mut current_spans),
830 heading_level: current_heading.take(),
831 list_style: current_list_style.clone(),
832 list_indent: current_list_indent,
833 list_prefix: String::new(),
834 list_suffix: String::new(),
835 marker: None,
836 is_code_block: false,
837 code_language: None,
838 blockquote_depth,
839 line_height: None,
840 non_breakable_lines: None,
841 page_break_before: None,
842 direction: None,
843 background_color: None,
844 alignment: None,
845 top_margin: None,
846 text_indent: None,
847 semantic_role: None,
848 }));
849 }
850 footnote_open = Some((label.to_string(), elements.len()));
851 }
852 Event::End(TagEnd::FootnoteDefinition) => {
853 if !current_spans.is_empty() {
854 elements.push(ParsedElement::Block(ParsedBlock {
855 spans: std::mem::take(&mut current_spans),
856 heading_level: current_heading.take(),
857 list_style: current_list_style.clone(),
858 list_indent: current_list_indent,
859 list_prefix: String::new(),
860 list_suffix: String::new(),
861 marker: None,
862 is_code_block: false,
863 code_language: None,
864 blockquote_depth,
865 line_height: None,
866 non_breakable_lines: None,
867 page_break_before: None,
868 direction: None,
869 background_color: None,
870 alignment: None,
871 top_margin: None,
872 text_indent: None,
873 semantic_role: None,
874 }));
875 }
876 if let Some((label, start)) = footnote_open.take() {
877 let blocks: Vec<ParsedBlock> = elements
878 .drain(start..)
879 .filter_map(|e| match e {
880 ParsedElement::Block(b) => Some(b),
881 _ => None,
884 })
885 .collect();
886 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
887 }
888 }
889 Event::FootnoteReference(label) => {
895 let span = ParsedSpan {
896 text: String::new(),
897 bold,
898 italic,
899 underline: false,
900 strikeout,
901 code: false,
902 superscript: false,
903 subscript: false,
904 link_href: link_href.clone(),
905 image: None,
906 footnote_ref: Some(label.to_string()),
907 };
908 if in_table {
909 current_cell_spans.push(span);
910 } else {
911 if !in_block {
912 in_block = true;
913 }
914 current_spans.push(span);
915 }
916 }
917 _ => {}
918 }
919 }
920
921 if !current_spans.is_empty() {
923 elements.push(ParsedElement::Block(ParsedBlock {
924 spans: std::mem::take(&mut current_spans),
925 heading_level: current_heading,
926 list_style: current_list_style,
927 list_indent: current_list_indent,
928 list_prefix: String::new(),
929 list_suffix: String::new(),
930 marker: None,
931 is_code_block,
932 code_language: code_language.take(),
933 blockquote_depth,
934 line_height: None,
935 non_breakable_lines: None,
936 page_break_before: None,
937 direction: None,
938 background_color: None,
939 alignment: None,
940 top_margin: None,
941 text_indent: None,
942 semantic_role: None,
943 }));
944 }
945
946 if !dangling.is_empty() {
953 elements.retain(
954 |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
955 );
956 }
957
958 if elements.is_empty() {
960 elements.push(ParsedElement::Block(ParsedBlock {
961 spans: vec![ParsedSpan {
962 text: String::new(),
963 ..Default::default()
964 }],
965 heading_level: None,
966 list_style: None,
967 list_indent: 0,
968 list_prefix: String::new(),
969 list_suffix: String::new(),
970 marker: None,
971 is_code_block: false,
972 code_language: None,
973 blockquote_depth: 0,
974 line_height: None,
975 non_breakable_lines: None,
976 page_break_before: None,
977 direction: None,
978 background_color: None,
979 alignment: None,
980 top_margin: None,
981 text_indent: None,
982 semantic_role: None,
983 }));
984 }
985
986 elements
987}
988
989fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
990 use pulldown_cmark::HeadingLevel;
991 match level {
992 HeadingLevel::H1 => 1,
993 HeadingLevel::H2 => 2,
994 HeadingLevel::H3 => 3,
995 HeadingLevel::H4 => 4,
996 HeadingLevel::H5 => 5,
997 HeadingLevel::H6 => 6,
998 }
999}
1000
1001use scraper::Node;
1004
1005#[derive(Debug, Clone, Default)]
1007struct BlockStyles {
1008 line_height: Option<i64>,
1009 non_breakable_lines: Option<bool>,
1010 page_break_before: Option<bool>,
1011 direction: Option<TextDirection>,
1012 background_color: Option<String>,
1013 preserve_whitespace: Option<bool>,
1018}
1019
1020fn parse_block_styles(style: &str) -> BlockStyles {
1024 let mut result = BlockStyles::default();
1025 for part in style.split(';') {
1026 let part = part.trim();
1027 if let Some((prop, val)) = part.split_once(':') {
1028 let prop = prop.trim().to_ascii_lowercase();
1029 let val = val.trim();
1030 match prop.as_str() {
1031 "line-height" => {
1032 if let Ok(v) = val.parse::<f64>() {
1034 result.line_height = Some((v * 1000.0) as i64);
1035 }
1036 }
1037 "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1038 result.non_breakable_lines = Some(true);
1039 if val != "nowrap" {
1040 result.preserve_whitespace = Some(true);
1041 }
1042 }
1043 "break-before" | "page-break-before" => {
1047 result.page_break_before = match val.to_ascii_lowercase().as_str() {
1048 "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1049 "avoid" | "auto" => Some(false),
1050 _ => None,
1051 };
1052 }
1053 "direction" => {
1054 if val.eq_ignore_ascii_case("rtl") {
1055 result.direction = Some(TextDirection::RightToLeft);
1056 } else if val.eq_ignore_ascii_case("ltr") {
1057 result.direction = Some(TextDirection::LeftToRight);
1058 }
1059 }
1060 "background-color" | "background" => {
1061 result.background_color = Some(val.to_string());
1062 }
1063 _ => {}
1064 }
1065 }
1066 }
1067 result
1068}
1069
1070fn is_html_space(ch: char) -> bool {
1077 matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1078}
1079
1080fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1082 spans
1083 .iter()
1084 .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1085}
1086
1087fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1101 fn is_replaced(span: &ParsedSpan) -> bool {
1102 span.image.is_some() || span.footnote_ref.is_some()
1103 }
1104
1105 for span in spans.iter_mut() {
1109 if is_replaced(span) {
1110 continue;
1111 }
1112 let mut out = String::with_capacity(span.text.len());
1113 let mut in_run = false;
1114 for ch in span.text.chars() {
1115 if is_html_space(ch) {
1116 if !in_run {
1117 out.push(' ');
1118 in_run = true;
1119 }
1120 } else {
1121 out.push(ch);
1122 in_run = false;
1123 }
1124 }
1125 span.text = out;
1126 }
1127
1128 let mut prev_ends_with_space = true;
1131 for span in spans.iter_mut() {
1132 if is_replaced(span) {
1133 prev_ends_with_space = false;
1134 continue;
1135 }
1136 if span.text.is_empty() {
1137 prev_ends_with_space = false;
1141 continue;
1142 }
1143 if prev_ends_with_space && span.text.starts_with(' ') {
1144 span.text.remove(0);
1145 }
1146 if !span.text.is_empty() {
1149 prev_ends_with_space = span.text.ends_with(' ');
1150 }
1151 }
1152
1153 for span in spans.iter_mut().rev() {
1157 if is_replaced(span) {
1158 break;
1159 }
1160 if span.text.is_empty() {
1161 continue;
1162 }
1163 if span.text.ends_with(' ') {
1164 span.text.pop();
1165 }
1166 break;
1167 }
1168}
1169
1170pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
1191 let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
1192 if !block.spans.iter().any(breaks) {
1193 return vec![block];
1194 }
1195 let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
1196 for mut span in std::mem::take(&mut block.spans) {
1197 if !breaks(&span) {
1198 push_to_last_line(&mut lines, span);
1199 continue;
1200 }
1201 let text = std::mem::take(&mut span.text);
1202 for (i, piece) in text.split('\n').enumerate() {
1203 if i > 0 {
1204 lines.push(Vec::new());
1205 }
1206 if !piece.is_empty() {
1207 push_to_last_line(
1208 &mut lines,
1209 ParsedSpan {
1210 text: piece.to_string(),
1211 ..span.clone()
1212 },
1213 );
1214 }
1215 }
1216 }
1217 if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
1218 lines.pop();
1219 }
1220 let later_line = ParsedBlock {
1221 page_break_before: None,
1222 ..block.clone()
1223 };
1224 let mut first_line = Some(block);
1225 lines
1226 .into_iter()
1227 .map(|spans| {
1228 let spans = if spans.is_empty() {
1229 vec![ParsedSpan::default()]
1230 } else {
1231 spans
1232 };
1233 match first_line.take() {
1234 Some(first) => ParsedBlock { spans, ..first },
1235 None => ParsedBlock {
1236 spans,
1237 ..later_line.clone()
1238 },
1239 }
1240 })
1241 .collect()
1242}
1243
1244fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
1247 match lines.last_mut() {
1248 Some(line) => line.push(span),
1249 None => lines.push(vec![span]),
1250 }
1251}
1252
1253pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
1254 ParsedElement::flatten_to_blocks(parse_html_elements(html))
1255}
1256
1257pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
1271
1272fn html_footnote_span(
1275 el: &scraper::node::Element,
1276 link_href: Option<String>,
1277) -> Option<ParsedSpan> {
1278 let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
1279 if label.is_empty() {
1280 return None;
1281 }
1282 Some(ParsedSpan {
1283 text: String::new(),
1284 link_href,
1285 footnote_ref: Some(label.to_string()),
1286 ..Default::default()
1287 })
1288}
1289
1290fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
1298 let src = el.attr("src")?;
1299 if src.is_empty() {
1300 return None;
1301 }
1302 let dim = |name: &str| -> i64 {
1303 el.attr(name)
1304 .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
1305 .filter(|n| *n > 0)
1306 .unwrap_or(0)
1307 };
1308 Some(ParsedSpan {
1309 text: String::new(),
1310 link_href,
1311 image: Some(ParsedImage {
1312 src: src.to_string(),
1313 alt: el.attr("alt").unwrap_or_default().to_string(),
1314 width: dim("width"),
1315 height: dim("height"),
1316 }),
1317 ..Default::default()
1318 })
1319}
1320
1321fn is_html_list(node: &Node) -> bool {
1323 node.as_element()
1324 .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
1325}
1326
1327fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
1340 use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
1341 use ego_tree::iter::Edge;
1342
1343 let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
1345 let mut depth = 0usize;
1346 for edge in tree.root().traverse() {
1347 match edge {
1348 Edge::Open(node) if is_html_list(node.value()) => {
1349 if depth == MAX_LIST_LEVELS {
1350 too_deep.push(node.id());
1351 }
1352 depth += 1;
1353 }
1354 Edge::Close(node) if is_html_list(node.value()) => {
1355 depth = depth.saturating_sub(1);
1356 }
1357 _ => {}
1358 }
1359 }
1360
1361 for list in too_deep {
1362 let Some(list_ref) = tree.get(list) else {
1366 continue;
1367 };
1368 let mut holder = None;
1369 let mut current = list_ref;
1370 while let Some(parent) = current.parent() {
1371 if is_html_list(parent.value()) {
1372 holder = Some(current.id());
1373 break;
1374 }
1375 current = parent;
1376 }
1377 let Some(holder) = holder else {
1378 continue;
1379 };
1380 let mut after = holder;
1381 let items: Vec<ego_tree::NodeId> = list_ref
1382 .descendants()
1383 .filter(|node| {
1384 node.value()
1385 .as_element()
1386 .is_some_and(|el| el.name() == "li")
1387 })
1388 .map(|node| node.id())
1389 .collect();
1390 for item in items {
1391 if tree.get(after).and_then(|node| node.parent()).is_none() {
1394 break;
1395 }
1396 if let Some(mut node) = tree.get_mut(after) {
1397 node.insert_id_after(item);
1398 after = item;
1399 }
1400 }
1401 let holds_text = tree.get(list).is_some_and(|node| {
1406 node.descendants()
1407 .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
1408 });
1409 if !holds_text {
1410 if let Some(mut node) = tree.get_mut(list) {
1411 node.detach();
1412 }
1413 } else if holder != list
1414 && tree.get(after).and_then(|node| node.parent()).is_some()
1415 && let Some(mut node) = tree.get_mut(after)
1416 {
1417 node.insert_id_after(list);
1418 }
1419 }
1420}
1421
1422pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
1423 use scraper::Html;
1424
1425 let mut fragment = Html::parse_fragment(html);
1426 flatten_deep_html_lists(&mut fragment.tree);
1427 let mut elements: Vec<ParsedElement> = Vec::new();
1428
1429 let root = fragment.root_element();
1431
1432 #[derive(Clone, Default)]
1433 struct FmtState {
1434 bold: bool,
1435 italic: bool,
1436 underline: bool,
1437 strikeout: bool,
1438 code: bool,
1439 superscript: bool,
1440 subscript: bool,
1441 link_href: Option<String>,
1442 }
1443
1444 const MAX_RECURSION_DEPTH: usize = 256;
1445
1446 fn is_metadata_tag(tag: &str) -> bool {
1456 matches!(
1457 tag,
1458 "head"
1459 | "style"
1460 | "script"
1461 | "title"
1462 | "meta"
1463 | "link"
1464 | "base"
1465 | "noscript"
1466 | "template"
1467 )
1468 }
1469
1470 fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
1479 use ego_tree::iter::Edge;
1480 let mut text = String::new();
1481 let mut hidden = 0usize;
1483 for edge in node.traverse() {
1484 match edge {
1485 Edge::Open(open) => match open.value() {
1486 Node::Element(el) => {
1487 if hidden > 0 || is_metadata_tag(el.name()) {
1488 hidden += 1;
1489 } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
1490 text.push(' ');
1491 }
1492 }
1493 Node::Text(t) if hidden == 0 => text.push_str(t),
1494 _ => {}
1495 },
1496 Edge::Close(close) => {
1497 if hidden > 0 && close.value().is_element() {
1498 hidden -= 1;
1499 }
1500 }
1501 }
1502 }
1503 text
1504 }
1505
1506 fn text_span(text: String, state: &FmtState) -> ParsedSpan {
1508 ParsedSpan {
1509 text,
1510 bold: state.bold,
1511 italic: state.italic,
1512 underline: state.underline,
1513 strikeout: state.strikeout,
1514 code: state.code,
1515 superscript: state.superscript,
1516 subscript: state.subscript,
1517 link_href: state.link_href.clone(),
1518 image: None,
1519 footnote_ref: None,
1520 }
1521 }
1522
1523 fn collect_cell_spans(
1525 node: ego_tree::NodeRef<Node>,
1526 state: &FmtState,
1527 spans: &mut Vec<ParsedSpan>,
1528 depth: usize,
1529 ) {
1530 if depth > MAX_RECURSION_DEPTH {
1531 let text = subtree_text(node);
1532 if !text.is_empty() {
1533 spans.push(text_span(text, state));
1534 }
1535 return;
1536 }
1537 for child in node.children() {
1538 match child.value() {
1539 Node::Text(text) => {
1540 let t = text.text.to_string();
1541 if !t.is_empty() {
1542 spans.push(ParsedSpan {
1543 text: t,
1544 bold: state.bold,
1545 italic: state.italic,
1546 underline: state.underline,
1547 strikeout: state.strikeout,
1548 code: state.code,
1549 superscript: state.superscript,
1550 subscript: state.subscript,
1551 link_href: state.link_href.clone(),
1552 image: None,
1553 footnote_ref: None,
1554 });
1555 }
1556 }
1557 Node::Element(el) => {
1558 let tag = el.name();
1559 if is_metadata_tag(tag) {
1560 continue;
1561 }
1562 let mut new_state = state.clone();
1563 match tag {
1564 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
1568 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
1569 {
1570 spans.push(span);
1571 }
1572 continue;
1573 }
1574 "b" | "strong" => new_state.bold = true,
1575 "i" | "em" => new_state.italic = true,
1576 "u" | "ins" => new_state.underline = true,
1577 "s" | "del" | "strike" => new_state.strikeout = true,
1578 "code" => new_state.code = true,
1579 "sup" => new_state.superscript = true,
1580 "sub" => new_state.subscript = true,
1581 "a" => {
1582 if let Some(href) = el.attr("href") {
1583 new_state.link_href = Some(href.to_string());
1584 }
1585 }
1586 "img" => {
1587 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
1588 spans.push(span);
1589 }
1590 continue;
1591 }
1592 "br" => {
1598 spans.push(text_span(" ".to_string(), state));
1599 continue;
1600 }
1601 "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
1602 | "h3" | "h4" | "h5" | "h6" => {
1603 spans.push(text_span(" ".to_string(), state));
1604 }
1605 _ => {}
1606 }
1607 collect_cell_spans(child, &new_state, spans, depth + 1);
1608 }
1609 _ => {}
1610 }
1611 }
1612 }
1613
1614 const MAX_HTML_COLSPAN: usize = 1000;
1617
1618 fn parse_table_element(table_node: ego_tree::NodeRef<Node>) -> (ParsedTable, usize) {
1621 let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
1622 let mut header_rows: usize = 0;
1623 let mut own_cells: usize = 0;
1624
1625 fn collect_rows(
1626 node: ego_tree::NodeRef<Node>,
1627 rows: &mut Vec<Vec<ParsedTableCell>>,
1628 header_rows: &mut usize,
1629 own_cells: &mut usize,
1630 in_thead: bool,
1631 ) {
1632 for child in node.children() {
1633 if let Node::Element(el) = child.value() {
1634 match el.name() {
1635 "thead" => collect_rows(child, rows, header_rows, own_cells, true),
1636 "tbody" | "tfoot" => {
1637 collect_rows(child, rows, header_rows, own_cells, false)
1638 }
1639 "tr" => {
1640 let mut cells: Vec<ParsedTableCell> = Vec::new();
1641 let mut own_end = 0;
1644 for td in child.children() {
1645 if let Node::Element(td_el) = td.value()
1646 && matches!(td_el.name(), "td" | "th")
1647 {
1648 let mut spans = Vec::new();
1649 let state = FmtState::default();
1650 collect_cell_spans(td, &state, &mut spans, 0);
1651 collapse_inline_whitespace(&mut spans);
1652 if spans.is_empty() {
1653 spans.push(ParsedSpan::default());
1654 }
1655 cells.push(ParsedTableCell { spans });
1656 *own_cells += 1;
1657 own_end = cells.len();
1658 let span = td_el
1663 .attr("colspan")
1664 .and_then(|value| value.trim().parse::<usize>().ok())
1665 .unwrap_or(1)
1666 .clamp(1, MAX_HTML_COLSPAN);
1667 for _ in 1..span {
1668 if cells.len() >= MAX_HTML_COLSPAN {
1669 break;
1670 }
1671 cells.push(ParsedTableCell {
1672 spans: vec![ParsedSpan::default()],
1673 });
1674 }
1675 }
1676 }
1677 cells.truncate(own_end);
1681 if !cells.is_empty() {
1682 rows.push(cells);
1683 if in_thead {
1684 *header_rows += 1;
1685 }
1686 }
1687 }
1688 _ => {}
1689 }
1690 }
1691 }
1692 }
1693
1694 collect_rows(
1695 table_node,
1696 &mut rows,
1697 &mut header_rows,
1698 &mut own_cells,
1699 false,
1700 );
1701
1702 if header_rows == 0 && !rows.is_empty() {
1704 header_rows = 1;
1705 }
1706
1707 (
1708 ParsedTable {
1709 header_rows,
1710 rows,
1711 blockquote_depth: 0,
1714 },
1715 own_cells,
1716 )
1717 }
1718
1719 fn walk_node(
1720 node: ego_tree::NodeRef<Node>,
1721 state: &FmtState,
1722 elements: &mut Vec<ParsedElement>,
1723 current_list_style: &Option<ListStyle>,
1724 blockquote_depth: u32,
1725 list_depth: u32,
1726 depth: usize,
1727 ) {
1728 if depth > MAX_RECURSION_DEPTH {
1729 let text = subtree_text(node);
1730 if !text.trim().is_empty() {
1731 let mut spans = vec![text_span(text, state)];
1732 collapse_inline_whitespace(&mut spans);
1733 elements.push(ParsedElement::Block(ParsedBlock {
1734 spans,
1735 blockquote_depth,
1736 ..ParsedBlock::default()
1737 }));
1738 }
1739 return;
1740 }
1741 match node.value() {
1742 Node::Element(el) => {
1743 let tag = el.name();
1744 if is_metadata_tag(tag) {
1745 return;
1746 }
1747 let mut new_state = state.clone();
1748 let mut new_list_style = current_list_style.clone();
1749 let mut bq_depth = blockquote_depth;
1750 let mut new_list_depth = list_depth;
1751
1752 let is_block_tag = matches!(
1754 tag,
1755 "p" | "div"
1756 | "h1"
1757 | "h2"
1758 | "h3"
1759 | "h4"
1760 | "h5"
1761 | "h6"
1762 | "li"
1763 | "pre"
1764 | "br"
1765 | "blockquote"
1766 | "body"
1767 | "html"
1768 );
1769
1770 match tag {
1772 "b" | "strong" => new_state.bold = true,
1773 "i" | "em" => new_state.italic = true,
1774 "u" | "ins" => new_state.underline = true,
1775 "s" | "del" | "strike" => new_state.strikeout = true,
1776 "code" => new_state.code = true,
1777 "sup" => new_state.superscript = true,
1778 "sub" => new_state.subscript = true,
1779 "a" => {
1780 if let Some(href) = el.attr("href") {
1781 new_state.link_href = Some(href.to_string());
1782 }
1783 }
1784 "ul" => {
1785 new_list_style = Some(ListStyle::Disc);
1786 new_list_depth = list_depth + 1;
1787 }
1788 "ol" => {
1789 new_list_style = Some(ListStyle::Decimal);
1790 new_list_depth = list_depth + 1;
1791 }
1792 "blockquote" => {
1793 bq_depth += 1;
1794 }
1795 _ => {}
1796 }
1797
1798 let heading_level = match tag {
1800 "h1" => Some(1),
1801 "h2" => Some(2),
1802 "h3" => Some(3),
1803 "h4" => Some(4),
1804 "h5" => Some(5),
1805 "h6" => Some(6),
1806 _ => None,
1807 };
1808
1809 let is_code_block = tag == "pre";
1810
1811 let code_language = if is_code_block {
1813 node.children().find_map(|child| {
1814 if let Node::Element(cel) = child.value()
1815 && cel.name() == "code"
1816 && let Some(cls) = cel.attr("class")
1817 {
1818 return cls
1819 .split_whitespace()
1820 .find_map(|c| c.strip_prefix("language-"))
1821 .map(|l| l.to_string());
1822 }
1823 None
1824 })
1825 } else {
1826 None
1827 };
1828
1829 let css = if is_block_tag {
1831 el.attr("style").map(parse_block_styles).unwrap_or_default()
1832 } else {
1833 BlockStyles::default()
1834 };
1835
1836 if tag == "table" {
1837 let (mut parsed_table, own_cells) = parse_table_element(node);
1839 if !parsed_table.rows.is_empty() {
1840 parsed_table.blockquote_depth = bq_depth;
1841 push_table(elements, parsed_table, own_cells);
1842 }
1843 return;
1844 }
1845
1846 if tag == "br" {
1847 elements.push(ParsedElement::Block(ParsedBlock {
1849 spans: vec![ParsedSpan {
1850 text: String::new(),
1851 ..Default::default()
1852 }],
1853 heading_level: None,
1854 list_style: None,
1855 list_indent: 0,
1856 list_prefix: String::new(),
1857 list_suffix: String::new(),
1858 marker: None,
1859 is_code_block: false,
1860 code_language: None,
1861 blockquote_depth: bq_depth,
1862 line_height: None,
1863 non_breakable_lines: None,
1864 page_break_before: None,
1865 direction: None,
1866 background_color: None,
1867 alignment: None,
1868 top_margin: None,
1869 text_indent: None,
1870 semantic_role: None,
1871 }));
1872 return;
1873 }
1874
1875 if tag == "blockquote" {
1876 for child in node.children() {
1878 walk_node(
1879 child,
1880 &new_state,
1881 elements,
1882 &new_list_style,
1883 bq_depth,
1884 new_list_depth,
1885 depth + 1,
1886 );
1887 }
1888 } else if is_block_tag && tag != "br" {
1889 let mut spans: Vec<ParsedSpan> = Vec::new();
1894 let mut nested_elements: Vec<ParsedElement> = Vec::new();
1895 collect_inline_spans(
1896 node,
1897 &new_state,
1898 &mut spans,
1899 &new_list_style,
1900 &mut nested_elements,
1901 bq_depth,
1902 new_list_depth,
1903 depth + 1,
1904 );
1905
1906 let list_style_for_block = if tag == "li" {
1907 new_list_style.clone()
1908 } else {
1909 None
1910 };
1911
1912 let list_indent_for_block = if tag == "li" {
1913 new_list_depth.saturating_sub(1)
1914 } else {
1915 0
1916 };
1917
1918 if !(is_code_block || css.preserve_whitespace == Some(true)) {
1919 collapse_inline_whitespace(&mut spans);
1920 }
1921
1922 let own_run_is_content =
1930 spans_carry_content(&spans) || nested_elements.is_empty();
1931
1932 if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
1933 let is_code_block = is_code_block
1937 && !spans
1938 .iter()
1939 .any(|span| span.image.is_some() || span.footnote_ref.is_some());
1940 let block = ParsedBlock {
1941 spans,
1942 heading_level,
1943 list_style: list_style_for_block,
1944 list_indent: list_indent_for_block,
1945 list_prefix: String::new(),
1946 list_suffix: String::new(),
1947 marker: None,
1948 is_code_block,
1949 code_language: code_language.filter(|_| is_code_block),
1950 blockquote_depth: bq_depth,
1951 line_height: css.line_height,
1952 non_breakable_lines: css.non_breakable_lines,
1953 page_break_before: css.page_break_before,
1954 direction: css.direction,
1955 background_color: css.background_color,
1956 alignment: None,
1957 top_margin: None,
1958 text_indent: None,
1959 semantic_role: None,
1960 };
1961 if is_code_block {
1965 elements.push(ParsedElement::Block(block));
1966 } else {
1967 elements.extend(
1968 split_block_at_line_breaks(block)
1969 .into_iter()
1970 .map(ParsedElement::Block),
1971 );
1972 }
1973 }
1974 elements.append(&mut nested_elements);
1976 } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
1977 for child in node.children() {
1979 walk_node(
1980 child,
1981 &new_state,
1982 elements,
1983 &new_list_style,
1984 bq_depth,
1985 new_list_depth,
1986 depth + 1,
1987 );
1988 }
1989 } else {
1990 for child in node.children() {
1992 walk_node(
1993 child,
1994 &new_state,
1995 elements,
1996 current_list_style,
1997 bq_depth,
1998 list_depth,
1999 depth + 1,
2000 );
2001 }
2002 }
2003 }
2004 Node::Text(text) => {
2005 let t = text.text.to_string();
2006 let trimmed = t.trim();
2007 if !trimmed.is_empty() {
2008 elements.push(ParsedElement::Block(ParsedBlock {
2010 spans: vec![ParsedSpan {
2011 text: trimmed.to_string(),
2012 bold: state.bold,
2013 italic: state.italic,
2014 underline: state.underline,
2015 strikeout: state.strikeout,
2016 code: state.code,
2017 superscript: state.superscript,
2018 subscript: state.subscript,
2019 link_href: state.link_href.clone(),
2020 image: None,
2021 footnote_ref: None,
2022 }],
2023 heading_level: None,
2024 list_style: None,
2025 list_indent: 0,
2026 list_prefix: String::new(),
2027 list_suffix: String::new(),
2028 marker: None,
2029 is_code_block: false,
2030 code_language: None,
2031 blockquote_depth,
2032 line_height: None,
2033 non_breakable_lines: None,
2034 page_break_before: None,
2035 direction: None,
2036 background_color: None,
2037 alignment: None,
2038 top_margin: None,
2039 text_indent: None,
2040 semantic_role: None,
2041 }));
2042 }
2043 }
2044 _ => {
2045 for child in node.children() {
2047 walk_node(
2048 child,
2049 state,
2050 elements,
2051 current_list_style,
2052 blockquote_depth,
2053 list_depth,
2054 depth + 1,
2055 );
2056 }
2057 }
2058 }
2059 }
2060
2061 #[allow(clippy::too_many_arguments)]
2065 fn collect_inline_spans(
2066 node: ego_tree::NodeRef<Node>,
2067 state: &FmtState,
2068 spans: &mut Vec<ParsedSpan>,
2069 current_list_style: &Option<ListStyle>,
2070 elements: &mut Vec<ParsedElement>,
2071 blockquote_depth: u32,
2072 list_depth: u32,
2073 depth: usize,
2074 ) {
2075 if depth > MAX_RECURSION_DEPTH {
2076 let text = subtree_text(node);
2077 if !text.is_empty() {
2078 spans.push(text_span(text, state));
2079 }
2080 return;
2081 }
2082 for child in node.children() {
2083 match child.value() {
2084 Node::Text(text) => {
2085 let t = text.text.to_string();
2086 if !t.is_empty() {
2087 spans.push(ParsedSpan {
2088 text: t,
2089 bold: state.bold,
2090 italic: state.italic,
2091 underline: state.underline,
2092 strikeout: state.strikeout,
2093 code: state.code,
2094 superscript: state.superscript,
2095 subscript: state.subscript,
2096 link_href: state.link_href.clone(),
2097 image: None,
2098 footnote_ref: None,
2099 });
2100 }
2101 }
2102 Node::Element(el) => {
2103 let tag = el.name();
2104 if is_metadata_tag(tag) {
2105 continue;
2106 }
2107 let mut new_state = state.clone();
2108
2109 match tag {
2110 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2114 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2115 {
2116 spans.push(span);
2117 }
2118 continue;
2119 }
2120 "b" | "strong" => new_state.bold = true,
2121 "i" | "em" => new_state.italic = true,
2122 "u" | "ins" => new_state.underline = true,
2123 "s" | "del" | "strike" => new_state.strikeout = true,
2124 "code" => new_state.code = true,
2125 "sup" => new_state.superscript = true,
2126 "sub" => new_state.subscript = true,
2127 "a" => {
2128 if let Some(href) = el.attr("href") {
2129 new_state.link_href = Some(href.to_string());
2130 }
2131 }
2132 "img" => {
2133 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2134 spans.push(span);
2135 }
2136 continue;
2137 }
2138 _ => {}
2139 }
2140
2141 let nested_block = matches!(
2143 tag,
2144 "p" | "div"
2145 | "h1"
2146 | "h2"
2147 | "h3"
2148 | "h4"
2149 | "h5"
2150 | "h6"
2151 | "li"
2152 | "pre"
2153 | "blockquote"
2154 | "ul"
2155 | "ol"
2156 );
2157
2158 if tag == "br" {
2159 spans.push(ParsedSpan {
2162 text: String::new(),
2163 ..Default::default()
2164 });
2165 } else if nested_block || tag == "table" {
2166 walk_node(
2168 child,
2169 &new_state,
2170 elements,
2171 current_list_style,
2172 blockquote_depth,
2173 list_depth,
2174 depth + 1,
2175 );
2176 } else {
2177 collect_inline_spans(
2179 child,
2180 &new_state,
2181 spans,
2182 current_list_style,
2183 elements,
2184 blockquote_depth,
2185 list_depth,
2186 depth + 1,
2187 );
2188 }
2189 }
2190 _ => {}
2191 }
2192 }
2193 }
2194
2195 let initial_state = FmtState::default();
2196 let mut root_spans: Vec<ParsedSpan> = Vec::new();
2200 collect_inline_spans(
2201 *root,
2202 &initial_state,
2203 &mut root_spans,
2204 &None,
2205 &mut elements,
2206 0,
2207 0,
2208 0,
2209 );
2210 collapse_inline_whitespace(&mut root_spans);
2211 if spans_carry_content(&root_spans) {
2214 elements.push(ParsedElement::Block(ParsedBlock {
2215 spans: root_spans,
2216 heading_level: None,
2217 list_style: None,
2218 list_indent: 0,
2219 list_prefix: String::new(),
2220 list_suffix: String::new(),
2221 marker: None,
2222 is_code_block: false,
2223 code_language: None,
2224 blockquote_depth: 0,
2225 line_height: None,
2226 non_breakable_lines: None,
2227 page_break_before: None,
2228 direction: None,
2229 background_color: None,
2230 alignment: None,
2231 top_margin: None,
2232 text_indent: None,
2233 semantic_role: None,
2234 }));
2235 }
2236
2237 if elements.is_empty() {
2239 elements.push(ParsedElement::Block(ParsedBlock {
2240 spans: vec![ParsedSpan {
2241 text: String::new(),
2242 ..Default::default()
2243 }],
2244 heading_level: None,
2245 list_style: None,
2246 list_indent: 0,
2247 list_prefix: String::new(),
2248 list_suffix: String::new(),
2249 marker: None,
2250 is_code_block: false,
2251 code_language: None,
2252 blockquote_depth: 0,
2253 line_height: None,
2254 non_breakable_lines: None,
2255 page_break_before: None,
2256 direction: None,
2257 background_color: None,
2258 alignment: None,
2259 top_margin: None,
2260 text_indent: None,
2261 semantic_role: None,
2262 }));
2263 }
2264
2265 elements
2266}
2267
2268pub fn character_format_from_span(
2272 span: &ParsedSpan,
2273 is_code_block: bool,
2274) -> crate::format_runs::CharacterFormat {
2275 use crate::entities::CharVerticalAlignment;
2276 crate::format_runs::CharacterFormat {
2277 font_bold: if span.bold { Some(true) } else { None },
2278 font_italic: if span.italic { Some(true) } else { None },
2279 font_underline: if span.underline { Some(true) } else { None },
2280 font_strikeout: if span.strikeout { Some(true) } else { None },
2281 font_family: if span.code || is_code_block {
2282 Some("monospace".to_string())
2283 } else {
2284 None
2285 },
2286 anchor_href: span.link_href.clone(),
2287 is_anchor: if span.link_href.is_some() {
2288 Some(true)
2289 } else {
2290 None
2291 },
2292 vertical_alignment: if span.superscript {
2293 Some(CharVerticalAlignment::SuperScript)
2294 } else if span.subscript {
2295 Some(CharVerticalAlignment::SubScript)
2296 } else {
2297 None
2298 },
2299 ..Default::default()
2300 }
2301}
2302
2303pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
2315 use crate::format_runs::{
2316 CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
2317 };
2318
2319 let mut plain_text = String::new();
2320 let mut runs: Vec<FormatRun> = Vec::new();
2321 let mut images: Vec<ImageAnchor> = Vec::new();
2322 let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
2323 let default = CharacterFormat::default();
2324
2325 for span in spans {
2326 let byte_start = plain_text.len() as u32;
2327
2328 if let Some(label) = &span.footnote_ref {
2329 plain_text.push('\u{FFFC}');
2332 let mut format = character_format_from_span(span, is_code_block);
2347 format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
2348 footnote_refs.push(FootnoteRefAnchor {
2349 byte_offset: byte_start,
2350 label: label.clone(),
2351 format,
2352 });
2353 continue;
2354 }
2355
2356 if let Some(image) = &span.image {
2357 plain_text.push('\u{FFFC}');
2361 images.push(ImageAnchor {
2362 byte_offset: byte_start,
2363 name: image.src.clone(),
2364 alt: image.alt.clone(),
2365 width: image.width,
2366 height: image.height,
2367 quality: 100,
2368 format: character_format_from_span(span, is_code_block),
2369 });
2370 continue;
2371 }
2372
2373 plain_text.push_str(&span.text);
2374 let byte_end = plain_text.len() as u32;
2375 if byte_start == byte_end {
2376 continue;
2377 }
2378 let format = character_format_from_span(span, is_code_block);
2379 if format == default {
2380 continue;
2381 }
2382 runs.push(FormatRun {
2383 byte_start,
2384 byte_end,
2385 format,
2386 });
2387 }
2388 coalesce_in_place(&mut runs);
2389 ParsedInline {
2390 plain_text,
2391 runs,
2392 images,
2393 footnote_refs,
2394 }
2395}
2396
2397#[derive(Debug, Clone, Default)]
2404pub struct ParsedInline {
2405 pub plain_text: String,
2406 pub runs: Vec<crate::format_runs::FormatRun>,
2407 pub images: Vec<crate::format_runs::ImageAnchor>,
2408 pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
2409}
2410
2411fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
2419 use jotdown::ListBulletType as B;
2420 match b {
2421 B::Dash => ListStyle::Disc,
2422 B::Star => ListStyle::Circle,
2423 B::Plus => ListStyle::Square,
2424 }
2425}
2426
2427fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
2429 use jotdown::OrderedListNumbering as N;
2430 match n {
2431 N::Decimal => ListStyle::Decimal,
2432 N::AlphaLower => ListStyle::LowerAlpha,
2433 N::AlphaUpper => ListStyle::UpperAlpha,
2434 N::RomanLower => ListStyle::LowerRoman,
2435 N::RomanUpper => ListStyle::UpperRoman,
2436 }
2437}
2438
2439fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
2443 use jotdown::OrderedListStyle as S;
2444 match style {
2445 S::Period => (String::new(), ".".to_string()),
2446 S::Paren => (String::new(), ")".to_string()),
2447 S::ParenParen => ("(".to_string(), ")".to_string()),
2448 }
2449}
2450
2451#[derive(Debug, Clone, Default)]
2455struct DjotBlockStyle {
2456 alignment: Option<Alignment>,
2457 line_height: Option<i64>,
2458 non_breakable_lines: Option<bool>,
2459 page_break_before: Option<bool>,
2460 direction: Option<TextDirection>,
2461 background_color: Option<String>,
2462 top_margin: Option<i64>,
2463 text_indent: Option<i64>,
2464 semantic_role: Option<SemanticRole>,
2465}
2466
2467impl DjotBlockStyle {
2468 fn merge_from(&mut self, other: DjotBlockStyle) {
2472 if other.alignment.is_some() {
2473 self.alignment = other.alignment;
2474 }
2475 if other.line_height.is_some() {
2476 self.line_height = other.line_height;
2477 }
2478 if other.non_breakable_lines.is_some() {
2479 self.non_breakable_lines = other.non_breakable_lines;
2480 }
2481 if other.page_break_before.is_some() {
2482 self.page_break_before = other.page_break_before;
2483 }
2484 if other.direction.is_some() {
2485 self.direction = other.direction;
2486 }
2487 if other.background_color.is_some() {
2488 self.background_color = other.background_color;
2489 }
2490 if other.top_margin.is_some() {
2491 self.top_margin = other.top_margin;
2492 }
2493 if other.text_indent.is_some() {
2494 self.text_indent = other.text_indent;
2495 }
2496 if other.semantic_role.is_some() {
2497 self.semantic_role = other.semantic_role.clone();
2498 }
2499 }
2500}
2501
2502fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
2508 let mut style = DjotBlockStyle::default();
2509
2510 if opts.alignment
2511 && let Some(v) = attrs.get_value("alignment")
2512 {
2513 style.alignment = match v.to_string().as_str() {
2514 "left" => Some(Alignment::Left),
2515 "right" => Some(Alignment::Right),
2516 "center" => Some(Alignment::Center),
2517 "justify" => Some(Alignment::Justify),
2518 _ => None,
2519 };
2520 }
2521 if opts.line_height
2522 && let Some(v) = attrs.get_value("line_height")
2523 {
2524 style.line_height = v.to_string().parse::<i64>().ok();
2525 }
2526 if opts.direction
2527 && let Some(v) = attrs.get_value("direction")
2528 {
2529 style.direction = match v.to_string().as_str() {
2530 "ltr" => Some(TextDirection::LeftToRight),
2531 "rtl" => Some(TextDirection::RightToLeft),
2532 _ => None,
2533 };
2534 }
2535 if opts.non_breakable_lines
2536 && let Some(v) = attrs.get_value("non_breakable_lines")
2537 {
2538 style.non_breakable_lines = match v.to_string().as_str() {
2539 "true" => Some(true),
2540 "false" => Some(false),
2541 _ => None,
2542 };
2543 }
2544 if opts.page_break_before
2545 && let Some(v) = attrs.get_value("page_break_before")
2546 {
2547 style.page_break_before = match v.to_string().as_str() {
2548 "true" => Some(true),
2549 "false" => Some(false),
2550 _ => None,
2551 };
2552 }
2553 if opts.background_color
2554 && let Some(v) = attrs.get_value("background_color")
2555 {
2556 style.background_color = Some(v.to_string());
2557 }
2558 if opts.top_margin
2559 && let Some(v) = attrs.get_value("top_margin")
2560 {
2561 style.top_margin = v.to_string().parse::<i64>().ok();
2562 }
2563 if opts.text_indent
2564 && let Some(v) = attrs.get_value("text_indent")
2565 {
2566 style.text_indent = v.to_string().parse::<i64>().ok();
2567 }
2568 if opts.semantic_role
2569 && let Some(v) = attrs.get_value("semantic_role")
2570 {
2571 style.semantic_role = match v.to_string().as_str() {
2572 "epigraph" => Some(SemanticRole::Epigraph),
2573 _ => None,
2577 };
2578 }
2579
2580 style
2581}
2582
2583#[allow(clippy::too_many_arguments)]
2586fn djot_push_block(
2587 elements: &mut Vec<ParsedElement>,
2588 spans: Vec<ParsedSpan>,
2589 heading_level: Option<i64>,
2590 list_style: Option<ListStyle>,
2591 list_indent: u32,
2592 list_prefix: String,
2593 list_suffix: String,
2594 marker: Option<MarkerType>,
2595 is_code_block: bool,
2596 code_language: Option<String>,
2597 blockquote_depth: u32,
2598 style: DjotBlockStyle,
2599) {
2600 elements.push(ParsedElement::Block(ParsedBlock {
2601 spans,
2602 heading_level,
2603 list_style,
2604 list_indent,
2605 list_prefix,
2606 list_suffix,
2607 marker,
2608 is_code_block,
2609 code_language,
2610 blockquote_depth,
2611 line_height: style.line_height,
2612 non_breakable_lines: style.non_breakable_lines,
2613 page_break_before: style.page_break_before,
2614 direction: style.direction,
2615 background_color: style.background_color,
2616 alignment: style.alignment,
2617 top_margin: style.top_margin,
2618 text_indent: style.text_indent,
2619 semantic_role: style.semantic_role.clone(),
2620 }));
2621}
2622
2623fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
2636 djot.lines()
2637 .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
2638 .filter(|line| !line.is_empty())
2639 .map(|line| {
2640 ParsedElement::Block(ParsedBlock {
2641 spans: vec![ParsedSpan {
2642 text: line.to_string(),
2643 ..Default::default()
2644 }],
2645 ..Default::default()
2646 })
2647 })
2648 .collect()
2649}
2650
2651pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
2672 use jotdown::{Container as C, Event as E, ListKind};
2673
2674 let parsable = crate::parser_tools::djot_depth::parsable(djot);
2700 let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
2701 return djot_source_lines(djot);
2702 };
2703
2704 let mut elements: Vec<ParsedElement> = Vec::new();
2705 let mut current_spans: Vec<ParsedSpan> = Vec::new();
2706 let mut current_heading: Option<i64> = None;
2707 let mut is_code_block = false;
2708 let mut code_language: Option<String> = None;
2709 let mut blockquote_depth: u32 = 0;
2710 let mut pending_style = DjotBlockStyle::default();
2713
2714 let mut bold = false;
2716 let mut italic = false;
2717 let mut underline = false;
2718 let mut strikeout = false;
2719 let mut code = false;
2720 let mut superscript = false;
2721 let mut subscript = false;
2722 let mut link_href: Option<String> = None;
2723 let mut pending_image: Option<ParsedImage> = None;
2726
2727 let mut list_stack: Vec<(ListStyle, String, String)> = Vec::new();
2729 let mut cur_list_style: Option<ListStyle> = None;
2731 let mut cur_list_prefix = String::new();
2732 let mut cur_list_suffix = String::new();
2733 let mut cur_list_indent: u32 = 0;
2734 let mut cur_marker: Option<MarkerType> = None;
2735
2736 let mut in_table_cell = false;
2738 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
2739 let mut current_row: Vec<ParsedTableCell> = Vec::new();
2740 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
2741 let mut table_header_rows: usize = 0;
2742 let mut row_is_head = false;
2743
2744 let mut skip_depth: u32 = 0;
2748
2749 let mut footnote_open: Option<(String, usize)> = None;
2753
2754 macro_rules! push_text {
2759 ($t:expr) => {{
2760 if let Some(img) = pending_image.as_mut() {
2765 img.alt.push_str(($t).as_ref());
2766 } else {
2767 let sp = ParsedSpan {
2768 text: ($t).to_string(),
2769 bold,
2770 italic,
2771 underline,
2772 strikeout,
2773 code,
2774 superscript,
2775 subscript,
2776 link_href: link_href.clone(),
2777 image: None,
2778 footnote_ref: None,
2779 };
2780 if in_table_cell {
2781 current_cell_spans.push(sp);
2782 } else {
2783 current_spans.push(sp);
2784 }
2785 }
2786 }};
2787 }
2788
2789 macro_rules! push_image {
2791 ($img:expr) => {{
2792 let sp = ParsedSpan {
2793 text: String::new(),
2794 bold,
2795 italic,
2796 underline,
2797 strikeout,
2798 code,
2799 superscript,
2800 subscript,
2801 link_href: link_href.clone(),
2802 image: Some($img),
2803 footnote_ref: None,
2804 };
2805 if in_table_cell {
2806 current_cell_spans.push(sp);
2807 } else {
2808 current_spans.push(sp);
2809 }
2810 }};
2811 }
2812
2813 macro_rules! enter_item {
2816 ($marker:expr) => {{
2817 if !current_spans.is_empty() {
2818 djot_push_block(
2819 &mut elements,
2820 std::mem::take(&mut current_spans),
2821 None,
2822 cur_list_style.clone(),
2823 cur_list_indent,
2824 cur_list_prefix.clone(),
2825 cur_list_suffix.clone(),
2826 cur_marker.clone(),
2827 false,
2828 None,
2829 blockquote_depth,
2830 DjotBlockStyle::default(),
2831 );
2832 }
2833 let (style, prefix, suffix) = list_stack.last().cloned().unwrap_or((
2834 ListStyle::Disc,
2835 String::new(),
2836 String::new(),
2837 ));
2838 cur_list_style = Some(style);
2839 cur_list_prefix = prefix;
2840 cur_list_suffix = suffix;
2841 cur_list_indent = list_stack.len().saturating_sub(1) as u32;
2842 cur_marker = $marker;
2843 }};
2844 }
2845
2846 macro_rules! line_break {
2853 () => {{
2854 if in_table_cell {
2855 push_text!(" ");
2856 } else if !current_spans.is_empty() && current_heading.is_some() {
2857 djot_push_block(
2862 &mut elements,
2863 std::mem::take(&mut current_spans),
2864 current_heading,
2865 None,
2866 0,
2867 String::new(),
2868 String::new(),
2869 None,
2870 false,
2871 None,
2872 blockquote_depth,
2873 pending_style.clone(),
2874 );
2875 pending_style.page_break_before = None;
2876 } else if !current_spans.is_empty() {
2877 djot_push_block(
2880 &mut elements,
2881 std::mem::take(&mut current_spans),
2882 None,
2883 cur_list_style.clone(),
2884 cur_list_indent,
2885 cur_list_prefix.clone(),
2886 cur_list_suffix.clone(),
2887 cur_marker.clone(),
2888 is_code_block,
2889 code_language.clone(),
2890 blockquote_depth,
2891 pending_style.clone(),
2892 );
2893 pending_style.page_break_before = None;
2894 }
2895 }};
2896 }
2897
2898 for event in events {
2899 if skip_depth > 0 {
2900 match event {
2901 E::Start(..) => skip_depth += 1,
2902 E::End(_) => skip_depth -= 1,
2903 _ => {}
2904 }
2905 continue;
2906 }
2907
2908 match event {
2909 E::Start(C::Document, _) | E::End(C::Document) => {}
2911 E::Start(C::Section { .. }, attrs) => {
2912 if list_stack.is_empty() {
2915 pending_style.merge_from(block_attrs_to_style(&attrs, options));
2916 }
2917 }
2918 E::End(C::Section { .. }) => {}
2919 E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
2920
2921 E::Start(C::Blockquote, _) => blockquote_depth += 1,
2923 E::End(C::Blockquote) => blockquote_depth = blockquote_depth.saturating_sub(1),
2924
2925 E::Start(C::List { kind, .. }, _) => {
2927 let (style, prefix, suffix) = match kind {
2928 ListKind::Unordered(b) | ListKind::Task(b) => {
2929 (djot_bullet_style(b), String::new(), String::new())
2930 }
2931 ListKind::Ordered {
2932 numbering, style, ..
2933 } => {
2934 let (p, s) = djot_ordered_affixes(style);
2935 (djot_ordered_style(numbering), p, s)
2936 }
2937 };
2938 list_stack.push((style, prefix, suffix));
2939 }
2940 E::End(C::List { .. }) => {
2941 list_stack.pop();
2942 cur_list_style = None;
2943 cur_marker = None;
2944 }
2945 E::Start(C::ListItem, _) => enter_item!(None),
2946 E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
2947 MarkerType::Checked
2948 } else {
2949 MarkerType::Unchecked
2950 })),
2951 E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
2952 if !current_spans.is_empty() {
2954 djot_push_block(
2955 &mut elements,
2956 std::mem::take(&mut current_spans),
2957 None,
2958 cur_list_style.clone(),
2959 cur_list_indent,
2960 cur_list_prefix.clone(),
2961 cur_list_suffix.clone(),
2962 cur_marker.clone(),
2963 false,
2964 None,
2965 blockquote_depth,
2966 DjotBlockStyle::default(),
2967 );
2968 }
2969 cur_list_style = None;
2970 cur_marker = None;
2971 }
2972
2973 E::Start(C::Heading { level, .. }, attrs) => {
2975 current_heading = Some(level as i64);
2976 pending_style.merge_from(block_attrs_to_style(&attrs, options));
2979 }
2980 E::End(C::Heading { .. }) => {
2981 djot_push_block(
2982 &mut elements,
2983 std::mem::take(&mut current_spans),
2984 current_heading.take(),
2985 None,
2986 0,
2987 String::new(),
2988 String::new(),
2989 None,
2990 false,
2991 None,
2992 blockquote_depth,
2993 std::mem::take(&mut pending_style),
2994 );
2995 }
2996 E::Start(C::Paragraph, attrs) => {
2997 current_heading = None;
2998 pending_style = if list_stack.is_empty() {
3002 block_attrs_to_style(&attrs, options)
3003 } else {
3004 DjotBlockStyle::default()
3005 };
3006 }
3007 E::End(C::Paragraph) => {
3008 if !current_spans.is_empty() {
3009 djot_push_block(
3010 &mut elements,
3011 std::mem::take(&mut current_spans),
3012 None,
3013 cur_list_style.clone(),
3014 cur_list_indent,
3015 cur_list_prefix.clone(),
3016 cur_list_suffix.clone(),
3017 cur_marker.clone(),
3018 false,
3019 None,
3020 blockquote_depth,
3021 std::mem::take(&mut pending_style),
3022 );
3023 }
3024 cur_list_style = None;
3025 cur_marker = None;
3026 }
3027 E::Start(C::CodeBlock { language }, _) => {
3028 is_code_block = true;
3029 code_language = if language.is_empty() {
3030 None
3031 } else {
3032 Some(language.to_string())
3033 };
3034 }
3035 E::End(C::CodeBlock { .. }) => {
3036 if let Some(last) = current_spans.last_mut()
3038 && last.text.ends_with('\n')
3039 {
3040 last.text.pop();
3041 }
3042 djot_push_block(
3043 &mut elements,
3044 std::mem::take(&mut current_spans),
3045 None,
3046 None,
3047 0,
3048 String::new(),
3049 String::new(),
3050 None,
3051 true,
3052 code_language.take(),
3053 blockquote_depth,
3054 DjotBlockStyle::default(),
3055 );
3056 is_code_block = false;
3057 }
3058
3059 E::Start(C::Table, _) => {
3061 table_rows.clear();
3062 current_row.clear();
3063 current_cell_spans.clear();
3064 table_header_rows = 0;
3065 }
3066 E::End(C::Table) => {
3067 let own_cells = cells_in_rows(&table_rows);
3068 push_table(
3069 &mut elements,
3070 ParsedTable {
3071 header_rows: table_header_rows,
3072 rows: std::mem::take(&mut table_rows),
3073 blockquote_depth,
3074 },
3075 own_cells,
3076 );
3077 }
3078 E::Start(C::TableRow { head }, _) => {
3079 row_is_head = head;
3080 current_row.clear();
3081 }
3082 E::End(C::TableRow { .. }) => {
3083 if row_is_head {
3084 table_header_rows += 1;
3085 }
3086 table_rows.push(std::mem::take(&mut current_row));
3087 }
3088 E::Start(C::TableCell { .. }, _) => {
3089 in_table_cell = true;
3090 current_cell_spans.clear();
3091 }
3092 E::End(C::TableCell { .. }) => {
3093 in_table_cell = false;
3094 current_row.push(ParsedTableCell {
3095 spans: std::mem::take(&mut current_cell_spans),
3096 });
3097 }
3098
3099 E::Start(C::Strong, _) => bold = true,
3101 E::End(C::Strong) => bold = false,
3102 E::Start(C::Emphasis, _) => italic = true,
3103 E::End(C::Emphasis) => italic = false,
3104 E::Start(C::Verbatim, _) => code = true,
3105 E::End(C::Verbatim) => code = false,
3106 E::Start(C::Superscript, _) => superscript = true,
3107 E::End(C::Superscript) => superscript = false,
3108 E::Start(C::Subscript, _) => subscript = true,
3109 E::End(C::Subscript) => subscript = false,
3110 E::Start(C::Insert, _) => underline = true,
3111 E::End(C::Insert) => underline = false,
3112 E::Start(C::Delete, _) => strikeout = true,
3113 E::End(C::Delete) => strikeout = false,
3114 E::Start(C::Mark, _) | E::End(C::Mark) => {}
3116 E::Start(C::Span, _) | E::End(C::Span) => {}
3117 E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
3118 E::End(C::Link(..)) => link_href = None,
3119 E::Start(C::Image(src, _), attrs) => {
3124 let attr_num = |key: &str| -> i64 {
3125 attrs
3126 .get_value(key)
3127 .map(|v| v.to_string())
3128 .and_then(|v| v.trim().parse::<i64>().ok())
3129 .filter(|n| *n > 0)
3130 .unwrap_or(0)
3131 };
3132 pending_image = Some(ParsedImage {
3133 src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
3136 alt: String::new(),
3137 width: attr_num("width"),
3138 height: attr_num("height"),
3139 });
3140 }
3141 E::End(C::Image(..)) => {
3142 if let Some(img) = pending_image.take() {
3143 push_image!(img);
3144 }
3145 }
3146
3147 E::Start(C::Footnote { label }, _) => {
3156 if !current_spans.is_empty() {
3157 djot_push_block(
3158 &mut elements,
3159 std::mem::take(&mut current_spans),
3160 None,
3161 cur_list_style.clone(),
3162 cur_list_indent,
3163 cur_list_prefix.clone(),
3164 cur_list_suffix.clone(),
3165 cur_marker.clone(),
3166 false,
3167 None,
3168 blockquote_depth,
3169 DjotBlockStyle::default(),
3170 );
3171 }
3172 footnote_open = Some((label.to_string(), elements.len()));
3173 }
3174 E::End(C::Footnote { .. }) => {
3175 if !current_spans.is_empty() {
3176 djot_push_block(
3177 &mut elements,
3178 std::mem::take(&mut current_spans),
3179 None,
3180 cur_list_style.clone(),
3181 cur_list_indent,
3182 cur_list_prefix.clone(),
3183 cur_list_suffix.clone(),
3184 cur_marker.clone(),
3185 false,
3186 None,
3187 blockquote_depth,
3188 DjotBlockStyle::default(),
3189 );
3190 }
3191 if let Some((label, start)) = footnote_open.take() {
3192 let blocks: Vec<ParsedBlock> = elements
3193 .drain(start..)
3194 .filter_map(|e| match e {
3195 ParsedElement::Block(b) => Some(b),
3196 _ => None,
3200 })
3201 .collect();
3202 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
3203 }
3204 }
3205
3206 E::Start(
3208 C::Math { .. }
3209 | C::RawBlock { .. }
3210 | C::RawInline { .. }
3211 | C::DescriptionList
3212 | C::DescriptionDetails
3213 | C::DescriptionTerm
3214 | C::Caption
3215 | C::LinkDefinition { .. },
3216 _,
3217 ) => skip_depth = 1,
3218
3219 E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
3231 for (i, piece) in s.split('\n').enumerate() {
3232 if i > 0 {
3233 line_break!();
3234 }
3235 if !piece.is_empty() {
3236 push_text!(piece);
3237 }
3238 }
3239 }
3240 E::Str(s) => push_text!(s.as_ref()),
3241 E::Softbreak => push_text!(" "),
3242 E::LeftSingleQuote => push_text!("\u{2018}"),
3243 E::RightSingleQuote => push_text!("\u{2019}"),
3244 E::LeftDoubleQuote => push_text!("\u{201C}"),
3245 E::RightDoubleQuote => push_text!("\u{201D}"),
3246 E::Ellipsis => push_text!("\u{2026}"),
3247 E::EnDash => push_text!("\u{2013}"),
3248 E::EmDash => push_text!("\u{2014}"),
3249 E::NonBreakingSpace => push_text!("\u{00A0}"),
3250 E::Hardbreak => line_break!(),
3251 E::FootnoteReference(label) => {
3257 let sp = ParsedSpan {
3258 text: String::new(),
3259 bold,
3260 italic,
3261 underline,
3262 strikeout,
3263 code,
3264 superscript,
3265 subscript,
3266 link_href: link_href.clone(),
3267 image: None,
3268 footnote_ref: Some(label.to_string()),
3269 };
3270 if in_table_cell {
3271 current_cell_spans.push(sp);
3272 } else {
3273 current_spans.push(sp);
3274 }
3275 }
3276 E::Symbol(_) => {}
3279 E::Escape | E::Blankline => {}
3280 E::ThematicBreak(_) | E::Attributes(_) => {}
3281
3282 _ => {}
3285 }
3286 }
3287
3288 if !current_spans.is_empty() {
3290 djot_push_block(
3291 &mut elements,
3292 std::mem::take(&mut current_spans),
3293 current_heading.take(),
3294 cur_list_style.clone(),
3295 cur_list_indent,
3296 cur_list_prefix.clone(),
3297 cur_list_suffix.clone(),
3298 cur_marker.clone(),
3299 is_code_block,
3300 code_language.take(),
3301 blockquote_depth,
3302 std::mem::take(&mut pending_style),
3303 );
3304 }
3305
3306 if elements.is_empty() {
3309 djot_push_block(
3310 &mut elements,
3311 vec![ParsedSpan {
3312 text: String::new(),
3313 ..Default::default()
3314 }],
3315 None,
3316 None,
3317 0,
3318 String::new(),
3319 String::new(),
3320 None,
3321 false,
3322 None,
3323 0,
3324 DjotBlockStyle::default(),
3325 );
3326 }
3327
3328 elements
3329}
3330
3331#[cfg(test)]
3332mod tests {
3333 use super::*;
3334
3335 fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
3337 ParsedElement::flatten_to_blocks(parse_markdown(md))
3338 }
3339
3340 #[test]
3341 fn test_parse_markdown_simple_paragraph() {
3342 let blocks = parse_markdown_blocks("Hello **world**");
3343 assert_eq!(blocks.len(), 1);
3344 assert!(blocks[0].spans.len() >= 2);
3345 let plain_span = blocks[0]
3347 .spans
3348 .iter()
3349 .find(|s| s.text.contains("Hello"))
3350 .unwrap();
3351 assert!(!plain_span.bold);
3352 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3353 assert!(bold_span.bold);
3354 }
3355
3356 #[test]
3357 fn test_parse_markdown_heading() {
3358 let blocks = parse_markdown_blocks("# Title");
3359 assert_eq!(blocks.len(), 1);
3360 assert_eq!(blocks[0].heading_level, Some(1));
3361 assert_eq!(blocks[0].spans[0].text, "Title");
3362 }
3363
3364 #[test]
3365 fn test_parse_markdown_list() {
3366 let blocks = parse_markdown_blocks("- item1\n- item2");
3367 assert!(blocks.len() >= 2);
3368 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3369 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3370 }
3371
3372 fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
3374 elements
3375 .iter()
3376 .map(|e| match e {
3377 ParsedElement::Block(b) => (false, b.blockquote_depth),
3378 ParsedElement::Table(t) => (true, t.blockquote_depth),
3379 ParsedElement::FootnoteDefinition { .. } => (false, 0),
3382 })
3383 .collect()
3384 }
3385
3386 #[test]
3387 fn test_parse_markdown_table_in_blockquote_records_depth() {
3388 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3389 assert_eq!(element_depths(&elements), vec![(true, 1)]);
3390 }
3391
3392 #[test]
3393 fn test_parse_markdown_text_then_table_in_blockquote() {
3394 let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
3395 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
3396 }
3397
3398 #[test]
3399 fn test_parse_markdown_table_after_blockquote_closes() {
3400 let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
3401 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
3402 }
3403
3404 #[test]
3405 fn test_parse_markdown_table_in_nested_blockquote() {
3406 let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
3407 assert_eq!(element_depths(&elements), vec![(true, 2)]);
3408 }
3409
3410 #[test]
3411 fn test_parse_markdown_list_in_blockquote_records_depth() {
3412 let elements = parse_markdown("> - item1\n> - item2");
3413 let depths = element_depths(&elements);
3414 assert_eq!(depths, vec![(false, 1), (false, 1)]);
3415 for e in &elements {
3416 if let ParsedElement::Block(b) = e {
3417 assert_eq!(b.list_style, Some(ListStyle::Disc));
3418 }
3419 }
3420 }
3421
3422 #[test]
3423 fn test_parse_html_table_in_blockquote_records_depth() {
3424 let elements = parse_html_elements(
3425 "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
3426 );
3427 assert_eq!(element_depths(&elements), vec![(true, 1)]);
3428 }
3429
3430 #[test]
3431 fn test_parse_html_table_after_blockquote() {
3432 let elements = parse_html_elements(
3433 "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
3434 );
3435 let depths = element_depths(&elements);
3436 assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
3438 assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
3439 }
3440
3441 #[test]
3442 fn test_flatten_to_blocks_propagates_blockquote_depth() {
3443 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3444 let blocks = ParsedElement::flatten_to_blocks(elements);
3445 assert!(!blocks.is_empty());
3446 for b in &blocks {
3447 assert_eq!(b.blockquote_depth, 1);
3448 }
3449 }
3450
3451 #[test]
3452 fn test_parse_html_simple() {
3453 let blocks = parse_html("<p>Hello <b>world</b></p>");
3454 assert_eq!(blocks.len(), 1);
3455 assert!(blocks[0].spans.len() >= 2);
3456 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3457 assert!(bold_span.bold);
3458 }
3459
3460 #[test]
3461 fn test_parse_html_multiple_paragraphs() {
3462 let blocks = parse_html("<p>A</p><p>B</p>");
3463 assert_eq!(blocks.len(), 2);
3464 }
3465
3466 #[test]
3467 fn test_parse_html_heading() {
3468 let blocks = parse_html("<h2>Subtitle</h2>");
3469 assert_eq!(blocks.len(), 1);
3470 assert_eq!(blocks[0].heading_level, Some(2));
3471 }
3472
3473 #[test]
3474 fn test_parse_html_list() {
3475 let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
3476 assert!(blocks.len() >= 2);
3477 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3478 }
3479
3480 #[test]
3481 fn test_parse_markdown_code_block() {
3482 let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
3483 assert_eq!(blocks.len(), 1);
3484 assert!(blocks[0].is_code_block);
3485 assert!(blocks[0].spans[0].code);
3486 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3488 assert_eq!(
3489 text, "fn main() {}",
3490 "code block text should not have trailing newline"
3491 );
3492 }
3493
3494 #[test]
3495 fn test_parse_markdown_nested_formatting() {
3496 let blocks = parse_markdown_blocks("***bold italic***");
3497 assert_eq!(blocks.len(), 1);
3498 let span = &blocks[0].spans[0];
3499 assert!(span.bold);
3500 assert!(span.italic);
3501 }
3502
3503 #[test]
3504 fn test_parse_markdown_link() {
3505 let blocks = parse_markdown_blocks("[click](http://example.com)");
3506 assert_eq!(blocks.len(), 1);
3507 let span = &blocks[0].spans[0];
3508 assert_eq!(span.text, "click");
3509 assert_eq!(span.link_href, Some("http://example.com".to_string()));
3510 }
3511
3512 #[test]
3513 fn test_parse_markdown_empty() {
3514 let blocks = parse_markdown_blocks("");
3515 assert_eq!(blocks.len(), 1);
3516 assert!(blocks[0].spans[0].text.is_empty());
3517 }
3518
3519 #[test]
3520 fn test_parse_html_empty() {
3521 let blocks = parse_html("");
3522 assert_eq!(blocks.len(), 1);
3523 assert!(blocks[0].spans[0].text.is_empty());
3524 }
3525
3526 #[test]
3527 fn test_parse_html_nested_formatting() {
3528 let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
3529 assert_eq!(blocks.len(), 1);
3530 let span = &blocks[0].spans[0];
3531 assert!(span.bold);
3532 assert!(span.italic);
3533 }
3534
3535 #[test]
3536 fn test_parse_html_link() {
3537 let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
3538 assert_eq!(blocks.len(), 1);
3539 let span = &blocks[0].spans[0];
3540 assert_eq!(span.text, "click");
3541 assert_eq!(span.link_href, Some("http://example.com".to_string()));
3542 }
3543
3544 #[test]
3545 fn test_parse_html_ordered_list() {
3546 let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
3547 assert!(blocks.len() >= 2);
3548 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3549 }
3550
3551 #[test]
3552 fn test_parse_markdown_ordered_list() {
3553 let blocks = parse_markdown_blocks("1. first\n2. second");
3554 assert!(blocks.len() >= 2);
3555 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3556 }
3557
3558 #[test]
3559 fn test_parse_html_blockquote_nested() {
3560 let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
3561 assert!(blocks.len() >= 3);
3562 }
3563
3564 #[test]
3565 fn test_parse_block_styles_line_height() {
3566 let styles = parse_block_styles("line-height: 1.5");
3567 assert_eq!(styles.line_height, Some(1500));
3568 }
3569
3570 #[test]
3571 fn test_parse_block_styles_direction_rtl() {
3572 let styles = parse_block_styles("direction: rtl");
3573 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3574 }
3575
3576 #[test]
3577 fn test_parse_block_styles_background_color() {
3578 let styles = parse_block_styles("background-color: #ff0000");
3579 assert_eq!(styles.background_color, Some("#ff0000".to_string()));
3580 }
3581
3582 #[test]
3583 fn test_parse_block_styles_white_space_pre() {
3584 let styles = parse_block_styles("white-space: pre");
3585 assert_eq!(styles.non_breakable_lines, Some(true));
3586 }
3587
3588 #[test]
3589 fn test_parse_block_styles_multiple() {
3590 let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
3591 assert_eq!(styles.line_height, Some(2000));
3592 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3593 assert_eq!(styles.background_color, Some("blue".to_string()));
3594 }
3595
3596 #[test]
3597 fn test_parse_html_block_styles_extracted() {
3598 let blocks = parse_html(
3599 r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
3600 );
3601 assert_eq!(blocks.len(), 1);
3602 assert_eq!(blocks[0].line_height, Some(1500));
3603 assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
3604 assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
3605 }
3606
3607 #[test]
3611 fn test_parse_html_collapses_exporter_line_wrapping() {
3612 let blocks = parse_html(
3613 "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
3614 <p>\nsecond para</p>\n</body>",
3615 );
3616 let texts: Vec<String> = blocks
3617 .iter()
3618 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
3619 .collect();
3620 assert_eq!(texts, vec!["first line wrapped here", "second para"]);
3621 }
3622
3623 #[test]
3627 fn test_parse_html_drops_whitespace_between_blocks() {
3628 let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
3629 assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
3630 assert!(
3631 blocks
3632 .iter()
3633 .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
3634 "no block keeps a literal newline"
3635 );
3636 }
3637
3638 #[test]
3641 fn test_parse_html_collapses_across_span_boundaries() {
3642 let blocks = parse_html("<p> <b>bold</b>\n <i>italic</i> </p>");
3643 assert_eq!(blocks.len(), 1);
3644 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3645 assert_eq!(text, "bold italic");
3646 }
3647
3648 #[test]
3651 fn test_parse_html_keeps_no_break_space() {
3652 let blocks = parse_html("<p>\nAttention : ici</p>");
3653 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3654 assert_eq!(text, "Attention\u{a0}: ici");
3655 }
3656
3657 #[test]
3660 fn test_parse_html_keeps_empty_paragraph() {
3661 let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
3662 assert_eq!(blocks.len(), 3);
3663 let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
3664 assert_eq!(text, "");
3665 }
3666
3667 #[test]
3671 fn test_parse_html_preserves_whitespace_in_pre() {
3672 let blocks = parse_html("<pre>\nfn main() {\n let x = 1;\n}</pre>");
3675 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3676 assert_eq!(text, "fn main() {\n let x = 1;\n}");
3677
3678 let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n b</p>");
3681 let texts: Vec<String> = styled
3682 .iter()
3683 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
3684 .collect();
3685 assert_eq!(texts, ["a", " b"]);
3686
3687 let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n b</p>");
3688 let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
3689 assert_eq!(text, "a b");
3690 }
3691
3692 #[test]
3695 fn test_parse_html_collapses_table_cell_whitespace() {
3696 let elements = parse_html_elements(
3697 "<table>\n<tr>\n<td>\n one\n</td>\n<td>\n two\n</td>\n</tr>\n</table>",
3698 );
3699 let table = elements
3700 .iter()
3701 .find_map(|e| match e {
3702 ParsedElement::Table(t) => Some(t),
3703 _ => None,
3704 })
3705 .expect("a table");
3706 let cells: Vec<String> = table.rows[0]
3707 .iter()
3708 .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
3709 .collect();
3710 assert_eq!(cells, vec!["one", "two"]);
3711 }
3712
3713 #[test]
3714 fn test_parse_html_white_space_pre() {
3715 let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
3716 assert_eq!(blocks.len(), 1);
3717 assert_eq!(blocks[0].non_breakable_lines, Some(true));
3718 }
3719
3720 #[test]
3721 fn test_parse_html_no_styles_returns_none() {
3722 let blocks = parse_html("<p>plain</p>");
3723 assert_eq!(blocks.len(), 1);
3724 assert_eq!(blocks[0].line_height, None);
3725 assert_eq!(blocks[0].direction, None);
3726 assert_eq!(blocks[0].background_color, None);
3727 assert_eq!(blocks[0].non_breakable_lines, None);
3728 }
3729
3730 #[test]
3731 fn test_parse_markdown_nested_list_indent() {
3732 let md = "- top\n - nested\n - deep";
3733 let blocks = parse_markdown_blocks(md);
3734 assert_eq!(blocks.len(), 3);
3735 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3736 assert_eq!(blocks[0].list_indent, 0);
3737 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3738 assert_eq!(blocks[1].list_indent, 1);
3739 assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
3740 assert_eq!(blocks[2].list_indent, 2);
3741 }
3742
3743 #[test]
3744 fn test_parse_markdown_nested_ordered_list_indent() {
3745 let md = "1. first\n 1. nested\n 2. nested2";
3746 let blocks = parse_markdown_blocks(md);
3747 assert_eq!(blocks.len(), 3);
3748 assert_eq!(blocks[0].list_indent, 0);
3749 assert_eq!(blocks[1].list_indent, 1);
3750 assert_eq!(blocks[2].list_indent, 1);
3751 }
3752
3753 #[test]
3754 fn test_parse_html_nested_list_indent() {
3755 let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
3756 let blocks = parse_html(html);
3757 assert!(blocks.len() >= 2);
3758 assert_eq!(blocks[0].list_indent, 0);
3759 assert_eq!(blocks[1].list_indent, 1);
3760 }
3761
3762 #[test]
3763 fn test_parse_markdown_table() {
3764 let md = "| A | B |\n|---|---|\n| 1 | 2 |";
3765 let elements = parse_markdown(md);
3766 assert_eq!(elements.len(), 1);
3767 match &elements[0] {
3768 ParsedElement::Table(table) => {
3769 assert_eq!(table.header_rows, 1);
3770 assert_eq!(table.rows.len(), 2); assert_eq!(table.rows[0].len(), 2);
3773 assert_eq!(table.rows[0][0].spans[0].text, "A");
3774 assert_eq!(table.rows[0][1].spans[0].text, "B");
3775 assert_eq!(table.rows[1].len(), 2);
3777 assert_eq!(table.rows[1][0].spans[0].text, "1");
3778 assert_eq!(table.rows[1][1].spans[0].text, "2");
3779 }
3780 _ => panic!("Expected ParsedElement::Table"),
3781 }
3782 }
3783
3784 #[test]
3785 fn test_parse_markdown_table_with_formatting() {
3786 let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
3787 let elements = parse_markdown(md);
3788 assert_eq!(elements.len(), 1);
3789 match &elements[0] {
3790 ParsedElement::Table(table) => {
3791 assert_eq!(table.rows.len(), 2);
3792 assert!(table.rows[0][0].spans[0].bold);
3794 assert!(table.rows[0][1].spans[0].code);
3796 assert!(table.rows[0][2].spans[0].italic);
3798 assert!(table.rows[1][0].spans[0].strikeout);
3800 assert_eq!(
3802 table.rows[1][2].spans[0].link_href,
3803 Some("http://x.com".to_string())
3804 );
3805 }
3806 _ => panic!("Expected ParsedElement::Table"),
3807 }
3808 }
3809
3810 #[test]
3811 fn test_parse_markdown_mixed_content_with_table() {
3812 let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
3813 let elements = parse_markdown(md);
3814 assert_eq!(elements.len(), 3);
3815 assert!(matches!(&elements[0], ParsedElement::Block(_)));
3816 assert!(matches!(&elements[1], ParsedElement::Table(_)));
3817 assert!(matches!(&elements[2], ParsedElement::Block(_)));
3818 }
3819}
3820
3821#[cfg(test)]
3822mod djot_tests {
3823 use super::*;
3824 use crate::entities::MarkerType;
3825
3826 fn blocks(d: &str) -> Vec<ParsedBlock> {
3827 ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
3828 }
3829
3830 fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
3831 b.spans.iter().find(|s| pred(s)).expect("span not found")
3832 }
3833
3834 #[test]
3835 fn paragraph_bold_italic() {
3836 let b = blocks("normal *bold* _italic_");
3837 assert_eq!(b.len(), 1);
3838 assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
3839 assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
3840 }
3841
3842 #[test]
3843 fn heading_levels() {
3844 assert_eq!(blocks("# H1")[0].heading_level, Some(1));
3845 assert_eq!(blocks("### H3")[0].heading_level, Some(3));
3846 assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
3847 }
3848
3849 #[test]
3850 fn unordered_bullet_styles_are_distinct() {
3851 assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
3852 assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
3853 assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
3854 }
3855
3856 #[test]
3857 fn ordered_delimiters() {
3858 let period = blocks("1. a");
3859 assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
3860 assert_eq!(period[0].list_prefix, "");
3861 assert_eq!(period[0].list_suffix, ".");
3862
3863 let paren = blocks("1) a");
3864 assert_eq!(paren[0].list_suffix, ")");
3865 assert_eq!(paren[0].list_prefix, "");
3866
3867 let paren_paren = blocks("(1) a");
3868 assert_eq!(paren_paren[0].list_prefix, "(");
3869 assert_eq!(paren_paren[0].list_suffix, ")");
3870 }
3871
3872 #[test]
3873 fn task_list_markers() {
3874 let b = blocks("- [ ] a\n- [x] b");
3875 assert_eq!(b.len(), 2);
3876 assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
3877 assert_eq!(b[1].marker, Some(MarkerType::Checked));
3878 }
3879
3880 #[test]
3881 fn code_block_with_language() {
3882 let b = blocks("```rust\nfn main() {}\n```");
3883 assert_eq!(b.len(), 1);
3884 assert!(b[0].is_code_block);
3885 assert_eq!(b[0].code_language.as_deref(), Some("rust"));
3886 let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
3887 assert_eq!(text, "fn main() {}");
3888 }
3889
3890 #[test]
3891 fn link_href() {
3892 let b = blocks("[text](http://example.com)");
3893 let s = first_span_with(&b[0], |s| s.text == "text");
3894 assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
3895 }
3896
3897 #[test]
3898 fn superscript_subscript() {
3899 assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
3900 assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
3901 }
3902
3903 #[test]
3904 fn delete_insert_verbatim() {
3905 assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
3906 assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
3907 assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
3908 }
3909
3910 #[test]
3911 fn blockquote_depth() {
3912 let els = parse_djot("> quoted", &DjotImportOptions::default());
3913 match &els[0] {
3914 ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
3915 _ => panic!("expected block"),
3916 }
3917 }
3918
3919 #[test]
3920 fn nested_list_indent() {
3921 let b = blocks("- a\n\n - b\n\n - c");
3926 assert_eq!(b.len(), 3);
3927 assert_eq!(b[0].list_indent, 0);
3928 assert_eq!(b[1].list_indent, 1);
3929 assert_eq!(b[2].list_indent, 2);
3930 }
3931
3932 #[test]
3933 fn table_parsed_as_table() {
3934 let els = parse_djot(
3935 "| a | b |\n|---|---|\n| c | d |",
3936 &DjotImportOptions::default(),
3937 );
3938 assert_eq!(els.len(), 1);
3939 match &els[0] {
3940 ParsedElement::Table(t) => {
3941 assert_eq!(t.header_rows, 1);
3942 assert_eq!(t.rows.len(), 2);
3943 assert_eq!(t.rows[0][0].spans[0].text, "a");
3944 assert_eq!(t.rows[1][1].spans[0].text, "d");
3945 }
3946 _ => panic!("expected table"),
3947 }
3948 }
3949
3950 #[test]
3951 fn smart_punctuation_normalised_to_unicode() {
3952 let text: String = blocks("a... b---c")[0]
3953 .spans
3954 .iter()
3955 .map(|s| s.text.as_str())
3956 .collect();
3957 assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
3958 assert!(text.contains('\u{2014}'), "em dash: {text:?}");
3959 }
3960
3961 #[test]
3962 fn unrepresentable_constructs_dropped_without_leaking_text() {
3963 let b = blocks("para1\n\n---\n\npara2");
3965 assert_eq!(b.len(), 2);
3966 assert_eq!(
3967 b[0].spans
3968 .iter()
3969 .map(|s| s.text.as_str())
3970 .collect::<String>(),
3971 "para1"
3972 );
3973 assert_eq!(
3974 b[1].spans
3975 .iter()
3976 .map(|s| s.text.as_str())
3977 .collect::<String>(),
3978 "para2"
3979 );
3980
3981 let d = blocks(":::\ninside\n:::");
3983 let joined: String = d
3984 .iter()
3985 .flat_map(|b| b.spans.iter())
3986 .map(|s| s.text.as_str())
3987 .collect();
3988 assert_eq!(joined, "inside");
3989
3990 let m = blocks("before $`E=mc^2` after");
3992 let joined: String = m
3993 .iter()
3994 .flat_map(|b| b.spans.iter())
3995 .map(|s| s.text.as_str())
3996 .collect();
3997 assert!(joined.contains("before"), "{joined:?}");
3998 assert!(joined.contains("after"), "{joined:?}");
3999 assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
4000 }
4001
4002 #[test]
4003 fn empty_document_yields_one_empty_block() {
4004 let b = blocks("");
4005 assert_eq!(b.len(), 1);
4006 assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
4007 }
4008
4009 #[test]
4010 fn block_attributes_parse_into_block() {
4011 let b = blocks(
4012 "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
4013 );
4014 assert_eq!(b.len(), 1);
4015 assert_eq!(b[0].alignment, Some(Alignment::Center));
4016 assert_eq!(b[0].line_height, Some(1500));
4017 assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
4018 assert_eq!(b[0].non_breakable_lines, Some(true));
4019 assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
4020 }
4021
4022 #[test]
4023 fn spacing_block_attributes_parse_into_block() {
4024 let b = blocks("{top_margin=24 text_indent=0}\nhello");
4028 assert_eq!(b.len(), 1);
4029 assert_eq!(b[0].top_margin, Some(24));
4030 assert_eq!(b[0].text_indent, Some(0));
4031 }
4032
4033 #[test]
4034 fn a_zero_text_indent_is_distinct_from_an_absent_one() {
4035 let explicit = blocks("{text_indent=0}\nhello");
4039 let absent = blocks("hello");
4040 assert_eq!(explicit[0].text_indent, Some(0));
4041 assert_eq!(absent[0].text_indent, None);
4042 assert!(!explicit[0].is_inline_only());
4043 assert!(absent[0].is_inline_only());
4044 }
4045
4046 #[test]
4047 fn spacing_block_attributes_respect_import_options() {
4048 let src = "{top_margin=24 text_indent=0}\nhello";
4049 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4050 assert_eq!(b[0].top_margin, None);
4051 assert_eq!(b[0].text_indent, None);
4052 }
4053
4054 #[test]
4055 fn block_attributes_on_heading() {
4056 let b = blocks("{alignment=right}\n# Title");
4057 assert_eq!(b[0].heading_level, Some(1));
4058 assert_eq!(b[0].alignment, Some(Alignment::Right));
4059 }
4060
4061 #[test]
4062 fn block_attributes_respect_import_options() {
4063 let src = "{alignment=center line_height=1500}\nhello";
4066 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4067 assert_eq!(b[0].alignment, None);
4068 assert_eq!(b[0].line_height, None);
4069 assert_eq!(
4070 b[0].spans
4071 .iter()
4072 .map(|s| s.text.as_str())
4073 .collect::<String>(),
4074 "hello"
4075 );
4076 }
4077
4078 #[test]
4079 fn list_item_block_attributes_are_dropped() {
4080 let b = blocks("{alignment=center}\n- item");
4083 assert!(b.iter().all(|blk| blk.alignment.is_none()));
4084 }
4085
4086 #[test]
4087 fn unknown_alignment_value_is_ignored() {
4088 let b = blocks("{alignment=sideways}\nhello");
4089 assert_eq!(b[0].alignment, None);
4090 }
4091}
4092
4093pub const TABLE_ANCHOR: &str = "\u{FFFC}";
4105
4106fn span_prose(span: &ParsedSpan, out: &mut String) {
4114 if span.image.is_some() || span.footnote_ref.is_some() {
4115 out.push('\u{FFFC}');
4116 return;
4117 }
4118 out.push_str(&span.text);
4119}
4120
4121fn block_prose(block: &ParsedBlock) -> String {
4122 let mut prose = String::new();
4123 for span in &block.spans {
4124 span_prose(span, &mut prose);
4125 }
4126 prose
4127}
4128
4129fn cell_prose(cell: &ParsedTableCell) -> String {
4130 let mut prose = String::new();
4131 for span in &cell.spans {
4132 span_prose(span, &mut prose);
4133 }
4134 prose
4135}
4136
4137pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
4188 let elements = parse_djot(djot, options);
4192
4193 let mut out = String::with_capacity(djot.len());
4196
4197 let mut first = true;
4202 let push = |text: &str, out: &mut String, first: &mut bool| {
4203 if *first {
4204 *first = false;
4205 } else {
4206 out.push('\n');
4207 }
4208 out.push_str(text);
4209 };
4210
4211 for element in &elements {
4212 match element {
4213 ParsedElement::Block(block) => {
4214 push(&block_prose(block), &mut out, &mut first);
4215 }
4216 ParsedElement::FootnoteDefinition { .. } => {}
4226 ParsedElement::Table(table) => {
4227 push(TABLE_ANCHOR, &mut out, &mut first);
4234 let columns = table.column_count();
4237 for row in &table.rows {
4238 for column in 0..columns {
4239 let prose = row.get(column).map(cell_prose).unwrap_or_default();
4240 push(&prose, &mut out, &mut first);
4241 }
4242 }
4243 }
4244 }
4245 }
4246 out
4247}
4248
4249#[cfg(test)]
4250mod html_footnote_tests {
4251 use super::*;
4252
4253 #[test]
4257 fn a_literal_reference_in_text_would_be_escaped_into_prose() {
4258 let blocks = parse_html("<p>The ferry.[^1]</p>");
4259 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4260 assert!(
4261 text.contains("[^1]"),
4262 "the parser keeps it as characters: {text:?}"
4263 );
4264 assert!(
4265 blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
4266 "text is text — it must not become a reference by accident"
4267 );
4268 }
4269
4270 #[test]
4271 fn an_attributed_element_becomes_a_real_footnote_reference() {
4272 let blocks = parse_html(
4273 r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
4274 );
4275 let refs: Vec<&str> = blocks
4276 .iter()
4277 .flat_map(|b| b.spans.iter())
4278 .filter_map(|s| s.footnote_ref.as_deref())
4279 .collect();
4280 assert_eq!(refs, vec!["3"]);
4281
4282 let marker = blocks[0]
4285 .spans
4286 .iter()
4287 .find(|s| s.footnote_ref.is_some())
4288 .expect("the reference span");
4289 assert!(marker.text.is_empty(), "{marker:?}");
4290 }
4291
4292 #[test]
4295 fn any_element_carrying_the_attribute_works() {
4296 for html in [
4297 r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
4298 r#"<p>a<span data-footnote-ref="1"></span></p>"#,
4299 r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
4300 ] {
4301 let blocks = parse_html(html);
4302 assert_eq!(
4303 blocks
4304 .iter()
4305 .flat_map(|b| b.spans.iter())
4306 .filter_map(|s| s.footnote_ref.as_deref())
4307 .collect::<Vec<_>>(),
4308 vec!["1"],
4309 "failed for {html}"
4310 );
4311 }
4312 }
4313
4314 #[test]
4315 fn an_empty_or_absent_label_is_not_a_reference() {
4316 for html in [
4317 r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
4318 r#"<p>a<sup></sup></p>"#,
4319 ] {
4320 let blocks = parse_html(html);
4321 assert!(
4322 blocks
4323 .iter()
4324 .flat_map(|b| b.spans.iter())
4325 .all(|s| s.footnote_ref.is_none()),
4326 "failed for {html}"
4327 );
4328 }
4329 }
4330}
4331
4332#[cfg(test)]
4333mod html_depth_tests {
4334 use super::*;
4337
4338 fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
4340 blocks
4341 .iter()
4342 .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
4343 .collect()
4344 }
4345
4346 #[test]
4351 fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
4352 use crate::parser_tools::list_depth::MAX_LIST_INDENT;
4353 let depth = 300;
4354 let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
4355 html.push_str(&"</li></ol>".repeat(depth));
4356 let blocks = parse_html(&html);
4357 let items: Vec<(String, u32)> = blocks
4358 .iter()
4359 .filter(|block| block.list_style.is_some())
4360 .map(|block| {
4361 let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
4362 (text, block.list_indent)
4363 })
4364 .collect();
4365 let expected: Vec<(String, u32)> = (0..depth)
4366 .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
4367 .collect();
4368 assert_eq!(items, expected);
4369 }
4370
4371 #[test]
4373 fn text_in_a_deep_html_list_outside_its_items_is_kept() {
4374 let depth = 40;
4375 let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
4376 html.push_str("<ul>loose words<li>last</li></ul>");
4377 html.push_str(&"</li></ul>".repeat(depth));
4378 let texts = block_texts(&parse_html(&html)).join(" ");
4379 assert!(texts.contains("loose words"), "{texts:?}");
4380 assert!(texts.contains("last"), "{texts:?}");
4381 assert!(texts.contains("item 39"), "{texts:?}");
4382 }
4383
4384 #[test]
4387 fn html_nested_past_the_reader_depth_keeps_its_words() {
4388 for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
4389 let html = format!(
4390 "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
4391 open.repeat(300),
4392 close.repeat(300)
4393 );
4394 let texts = block_texts(&parse_html(&html)).join(" ");
4395 let words: Vec<&str> = texts.split_whitespace().collect();
4396 assert_eq!(
4397 words,
4398 ["Top.", "Deep", "words", "here.", "End."],
4399 "for {open}"
4400 );
4401 }
4402 }
4403
4404 #[test]
4406 fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
4407 let html = format!(
4408 "<table><tr><td>{}deep words{}</td></tr></table>",
4409 "<span>".repeat(300),
4410 "</span>".repeat(300)
4411 );
4412 let elements = parse_html_elements(&html);
4413 let Some(ParsedElement::Table(table)) = elements.first() else {
4414 panic!("a table: {elements:?}");
4415 };
4416 let text: String = table.rows[0][0]
4417 .spans
4418 .iter()
4419 .map(|span| span.text.as_str())
4420 .collect();
4421 assert_eq!(text, "deep words");
4422 }
4423
4424 #[test]
4428 fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
4429 for cell in [
4430 "One<br/>Two",
4431 "<p>One</p><p>Two</p>",
4432 "<ul><li>One</li><li>Two</li></ul>",
4433 ] {
4434 let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
4435 let Some(ParsedElement::Table(table)) = elements.first() else {
4436 panic!("a table: {elements:?}");
4437 };
4438 let text: String = table.rows[0][0]
4439 .spans
4440 .iter()
4441 .map(|span| span.text.as_str())
4442 .collect();
4443 assert_eq!(text, "One Two", "for {cell}");
4444 }
4445 }
4446}