1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3use crate::parser_tools::list_grouper::default_list_suffix;
4
5#[derive(Debug, Clone, Default, PartialEq, Eq)]
11pub struct ParsedImage {
12 pub src: String,
13 pub alt: String,
14 pub width: i64,
20 pub height: i64,
21}
22
23#[derive(Debug, Clone, Default)]
30pub struct ParsedSpan {
31 pub text: String,
32 pub bold: bool,
33 pub italic: bool,
34 pub underline: bool,
35 pub strikeout: bool,
36 pub code: bool,
37 pub superscript: bool,
39 pub subscript: bool,
41 pub link_href: Option<String>,
42 pub image: Option<ParsedImage>,
44 pub footnote_ref: Option<String>,
47 pub preserve_whitespace: bool,
51}
52
53#[derive(Debug, Clone)]
55pub struct ParsedTableCell {
56 pub spans: Vec<ParsedSpan>,
57}
58
59#[derive(Debug, Clone)]
61pub struct ParsedTable {
62 pub header_rows: usize,
64 pub rows: Vec<Vec<ParsedTableCell>>,
66 pub blockquote_depth: u32,
69 pub blockquote_opens: u32,
72}
73
74impl ParsedTable {
75 pub fn column_count(&self) -> usize {
85 self.rows.iter().map(Vec::len).max().unwrap_or(0)
86 }
87}
88
89const SQUARED_CELLS_ALWAYS: usize = 4096;
93
94const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
104
105fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
116 let squared = table.rows.len().saturating_mul(table.column_count());
117 let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
118 if squared <= allowed {
119 elements.push(ParsedElement::Table(table));
120 return;
121 }
122 elements.extend(
123 ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
124 .into_iter()
125 .filter(|block| spans_carry_content(&block.spans))
126 .map(ParsedElement::Block),
127 );
128}
129
130fn mark_quotation_opened(elements: &mut [ParsedElement], first: usize) {
134 let Some(rest) = elements.get_mut(first..) else {
135 return;
136 };
137 for element in rest {
138 match element {
139 ParsedElement::Block(block) => {
140 block.blockquote_opens += 1;
141 return;
142 }
143 ParsedElement::Table(table) => {
144 table.blockquote_opens += 1;
145 return;
146 }
147 ParsedElement::FootnoteDefinition { .. } => {}
148 }
149 }
150}
151
152fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
154 rows.iter().map(Vec::len).sum()
155}
156
157#[derive(Debug, Clone)]
159pub enum ParsedElement {
160 Block(ParsedBlock),
161 Table(ParsedTable),
162 FootnoteDefinition {
169 label: String,
170 blocks: Vec<ParsedBlock>,
171 },
172}
173
174impl ParsedElement {
175 pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
178 let mut blocks = Vec::new();
179 for elem in elements {
180 match elem {
181 ParsedElement::Block(b) => blocks.push(b),
182 ParsedElement::FootnoteDefinition { .. } => {}
187 ParsedElement::Table(t) => {
188 for row in t.rows {
189 for cell in row {
190 blocks.push(ParsedBlock {
191 spans: cell.spans,
192 heading_level: None,
193 list_style: None,
194 list_indent: 0,
195 list_prefix: String::new(),
196 list_suffix: String::new(),
197 list_start: None,
198 marker: None,
199 is_code_block: false,
200 code_language: None,
201 blockquote_depth: t.blockquote_depth,
202 blockquote_opens: 0,
203 line_height: None,
204 non_breakable_lines: None,
205 page_break_before: None,
206 front_matter: None,
207 direction: None,
208 background_color: None,
209 alignment: None,
210 top_margin: None,
211 text_indent: None,
212 semantic_role: None,
213 });
214 }
215 }
216 }
217 }
218 }
219 if blocks.is_empty() {
220 blocks.push(ParsedBlock {
221 spans: vec![ParsedSpan {
222 text: String::new(),
223 ..Default::default()
224 }],
225 heading_level: None,
226 list_style: None,
227 list_indent: 0,
228 list_prefix: String::new(),
229 list_suffix: String::new(),
230 list_start: None,
231 marker: None,
232 is_code_block: false,
233 code_language: None,
234 blockquote_depth: 0,
235 blockquote_opens: 0,
236 line_height: None,
237 non_breakable_lines: None,
238 page_break_before: None,
239 front_matter: None,
240 direction: None,
241 background_color: None,
242 alignment: None,
243 top_margin: None,
244 text_indent: None,
245 semantic_role: None,
246 });
247 }
248 blocks
249 }
250}
251
252#[derive(Debug, Clone, Default)]
259pub struct ParsedBlock {
260 pub spans: Vec<ParsedSpan>,
261 pub heading_level: Option<i64>,
262 pub list_style: Option<ListStyle>,
263 pub list_indent: u32,
264 pub list_prefix: String,
267 pub list_suffix: String,
272 pub list_start: Option<i64>,
276 pub marker: Option<MarkerType>,
279 pub is_code_block: bool,
280 pub code_language: Option<String>,
281 pub blockquote_depth: u32,
282 pub blockquote_opens: u32,
289 pub line_height: Option<i64>,
290 pub non_breakable_lines: Option<bool>,
291 pub page_break_before: Option<bool>,
294 pub front_matter: Option<bool>,
297 pub direction: Option<TextDirection>,
298 pub background_color: Option<String>,
299 pub alignment: Option<Alignment>,
302 pub top_margin: Option<i64>,
306 pub text_indent: Option<i64>,
310 pub semantic_role: Option<SemanticRole>,
314}
315
316impl ParsedBlock {
317 pub fn is_inline_only(&self) -> bool {
320 self.heading_level.is_none()
321 && self.list_style.is_none()
322 && !self.is_code_block
323 && self.blockquote_depth == 0
324 && self.line_height.is_none()
325 && self.non_breakable_lines.is_none()
326 && self.page_break_before.is_none()
327 && self.front_matter.is_none()
328 && self.direction.is_none()
329 && self.background_color.is_none()
330 && self.alignment.is_none()
331 && self.top_margin.is_none()
332 && self.text_indent.is_none()
333 }
334}
335
336fn dangling_footnote_labels(
365 markdown: &str,
366 options: pulldown_cmark::Options,
367) -> std::collections::BTreeSet<String> {
368 use pulldown_cmark::{Event, Parser, Tag};
369
370 let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
371 for event in Parser::new_ext(markdown, options) {
372 if let Event::Start(Tag::FootnoteDefinition(label)) = event {
373 defined.insert(label.to_string());
374 }
375 }
376
377 let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
378 let bytes = markdown.as_bytes();
379 let mut search_from = 0usize;
380 while let Some(rel) = markdown[search_from..].find("[^") {
381 let open = search_from + rel;
382 let label_start = open + 2;
383 let Some(close_rel) = markdown[label_start..].find(']') else {
384 break;
385 };
386 let close = label_start + close_rel;
387 let label = &markdown[label_start..close];
388 let looks_like_definition = bytes.get(close + 1) == Some(&b':');
393 if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
394 referenced.insert(label.to_string());
395 }
396 search_from = close + 1;
397 }
398
399 referenced.difference(&defined).cloned().collect()
400}
401
402fn markdown_ordered_suffix(source: &str, start: usize) -> &'static str {
412 let Some(rest) = source.get(start..) else {
413 return ".";
414 };
415 let line = rest.split('\n').next().unwrap_or("");
416 let after_blanks = line.trim_start_matches([' ', '\t', '>']);
417 let after_digits = after_blanks.trim_start_matches(|c: char| c.is_ascii_digit());
418 if after_digits.starts_with(')') {
419 ")"
420 } else {
421 "."
422 }
423}
424
425pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
433 use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
434
435 let options = Options::ENABLE_STRIKETHROUGH
446 | Options::ENABLE_TABLES
447 | Options::ENABLE_TASKLISTS
448 | Options::ENABLE_FOOTNOTES;
449
450 let dangling = dangling_footnote_labels(markdown, options);
457 let augmented_owner;
458 let source: &str = if dangling.is_empty() {
459 markdown
460 } else {
461 augmented_owner = dangling
462 .iter()
463 .fold(markdown.to_string(), |mut acc, label| {
464 acc.push_str("\n\n[^");
465 acc.push_str(label);
466 acc.push_str("]:\n");
467 acc
468 });
469 &augmented_owner
470 };
471 let parser = Parser::new_ext(source, options).into_offset_iter();
472
473 let mut elements: Vec<ParsedElement> = Vec::new();
474 let mut current_spans: Vec<ParsedSpan> = Vec::new();
475 let mut current_heading: Option<i64> = None;
476 let mut current_list_style: Option<ListStyle> = None;
477 let mut current_list_start: Option<i64> = None;
478 let mut current_list_suffix = String::new();
479 let mut current_marker: Option<MarkerType> = None;
481 let mut is_code_block = false;
482 let mut code_language: Option<String> = None;
483 let mut blockquote_depth: u32 = 0;
484 let mut quotations_open: Vec<usize> = Vec::new();
486 let mut in_block = false;
487
488 let mut bold = false;
490 let mut italic = false;
491 let mut strikeout = false;
492 let mut link_href: Option<String> = None;
493 let mut pending_image: Option<ParsedImage> = None;
495
496 let mut footnote_open: Option<(String, usize)> = None;
503
504 let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
507 let mut list_start_stack: Vec<Option<i64>> = Vec::new();
508 let mut list_suffix_stack: Vec<String> = Vec::new();
510 let mut current_list_indent: u32 = 0;
511
512 let mut in_table = false;
514 let mut in_table_head = false;
515 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
516 let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
517 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
518 let mut table_header_rows: usize = 0;
519
520 for (event, range) in parser {
521 match event {
522 Event::Start(Tag::Paragraph) => {
523 in_block = true;
524 current_heading = None;
525 is_code_block = false;
526 }
527 Event::End(TagEnd::Paragraph) => {
528 if !current_spans.is_empty() || in_block {
529 elements.push(ParsedElement::Block(ParsedBlock {
530 spans: std::mem::take(&mut current_spans),
531 heading_level: current_heading.take(),
532 list_style: current_list_style.clone(),
533 list_indent: current_list_indent,
534 list_prefix: String::new(),
535 list_suffix: current_list_suffix.clone(),
536 list_start: current_list_start,
537 marker: current_marker.clone(),
538 is_code_block: false,
539 code_language: None,
540 blockquote_depth,
541 blockquote_opens: 0,
542 line_height: None,
543 non_breakable_lines: None,
544 page_break_before: None,
545 front_matter: None,
546 direction: None,
547 background_color: None,
548 alignment: None,
549 top_margin: None,
550 text_indent: None,
551 semantic_role: None,
552 }));
553 }
554 in_block = false;
555 current_list_style = None;
556 current_list_start = None;
557 current_list_suffix.clear();
558 current_marker = None;
559 }
560 Event::Start(Tag::Heading { level, .. }) => {
561 in_block = true;
562 current_heading = Some(heading_level_to_i64(level));
563 is_code_block = false;
564 }
565 Event::End(TagEnd::Heading(_)) => {
566 elements.push(ParsedElement::Block(ParsedBlock {
567 spans: std::mem::take(&mut current_spans),
568 heading_level: current_heading.take(),
569 list_style: None,
570 list_indent: 0,
571 list_prefix: String::new(),
572 list_suffix: String::new(),
573 list_start: None,
574 marker: None,
575 is_code_block: false,
576 code_language: None,
577 blockquote_depth,
578 blockquote_opens: 0,
579 line_height: None,
580 non_breakable_lines: None,
581 page_break_before: None,
582 front_matter: None,
583 direction: None,
584 background_color: None,
585 alignment: None,
586 top_margin: None,
587 text_indent: None,
588 semantic_role: None,
589 }));
590 in_block = false;
591 }
592 Event::Start(Tag::List(ordered)) => {
593 let style = if ordered.is_some() {
594 Some(ListStyle::Decimal)
595 } else {
596 Some(ListStyle::Disc)
597 };
598 list_stack.push(style);
599 list_start_stack.push(ordered.and_then(list_start_of));
600 list_suffix_stack.push(if ordered.is_some() {
601 markdown_ordered_suffix(source, range.start).to_string()
602 } else {
603 String::new()
604 });
605 }
606 Event::TaskListMarker(checked) => {
607 current_marker = Some(if checked {
608 MarkerType::Checked
609 } else {
610 MarkerType::Unchecked
611 });
612 }
613 Event::End(TagEnd::List(_)) => {
614 list_stack.pop();
615 list_start_stack.pop();
616 list_suffix_stack.pop();
617 }
618 Event::Start(Tag::Item) => {
619 if !current_spans.is_empty() {
622 elements.push(ParsedElement::Block(ParsedBlock {
623 spans: std::mem::take(&mut current_spans),
624 heading_level: None,
625 list_style: current_list_style.clone(),
626 list_indent: current_list_indent,
627 list_prefix: String::new(),
628 list_suffix: current_list_suffix.clone(),
629 list_start: current_list_start,
630 marker: current_marker.clone(),
631 is_code_block: false,
632 code_language: None,
633 blockquote_depth,
634 blockquote_opens: 0,
635 line_height: None,
636 non_breakable_lines: None,
637 page_break_before: None,
638 front_matter: None,
639 direction: None,
640 background_color: None,
641 alignment: None,
642 top_margin: None,
643 text_indent: None,
644 semantic_role: None,
645 }));
646 }
647 in_block = true;
648 current_list_style = list_stack.last().cloned().flatten();
649 current_list_start = list_start_stack.last().copied().flatten();
650 current_list_suffix = list_suffix_stack.last().cloned().unwrap_or_default();
651 current_marker = None;
653 current_list_indent = if list_stack.is_empty() {
654 0
655 } else {
656 (list_stack.len() - 1) as u32
657 };
658 }
659 Event::End(TagEnd::Item) => {
660 if !current_spans.is_empty() {
663 elements.push(ParsedElement::Block(ParsedBlock {
664 spans: std::mem::take(&mut current_spans),
665 heading_level: None,
666 list_style: current_list_style.clone(),
667 list_indent: current_list_indent,
668 list_prefix: String::new(),
669 list_suffix: current_list_suffix.clone(),
670 list_start: current_list_start,
671 marker: current_marker.clone(),
672 is_code_block: false,
673 code_language: None,
674 blockquote_depth,
675 blockquote_opens: 0,
676 line_height: None,
677 non_breakable_lines: None,
678 page_break_before: None,
679 front_matter: None,
680 direction: None,
681 background_color: None,
682 alignment: None,
683 top_margin: None,
684 text_indent: None,
685 semantic_role: None,
686 }));
687 }
688 in_block = false;
689 current_list_style = None;
690 current_list_start = None;
691 current_list_suffix.clear();
692 current_marker = None;
693 }
694 Event::Start(Tag::CodeBlock(kind)) => {
695 in_block = true;
696 is_code_block = true;
697 code_language = match &kind {
698 pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
699 Some(lang.to_string())
700 }
701 _ => None,
702 };
703 }
704 Event::End(TagEnd::CodeBlock) => {
705 if let Some(last) = current_spans.last_mut()
707 && last.text.ends_with('\n')
708 {
709 last.text.truncate(last.text.len() - 1);
710 }
711 elements.push(ParsedElement::Block(ParsedBlock {
712 spans: std::mem::take(&mut current_spans),
713 heading_level: None,
714 list_style: None,
715 list_indent: 0,
716 list_prefix: String::new(),
717 list_suffix: String::new(),
718 list_start: None,
719 marker: None,
720 is_code_block: true,
721 code_language: code_language.take(),
722 blockquote_depth,
723 blockquote_opens: 0,
724 line_height: None,
725 non_breakable_lines: None,
726 page_break_before: None,
727 front_matter: None,
728 direction: None,
729 background_color: None,
730 alignment: None,
731 top_margin: None,
732 text_indent: None,
733 semantic_role: None,
734 }));
735 in_block = false;
736 is_code_block = false;
737 }
738 Event::Start(Tag::Table(_)) => {
740 in_table = true;
741 in_table_head = false;
742 table_rows.clear();
743 current_row_cells.clear();
744 current_cell_spans.clear();
745 table_header_rows = 0;
746 }
747 Event::End(TagEnd::Table) => {
748 let own_cells = cells_in_rows(&table_rows);
749 push_table(
750 &mut elements,
751 ParsedTable {
752 header_rows: table_header_rows,
753 rows: std::mem::take(&mut table_rows),
754 blockquote_depth,
755 blockquote_opens: 0,
756 },
757 own_cells,
758 );
759 in_table = false;
760 }
761 Event::Start(Tag::TableHead) => {
762 in_table_head = true;
763 current_row_cells.clear();
764 }
765 Event::End(TagEnd::TableHead) => {
766 table_rows.push(std::mem::take(&mut current_row_cells));
768 table_header_rows += 1;
769 in_table_head = false;
770 }
771 Event::Start(Tag::TableRow) => {
772 current_row_cells.clear();
773 }
774 Event::End(TagEnd::TableRow) if !in_table_head => {
775 table_rows.push(std::mem::take(&mut current_row_cells));
777 }
778 Event::Start(Tag::TableCell) => {
779 current_cell_spans.clear();
780 }
781 Event::End(TagEnd::TableCell) => {
782 current_row_cells.push(ParsedTableCell {
783 spans: std::mem::take(&mut current_cell_spans),
784 });
785 }
786 Event::Start(Tag::Emphasis) => {
788 italic = true;
789 }
790 Event::End(TagEnd::Emphasis) => {
791 italic = false;
792 }
793 Event::Start(Tag::Strong) => {
794 bold = true;
795 }
796 Event::End(TagEnd::Strong) => {
797 bold = false;
798 }
799 Event::Start(Tag::Strikethrough) => {
800 strikeout = true;
801 }
802 Event::End(TagEnd::Strikethrough) => {
803 strikeout = false;
804 }
805 Event::Start(Tag::Link { dest_url, .. }) => {
806 link_href = Some(dest_url.to_string());
807 }
808 Event::End(TagEnd::Link) => {
809 link_href = None;
810 }
811 Event::Start(Tag::Image { dest_url, .. }) => {
814 pending_image = Some(ParsedImage {
815 src: dest_url.to_string(),
816 alt: String::new(),
817 width: 0,
818 height: 0,
819 });
820 }
821 Event::End(TagEnd::Image) => {
822 if let Some(image) = pending_image.take() {
823 let span = ParsedSpan {
824 text: String::new(),
825 bold,
826 italic,
827 underline: false,
828 strikeout,
829 code: false,
830 superscript: false,
831 subscript: false,
832 link_href: link_href.clone(),
833 image: Some(image),
834 footnote_ref: None,
835 preserve_whitespace: false,
836 };
837 if in_table {
838 current_cell_spans.push(span);
839 } else {
840 if !in_block {
841 in_block = true;
842 }
843 current_spans.push(span);
844 }
845 }
846 }
847 Event::Text(text) => {
848 if let Some(img) = pending_image.as_mut() {
853 img.alt.push_str(&text);
854 continue;
855 }
856 let span = ParsedSpan {
857 text: text.to_string(),
858 bold,
859 italic,
860 underline: false,
861 strikeout,
862 code: is_code_block,
863 superscript: false,
864 subscript: false,
865 link_href: link_href.clone(),
866 image: None,
867 footnote_ref: None,
868 preserve_whitespace: false,
869 };
870 if in_table {
871 current_cell_spans.push(span);
872 } else {
873 if !in_block {
874 in_block = true;
875 }
876 current_spans.push(span);
877 }
878 }
879 Event::Code(text) => {
880 let span = ParsedSpan {
881 text: text.to_string(),
882 bold,
883 italic,
884 underline: false,
885 strikeout,
886 code: true,
887 superscript: false,
888 subscript: false,
889 link_href: link_href.clone(),
890 image: None,
891 footnote_ref: None,
892 preserve_whitespace: false,
893 };
894 if in_table {
895 current_cell_spans.push(span);
896 } else {
897 if !in_block {
898 in_block = true;
899 }
900 current_spans.push(span);
901 }
902 }
903 Event::SoftBreak => {
904 let span = ParsedSpan {
905 text: " ".to_string(),
906 bold,
907 italic,
908 underline: false,
909 strikeout,
910 code: false,
911 superscript: false,
912 subscript: false,
913 link_href: link_href.clone(),
914 image: None,
915 footnote_ref: None,
916 preserve_whitespace: false,
917 };
918 if in_table {
919 current_cell_spans.push(span);
920 } else {
921 current_spans.push(span);
922 }
923 }
924 Event::HardBreak if pending_image.is_some() => {
928 if let Some(img) = pending_image.as_mut() {
929 img.alt.push(' ');
930 }
931 }
932 Event::HardBreak => {
933 let span = ParsedSpan {
934 text: if in_table { " " } else { LINE_SEPARATOR_STR }.to_string(),
935 bold,
936 italic,
937 underline: false,
938 strikeout,
939 code: false,
940 superscript: false,
941 subscript: false,
942 link_href: link_href.clone(),
943 image: None,
944 footnote_ref: None,
945 preserve_whitespace: false,
946 };
947 if in_table {
948 current_cell_spans.push(span);
949 } else {
950 current_spans.push(span);
951 }
952 }
953 Event::Start(Tag::BlockQuote(_)) => {
954 blockquote_depth += 1;
955 quotations_open.push(elements.len());
956 }
957 Event::End(TagEnd::BlockQuote(_)) => {
958 blockquote_depth = blockquote_depth.saturating_sub(1);
959 if let Some(first) = quotations_open.pop() {
960 mark_quotation_opened(&mut elements, first);
961 }
962 }
963 Event::Start(Tag::FootnoteDefinition(label)) => {
974 if !current_spans.is_empty() {
975 elements.push(ParsedElement::Block(ParsedBlock {
976 spans: std::mem::take(&mut current_spans),
977 heading_level: current_heading.take(),
978 list_style: current_list_style.clone(),
979 list_indent: current_list_indent,
980 list_prefix: String::new(),
981 list_suffix: current_list_suffix.clone(),
982 list_start: current_list_start,
983 marker: current_marker.clone(),
984 is_code_block: false,
985 code_language: None,
986 blockquote_depth,
987 blockquote_opens: 0,
988 line_height: None,
989 non_breakable_lines: None,
990 page_break_before: None,
991 front_matter: None,
992 direction: None,
993 background_color: None,
994 alignment: None,
995 top_margin: None,
996 text_indent: None,
997 semantic_role: None,
998 }));
999 }
1000 footnote_open = Some((label.to_string(), elements.len()));
1001 }
1002 Event::End(TagEnd::FootnoteDefinition) => {
1003 if !current_spans.is_empty() {
1004 elements.push(ParsedElement::Block(ParsedBlock {
1005 spans: std::mem::take(&mut current_spans),
1006 heading_level: current_heading.take(),
1007 list_style: current_list_style.clone(),
1008 list_indent: current_list_indent,
1009 list_prefix: String::new(),
1010 list_suffix: current_list_suffix.clone(),
1011 list_start: current_list_start,
1012 marker: current_marker.clone(),
1013 is_code_block: false,
1014 code_language: None,
1015 blockquote_depth,
1016 blockquote_opens: 0,
1017 line_height: None,
1018 non_breakable_lines: None,
1019 page_break_before: None,
1020 front_matter: None,
1021 direction: None,
1022 background_color: None,
1023 alignment: None,
1024 top_margin: None,
1025 text_indent: None,
1026 semantic_role: None,
1027 }));
1028 }
1029 if let Some((label, start)) = footnote_open.take() {
1030 let blocks: Vec<ParsedBlock> = elements
1031 .drain(start..)
1032 .filter_map(|e| match e {
1033 ParsedElement::Block(b) => Some(b),
1034 _ => None,
1037 })
1038 .collect();
1039 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
1040 }
1041 }
1042 Event::FootnoteReference(label) => {
1048 let span = ParsedSpan {
1049 text: String::new(),
1050 bold,
1051 italic,
1052 underline: false,
1053 strikeout,
1054 code: false,
1055 superscript: false,
1056 subscript: false,
1057 link_href: link_href.clone(),
1058 image: None,
1059 footnote_ref: Some(label.to_string()),
1060 preserve_whitespace: false,
1061 };
1062 if in_table {
1063 current_cell_spans.push(span);
1064 } else {
1065 if !in_block {
1066 in_block = true;
1067 }
1068 current_spans.push(span);
1069 }
1070 }
1071 _ => {}
1072 }
1073 }
1074
1075 if !current_spans.is_empty() {
1077 elements.push(ParsedElement::Block(ParsedBlock {
1078 spans: std::mem::take(&mut current_spans),
1079 heading_level: current_heading,
1080 list_style: current_list_style,
1081 list_indent: current_list_indent,
1082 list_prefix: String::new(),
1083 list_suffix: current_list_suffix.clone(),
1084 list_start: current_list_start,
1085 marker: current_marker.clone(),
1086 is_code_block,
1087 code_language: code_language.take(),
1088 blockquote_depth,
1089 blockquote_opens: 0,
1090 line_height: None,
1091 non_breakable_lines: None,
1092 page_break_before: None,
1093 front_matter: None,
1094 direction: None,
1095 background_color: None,
1096 alignment: None,
1097 top_margin: None,
1098 text_indent: None,
1099 semantic_role: None,
1100 }));
1101 }
1102
1103 if !dangling.is_empty() {
1110 elements.retain(
1111 |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
1112 );
1113 }
1114
1115 if elements.is_empty() {
1117 elements.push(ParsedElement::Block(ParsedBlock {
1118 spans: vec![ParsedSpan {
1119 text: String::new(),
1120 ..Default::default()
1121 }],
1122 heading_level: None,
1123 list_style: None,
1124 list_indent: 0,
1125 list_prefix: String::new(),
1126 list_suffix: String::new(),
1127 list_start: None,
1128 marker: None,
1129 is_code_block: false,
1130 code_language: None,
1131 blockquote_depth: 0,
1132 blockquote_opens: 0,
1133 line_height: None,
1134 non_breakable_lines: None,
1135 page_break_before: None,
1136 front_matter: None,
1137 direction: None,
1138 background_color: None,
1139 alignment: None,
1140 top_margin: None,
1141 text_indent: None,
1142 semantic_role: None,
1143 }));
1144 }
1145
1146 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1149 elements
1150}
1151
1152fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
1153 use pulldown_cmark::HeadingLevel;
1154 match level {
1155 HeadingLevel::H1 => 1,
1156 HeadingLevel::H2 => 2,
1157 HeadingLevel::H3 => 3,
1158 HeadingLevel::H4 => 4,
1159 HeadingLevel::H5 => 5,
1160 HeadingLevel::H6 => 6,
1161 }
1162}
1163
1164use scraper::Node;
1167
1168#[derive(Debug, Clone, Default)]
1170struct BlockStyles {
1171 line_height: Option<i64>,
1172 non_breakable_lines: Option<bool>,
1173 page_break_before: Option<bool>,
1174 direction: Option<TextDirection>,
1175 background_color: Option<String>,
1176 preserve_whitespace: Option<bool>,
1181}
1182
1183fn parse_block_styles(style: &str) -> BlockStyles {
1187 let mut result = BlockStyles::default();
1188 for part in style.split(';') {
1189 let part = part.trim();
1190 if let Some((prop, val)) = part.split_once(':') {
1191 let prop = prop.trim().to_ascii_lowercase();
1192 let val = val.trim();
1193 match prop.as_str() {
1194 "line-height" => {
1195 if let Ok(v) = val.parse::<f64>() {
1197 result.line_height = Some((v * 1000.0) as i64);
1198 }
1199 }
1200 "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1201 result.non_breakable_lines = Some(true);
1202 if val != "nowrap" {
1203 result.preserve_whitespace = Some(true);
1204 }
1205 }
1206 "break-before" | "page-break-before" => {
1210 result.page_break_before = match val.to_ascii_lowercase().as_str() {
1211 "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1212 "avoid" | "auto" => Some(false),
1213 _ => None,
1214 };
1215 }
1216 "direction" => {
1217 if val.eq_ignore_ascii_case("rtl") {
1218 result.direction = Some(TextDirection::RightToLeft);
1219 } else if val.eq_ignore_ascii_case("ltr") {
1220 result.direction = Some(TextDirection::LeftToRight);
1221 }
1222 }
1223 "background-color" | "background" => {
1224 result.background_color = Some(val.to_string());
1225 }
1226 _ => {}
1227 }
1228 }
1229 }
1230 result
1231}
1232
1233fn style_keeps_whitespace(style: &str) -> Option<bool> {
1239 let mut kept = None;
1240 let mut important_kept = false;
1241 for part in style.split(';') {
1242 let Some((property, value)) = part.split_once(':') else {
1243 continue;
1244 };
1245 if !property.trim().eq_ignore_ascii_case("white-space") {
1246 continue;
1247 }
1248 let value = value.trim().to_ascii_lowercase();
1249 let (value, important) = value
1250 .strip_suffix("important")
1251 .and_then(|rest| rest.trim_end().strip_suffix('!'))
1252 .map_or((value.as_str(), false), |rest| (rest.trim_end(), true));
1253 let says = match value {
1254 "pre" | "pre-wrap" | "break-spaces" => true,
1255 "normal" | "nowrap" | "pre-line" => false,
1256 _ => continue,
1257 };
1258 if important || !important_kept {
1259 kept = Some(says);
1260 important_kept |= important;
1261 }
1262 }
1263 kept
1264}
1265
1266#[derive(Debug, Default)]
1277struct SheetWhitespace {
1278 universal: Option<(&'static str, bool)>,
1280 per_tag: std::collections::HashMap<String, (&'static str, bool)>,
1284}
1285
1286struct WhitespaceRule {
1289 selectors: Vec<String>,
1290 declarations: Vec<(&'static str, bool)>,
1291}
1292
1293fn cascade(
1296 current: Option<(&'static str, bool)>,
1297 next: (&'static str, bool),
1298) -> Option<(&'static str, bool)> {
1299 match current {
1300 Some((_, true)) if !next.1 => current,
1301 _ => Some(next),
1302 }
1303}
1304
1305impl SheetWhitespace {
1306 fn from_css(css: &str) -> Self {
1309 let rules = parse_whitespace_rules(css);
1310 let mut sheet = SheetWhitespace::default();
1311 for rule in rules
1312 .iter()
1313 .filter(|r| r.selectors.iter().any(|s| s == "*"))
1314 {
1315 for declaration in &rule.declarations {
1316 sheet.universal = cascade(sheet.universal, *declaration);
1317 }
1318 }
1319 for rule in &rules {
1320 for tag in rule.selectors.iter().filter(|s| *s != "*") {
1321 let winner = sheet
1322 .per_tag
1323 .entry(tag.clone())
1324 .or_insert_with(|| sheet.universal.unwrap_or(("", false)));
1325 let mut current = (!winner.0.is_empty()).then_some(*winner);
1326 for declaration in &rule.declarations {
1327 current = cascade(current, *declaration);
1328 }
1329 if let Some(resolved) = current {
1330 *winner = resolved;
1331 }
1332 }
1333 }
1334 sheet.per_tag.retain(|_, (keyword, _)| !keyword.is_empty());
1335 sheet
1336 }
1337
1338 fn declaration_for(&self, tag: &str) -> Option<(&'static str, bool)> {
1340 self.per_tag.get(tag).copied().or(self.universal)
1341 }
1342
1343 fn is_empty(&self) -> bool {
1344 self.universal.is_none() && self.per_tag.is_empty()
1345 }
1346
1347 fn initial_keeps_whitespace(&self) -> Option<bool> {
1352 let of = |tag: &str| {
1353 self.declaration_for(tag).and_then(|(keyword, important)| {
1354 style_keeps_whitespace(&format!(
1355 "white-space: {keyword}{};",
1356 if important { " !important" } else { "" }
1357 ))
1358 })
1359 };
1360 of("body").or_else(|| of("html"))
1361 }
1362}
1363
1364fn parse_whitespace_rules(css: &str) -> Vec<WhitespaceRule> {
1370 let mut text = String::with_capacity(css.len());
1373 let mut rest = css;
1374 while let Some(start) = rest.find("/*") {
1375 text.push_str(&rest[..start]);
1376 text.push(' ');
1377 match rest[start + 2..].find("*/") {
1378 Some(end) => rest = &rest[start + 2 + end + 2..],
1379 None => {
1380 rest = "";
1381 break;
1382 }
1383 }
1384 }
1385 text.push_str(rest);
1386 let text = text.replace("<!--", " ").replace("-->", " ");
1387 let bytes = text.as_bytes();
1388
1389 let scan = |from: usize, stops: &[u8]| -> usize {
1391 let mut i = from;
1392 while i < bytes.len() {
1393 match bytes[i] {
1394 b'\\' => i += 1,
1395 quote @ (b'"' | b'\'') => {
1396 i += 1;
1397 while i < bytes.len() && bytes[i] != quote {
1398 if bytes[i] == b'\\' {
1399 i += 1;
1400 }
1401 i += 1;
1402 }
1403 }
1404 b if stops.contains(&b) => return i,
1405 _ => {}
1406 }
1407 i += 1;
1408 }
1409 bytes.len()
1410 };
1411 let block_end = |open: usize| -> usize {
1413 let mut depth = 0usize;
1414 let mut i = open;
1415 while i < bytes.len() {
1416 i = scan(i, b"{}");
1417 match bytes.get(i) {
1418 Some(b'{') => depth += 1,
1419 Some(b'}') => {
1420 depth = depth.saturating_sub(1);
1421 if depth == 0 {
1422 return i;
1423 }
1424 }
1425 _ => return bytes.len(),
1426 }
1427 i += 1;
1428 }
1429 bytes.len()
1430 };
1431
1432 let mut rules = Vec::new();
1433 let mut pos = 0usize;
1434 while pos < bytes.len() {
1435 let prelude_end = scan(pos, b"{;}");
1436 let prelude = text[pos..prelude_end].trim();
1437 match bytes.get(prelude_end) {
1438 Some(b';') | Some(b'}') => pos = prelude_end + 1,
1440 Some(b'{') => {
1441 let close = block_end(prelude_end);
1442 if !prelude.starts_with('@') {
1443 let body = &text[prelude_end + 1..close];
1444 if let Some(rule) = whitespace_rule(prelude, body) {
1445 rules.push(rule);
1446 }
1447 }
1448 pos = close + 1;
1449 }
1450 _ => break,
1451 }
1452 }
1453 rules
1454}
1455
1456fn whitespace_rule(selectors: &str, body: &str) -> Option<WhitespaceRule> {
1459 let selectors: Vec<String> = selectors
1460 .split(',')
1461 .map(|s| s.trim().to_ascii_lowercase())
1462 .collect();
1463 let bare = |s: &String| {
1464 s == "*" || (!s.is_empty() && s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-'))
1465 };
1466 if !selectors.iter().all(bare) {
1467 return None;
1468 }
1469 let mut declarations = Vec::new();
1470 let own;
1473 let body = if body.contains('{') {
1474 let mut text = String::with_capacity(body.len());
1475 let mut depth = 0usize;
1476 for ch in body.chars() {
1477 match ch {
1478 '{' => depth += 1,
1479 '}' if depth > 0 => {
1480 depth -= 1;
1481 if depth == 0 {
1482 text.push(';');
1484 }
1485 }
1486 _ if depth == 0 => text.push(ch),
1487 _ => {}
1488 }
1489 }
1490 own = text;
1491 own.as_str()
1492 } else {
1493 body
1494 };
1495 for part in body.split(';') {
1496 let Some((property, value)) = part.split_once(':') else {
1497 continue;
1498 };
1499 if !property.trim().eq_ignore_ascii_case("white-space") {
1500 continue;
1501 }
1502 let value = value.trim().to_ascii_lowercase();
1503 let (value, important) = value
1504 .strip_suffix("important")
1505 .and_then(|rest| rest.trim_end().strip_suffix('!'))
1506 .map_or((value.as_str(), false), |rest| (rest.trim_end(), true));
1507 let keyword = match value {
1509 "normal" => "normal",
1510 "nowrap" => "nowrap",
1511 "pre" => "pre",
1512 "pre-wrap" => "pre-wrap",
1513 "pre-line" => "pre-line",
1514 "break-spaces" => "break-spaces",
1515 _ => continue,
1516 };
1517 declarations.push((keyword, important));
1518 }
1519 (!declarations.is_empty()).then_some(WhitespaceRule {
1520 selectors,
1521 declarations,
1522 })
1523}
1524
1525fn element_keeps_whitespace(
1529 el: &scraper::node::Element,
1530 sheet: Option<&SheetWhitespace>,
1531) -> Option<bool> {
1532 let inline = el.attr("style");
1533 match sheet.and_then(|s| s.declaration_for(el.name())) {
1534 None => inline.and_then(style_keeps_whitespace),
1535 Some((keyword, important)) => style_keeps_whitespace(&format!(
1536 "white-space: {keyword}{};{}",
1537 if important { " !important" } else { "" },
1538 inline.unwrap_or("")
1539 )),
1540 }
1541}
1542
1543fn html_sheet(fragment: &scraper::Html) -> Option<SheetWhitespace> {
1546 let mut css = String::new();
1547 for node in fragment.tree.nodes() {
1548 let Node::Element(el) = node.value() else {
1549 continue;
1550 };
1551 if el.name() != "style" {
1552 continue;
1553 }
1554 let type_ok = el
1555 .attr("type")
1556 .is_none_or(|t| t.trim().is_empty() || t.trim().eq_ignore_ascii_case("text/css"));
1557 let media_ok = el.attr("media").is_none_or(|m| {
1558 let m = m.trim();
1559 m.is_empty() || m.eq_ignore_ascii_case("all") || m.eq_ignore_ascii_case("screen")
1560 });
1561 if !(type_ok && media_ok) {
1562 continue;
1563 }
1564 for child in node.children() {
1565 if let Node::Text(text) = child.value() {
1566 css.push_str(text);
1567 css.push('\n');
1568 }
1569 }
1570 }
1571 if css.is_empty() {
1572 return None;
1573 }
1574 let sheet = SheetWhitespace::from_css(&css);
1575 (!sheet.is_empty()).then_some(sheet)
1576}
1577
1578fn is_block_level_name(name: &str) -> bool {
1581 matches!(
1582 name,
1583 "address"
1584 | "article"
1585 | "aside"
1586 | "blockquote"
1587 | "caption"
1588 | "center"
1589 | "dd"
1590 | "details"
1591 | "dialog"
1592 | "dir"
1593 | "div"
1594 | "dl"
1595 | "dt"
1596 | "fieldset"
1597 | "figcaption"
1598 | "figure"
1599 | "footer"
1600 | "form"
1601 | "h1"
1602 | "h2"
1603 | "h3"
1604 | "h4"
1605 | "h5"
1606 | "h6"
1607 | "header"
1608 | "hgroup"
1609 | "hr"
1610 | "li"
1611 | "main"
1612 | "menu"
1613 | "nav"
1614 | "ol"
1615 | "p"
1616 | "pre"
1617 | "search"
1618 | "section"
1619 | "summary"
1620 | "table"
1621 | "tbody"
1622 | "td"
1623 | "tfoot"
1624 | "th"
1625 | "thead"
1626 | "tr"
1627 | "ul"
1628 )
1629}
1630
1631fn is_block_in_the_reader(name: &str) -> bool {
1635 matches!(
1636 name,
1637 "p" | "div"
1638 | "h1"
1639 | "h2"
1640 | "h3"
1641 | "h4"
1642 | "h5"
1643 | "h6"
1644 | "li"
1645 | "pre"
1646 | "blockquote"
1647 | "ul"
1648 | "ol"
1649 | "table"
1650 )
1651}
1652
1653#[derive(Debug, PartialEq, Eq)]
1655enum BlankText {
1656 Content,
1658 Layout,
1660 Boundary,
1666}
1667
1668fn blank_text_kind(text: &str, node: ego_tree::NodeRef<Node>) -> BlankText {
1675 if !text.chars().all(is_html_space) {
1676 return BlankText::Content;
1677 }
1678 let neighbour = |next: bool| {
1680 let mut at = node;
1681 loop {
1682 let sibling = if next {
1683 at.next_sibling()
1684 } else {
1685 at.prev_sibling()
1686 };
1687 match sibling {
1688 Some(s) if matches!(s.value(), Node::Comment(_)) => at = s,
1689 other => return other,
1690 }
1691 }
1692 };
1693 let side = |sibling: Option<ego_tree::NodeRef<Node>>| match sibling {
1697 None => Some((false, false, true)),
1698 Some(s) => match s.value() {
1699 Node::Element(el) => Some((
1700 is_block_level_name(el.name()),
1701 is_block_in_the_reader(el.name()),
1702 false,
1703 )),
1704 _ => None,
1705 },
1706 };
1707 match (side(neighbour(false)), side(neighbour(true))) {
1708 (Some(before), Some(after)) if before.0 || after.0 => {
1709 if before.1 || after.1 || before.2 || after.2 {
1710 BlankText::Layout
1711 } else {
1712 BlankText::Boundary
1713 }
1714 }
1715 _ => BlankText::Content,
1716 }
1717}
1718
1719pub fn push_html_text(
1734 out: &mut String,
1735 text: &str,
1736 after_space: bool,
1737 ends_line: bool,
1738 escape: impl Fn(&str) -> String,
1739 line_break: &str,
1740) {
1741 let mut lines = text.split(LINE_SEPARATOR).peekable();
1742 let mut first = true;
1743 while let Some(line) = lines.next() {
1744 if !first {
1745 out.push_str(line_break);
1746 }
1747 let opens_on_space = if first { after_space } else { true };
1748 let ends = ends_line || lines.peek().is_some();
1749 first = false;
1750 let collapses = line.chars().any(|ch| ch != ' ' && is_html_space(ch))
1751 || line.contains(" ")
1752 || (opens_on_space && line.starts_with(' '))
1753 || (ends && line.ends_with(' '));
1754 if collapses {
1755 out.push_str("<span style=\"white-space: pre-wrap\">");
1756 out.push_str(&escape(line));
1757 out.push_str("</span>");
1758 } else {
1759 out.push_str(&escape(line));
1760 }
1761 }
1762}
1763
1764pub fn html_line_ends_before<'a>(
1768 rest: impl IntoIterator<Item = &'a crate::format_runs::InlineContent>,
1769) -> bool {
1770 use crate::format_runs::InlineContent;
1771 for content in rest {
1772 match content {
1773 InlineContent::Empty => {}
1774 InlineContent::Text(text) if text.is_empty() => {}
1775 InlineContent::Text(text) => return text.starts_with(LINE_SEPARATOR),
1776 _ => return false,
1777 }
1778 }
1779 true
1780}
1781
1782pub fn html_text_ends_open(text: &str, after_space: bool) -> bool {
1787 match text.chars().next_back() {
1788 None => after_space,
1789 Some(ch) => ch == LINE_SEPARATOR || is_html_space(ch),
1790 }
1791}
1792
1793fn is_html_space(ch: char) -> bool {
1800 matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1801}
1802
1803pub fn task_box_markdown(marker: Option<&MarkerType>) -> &'static str {
1806 match marker {
1807 Some(MarkerType::Checked) => "[x] ",
1808 Some(MarkerType::Unchecked) => "[ ] ",
1809 _ => "",
1810 }
1811}
1812
1813pub fn task_checkbox_html(marker: Option<&MarkerType>) -> &'static str {
1821 match marker {
1822 Some(MarkerType::Checked) => {
1823 "<label><input type=\"checkbox\" disabled=\"disabled\" checked=\"checked\"/> "
1824 }
1825 Some(MarkerType::Unchecked) => "<label><input type=\"checkbox\" disabled=\"disabled\"/> ",
1826 _ => "",
1827 }
1828}
1829
1830pub fn task_checkbox_end_html(marker: Option<&MarkerType>) -> &'static str {
1832 match marker {
1833 Some(MarkerType::Checked | MarkerType::Unchecked) => "</label>",
1834 _ => "",
1835 }
1836}
1837
1838fn html_item_checkbox(item: ego_tree::NodeRef<Node>) -> Option<MarkerType> {
1845 fn scan(node: ego_tree::NodeRef<Node>, depth: usize) -> Option<Option<MarkerType>> {
1847 for child in node.children() {
1848 match child.value() {
1849 Node::Text(text) => {
1850 if !text.chars().all(is_html_space) {
1851 return Some(None);
1852 }
1853 }
1854 Node::Element(el) => {
1855 let name = el.name();
1856 if name == "input" {
1857 let is_checkbox = el
1858 .attr("type")
1859 .is_some_and(|t| t.trim().eq_ignore_ascii_case("checkbox"));
1860 return Some(is_checkbox.then(|| {
1861 if el.attr("checked").is_some() {
1862 MarkerType::Checked
1863 } else {
1864 MarkerType::Unchecked
1865 }
1866 }));
1867 }
1868 if matches!(name, "label" | "span") && depth < 8 {
1869 if let Some(decided) = scan(child, depth + 1) {
1870 return Some(decided);
1871 }
1872 } else {
1873 return Some(None);
1874 }
1875 }
1876 _ => {}
1877 }
1878 }
1879 None
1880 }
1881 scan(item, 0).flatten()
1882}
1883
1884fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1886 spans
1887 .iter()
1888 .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1889}
1890
1891fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1905 fn is_replaced(span: &ParsedSpan) -> bool {
1906 span.image.is_some() || span.footnote_ref.is_some()
1907 }
1908
1909 for span in spans.iter_mut() {
1913 if is_replaced(span) || span.preserve_whitespace {
1914 continue;
1915 }
1916 let mut out = String::with_capacity(span.text.len());
1917 let mut in_run = false;
1918 for ch in span.text.chars() {
1919 if is_html_space(ch) {
1920 if !in_run {
1921 out.push(' ');
1922 in_run = true;
1923 }
1924 } else {
1925 out.push(ch);
1926 in_run = false;
1927 }
1928 }
1929 span.text = out;
1930 }
1931
1932 let mut prev_ends_with_space = true;
1937 let mut line_end: Option<usize> = None;
1940 for i in 0..spans.len() {
1941 let span = &mut spans[i];
1942 if is_replaced(span) {
1943 prev_ends_with_space = false;
1944 line_end = None;
1945 continue;
1946 }
1947 if span.text.is_empty() {
1948 prev_ends_with_space = false;
1951 continue;
1952 }
1953 if span.text == LINE_SEPARATOR_STR {
1954 if let Some(end) = line_end.take()
1955 && spans[end].text.ends_with(' ')
1956 {
1957 spans[end].text.pop();
1958 }
1959 prev_ends_with_space = true;
1960 continue;
1961 }
1962 if span.preserve_whitespace {
1965 prev_ends_with_space = false;
1966 line_end = None;
1967 continue;
1968 }
1969 if prev_ends_with_space && span.text.starts_with(' ') {
1970 span.text.remove(0);
1971 }
1972 if !span.text.is_empty() {
1975 prev_ends_with_space = span.text.ends_with(' ');
1976 line_end = Some(i);
1977 }
1978 }
1979
1980 let mut trailing_break_dropped = false;
1986 for span in spans.iter_mut().rev() {
1987 if is_replaced(span) {
1988 break;
1989 }
1990 if span.text.is_empty() {
1991 continue;
1992 }
1993 if span.text == LINE_SEPARATOR_STR {
1994 if trailing_break_dropped {
1995 break;
1996 }
1997 span.text.clear();
1998 trailing_break_dropped = true;
1999 continue;
2000 }
2001 if span.preserve_whitespace {
2002 break;
2003 }
2004 if span.text.ends_with(' ') {
2005 span.text.pop();
2006 }
2007 break;
2008 }
2009}
2010
2011pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
2032 let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
2033 if !block.spans.iter().any(breaks) {
2034 return vec![block];
2035 }
2036 let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
2037 for mut span in std::mem::take(&mut block.spans) {
2038 if !breaks(&span) {
2039 push_to_last_line(&mut lines, span);
2040 continue;
2041 }
2042 let text = std::mem::take(&mut span.text);
2043 for (i, piece) in text.split('\n').enumerate() {
2044 if i > 0 {
2045 lines.push(Vec::new());
2046 }
2047 if !piece.is_empty() {
2048 push_to_last_line(
2049 &mut lines,
2050 ParsedSpan {
2051 text: piece.to_string(),
2052 ..span.clone()
2053 },
2054 );
2055 }
2056 }
2057 }
2058 if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
2059 lines.pop();
2060 }
2061 let later_line = ParsedBlock {
2062 page_break_before: None,
2063 blockquote_opens: 0,
2064 marker: None,
2066 ..block.clone()
2067 };
2068 let mut first_line = Some(block);
2069 lines
2070 .into_iter()
2071 .map(|spans| {
2072 let spans = if spans.is_empty() {
2073 vec![ParsedSpan::default()]
2074 } else {
2075 spans
2076 };
2077 match first_line.take() {
2078 Some(first) => ParsedBlock { spans, ..first },
2079 None => ParsedBlock {
2080 spans,
2081 ..later_line.clone()
2082 },
2083 }
2084 })
2085 .collect()
2086}
2087
2088fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
2091 match lines.last_mut() {
2092 Some(line) => line.push(span),
2093 None => lines.push(vec![span]),
2094 }
2095}
2096
2097pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
2098 ParsedElement::flatten_to_blocks(parse_html_elements(html))
2099}
2100
2101pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
2115
2116fn html_footnote_span(
2119 el: &scraper::node::Element,
2120 link_href: Option<String>,
2121) -> Option<ParsedSpan> {
2122 let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
2123 if label.is_empty() {
2124 return None;
2125 }
2126 Some(ParsedSpan {
2127 text: String::new(),
2128 link_href,
2129 footnote_ref: Some(label.to_string()),
2130 ..Default::default()
2131 })
2132}
2133
2134fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
2142 let src = el.attr("src")?;
2143 if src.is_empty() {
2144 return None;
2145 }
2146 let dim = |name: &str| -> i64 {
2147 el.attr(name)
2148 .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
2149 .filter(|n| *n > 0)
2150 .unwrap_or(0)
2151 };
2152 Some(ParsedSpan {
2153 text: String::new(),
2154 link_href,
2155 image: Some(ParsedImage {
2156 src: src.to_string(),
2157 alt: el.attr("alt").unwrap_or_default().to_string(),
2158 width: dim("width"),
2159 height: dim("height"),
2160 }),
2161 ..Default::default()
2162 })
2163}
2164
2165fn is_html_list(node: &Node) -> bool {
2167 node.as_element()
2168 .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
2169}
2170
2171fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
2184 use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
2185 use ego_tree::iter::Edge;
2186
2187 let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
2189 let mut depth = 0usize;
2190 for edge in tree.root().traverse() {
2191 match edge {
2192 Edge::Open(node) if is_html_list(node.value()) => {
2193 if depth == MAX_LIST_LEVELS {
2194 too_deep.push(node.id());
2195 }
2196 depth += 1;
2197 }
2198 Edge::Close(node) if is_html_list(node.value()) => {
2199 depth = depth.saturating_sub(1);
2200 }
2201 _ => {}
2202 }
2203 }
2204
2205 for list in too_deep {
2206 let Some(list_ref) = tree.get(list) else {
2210 continue;
2211 };
2212 let mut holder = None;
2213 let mut current = list_ref;
2214 while let Some(parent) = current.parent() {
2215 if is_html_list(parent.value()) {
2216 holder = Some(current.id());
2217 break;
2218 }
2219 current = parent;
2220 }
2221 let Some(holder) = holder else {
2222 continue;
2223 };
2224 let mut after = holder;
2225 let items: Vec<ego_tree::NodeId> = list_ref
2226 .descendants()
2227 .filter(|node| {
2228 node.value()
2229 .as_element()
2230 .is_some_and(|el| el.name() == "li")
2231 })
2232 .map(|node| node.id())
2233 .collect();
2234 for item in items {
2235 if tree.get(after).and_then(|node| node.parent()).is_none() {
2238 break;
2239 }
2240 if let Some(mut node) = tree.get_mut(after) {
2241 node.insert_id_after(item);
2242 after = item;
2243 }
2244 }
2245 let holds_text = tree.get(list).is_some_and(|node| {
2250 node.descendants()
2251 .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
2252 });
2253 if !holds_text {
2254 if let Some(mut node) = tree.get_mut(list) {
2255 node.detach();
2256 }
2257 } else if holder != list
2258 && tree.get(after).and_then(|node| node.parent()).is_some()
2259 && let Some(mut node) = tree.get_mut(after)
2260 {
2261 node.insert_id_after(list);
2262 }
2263 }
2264}
2265
2266const HTML_DEPTH_ON_THE_CALLERS_STACK: usize = 32;
2270
2271const MAX_HTML_READ_DEPTH: usize = 256;
2274
2275const HTML_READER_STACK: usize = 2 << 20;
2278
2279const HTML_STACK_PER_LEVEL: usize = 16 << 10;
2282
2283fn html_integer(value: &str) -> Option<i64> {
2289 let value = value.trim_start_matches(|c: char| c.is_ascii_whitespace());
2290 let (negative, digits) = match value.as_bytes().first() {
2291 Some(b'-') => (true, &value[1..]),
2292 Some(b'+') => (false, &value[1..]),
2293 _ => (false, value),
2294 };
2295 let digits: &str = digits
2296 .split(|c: char| !c.is_ascii_digit())
2297 .next()
2298 .unwrap_or("");
2299 if digits.is_empty() {
2300 return None;
2301 }
2302 let magnitude = digits.bytes().fold(0i64, |number, digit| {
2303 number
2304 .saturating_mul(10)
2305 .saturating_add(i64::from(digit - b'0'))
2306 });
2307 Some(if negative { -magnitude } else { magnitude })
2308}
2309
2310fn html_tree_depth(tree: &ego_tree::Tree<Node>) -> usize {
2313 use ego_tree::iter::Edge;
2314 let mut depth = 0usize;
2315 let mut deepest = 0usize;
2316 for edge in tree.root().traverse() {
2317 match edge {
2318 Edge::Open(node) if node.value().is_element() => {
2319 depth += 1;
2320 deepest = deepest.max(depth);
2321 }
2322 Edge::Close(node) if node.value().is_element() => depth = depth.saturating_sub(1),
2323 _ => {}
2324 }
2325 }
2326 deepest
2327}
2328
2329pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
2353 use scraper::Html;
2354
2355 let read = |depth_limit: usize, html: &str| {
2356 let mut fragment = Html::parse_fragment(html);
2357 flatten_deep_html_lists(&mut fragment.tree);
2358 let mut elements = read_html_tree(&fragment, depth_limit);
2359 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
2360 elements
2361 };
2362 let mut fragment = Html::parse_fragment(html);
2363 flatten_deep_html_lists(&mut fragment.tree);
2364 let depth = html_tree_depth(&fragment.tree);
2365 if depth <= HTML_DEPTH_ON_THE_CALLERS_STACK {
2366 let mut elements = read_html_tree(&fragment, MAX_HTML_READ_DEPTH);
2367 crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
2368 return elements;
2369 }
2370 drop(fragment);
2371 let stack = HTML_READER_STACK + depth.min(MAX_HTML_READ_DEPTH + 1) * HTML_STACK_PER_LEVEL;
2372 let owned = html.to_owned();
2373 let on_its_own_thread = std::thread::Builder::new()
2374 .stack_size(stack)
2375 .spawn(move || read(MAX_HTML_READ_DEPTH, &owned))
2376 .ok()
2377 .and_then(|reader| reader.join().ok());
2378 on_its_own_thread.unwrap_or_else(|| read(HTML_DEPTH_ON_THE_CALLERS_STACK, html))
2379}
2380
2381fn read_html_tree(fragment: &scraper::Html, depth_limit: usize) -> Vec<ParsedElement> {
2385 let mut elements: Vec<ParsedElement> = Vec::new();
2386
2387 let root = fragment.root_element();
2389
2390 #[derive(Clone, Default)]
2391 struct FmtState {
2392 bold: bool,
2393 italic: bool,
2394 underline: bool,
2395 strikeout: bool,
2396 code: bool,
2397 superscript: bool,
2398 subscript: bool,
2399 link_href: Option<String>,
2400 preserve_whitespace: bool,
2402 sheet: Option<std::rc::Rc<SheetWhitespace>>,
2404 }
2405
2406 type HtmlList = (Option<ListStyle>, Option<i64>);
2409
2410 fn is_metadata_tag(tag: &str) -> bool {
2420 matches!(
2421 tag,
2422 "head"
2423 | "style"
2424 | "script"
2425 | "title"
2426 | "meta"
2427 | "link"
2428 | "base"
2429 | "noscript"
2430 | "template"
2431 )
2432 }
2433
2434 fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
2443 use ego_tree::iter::Edge;
2444 let mut text = String::new();
2445 let mut hidden = 0usize;
2447 for edge in node.traverse() {
2448 match edge {
2449 Edge::Open(open) => match open.value() {
2450 Node::Element(el) => {
2451 if hidden > 0 || is_metadata_tag(el.name()) {
2452 hidden += 1;
2453 } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
2454 text.push(' ');
2455 }
2456 }
2457 Node::Text(t) if hidden == 0 => text.push_str(t),
2458 _ => {}
2459 },
2460 Edge::Close(close) => {
2461 if hidden > 0 && close.value().is_element() {
2462 hidden -= 1;
2463 }
2464 }
2465 }
2466 }
2467 text
2468 }
2469
2470 fn text_span(text: String, state: &FmtState) -> ParsedSpan {
2472 ParsedSpan {
2473 text,
2474 bold: state.bold,
2475 italic: state.italic,
2476 underline: state.underline,
2477 strikeout: state.strikeout,
2478 code: state.code,
2479 superscript: state.superscript,
2480 subscript: state.subscript,
2481 link_href: state.link_href.clone(),
2482 image: None,
2483 footnote_ref: None,
2484 preserve_whitespace: state.preserve_whitespace,
2485 }
2486 }
2487
2488 fn separator_span(state: &FmtState) -> ParsedSpan {
2491 let mut span = text_span(" ".to_string(), state);
2492 span.preserve_whitespace = false;
2493 span
2494 }
2495
2496 fn collect_cell_spans(
2498 node: ego_tree::NodeRef<Node>,
2499 state: &FmtState,
2500 spans: &mut Vec<ParsedSpan>,
2501 depth: usize,
2502 limit: usize,
2503 ) {
2504 if depth > limit {
2505 let text = subtree_text(node);
2506 if !text.is_empty() {
2507 spans.push(text_span(text, state));
2508 }
2509 return;
2510 }
2511 for child in node.children() {
2512 match child.value() {
2513 Node::Text(text) => {
2514 let t = text.text.to_string();
2515 let blank = if state.preserve_whitespace {
2517 blank_text_kind(&t, child)
2518 } else {
2519 BlankText::Content
2520 };
2521 if blank == BlankText::Boundary {
2522 if spans
2525 .last()
2526 .is_some_and(|last| !last.text.ends_with(is_html_space))
2527 {
2528 spans.push(separator_span(state));
2529 }
2530 } else if !t.is_empty() && blank == BlankText::Content {
2531 spans.push(ParsedSpan {
2532 text: t,
2533 bold: state.bold,
2534 italic: state.italic,
2535 underline: state.underline,
2536 strikeout: state.strikeout,
2537 code: state.code,
2538 superscript: state.superscript,
2539 subscript: state.subscript,
2540 link_href: state.link_href.clone(),
2541 image: None,
2542 footnote_ref: None,
2543 preserve_whitespace: state.preserve_whitespace,
2544 });
2545 }
2546 }
2547 Node::Element(el) => {
2548 let tag = el.name();
2549 if is_metadata_tag(tag) {
2550 continue;
2551 }
2552 let mut new_state = state.clone();
2553 if let Some(keep) = element_keeps_whitespace(el, state.sheet.as_deref()) {
2554 new_state.preserve_whitespace = keep;
2555 }
2556 match tag {
2557 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2561 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2562 {
2563 spans.push(span);
2564 }
2565 continue;
2566 }
2567 "b" | "strong" => new_state.bold = true,
2568 "i" | "em" => new_state.italic = true,
2569 "u" | "ins" => new_state.underline = true,
2570 "s" | "del" | "strike" => new_state.strikeout = true,
2571 "code" => new_state.code = true,
2572 "sup" => new_state.superscript = true,
2573 "sub" => new_state.subscript = true,
2574 "a" => {
2575 if let Some(href) = el.attr("href") {
2576 new_state.link_href = Some(href.to_string());
2577 }
2578 }
2579 "img" => {
2580 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2581 spans.push(span);
2582 }
2583 continue;
2584 }
2585 "br" => {
2591 spans.push(separator_span(state));
2592 continue;
2593 }
2594 "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
2595 | "h3" | "h4" | "h5" | "h6" => {
2596 spans.push(separator_span(state));
2597 }
2598 _ => {}
2599 }
2600 collect_cell_spans(child, &new_state, spans, depth + 1, limit);
2601 }
2602 _ => {}
2603 }
2604 }
2605 }
2606
2607 const MAX_HTML_COLSPAN: usize = 1000;
2610
2611 fn parse_table_element(
2615 table_node: ego_tree::NodeRef<Node>,
2616 state: &FmtState,
2617 depth: usize,
2618 limit: usize,
2619 ) -> (ParsedTable, usize) {
2620 let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
2621 let mut header_rows: usize = 0;
2622 let mut own_cells: usize = 0;
2623
2624 fn collect_rows(
2626 node: ego_tree::NodeRef<Node>,
2627 state: &FmtState,
2628 rows: &mut Vec<Vec<ParsedTableCell>>,
2629 header_rows: &mut usize,
2630 own_cells: &mut usize,
2631 in_thead: bool,
2632 walk: (usize, usize),
2633 ) {
2634 let (depth, limit) = walk;
2635 for child in node.children() {
2636 if let Node::Element(el) = child.value() {
2637 match el.name() {
2638 "thead" => collect_rows(
2639 child,
2640 state,
2641 rows,
2642 header_rows,
2643 own_cells,
2644 true,
2645 (depth + 1, limit),
2646 ),
2647 "tbody" | "tfoot" => collect_rows(
2648 child,
2649 state,
2650 rows,
2651 header_rows,
2652 own_cells,
2653 false,
2654 (depth + 1, limit),
2655 ),
2656 "tr" => {
2657 let mut cells: Vec<ParsedTableCell> = Vec::new();
2658 let mut own_end = 0;
2661 for td in child.children() {
2662 if let Node::Element(td_el) = td.value()
2663 && matches!(td_el.name(), "td" | "th")
2664 {
2665 let mut spans = Vec::new();
2666 let mut cell_state = FmtState {
2671 preserve_whitespace: state.preserve_whitespace,
2672 sheet: state.sheet.clone(),
2673 ..FmtState::default()
2674 };
2675 if let Some(keep) =
2676 element_keeps_whitespace(td_el, state.sheet.as_deref())
2677 {
2678 cell_state.preserve_whitespace = keep;
2679 }
2680 collect_cell_spans(
2681 td,
2682 &cell_state,
2683 &mut spans,
2684 depth + 2,
2685 limit,
2686 );
2687 collapse_inline_whitespace(&mut spans);
2688 if spans.is_empty() {
2689 spans.push(ParsedSpan::default());
2690 }
2691 cells.push(ParsedTableCell { spans });
2692 *own_cells += 1;
2693 own_end = cells.len();
2694 let span = td_el
2699 .attr("colspan")
2700 .and_then(|value| value.trim().parse::<usize>().ok())
2701 .unwrap_or(1)
2702 .clamp(1, MAX_HTML_COLSPAN);
2703 for _ in 1..span {
2704 if cells.len() >= MAX_HTML_COLSPAN {
2705 break;
2706 }
2707 cells.push(ParsedTableCell {
2708 spans: vec![ParsedSpan::default()],
2709 });
2710 }
2711 }
2712 }
2713 cells.truncate(own_end);
2717 if !cells.is_empty() {
2718 rows.push(cells);
2719 if in_thead {
2720 *header_rows += 1;
2721 }
2722 }
2723 }
2724 _ => {}
2725 }
2726 }
2727 }
2728 }
2729
2730 collect_rows(
2731 table_node,
2732 state,
2733 &mut rows,
2734 &mut header_rows,
2735 &mut own_cells,
2736 false,
2737 (depth, limit),
2738 );
2739
2740 if header_rows == 0 && !rows.is_empty() {
2742 header_rows = 1;
2743 }
2744
2745 (
2746 ParsedTable {
2747 header_rows,
2748 rows,
2749 blockquote_depth: 0,
2752 blockquote_opens: 0,
2753 },
2754 own_cells,
2755 )
2756 }
2757
2758 #[allow(clippy::too_many_arguments)]
2759 fn walk_node(
2760 node: ego_tree::NodeRef<Node>,
2761 state: &FmtState,
2762 elements: &mut Vec<ParsedElement>,
2763 current_list: &HtmlList,
2764 blockquote_depth: u32,
2765 list_depth: u32,
2766 depth: usize,
2767 limit: usize,
2768 ) {
2769 if depth > limit {
2770 let text = subtree_text(node);
2771 if !text.trim().is_empty() {
2772 let mut spans = vec![text_span(text, state)];
2773 collapse_inline_whitespace(&mut spans);
2774 elements.push(ParsedElement::Block(ParsedBlock {
2775 spans,
2776 blockquote_depth,
2777 blockquote_opens: 0,
2778 ..ParsedBlock::default()
2779 }));
2780 }
2781 return;
2782 }
2783 match node.value() {
2784 Node::Element(el) => {
2785 let tag = el.name();
2786 if is_metadata_tag(tag) {
2787 return;
2788 }
2789 let mut new_state = state.clone();
2790 if let Some(keep) = element_keeps_whitespace(el, state.sheet.as_deref()) {
2791 new_state.preserve_whitespace = keep;
2792 }
2793 let mut new_list = current_list.clone();
2794 let mut bq_depth = blockquote_depth;
2795 let mut new_list_depth = list_depth;
2796
2797 let is_block_tag = matches!(
2799 tag,
2800 "p" | "div"
2801 | "h1"
2802 | "h2"
2803 | "h3"
2804 | "h4"
2805 | "h5"
2806 | "h6"
2807 | "li"
2808 | "pre"
2809 | "br"
2810 | "blockquote"
2811 | "body"
2812 | "html"
2813 );
2814
2815 match tag {
2817 "b" | "strong" => new_state.bold = true,
2818 "i" | "em" => new_state.italic = true,
2819 "u" | "ins" => new_state.underline = true,
2820 "s" | "del" | "strike" => new_state.strikeout = true,
2821 "code" => new_state.code = true,
2822 "sup" => new_state.superscript = true,
2823 "sub" => new_state.subscript = true,
2824 "a" => {
2825 if let Some(href) = el.attr("href") {
2826 new_state.link_href = Some(href.to_string());
2827 }
2828 }
2829 "ul" => {
2830 new_list = (Some(ListStyle::Disc), None);
2831 new_list_depth = list_depth + 1;
2832 }
2833 "ol" => {
2834 let start = el
2838 .attr("start")
2839 .and_then(html_integer)
2840 .and_then(crate::parser_tools::list_start::list_start);
2841 new_list = (Some(ListStyle::Decimal), start);
2842 new_list_depth = list_depth + 1;
2843 }
2844 "blockquote" => {
2845 bq_depth += 1;
2846 }
2847 _ => {}
2848 }
2849
2850 let heading_level = match tag {
2852 "h1" => Some(1),
2853 "h2" => Some(2),
2854 "h3" => Some(3),
2855 "h4" => Some(4),
2856 "h5" => Some(5),
2857 "h6" => Some(6),
2858 _ => None,
2859 };
2860
2861 let is_code_block = tag == "pre";
2862
2863 let code_language = if is_code_block {
2865 node.children().find_map(|child| {
2866 if let Node::Element(cel) = child.value()
2867 && cel.name() == "code"
2868 && let Some(cls) = cel.attr("class")
2869 {
2870 return cls
2871 .split_whitespace()
2872 .find_map(|c| c.strip_prefix("language-"))
2873 .map(|l| l.to_string());
2874 }
2875 None
2876 })
2877 } else {
2878 None
2879 };
2880
2881 let css = if is_block_tag {
2883 el.attr("style").map(parse_block_styles).unwrap_or_default()
2884 } else {
2885 BlockStyles::default()
2886 };
2887
2888 if tag == "table" {
2889 let (mut parsed_table, own_cells) =
2891 parse_table_element(node, &new_state, depth, limit);
2892 if !parsed_table.rows.is_empty() {
2893 parsed_table.blockquote_depth = bq_depth;
2894 push_table(elements, parsed_table, own_cells);
2895 }
2896 return;
2897 }
2898
2899 if tag == "br" {
2900 elements.push(ParsedElement::Block(ParsedBlock {
2902 spans: vec![ParsedSpan {
2903 text: String::new(),
2904 ..Default::default()
2905 }],
2906 heading_level: None,
2907 list_style: None,
2908 list_indent: 0,
2909 list_prefix: String::new(),
2910 list_suffix: String::new(),
2911 list_start: None,
2912 marker: None,
2913 is_code_block: false,
2914 code_language: None,
2915 blockquote_depth: bq_depth,
2916 blockquote_opens: 0,
2917 line_height: None,
2918 non_breakable_lines: None,
2919 page_break_before: None,
2920 front_matter: None,
2921 direction: None,
2922 background_color: None,
2923 alignment: None,
2924 top_margin: None,
2925 text_indent: None,
2926 semantic_role: None,
2927 }));
2928 return;
2929 }
2930
2931 if tag == "blockquote" {
2932 let first = elements.len();
2933 let inline_only = node.children().all(|child| match child.value() {
2938 Node::Element(el) => !matches!(
2939 el.name(),
2940 "p" | "div"
2941 | "h1"
2942 | "h2"
2943 | "h3"
2944 | "h4"
2945 | "h5"
2946 | "h6"
2947 | "li"
2948 | "pre"
2949 | "blockquote"
2950 | "ul"
2951 | "ol"
2952 | "table"
2953 ),
2954 _ => true,
2955 });
2956 if inline_only {
2957 let mut spans: Vec<ParsedSpan> = Vec::new();
2958 let mut nested_elements: Vec<ParsedElement> = Vec::new();
2959 collect_inline_spans(
2960 node,
2961 &new_state,
2962 &mut spans,
2963 &new_list,
2964 &mut nested_elements,
2965 bq_depth,
2966 new_list_depth,
2967 depth + 1,
2968 limit,
2969 );
2970 collapse_inline_whitespace(&mut spans);
2971 if spans_carry_content(&spans) {
2972 elements.push(ParsedElement::Block(ParsedBlock {
2973 spans,
2974 blockquote_depth: bq_depth,
2975 blockquote_opens: 0,
2976 ..ParsedBlock::default()
2977 }));
2978 }
2979 elements.append(&mut nested_elements);
2980 } else {
2981 for child in node.children() {
2983 walk_node(
2984 child,
2985 &new_state,
2986 elements,
2987 &new_list,
2988 bq_depth,
2989 new_list_depth,
2990 depth + 1,
2991 limit,
2992 );
2993 }
2994 }
2995 mark_quotation_opened(elements, first);
2996 } else if is_block_tag && tag != "br" {
2997 let mut spans: Vec<ParsedSpan> = Vec::new();
3002 let mut nested_elements: Vec<ParsedElement> = Vec::new();
3003 collect_inline_spans(
3004 node,
3005 &new_state,
3006 &mut spans,
3007 &new_list,
3008 &mut nested_elements,
3009 bq_depth,
3010 new_list_depth,
3011 depth + 1,
3012 limit,
3013 );
3014
3015 let (list_style_for_block, list_start_for_block) = if tag == "li" {
3016 new_list.clone()
3017 } else {
3018 (None, None)
3019 };
3020
3021 let list_indent_for_block = if tag == "li" {
3022 new_list_depth.saturating_sub(1)
3023 } else {
3024 0
3025 };
3026
3027 if !(is_code_block || css.preserve_whitespace == Some(true)) {
3028 collapse_inline_whitespace(&mut spans);
3029 }
3030
3031 let own_run_is_content =
3039 spans_carry_content(&spans) || nested_elements.is_empty();
3040
3041 if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
3042 let is_code_block = is_code_block
3046 && !spans
3047 .iter()
3048 .any(|span| span.image.is_some() || span.footnote_ref.is_some());
3049 let list_style_for_block_is_some = list_style_for_block.is_some();
3050 let block = ParsedBlock {
3051 spans,
3052 heading_level,
3053 list_suffix: list_style_for_block
3054 .as_ref()
3055 .map(|style| default_list_suffix(style).to_string())
3056 .unwrap_or_default(),
3057 list_style: list_style_for_block,
3058 list_indent: list_indent_for_block,
3059 list_prefix: String::new(),
3060 list_start: list_start_for_block,
3061 marker: if list_style_for_block_is_some {
3064 html_item_checkbox(node)
3065 } else {
3066 None
3067 },
3068 is_code_block,
3069 code_language: code_language.filter(|_| is_code_block),
3070 blockquote_depth: bq_depth,
3071 blockquote_opens: 0,
3072 line_height: css.line_height,
3073 non_breakable_lines: css.non_breakable_lines,
3074 page_break_before: css.page_break_before,
3075 front_matter: None,
3076 direction: css.direction,
3077 background_color: css.background_color,
3078 alignment: None,
3079 top_margin: None,
3080 text_indent: None,
3081 semantic_role: None,
3082 };
3083 if is_code_block {
3087 elements.push(ParsedElement::Block(block));
3088 } else {
3089 elements.extend(
3090 split_block_at_line_breaks(block)
3091 .into_iter()
3092 .map(ParsedElement::Block),
3093 );
3094 }
3095 }
3096 elements.append(&mut nested_elements);
3098 } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
3099 for child in node.children() {
3101 walk_node(
3102 child,
3103 &new_state,
3104 elements,
3105 &new_list,
3106 bq_depth,
3107 new_list_depth,
3108 depth + 1,
3109 limit,
3110 );
3111 }
3112 } else {
3113 for child in node.children() {
3115 walk_node(
3116 child,
3117 &new_state,
3118 elements,
3119 current_list,
3120 bq_depth,
3121 list_depth,
3122 depth + 1,
3123 limit,
3124 );
3125 }
3126 }
3127 }
3128 Node::Text(text) => {
3129 let mut spans = vec![text_span(text.text.to_string(), state)];
3136 collapse_inline_whitespace(&mut spans);
3137 if spans_carry_content(&spans) && !text.chars().all(is_html_space) {
3138 let block = ParsedBlock {
3139 spans,
3140 blockquote_depth,
3141 blockquote_opens: 0,
3142 ..ParsedBlock::default()
3143 };
3144 elements.extend(
3145 split_block_at_line_breaks(block)
3146 .into_iter()
3147 .map(ParsedElement::Block),
3148 );
3149 }
3150 }
3151 _ => {
3152 for child in node.children() {
3154 walk_node(
3155 child,
3156 state,
3157 elements,
3158 current_list,
3159 blockquote_depth,
3160 list_depth,
3161 depth + 1,
3162 limit,
3163 );
3164 }
3165 }
3166 }
3167 }
3168
3169 #[allow(clippy::too_many_arguments)]
3173 fn collect_inline_spans(
3174 node: ego_tree::NodeRef<Node>,
3175 state: &FmtState,
3176 spans: &mut Vec<ParsedSpan>,
3177 current_list: &HtmlList,
3178 elements: &mut Vec<ParsedElement>,
3179 blockquote_depth: u32,
3180 list_depth: u32,
3181 depth: usize,
3182 limit: usize,
3183 ) {
3184 if depth > limit {
3185 let text = subtree_text(node);
3186 if !text.is_empty() {
3187 spans.push(text_span(text, state));
3188 }
3189 return;
3190 }
3191 for child in node.children() {
3192 match child.value() {
3193 Node::Text(text) => {
3194 let t = text.text.to_string();
3195 let blank = if state.preserve_whitespace {
3197 blank_text_kind(&t, child)
3198 } else {
3199 BlankText::Content
3200 };
3201 if blank == BlankText::Boundary {
3202 if spans.last().is_some_and(|last| !last.text.ends_with('\n')) {
3208 spans.push(text_span("\n".to_string(), state));
3209 }
3210 } else if !t.is_empty() && blank == BlankText::Content {
3211 spans.push(ParsedSpan {
3212 text: t,
3213 bold: state.bold,
3214 italic: state.italic,
3215 underline: state.underline,
3216 strikeout: state.strikeout,
3217 code: state.code,
3218 superscript: state.superscript,
3219 subscript: state.subscript,
3220 link_href: state.link_href.clone(),
3221 image: None,
3222 footnote_ref: None,
3223 preserve_whitespace: state.preserve_whitespace,
3224 });
3225 }
3226 }
3227 Node::Element(el) => {
3228 let tag = el.name();
3229 if is_metadata_tag(tag) {
3230 continue;
3231 }
3232 let mut new_state = state.clone();
3233 if let Some(keep) = element_keeps_whitespace(el, state.sheet.as_deref()) {
3234 new_state.preserve_whitespace = keep;
3235 }
3236
3237 match tag {
3238 _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
3242 if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
3243 {
3244 spans.push(span);
3245 }
3246 continue;
3247 }
3248 "b" | "strong" => new_state.bold = true,
3249 "i" | "em" => new_state.italic = true,
3250 "u" | "ins" => new_state.underline = true,
3251 "s" | "del" | "strike" => new_state.strikeout = true,
3252 "code" => new_state.code = true,
3253 "sup" => new_state.superscript = true,
3254 "sub" => new_state.subscript = true,
3255 "a" => {
3256 if let Some(href) = el.attr("href") {
3257 new_state.link_href = Some(href.to_string());
3258 }
3259 }
3260 "img" => {
3261 if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
3262 spans.push(span);
3263 }
3264 continue;
3265 }
3266 _ => {}
3267 }
3268
3269 let nested_block = matches!(
3271 tag,
3272 "p" | "div"
3273 | "h1"
3274 | "h2"
3275 | "h3"
3276 | "h4"
3277 | "h5"
3278 | "h6"
3279 | "li"
3280 | "pre"
3281 | "blockquote"
3282 | "ul"
3283 | "ol"
3284 );
3285
3286 if tag == "br" {
3287 spans.push(text_span(LINE_SEPARATOR_STR.to_string(), state));
3291 } else if nested_block || tag == "table" {
3292 walk_node(
3294 child,
3295 &new_state,
3296 elements,
3297 current_list,
3298 blockquote_depth,
3299 list_depth,
3300 depth + 1,
3301 limit,
3302 );
3303 } else {
3304 collect_inline_spans(
3306 child,
3307 &new_state,
3308 spans,
3309 current_list,
3310 elements,
3311 blockquote_depth,
3312 list_depth,
3313 depth + 1,
3314 limit,
3315 );
3316 }
3317 }
3318 _ => {}
3319 }
3320 }
3321 }
3322
3323 let sheet = html_sheet(fragment).map(std::rc::Rc::new);
3327 let initial_state = FmtState {
3328 preserve_whitespace: sheet
3329 .as_deref()
3330 .and_then(SheetWhitespace::initial_keeps_whitespace)
3331 .unwrap_or(false),
3332 sheet,
3333 ..FmtState::default()
3334 };
3335 let mut root_spans: Vec<ParsedSpan> = Vec::new();
3339 collect_inline_spans(
3340 *root,
3341 &initial_state,
3342 &mut root_spans,
3343 &(None, None),
3344 &mut elements,
3345 0,
3346 0,
3347 0,
3348 depth_limit,
3349 );
3350 collapse_inline_whitespace(&mut root_spans);
3351 if spans_carry_content(&root_spans) {
3354 let root_block = ParsedBlock {
3356 spans: root_spans,
3357 heading_level: None,
3358 list_style: None,
3359 list_indent: 0,
3360 list_prefix: String::new(),
3361 list_suffix: String::new(),
3362 list_start: None,
3363 marker: None,
3364 is_code_block: false,
3365 code_language: None,
3366 blockquote_depth: 0,
3367 blockquote_opens: 0,
3368 line_height: None,
3369 non_breakable_lines: None,
3370 page_break_before: None,
3371 front_matter: None,
3372 direction: None,
3373 background_color: None,
3374 alignment: None,
3375 top_margin: None,
3376 text_indent: None,
3377 semantic_role: None,
3378 };
3379 elements.extend(
3380 split_block_at_line_breaks(root_block)
3381 .into_iter()
3382 .map(ParsedElement::Block),
3383 );
3384 }
3385
3386 if elements.is_empty() {
3388 elements.push(ParsedElement::Block(ParsedBlock {
3389 spans: vec![ParsedSpan {
3390 text: String::new(),
3391 ..Default::default()
3392 }],
3393 heading_level: None,
3394 list_style: None,
3395 list_indent: 0,
3396 list_prefix: String::new(),
3397 list_suffix: String::new(),
3398 list_start: None,
3399 marker: None,
3400 is_code_block: false,
3401 code_language: None,
3402 blockquote_depth: 0,
3403 blockquote_opens: 0,
3404 line_height: None,
3405 non_breakable_lines: None,
3406 page_break_before: None,
3407 front_matter: None,
3408 direction: None,
3409 background_color: None,
3410 alignment: None,
3411 top_margin: None,
3412 text_indent: None,
3413 semantic_role: None,
3414 }));
3415 }
3416
3417 elements
3418}
3419
3420pub fn character_format_from_span(
3424 span: &ParsedSpan,
3425 is_code_block: bool,
3426) -> crate::format_runs::CharacterFormat {
3427 use crate::entities::CharVerticalAlignment;
3428 crate::format_runs::CharacterFormat {
3429 font_bold: if span.bold { Some(true) } else { None },
3430 font_italic: if span.italic { Some(true) } else { None },
3431 font_underline: if span.underline { Some(true) } else { None },
3432 font_strikeout: if span.strikeout { Some(true) } else { None },
3433 font_family: if span.code || is_code_block {
3434 Some("monospace".to_string())
3435 } else {
3436 None
3437 },
3438 anchor_href: span.link_href.clone(),
3439 is_anchor: if span.link_href.is_some() {
3440 Some(true)
3441 } else {
3442 None
3443 },
3444 vertical_alignment: if span.superscript {
3445 Some(CharVerticalAlignment::SuperScript)
3446 } else if span.subscript {
3447 Some(CharVerticalAlignment::SubScript)
3448 } else {
3449 None
3450 },
3451 ..Default::default()
3452 }
3453}
3454
3455pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
3467 use crate::format_runs::{
3468 CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
3469 };
3470
3471 let mut plain_text = String::new();
3472 let mut runs: Vec<FormatRun> = Vec::new();
3473 let mut images: Vec<ImageAnchor> = Vec::new();
3474 let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
3475 let default = CharacterFormat::default();
3476
3477 for span in spans {
3478 let byte_start = plain_text.len() as u32;
3479
3480 if let Some(label) = &span.footnote_ref {
3481 plain_text.push('\u{FFFC}');
3484 let mut format = character_format_from_span(span, is_code_block);
3499 format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
3500 footnote_refs.push(FootnoteRefAnchor {
3501 byte_offset: byte_start,
3502 label: label.clone(),
3503 format,
3504 });
3505 continue;
3506 }
3507
3508 if let Some(image) = &span.image {
3509 plain_text.push('\u{FFFC}');
3513 images.push(ImageAnchor {
3514 byte_offset: byte_start,
3515 name: image.src.clone(),
3516 alt: image.alt.clone(),
3517 width: image.width,
3518 height: image.height,
3519 quality: 100,
3520 format: character_format_from_span(span, is_code_block),
3521 });
3522 continue;
3523 }
3524
3525 plain_text.push_str(&span.text);
3526 let byte_end = plain_text.len() as u32;
3527 if byte_start == byte_end {
3528 continue;
3529 }
3530 let format = character_format_from_span(span, is_code_block);
3531 if format == default {
3532 continue;
3533 }
3534 runs.push(FormatRun {
3535 byte_start,
3536 byte_end,
3537 format,
3538 });
3539 }
3540 coalesce_in_place(&mut runs);
3541 ParsedInline {
3542 plain_text,
3543 runs,
3544 images,
3545 footnote_refs,
3546 }
3547}
3548
3549#[derive(Debug, Clone, Default)]
3556pub struct ParsedInline {
3557 pub plain_text: String,
3558 pub runs: Vec<crate::format_runs::FormatRun>,
3559 pub images: Vec<crate::format_runs::ImageAnchor>,
3560 pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
3561}
3562
3563fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
3571 use jotdown::ListBulletType as B;
3572 match b {
3573 B::Dash => ListStyle::Disc,
3574 B::Star => ListStyle::Circle,
3575 B::Plus => ListStyle::Square,
3576 }
3577}
3578
3579fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
3581 use jotdown::OrderedListNumbering as N;
3582 match n {
3583 N::Decimal => ListStyle::Decimal,
3584 N::AlphaLower => ListStyle::LowerAlpha,
3585 N::AlphaUpper => ListStyle::UpperAlpha,
3586 N::RomanLower => ListStyle::LowerRoman,
3587 N::RomanUpper => ListStyle::UpperRoman,
3588 }
3589}
3590
3591fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
3595 use jotdown::OrderedListStyle as S;
3596 match style {
3597 S::Period => (String::new(), ".".to_string()),
3598 S::Paren => (String::new(), ")".to_string()),
3599 S::ParenParen => ("(".to_string(), ")".to_string()),
3600 }
3601}
3602
3603#[derive(Debug, Clone, Default)]
3607struct DjotBlockStyle {
3608 alignment: Option<Alignment>,
3609 line_height: Option<i64>,
3610 non_breakable_lines: Option<bool>,
3611 page_break_before: Option<bool>,
3612 front_matter: Option<bool>,
3613 direction: Option<TextDirection>,
3614 background_color: Option<String>,
3615 top_margin: Option<i64>,
3616 text_indent: Option<i64>,
3617 semantic_role: Option<SemanticRole>,
3618}
3619
3620impl DjotBlockStyle {
3621 fn merge_from(&mut self, other: DjotBlockStyle) {
3625 if other.alignment.is_some() {
3626 self.alignment = other.alignment;
3627 }
3628 if other.line_height.is_some() {
3629 self.line_height = other.line_height;
3630 }
3631 if other.non_breakable_lines.is_some() {
3632 self.non_breakable_lines = other.non_breakable_lines;
3633 }
3634 if other.page_break_before.is_some() {
3635 self.page_break_before = other.page_break_before;
3636 }
3637 if other.front_matter.is_some() {
3638 self.front_matter = other.front_matter;
3639 }
3640 if other.direction.is_some() {
3641 self.direction = other.direction;
3642 }
3643 if other.background_color.is_some() {
3644 self.background_color = other.background_color;
3645 }
3646 if other.top_margin.is_some() {
3647 self.top_margin = other.top_margin;
3648 }
3649 if other.text_indent.is_some() {
3650 self.text_indent = other.text_indent;
3651 }
3652 if other.semantic_role.is_some() {
3653 self.semantic_role = other.semantic_role.clone();
3654 }
3655 }
3656}
3657
3658fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
3664 let mut style = DjotBlockStyle::default();
3665
3666 if opts.alignment
3667 && let Some(v) = attrs.get_value("alignment")
3668 {
3669 style.alignment = match v.to_string().as_str() {
3670 "left" => Some(Alignment::Left),
3671 "right" => Some(Alignment::Right),
3672 "center" => Some(Alignment::Center),
3673 "justify" => Some(Alignment::Justify),
3674 _ => None,
3675 };
3676 }
3677 if opts.line_height
3678 && let Some(v) = attrs.get_value("line_height")
3679 {
3680 style.line_height = v.to_string().parse::<i64>().ok();
3681 }
3682 if opts.direction
3683 && let Some(v) = attrs.get_value("direction")
3684 {
3685 style.direction = match v.to_string().as_str() {
3686 "ltr" => Some(TextDirection::LeftToRight),
3687 "rtl" => Some(TextDirection::RightToLeft),
3688 _ => None,
3689 };
3690 }
3691 if opts.non_breakable_lines
3692 && let Some(v) = attrs.get_value("non_breakable_lines")
3693 {
3694 style.non_breakable_lines = match v.to_string().as_str() {
3695 "true" => Some(true),
3696 "false" => Some(false),
3697 _ => None,
3698 };
3699 }
3700 if opts.page_break_before
3701 && let Some(v) = attrs.get_value("page_break_before")
3702 {
3703 style.page_break_before = match v.to_string().as_str() {
3704 "true" => Some(true),
3705 "false" => Some(false),
3706 _ => None,
3707 };
3708 }
3709 if opts.front_matter
3710 && let Some(v) = attrs.get_value("front_matter")
3711 {
3712 style.front_matter = match v.to_string().as_str() {
3713 "true" => Some(true),
3714 "false" => Some(false),
3715 _ => None,
3716 };
3717 }
3718 if opts.background_color
3719 && let Some(v) = attrs.get_value("background_color")
3720 {
3721 style.background_color = Some(v.to_string());
3722 }
3723 if opts.top_margin
3724 && let Some(v) = attrs.get_value("top_margin")
3725 {
3726 style.top_margin = v.to_string().parse::<i64>().ok();
3727 }
3728 if opts.text_indent
3729 && let Some(v) = attrs.get_value("text_indent")
3730 {
3731 style.text_indent = v.to_string().parse::<i64>().ok();
3732 }
3733 if opts.semantic_role
3734 && let Some(v) = attrs.get_value("semantic_role")
3735 {
3736 style.semantic_role = match v.to_string().as_str() {
3737 "epigraph" => Some(SemanticRole::Epigraph),
3738 _ => None,
3742 };
3743 }
3744
3745 style
3746}
3747
3748fn list_start_of(start: u64) -> Option<i64> {
3752 crate::parser_tools::list_start::list_start_from_unsigned(start)
3753}
3754
3755#[allow(clippy::too_many_arguments)]
3758fn djot_push_block(
3759 elements: &mut Vec<ParsedElement>,
3760 spans: Vec<ParsedSpan>,
3761 heading_level: Option<i64>,
3762 list_style: Option<ListStyle>,
3763 list_indent: u32,
3764 list_prefix: String,
3765 list_suffix: String,
3766 list_start: Option<i64>,
3767 marker: Option<MarkerType>,
3768 is_code_block: bool,
3769 code_language: Option<String>,
3770 blockquote_depth: u32,
3771 style: DjotBlockStyle,
3772) {
3773 elements.push(ParsedElement::Block(ParsedBlock {
3774 spans,
3775 heading_level,
3776 list_style,
3777 list_indent,
3778 list_prefix,
3779 list_suffix,
3780 list_start,
3781 marker,
3782 is_code_block,
3783 code_language,
3784 blockquote_depth,
3785 blockquote_opens: 0,
3786 line_height: style.line_height,
3787 non_breakable_lines: style.non_breakable_lines,
3788 page_break_before: style.page_break_before,
3789 front_matter: style.front_matter,
3790 direction: style.direction,
3791 background_color: style.background_color,
3792 alignment: style.alignment,
3793 top_margin: style.top_margin,
3794 text_indent: style.text_indent,
3795 semantic_role: style.semantic_role.clone(),
3796 }));
3797}
3798
3799fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
3812 djot.lines()
3813 .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
3814 .filter(|line| !line.is_empty())
3815 .map(|line| {
3816 ParsedElement::Block(ParsedBlock {
3817 spans: vec![ParsedSpan {
3818 text: line.to_string(),
3819 ..Default::default()
3820 }],
3821 ..Default::default()
3822 })
3823 })
3824 .collect()
3825}
3826
3827pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
3853 use jotdown::{Container as C, Event as E, ListKind};
3854
3855 let djot = crate::parser_tools::djot_depth::crlf_to_lf(djot);
3856 let djot: &str = &djot;
3857
3858 let parsable = crate::parser_tools::djot_depth::parsable(djot);
3884 let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
3885 return djot_source_lines(djot);
3886 };
3887
3888 let mut elements: Vec<ParsedElement> = Vec::new();
3889 let mut current_spans: Vec<ParsedSpan> = Vec::new();
3890 let mut current_heading: Option<i64> = None;
3891 let mut is_code_block = false;
3892 let mut code_language: Option<String> = None;
3893 let mut blockquote_depth: u32 = 0;
3894 let mut quotations_open: Vec<usize> = Vec::new();
3896 let mut pending_style = DjotBlockStyle::default();
3899
3900 let mut bold = false;
3902 let mut italic = false;
3903 let mut underline = false;
3904 let mut strikeout = false;
3905 let mut code = false;
3906 let mut superscript = false;
3907 let mut subscript = false;
3908 let mut link_href: Option<String> = None;
3909 let mut pending_image: Option<ParsedImage> = None;
3912
3913 let mut list_stack: Vec<(ListStyle, String, String, Option<i64>)> = Vec::new();
3915 let mut cur_list_style: Option<ListStyle> = None;
3917 let mut cur_list_prefix = String::new();
3918 let mut cur_list_suffix = String::new();
3919 let mut cur_list_start: Option<i64> = None;
3920 let mut cur_list_indent: u32 = 0;
3921 let mut cur_marker: Option<MarkerType> = None;
3922
3923 let mut in_table_cell = false;
3925 let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
3926 let mut current_row: Vec<ParsedTableCell> = Vec::new();
3927 let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
3928 let mut table_header_rows: usize = 0;
3929 let mut row_is_head = false;
3930
3931 let mut skip_depth: u32 = 0;
3935
3936 let mut footnote_open: Option<(String, usize)> = None;
3940
3941 macro_rules! push_text {
3946 ($t:expr) => {{
3947 if let Some(img) = pending_image.as_mut() {
3952 img.alt.push_str(($t).as_ref());
3953 } else {
3954 let sp = ParsedSpan {
3955 text: ($t).to_string(),
3956 bold,
3957 italic,
3958 underline,
3959 strikeout,
3960 code,
3961 superscript,
3962 subscript,
3963 link_href: link_href.clone(),
3964 image: None,
3965 footnote_ref: None,
3966 preserve_whitespace: false,
3967 };
3968 if in_table_cell {
3969 current_cell_spans.push(sp);
3970 } else {
3971 current_spans.push(sp);
3972 }
3973 }
3974 }};
3975 }
3976
3977 macro_rules! push_image {
3979 ($img:expr) => {{
3980 let sp = ParsedSpan {
3981 text: String::new(),
3982 bold,
3983 italic,
3984 underline,
3985 strikeout,
3986 code,
3987 superscript,
3988 subscript,
3989 link_href: link_href.clone(),
3990 image: Some($img),
3991 footnote_ref: None,
3992 preserve_whitespace: false,
3993 };
3994 if in_table_cell {
3995 current_cell_spans.push(sp);
3996 } else {
3997 current_spans.push(sp);
3998 }
3999 }};
4000 }
4001
4002 macro_rules! enter_item {
4005 ($marker:expr) => {{
4006 if !current_spans.is_empty() {
4007 djot_push_block(
4008 &mut elements,
4009 std::mem::take(&mut current_spans),
4010 None,
4011 cur_list_style.clone(),
4012 cur_list_indent,
4013 cur_list_prefix.clone(),
4014 cur_list_suffix.clone(),
4015 cur_list_start,
4016 cur_marker.clone(),
4017 false,
4018 None,
4019 blockquote_depth,
4020 DjotBlockStyle::default(),
4021 );
4022 }
4023 let (style, prefix, suffix, start) = list_stack.last().cloned().unwrap_or((
4024 ListStyle::Disc,
4025 String::new(),
4026 String::new(),
4027 None,
4028 ));
4029 cur_list_style = Some(style);
4030 cur_list_prefix = prefix;
4031 cur_list_suffix = suffix;
4032 cur_list_start = start;
4033 cur_list_indent = list_stack.len().saturating_sub(1) as u32;
4034 cur_marker = $marker;
4035 }};
4036 }
4037
4038 macro_rules! line_break {
4042 () => {{
4043 if in_table_cell || pending_image.is_some() {
4044 push_text!(" ");
4045 } else {
4046 push_text!(LINE_SEPARATOR_STR);
4047 }
4048 }};
4049 }
4050
4051 macro_rules! split_line {
4058 () => {{
4059 if in_table_cell {
4060 push_text!(" ");
4061 } else if !current_spans.is_empty() && current_heading.is_some() {
4062 djot_push_block(
4067 &mut elements,
4068 std::mem::take(&mut current_spans),
4069 current_heading,
4070 None,
4071 0,
4072 String::new(),
4073 String::new(),
4074 None,
4075 None,
4076 false,
4077 None,
4078 blockquote_depth,
4079 pending_style.clone(),
4080 );
4081 pending_style.page_break_before = None;
4082 } else if !current_spans.is_empty() {
4083 djot_push_block(
4084 &mut elements,
4085 std::mem::take(&mut current_spans),
4086 None,
4087 cur_list_style.clone(),
4088 cur_list_indent,
4089 cur_list_prefix.clone(),
4090 cur_list_suffix.clone(),
4091 cur_list_start,
4092 cur_marker.clone(),
4093 is_code_block,
4094 code_language.clone(),
4095 blockquote_depth,
4096 pending_style.clone(),
4097 );
4098 pending_style.page_break_before = None;
4099 }
4100 }};
4101 }
4102
4103 for event in events {
4104 if skip_depth > 0 {
4105 match event {
4106 E::Start(..) => skip_depth += 1,
4107 E::End(_) => skip_depth -= 1,
4108 _ => {}
4109 }
4110 continue;
4111 }
4112
4113 match event {
4114 E::Start(C::Document, _) | E::End(C::Document) => {}
4116 E::Start(C::Section { .. }, attrs) => {
4117 if list_stack.is_empty() {
4120 pending_style.merge_from(block_attrs_to_style(&attrs, options));
4121 }
4122 }
4123 E::End(C::Section { .. }) => {}
4124 E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
4125
4126 E::Start(C::Blockquote, _) => {
4128 blockquote_depth += 1;
4129 quotations_open.push(elements.len());
4130 }
4131 E::End(C::Blockquote) => {
4132 blockquote_depth = blockquote_depth.saturating_sub(1);
4133 if let Some(first) = quotations_open.pop() {
4134 mark_quotation_opened(&mut elements, first);
4135 }
4136 }
4137
4138 E::Start(C::List { kind, .. }, _) => {
4140 let (style, prefix, suffix, start) = match kind {
4141 ListKind::Unordered(b) | ListKind::Task(b) => {
4142 (djot_bullet_style(b), String::new(), String::new(), None)
4143 }
4144 ListKind::Ordered {
4145 numbering,
4146 style,
4147 start,
4148 } => {
4149 let (p, s) = djot_ordered_affixes(style);
4150 (djot_ordered_style(numbering), p, s, list_start_of(start))
4151 }
4152 };
4153 list_stack.push((style, prefix, suffix, start));
4154 }
4155 E::End(C::List { .. }) => {
4156 list_stack.pop();
4157 cur_list_style = None;
4158 cur_list_start = None;
4159 cur_marker = None;
4160 }
4161 E::Start(C::ListItem, _) => enter_item!(None),
4162 E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
4163 MarkerType::Checked
4164 } else {
4165 MarkerType::Unchecked
4166 })),
4167 E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
4168 if !current_spans.is_empty() {
4170 djot_push_block(
4171 &mut elements,
4172 std::mem::take(&mut current_spans),
4173 None,
4174 cur_list_style.clone(),
4175 cur_list_indent,
4176 cur_list_prefix.clone(),
4177 cur_list_suffix.clone(),
4178 cur_list_start,
4179 cur_marker.clone(),
4180 false,
4181 None,
4182 blockquote_depth,
4183 DjotBlockStyle::default(),
4184 );
4185 }
4186 cur_list_style = None;
4187 cur_list_start = None;
4188 cur_marker = None;
4189 }
4190
4191 E::Start(C::Heading { level, .. }, attrs) => {
4193 current_heading = Some(level as i64);
4194 pending_style.merge_from(block_attrs_to_style(&attrs, options));
4197 }
4198 E::End(C::Heading { .. }) => {
4199 djot_push_block(
4200 &mut elements,
4201 std::mem::take(&mut current_spans),
4202 current_heading.take(),
4203 None,
4204 0,
4205 String::new(),
4206 String::new(),
4207 None,
4208 None,
4209 false,
4210 None,
4211 blockquote_depth,
4212 std::mem::take(&mut pending_style),
4213 );
4214 }
4215 E::Start(C::Paragraph, attrs) => {
4216 current_heading = None;
4217 pending_style = if list_stack.is_empty() {
4221 block_attrs_to_style(&attrs, options)
4222 } else {
4223 DjotBlockStyle::default()
4224 };
4225 }
4226 E::End(C::Paragraph) => {
4227 if !current_spans.is_empty() {
4228 djot_push_block(
4229 &mut elements,
4230 std::mem::take(&mut current_spans),
4231 None,
4232 cur_list_style.clone(),
4233 cur_list_indent,
4234 cur_list_prefix.clone(),
4235 cur_list_suffix.clone(),
4236 cur_list_start,
4237 cur_marker.clone(),
4238 false,
4239 None,
4240 blockquote_depth,
4241 std::mem::take(&mut pending_style),
4242 );
4243 }
4244 cur_list_style = None;
4245 cur_list_start = None;
4246 cur_marker = None;
4247 }
4248 E::Start(C::CodeBlock { language }, _) => {
4249 is_code_block = true;
4250 code_language = if language.is_empty() {
4251 None
4252 } else {
4253 Some(language.to_string())
4254 };
4255 }
4256 E::End(C::CodeBlock { .. }) => {
4257 if let Some(last) = current_spans.last_mut()
4259 && last.text.ends_with('\n')
4260 {
4261 last.text.pop();
4262 }
4263 djot_push_block(
4264 &mut elements,
4265 std::mem::take(&mut current_spans),
4266 None,
4267 None,
4268 0,
4269 String::new(),
4270 String::new(),
4271 None,
4272 None,
4273 true,
4274 code_language.take(),
4275 blockquote_depth,
4276 DjotBlockStyle::default(),
4277 );
4278 is_code_block = false;
4279 }
4280
4281 E::Start(C::Table, _) => {
4283 table_rows.clear();
4284 current_row.clear();
4285 current_cell_spans.clear();
4286 table_header_rows = 0;
4287 }
4288 E::End(C::Table) => {
4289 let own_cells = cells_in_rows(&table_rows);
4290 push_table(
4291 &mut elements,
4292 ParsedTable {
4293 header_rows: table_header_rows,
4294 rows: std::mem::take(&mut table_rows),
4295 blockquote_depth,
4296 blockquote_opens: 0,
4297 },
4298 own_cells,
4299 );
4300 }
4301 E::Start(C::TableRow { head }, _) => {
4302 row_is_head = head;
4303 current_row.clear();
4304 }
4305 E::End(C::TableRow { .. }) => {
4306 if row_is_head {
4307 table_header_rows += 1;
4308 }
4309 table_rows.push(std::mem::take(&mut current_row));
4310 }
4311 E::Start(C::TableCell { .. }, _) => {
4312 in_table_cell = true;
4313 current_cell_spans.clear();
4314 }
4315 E::End(C::TableCell { .. }) => {
4316 in_table_cell = false;
4317 current_row.push(ParsedTableCell {
4318 spans: std::mem::take(&mut current_cell_spans),
4319 });
4320 }
4321
4322 E::Start(C::Strong, _) => bold = true,
4324 E::End(C::Strong) => bold = false,
4325 E::Start(C::Emphasis, _) => italic = true,
4326 E::End(C::Emphasis) => italic = false,
4327 E::Start(C::Verbatim, _) => code = true,
4328 E::End(C::Verbatim) => code = false,
4329 E::Start(C::Superscript, _) => superscript = true,
4330 E::End(C::Superscript) => superscript = false,
4331 E::Start(C::Subscript, _) => subscript = true,
4332 E::End(C::Subscript) => subscript = false,
4333 E::Start(C::Insert, _) => underline = true,
4334 E::End(C::Insert) => underline = false,
4335 E::Start(C::Delete, _) => strikeout = true,
4336 E::End(C::Delete) => strikeout = false,
4337 E::Start(C::Mark, _) | E::End(C::Mark) => {}
4339 E::Start(C::Span, _) | E::End(C::Span) => {}
4340 E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
4341 E::End(C::Link(..)) => link_href = None,
4342 E::Start(C::Image(src, _), attrs) => {
4347 let attr_num = |key: &str| -> i64 {
4348 attrs
4349 .get_value(key)
4350 .map(|v| v.to_string())
4351 .and_then(|v| v.trim().parse::<i64>().ok())
4352 .filter(|n| *n > 0)
4353 .unwrap_or(0)
4354 };
4355 pending_image = Some(ParsedImage {
4356 src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
4359 alt: String::new(),
4360 width: attr_num("width"),
4361 height: attr_num("height"),
4362 });
4363 }
4364 E::End(C::Image(..)) => {
4365 if let Some(img) = pending_image.take() {
4366 push_image!(img);
4367 }
4368 }
4369
4370 E::Start(C::Footnote { label }, _) => {
4379 if !current_spans.is_empty() {
4380 djot_push_block(
4381 &mut elements,
4382 std::mem::take(&mut current_spans),
4383 None,
4384 cur_list_style.clone(),
4385 cur_list_indent,
4386 cur_list_prefix.clone(),
4387 cur_list_suffix.clone(),
4388 cur_list_start,
4389 cur_marker.clone(),
4390 false,
4391 None,
4392 blockquote_depth,
4393 DjotBlockStyle::default(),
4394 );
4395 }
4396 footnote_open = Some((label.to_string(), elements.len()));
4397 }
4398 E::End(C::Footnote { .. }) => {
4399 if !current_spans.is_empty() {
4400 djot_push_block(
4401 &mut elements,
4402 std::mem::take(&mut current_spans),
4403 None,
4404 cur_list_style.clone(),
4405 cur_list_indent,
4406 cur_list_prefix.clone(),
4407 cur_list_suffix.clone(),
4408 cur_list_start,
4409 cur_marker.clone(),
4410 false,
4411 None,
4412 blockquote_depth,
4413 DjotBlockStyle::default(),
4414 );
4415 }
4416 if let Some((label, start)) = footnote_open.take() {
4417 let blocks: Vec<ParsedBlock> = elements
4418 .drain(start..)
4419 .filter_map(|e| match e {
4420 ParsedElement::Block(b) => Some(b),
4421 _ => None,
4425 })
4426 .collect();
4427 elements.push(ParsedElement::FootnoteDefinition { label, blocks });
4428 }
4429 }
4430
4431 E::Start(
4433 C::Math { .. }
4434 | C::RawBlock { .. }
4435 | C::RawInline { .. }
4436 | C::DescriptionList
4437 | C::DescriptionDetails
4438 | C::DescriptionTerm
4439 | C::Caption
4440 | C::LinkDefinition { .. },
4441 _,
4442 ) => skip_depth = 1,
4443
4444 E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
4456 for (i, piece) in s.split('\n').enumerate() {
4457 if i > 0 {
4458 split_line!();
4459 }
4460 if !piece.is_empty() {
4461 push_text!(piece);
4462 }
4463 }
4464 }
4465 E::Str(s) => push_text!(s.as_ref()),
4466 E::Softbreak => push_text!(" "),
4467 E::LeftSingleQuote => push_text!("\u{2018}"),
4468 E::RightSingleQuote => push_text!("\u{2019}"),
4469 E::LeftDoubleQuote => push_text!("\u{201C}"),
4470 E::RightDoubleQuote => push_text!("\u{201D}"),
4471 E::Ellipsis => push_text!("\u{2026}"),
4472 E::EnDash => push_text!("\u{2013}"),
4473 E::EmDash => push_text!("\u{2014}"),
4474 E::NonBreakingSpace => push_text!("\u{00A0}"),
4475 E::Hardbreak => line_break!(),
4476 E::FootnoteReference(label) => {
4482 let sp = ParsedSpan {
4483 text: String::new(),
4484 bold,
4485 italic,
4486 underline,
4487 strikeout,
4488 code,
4489 superscript,
4490 subscript,
4491 link_href: link_href.clone(),
4492 image: None,
4493 footnote_ref: Some(label.to_string()),
4494 preserve_whitespace: false,
4495 };
4496 if in_table_cell {
4497 current_cell_spans.push(sp);
4498 } else {
4499 current_spans.push(sp);
4500 }
4501 }
4502 E::Symbol(_) => {}
4505 E::Escape | E::Blankline => {}
4506 E::ThematicBreak(_) | E::Attributes(_) => {}
4507
4508 _ => {}
4511 }
4512 }
4513
4514 if !current_spans.is_empty() {
4516 djot_push_block(
4517 &mut elements,
4518 std::mem::take(&mut current_spans),
4519 current_heading.take(),
4520 cur_list_style.clone(),
4521 cur_list_indent,
4522 cur_list_prefix.clone(),
4523 cur_list_suffix.clone(),
4524 cur_list_start,
4525 cur_marker.clone(),
4526 is_code_block,
4527 code_language.take(),
4528 blockquote_depth,
4529 std::mem::take(&mut pending_style),
4530 );
4531 }
4532
4533 if elements.is_empty() {
4536 djot_push_block(
4537 &mut elements,
4538 vec![ParsedSpan {
4539 text: String::new(),
4540 ..Default::default()
4541 }],
4542 None,
4543 None,
4544 0,
4545 String::new(),
4546 String::new(),
4547 None,
4548 None,
4549 false,
4550 None,
4551 0,
4552 DjotBlockStyle::default(),
4553 );
4554 }
4555
4556 elements
4557}
4558
4559#[cfg(test)]
4560mod tests {
4561 use super::*;
4562
4563 #[test]
4564 fn test_style_keeps_whitespace_reads_important() {
4565 assert_eq!(style_keeps_whitespace("white-space: pre-wrap"), Some(true));
4566 assert_eq!(
4567 style_keeps_whitespace("white-space: pre-wrap !important"),
4568 Some(true)
4569 );
4570 assert_eq!(
4571 style_keeps_whitespace("white-space:PRE!IMPORTANT; color: red"),
4572 Some(true)
4573 );
4574 assert_eq!(
4575 style_keeps_whitespace("white-space: pre ! important; white-space: normal"),
4576 Some(true),
4577 "a later declaration that is not important does not win"
4578 );
4579 assert_eq!(
4580 style_keeps_whitespace("white-space: pre; white-space: normal !important"),
4581 Some(false)
4582 );
4583 assert_eq!(
4584 style_keeps_whitespace("white-space: pre; white-space: normal"),
4585 Some(false)
4586 );
4587 assert_eq!(style_keeps_whitespace("white-space: inherit"), None);
4588 }
4589
4590 fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
4592 ParsedElement::flatten_to_blocks(parse_markdown(md))
4593 }
4594
4595 #[test]
4596 fn test_parse_markdown_simple_paragraph() {
4597 let blocks = parse_markdown_blocks("Hello **world**");
4598 assert_eq!(blocks.len(), 1);
4599 assert!(blocks[0].spans.len() >= 2);
4600 let plain_span = blocks[0]
4602 .spans
4603 .iter()
4604 .find(|s| s.text.contains("Hello"))
4605 .unwrap();
4606 assert!(!plain_span.bold);
4607 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
4608 assert!(bold_span.bold);
4609 }
4610
4611 #[test]
4612 fn test_parse_markdown_heading() {
4613 let blocks = parse_markdown_blocks("# Title");
4614 assert_eq!(blocks.len(), 1);
4615 assert_eq!(blocks[0].heading_level, Some(1));
4616 assert_eq!(blocks[0].spans[0].text, "Title");
4617 }
4618
4619 #[test]
4620 fn test_parse_markdown_list() {
4621 let blocks = parse_markdown_blocks("- item1\n- item2");
4622 assert!(blocks.len() >= 2);
4623 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4624 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4625 }
4626
4627 fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
4629 elements
4630 .iter()
4631 .map(|e| match e {
4632 ParsedElement::Block(b) => (false, b.blockquote_depth),
4633 ParsedElement::Table(t) => (true, t.blockquote_depth),
4634 ParsedElement::FootnoteDefinition { .. } => (false, 0),
4637 })
4638 .collect()
4639 }
4640
4641 #[test]
4642 fn test_parse_markdown_table_in_blockquote_records_depth() {
4643 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
4644 assert_eq!(element_depths(&elements), vec![(true, 1)]);
4645 }
4646
4647 #[test]
4648 fn test_parse_markdown_text_then_table_in_blockquote() {
4649 let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
4650 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
4651 }
4652
4653 #[test]
4654 fn test_parse_markdown_table_after_blockquote_closes() {
4655 let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
4656 assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
4657 }
4658
4659 #[test]
4660 fn test_parse_markdown_table_in_nested_blockquote() {
4661 let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
4662 assert_eq!(element_depths(&elements), vec![(true, 2)]);
4663 }
4664
4665 #[test]
4666 fn test_parse_markdown_list_in_blockquote_records_depth() {
4667 let elements = parse_markdown("> - item1\n> - item2");
4668 let depths = element_depths(&elements);
4669 assert_eq!(depths, vec![(false, 1), (false, 1)]);
4670 for e in &elements {
4671 if let ParsedElement::Block(b) = e {
4672 assert_eq!(b.list_style, Some(ListStyle::Disc));
4673 }
4674 }
4675 }
4676
4677 #[test]
4678 fn test_parse_html_table_in_blockquote_records_depth() {
4679 let elements = parse_html_elements(
4680 "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
4681 );
4682 assert_eq!(element_depths(&elements), vec![(true, 1)]);
4683 }
4684
4685 #[test]
4686 fn test_parse_html_table_after_blockquote() {
4687 let elements = parse_html_elements(
4688 "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
4689 );
4690 let depths = element_depths(&elements);
4691 assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
4693 assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
4694 }
4695
4696 #[test]
4697 fn test_flatten_to_blocks_propagates_blockquote_depth() {
4698 let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
4699 let blocks = ParsedElement::flatten_to_blocks(elements);
4700 assert!(!blocks.is_empty());
4701 for b in &blocks {
4702 assert_eq!(b.blockquote_depth, 1);
4703 }
4704 }
4705
4706 #[test]
4707 fn test_parse_html_simple() {
4708 let blocks = parse_html("<p>Hello <b>world</b></p>");
4709 assert_eq!(blocks.len(), 1);
4710 assert!(blocks[0].spans.len() >= 2);
4711 let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
4712 assert!(bold_span.bold);
4713 }
4714
4715 #[test]
4716 fn test_parse_html_multiple_paragraphs() {
4717 let blocks = parse_html("<p>A</p><p>B</p>");
4718 assert_eq!(blocks.len(), 2);
4719 }
4720
4721 #[test]
4722 fn test_parse_html_heading() {
4723 let blocks = parse_html("<h2>Subtitle</h2>");
4724 assert_eq!(blocks.len(), 1);
4725 assert_eq!(blocks[0].heading_level, Some(2));
4726 }
4727
4728 #[test]
4729 fn test_parse_html_list() {
4730 let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
4731 assert!(blocks.len() >= 2);
4732 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4733 }
4734
4735 #[test]
4736 fn test_parse_markdown_code_block() {
4737 let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
4738 assert_eq!(blocks.len(), 1);
4739 assert!(blocks[0].is_code_block);
4740 assert!(blocks[0].spans[0].code);
4741 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4743 assert_eq!(
4744 text, "fn main() {}",
4745 "code block text should not have trailing newline"
4746 );
4747 }
4748
4749 #[test]
4750 fn test_parse_markdown_nested_formatting() {
4751 let blocks = parse_markdown_blocks("***bold italic***");
4752 assert_eq!(blocks.len(), 1);
4753 let span = &blocks[0].spans[0];
4754 assert!(span.bold);
4755 assert!(span.italic);
4756 }
4757
4758 #[test]
4759 fn test_parse_markdown_link() {
4760 let blocks = parse_markdown_blocks("[click](http://example.com)");
4761 assert_eq!(blocks.len(), 1);
4762 let span = &blocks[0].spans[0];
4763 assert_eq!(span.text, "click");
4764 assert_eq!(span.link_href, Some("http://example.com".to_string()));
4765 }
4766
4767 #[test]
4768 fn test_parse_markdown_empty() {
4769 let blocks = parse_markdown_blocks("");
4770 assert_eq!(blocks.len(), 1);
4771 assert!(blocks[0].spans[0].text.is_empty());
4772 }
4773
4774 #[test]
4775 fn test_parse_html_empty() {
4776 let blocks = parse_html("");
4777 assert_eq!(blocks.len(), 1);
4778 assert!(blocks[0].spans[0].text.is_empty());
4779 }
4780
4781 #[test]
4782 fn test_parse_html_nested_formatting() {
4783 let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
4784 assert_eq!(blocks.len(), 1);
4785 let span = &blocks[0].spans[0];
4786 assert!(span.bold);
4787 assert!(span.italic);
4788 }
4789
4790 #[test]
4791 fn test_parse_html_link() {
4792 let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
4793 assert_eq!(blocks.len(), 1);
4794 let span = &blocks[0].spans[0];
4795 assert_eq!(span.text, "click");
4796 assert_eq!(span.link_href, Some("http://example.com".to_string()));
4797 }
4798
4799 #[test]
4800 fn test_parse_html_ordered_list() {
4801 let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
4802 assert!(blocks.len() >= 2);
4803 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
4804 }
4805
4806 #[test]
4807 fn test_parse_markdown_ordered_list() {
4808 let blocks = parse_markdown_blocks("1. first\n2. second");
4809 assert!(blocks.len() >= 2);
4810 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
4811 }
4812
4813 #[test]
4814 fn test_parse_html_blockquote_nested() {
4815 let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
4816 assert!(blocks.len() >= 3);
4817 }
4818
4819 #[test]
4820 fn test_parse_block_styles_line_height() {
4821 let styles = parse_block_styles("line-height: 1.5");
4822 assert_eq!(styles.line_height, Some(1500));
4823 }
4824
4825 #[test]
4826 fn test_parse_block_styles_direction_rtl() {
4827 let styles = parse_block_styles("direction: rtl");
4828 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
4829 }
4830
4831 #[test]
4832 fn test_parse_block_styles_background_color() {
4833 let styles = parse_block_styles("background-color: #ff0000");
4834 assert_eq!(styles.background_color, Some("#ff0000".to_string()));
4835 }
4836
4837 #[test]
4838 fn test_parse_block_styles_white_space_pre() {
4839 let styles = parse_block_styles("white-space: pre");
4840 assert_eq!(styles.non_breakable_lines, Some(true));
4841 }
4842
4843 #[test]
4844 fn test_parse_block_styles_multiple() {
4845 let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
4846 assert_eq!(styles.line_height, Some(2000));
4847 assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
4848 assert_eq!(styles.background_color, Some("blue".to_string()));
4849 }
4850
4851 #[test]
4852 fn test_parse_html_block_styles_extracted() {
4853 let blocks = parse_html(
4854 r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
4855 );
4856 assert_eq!(blocks.len(), 1);
4857 assert_eq!(blocks[0].line_height, Some(1500));
4858 assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
4859 assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
4860 }
4861
4862 #[test]
4866 fn test_parse_html_collapses_exporter_line_wrapping() {
4867 let blocks = parse_html(
4868 "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
4869 <p>\nsecond para</p>\n</body>",
4870 );
4871 let texts: Vec<String> = blocks
4872 .iter()
4873 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4874 .collect();
4875 assert_eq!(texts, vec!["first line wrapped here", "second para"]);
4876 }
4877
4878 #[test]
4882 fn test_parse_html_drops_whitespace_between_blocks() {
4883 let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
4884 assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
4885 assert!(
4886 blocks
4887 .iter()
4888 .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
4889 "no block keeps a literal newline"
4890 );
4891 }
4892
4893 #[test]
4896 fn test_parse_html_collapses_across_span_boundaries() {
4897 let blocks = parse_html("<p> <b>bold</b>\n <i>italic</i> </p>");
4898 assert_eq!(blocks.len(), 1);
4899 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4900 assert_eq!(text, "bold italic");
4901 }
4902
4903 #[test]
4906 fn test_parse_html_keeps_no_break_space() {
4907 let blocks = parse_html("<p>\nAttention : ici</p>");
4908 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4909 assert_eq!(text, "Attention\u{a0}: ici");
4910 }
4911
4912 #[test]
4915 fn test_parse_html_keeps_empty_paragraph() {
4916 let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
4917 assert_eq!(blocks.len(), 3);
4918 let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
4919 assert_eq!(text, "");
4920 }
4921
4922 #[test]
4926 fn test_parse_html_preserves_whitespace_in_pre() {
4927 let blocks = parse_html("<pre>\nfn main() {\n let x = 1;\n}</pre>");
4930 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4931 assert_eq!(text, "fn main() {\n let x = 1;\n}");
4932
4933 let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n b</p>");
4936 let texts: Vec<String> = styled
4937 .iter()
4938 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4939 .collect();
4940 assert_eq!(texts, ["a", " b"]);
4941
4942 let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n b</p>");
4943 let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
4944 assert_eq!(text, "a b");
4945 }
4946
4947 #[test]
4950 fn test_parse_html_collapses_table_cell_whitespace() {
4951 let elements = parse_html_elements(
4952 "<table>\n<tr>\n<td>\n one\n</td>\n<td>\n two\n</td>\n</tr>\n</table>",
4953 );
4954 let table = elements
4955 .iter()
4956 .find_map(|e| match e {
4957 ParsedElement::Table(t) => Some(t),
4958 _ => None,
4959 })
4960 .expect("a table");
4961 let cells: Vec<String> = table.rows[0]
4962 .iter()
4963 .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
4964 .collect();
4965 assert_eq!(cells, vec!["one", "two"]);
4966 }
4967
4968 #[test]
4969 fn test_parse_html_white_space_pre() {
4970 let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
4971 assert_eq!(blocks.len(), 1);
4972 assert_eq!(blocks[0].non_breakable_lines, Some(true));
4973 }
4974
4975 #[test]
4976 fn test_parse_html_no_styles_returns_none() {
4977 let blocks = parse_html("<p>plain</p>");
4978 assert_eq!(blocks.len(), 1);
4979 assert_eq!(blocks[0].line_height, None);
4980 assert_eq!(blocks[0].direction, None);
4981 assert_eq!(blocks[0].background_color, None);
4982 assert_eq!(blocks[0].non_breakable_lines, None);
4983 }
4984
4985 #[test]
4986 fn test_parse_markdown_nested_list_indent() {
4987 let md = "- top\n - nested\n - deep";
4988 let blocks = parse_markdown_blocks(md);
4989 assert_eq!(blocks.len(), 3);
4990 assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4991 assert_eq!(blocks[0].list_indent, 0);
4992 assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4993 assert_eq!(blocks[1].list_indent, 1);
4994 assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
4995 assert_eq!(blocks[2].list_indent, 2);
4996 }
4997
4998 #[test]
4999 fn test_parse_markdown_nested_ordered_list_indent() {
5000 let md = "1. first\n 1. nested\n 2. nested2";
5001 let blocks = parse_markdown_blocks(md);
5002 assert_eq!(blocks.len(), 3);
5003 assert_eq!(blocks[0].list_indent, 0);
5004 assert_eq!(blocks[1].list_indent, 1);
5005 assert_eq!(blocks[2].list_indent, 1);
5006 }
5007
5008 #[test]
5009 fn test_parse_html_nested_list_indent() {
5010 let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
5011 let blocks = parse_html(html);
5012 assert!(blocks.len() >= 2);
5013 assert_eq!(blocks[0].list_indent, 0);
5014 assert_eq!(blocks[1].list_indent, 1);
5015 }
5016
5017 #[test]
5018 fn test_parse_markdown_table() {
5019 let md = "| A | B |\n|---|---|\n| 1 | 2 |";
5020 let elements = parse_markdown(md);
5021 assert_eq!(elements.len(), 1);
5022 match &elements[0] {
5023 ParsedElement::Table(table) => {
5024 assert_eq!(table.header_rows, 1);
5025 assert_eq!(table.rows.len(), 2); assert_eq!(table.rows[0].len(), 2);
5028 assert_eq!(table.rows[0][0].spans[0].text, "A");
5029 assert_eq!(table.rows[0][1].spans[0].text, "B");
5030 assert_eq!(table.rows[1].len(), 2);
5032 assert_eq!(table.rows[1][0].spans[0].text, "1");
5033 assert_eq!(table.rows[1][1].spans[0].text, "2");
5034 }
5035 _ => panic!("Expected ParsedElement::Table"),
5036 }
5037 }
5038
5039 #[test]
5040 fn test_parse_markdown_table_with_formatting() {
5041 let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
5042 let elements = parse_markdown(md);
5043 assert_eq!(elements.len(), 1);
5044 match &elements[0] {
5045 ParsedElement::Table(table) => {
5046 assert_eq!(table.rows.len(), 2);
5047 assert!(table.rows[0][0].spans[0].bold);
5049 assert!(table.rows[0][1].spans[0].code);
5051 assert!(table.rows[0][2].spans[0].italic);
5053 assert!(table.rows[1][0].spans[0].strikeout);
5055 assert_eq!(
5057 table.rows[1][2].spans[0].link_href,
5058 Some("http://x.com".to_string())
5059 );
5060 }
5061 _ => panic!("Expected ParsedElement::Table"),
5062 }
5063 }
5064
5065 #[test]
5066 fn test_parse_markdown_mixed_content_with_table() {
5067 let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
5068 let elements = parse_markdown(md);
5069 assert_eq!(elements.len(), 3);
5070 assert!(matches!(&elements[0], ParsedElement::Block(_)));
5071 assert!(matches!(&elements[1], ParsedElement::Table(_)));
5072 assert!(matches!(&elements[2], ParsedElement::Block(_)));
5073 }
5074}
5075
5076#[cfg(test)]
5077mod djot_tests {
5078 use super::*;
5079 use crate::entities::MarkerType;
5080
5081 fn blocks(d: &str) -> Vec<ParsedBlock> {
5082 ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
5083 }
5084
5085 fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
5086 b.spans.iter().find(|s| pred(s)).expect("span not found")
5087 }
5088
5089 #[test]
5090 fn paragraph_bold_italic() {
5091 let b = blocks("normal *bold* _italic_");
5092 assert_eq!(b.len(), 1);
5093 assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
5094 assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
5095 }
5096
5097 #[test]
5098 fn heading_levels() {
5099 assert_eq!(blocks("# H1")[0].heading_level, Some(1));
5100 assert_eq!(blocks("### H3")[0].heading_level, Some(3));
5101 assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
5102 }
5103
5104 #[test]
5105 fn unordered_bullet_styles_are_distinct() {
5106 assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
5107 assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
5108 assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
5109 }
5110
5111 #[test]
5112 fn ordered_delimiters() {
5113 let period = blocks("1. a");
5114 assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
5115 assert_eq!(period[0].list_prefix, "");
5116 assert_eq!(period[0].list_suffix, ".");
5117
5118 let paren = blocks("1) a");
5119 assert_eq!(paren[0].list_suffix, ")");
5120 assert_eq!(paren[0].list_prefix, "");
5121
5122 let paren_paren = blocks("(1) a");
5123 assert_eq!(paren_paren[0].list_prefix, "(");
5124 assert_eq!(paren_paren[0].list_suffix, ")");
5125 }
5126
5127 #[test]
5128 fn task_list_markers() {
5129 let b = blocks("- [ ] a\n- [x] b");
5130 assert_eq!(b.len(), 2);
5131 assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
5132 assert_eq!(b[1].marker, Some(MarkerType::Checked));
5133 }
5134
5135 #[test]
5136 fn code_block_with_language() {
5137 let b = blocks("```rust\nfn main() {}\n```");
5138 assert_eq!(b.len(), 1);
5139 assert!(b[0].is_code_block);
5140 assert_eq!(b[0].code_language.as_deref(), Some("rust"));
5141 let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
5142 assert_eq!(text, "fn main() {}");
5143 }
5144
5145 #[test]
5146 fn link_href() {
5147 let b = blocks("[text](http://example.com)");
5148 let s = first_span_with(&b[0], |s| s.text == "text");
5149 assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
5150 }
5151
5152 #[test]
5153 fn superscript_subscript() {
5154 assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
5155 assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
5156 }
5157
5158 #[test]
5159 fn delete_insert_verbatim() {
5160 assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
5161 assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
5162 assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
5163 }
5164
5165 #[test]
5166 fn blockquote_depth() {
5167 let els = parse_djot("> quoted", &DjotImportOptions::default());
5168 match &els[0] {
5169 ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
5170 _ => panic!("expected block"),
5171 }
5172 }
5173
5174 #[test]
5175 fn nested_list_indent() {
5176 let b = blocks("- a\n\n - b\n\n - c");
5181 assert_eq!(b.len(), 3);
5182 assert_eq!(b[0].list_indent, 0);
5183 assert_eq!(b[1].list_indent, 1);
5184 assert_eq!(b[2].list_indent, 2);
5185 }
5186
5187 #[test]
5188 fn table_parsed_as_table() {
5189 let els = parse_djot(
5190 "| a | b |\n|---|---|\n| c | d |",
5191 &DjotImportOptions::default(),
5192 );
5193 assert_eq!(els.len(), 1);
5194 match &els[0] {
5195 ParsedElement::Table(t) => {
5196 assert_eq!(t.header_rows, 1);
5197 assert_eq!(t.rows.len(), 2);
5198 assert_eq!(t.rows[0][0].spans[0].text, "a");
5199 assert_eq!(t.rows[1][1].spans[0].text, "d");
5200 }
5201 _ => panic!("expected table"),
5202 }
5203 }
5204
5205 #[test]
5206 fn smart_punctuation_normalised_to_unicode() {
5207 let text: String = blocks("a... b---c")[0]
5208 .spans
5209 .iter()
5210 .map(|s| s.text.as_str())
5211 .collect();
5212 assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
5213 assert!(text.contains('\u{2014}'), "em dash: {text:?}");
5214 }
5215
5216 #[test]
5217 fn unrepresentable_constructs_dropped_without_leaking_text() {
5218 let b = blocks("para1\n\n---\n\npara2");
5220 assert_eq!(b.len(), 2);
5221 assert_eq!(
5222 b[0].spans
5223 .iter()
5224 .map(|s| s.text.as_str())
5225 .collect::<String>(),
5226 "para1"
5227 );
5228 assert_eq!(
5229 b[1].spans
5230 .iter()
5231 .map(|s| s.text.as_str())
5232 .collect::<String>(),
5233 "para2"
5234 );
5235
5236 let d = blocks(":::\ninside\n:::");
5238 let joined: String = d
5239 .iter()
5240 .flat_map(|b| b.spans.iter())
5241 .map(|s| s.text.as_str())
5242 .collect();
5243 assert_eq!(joined, "inside");
5244
5245 let m = blocks("before $`E=mc^2` after");
5247 let joined: String = m
5248 .iter()
5249 .flat_map(|b| b.spans.iter())
5250 .map(|s| s.text.as_str())
5251 .collect();
5252 assert!(joined.contains("before"), "{joined:?}");
5253 assert!(joined.contains("after"), "{joined:?}");
5254 assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
5255 }
5256
5257 #[test]
5258 fn empty_document_yields_one_empty_block() {
5259 let b = blocks("");
5260 assert_eq!(b.len(), 1);
5261 assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
5262 }
5263
5264 #[test]
5265 fn block_attributes_parse_into_block() {
5266 let b = blocks(
5267 "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
5268 );
5269 assert_eq!(b.len(), 1);
5270 assert_eq!(b[0].alignment, Some(Alignment::Center));
5271 assert_eq!(b[0].line_height, Some(1500));
5272 assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
5273 assert_eq!(b[0].non_breakable_lines, Some(true));
5274 assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
5275 }
5276
5277 #[test]
5278 fn spacing_block_attributes_parse_into_block() {
5279 let b = blocks("{top_margin=24 text_indent=0}\nhello");
5283 assert_eq!(b.len(), 1);
5284 assert_eq!(b[0].top_margin, Some(24));
5285 assert_eq!(b[0].text_indent, Some(0));
5286 }
5287
5288 #[test]
5289 fn a_zero_text_indent_is_distinct_from_an_absent_one() {
5290 let explicit = blocks("{text_indent=0}\nhello");
5294 let absent = blocks("hello");
5295 assert_eq!(explicit[0].text_indent, Some(0));
5296 assert_eq!(absent[0].text_indent, None);
5297 assert!(!explicit[0].is_inline_only());
5298 assert!(absent[0].is_inline_only());
5299 }
5300
5301 #[test]
5302 fn spacing_block_attributes_respect_import_options() {
5303 let src = "{top_margin=24 text_indent=0}\nhello";
5304 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
5305 assert_eq!(b[0].top_margin, None);
5306 assert_eq!(b[0].text_indent, None);
5307 }
5308
5309 #[test]
5310 fn block_attributes_on_heading() {
5311 let b = blocks("{alignment=right}\n# Title");
5312 assert_eq!(b[0].heading_level, Some(1));
5313 assert_eq!(b[0].alignment, Some(Alignment::Right));
5314 }
5315
5316 #[test]
5317 fn block_attributes_respect_import_options() {
5318 let src = "{alignment=center line_height=1500}\nhello";
5321 let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
5322 assert_eq!(b[0].alignment, None);
5323 assert_eq!(b[0].line_height, None);
5324 assert_eq!(
5325 b[0].spans
5326 .iter()
5327 .map(|s| s.text.as_str())
5328 .collect::<String>(),
5329 "hello"
5330 );
5331 }
5332
5333 #[test]
5334 fn list_item_block_attributes_are_dropped() {
5335 let b = blocks("{alignment=center}\n- item");
5338 assert!(b.iter().all(|blk| blk.alignment.is_none()));
5339 }
5340
5341 #[test]
5342 fn unknown_alignment_value_is_ignored() {
5343 let b = blocks("{alignment=sideways}\nhello");
5344 assert_eq!(b[0].alignment, None);
5345 }
5346}
5347
5348pub const TABLE_ANCHOR: &str = "\u{FFFC}";
5360
5361pub const LINE_SEPARATOR: char = '\u{2028}';
5375
5376const LINE_SEPARATOR_STR: &str = "\u{2028}";
5378
5379fn span_prose(span: &ParsedSpan, out: &mut String) {
5387 if span.image.is_some() || span.footnote_ref.is_some() {
5388 out.push('\u{FFFC}');
5389 return;
5390 }
5391 out.push_str(&span.text);
5392}
5393
5394fn block_prose(block: &ParsedBlock) -> String {
5395 let mut prose = String::new();
5396 for span in &block.spans {
5397 span_prose(span, &mut prose);
5398 }
5399 prose
5400}
5401
5402fn cell_prose(cell: &ParsedTableCell) -> String {
5403 let mut prose = String::new();
5404 for span in &cell.spans {
5405 span_prose(span, &mut prose);
5406 }
5407 prose
5408}
5409
5410pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
5461 let elements = parse_djot(djot, options);
5465
5466 let mut out = String::with_capacity(djot.len());
5469
5470 let mut first = true;
5475 let push = |text: &str, out: &mut String, first: &mut bool| {
5476 if *first {
5477 *first = false;
5478 } else {
5479 out.push('\n');
5480 }
5481 out.push_str(text);
5482 };
5483
5484 for element in &elements {
5485 match element {
5486 ParsedElement::Block(block) => {
5487 push(&block_prose(block), &mut out, &mut first);
5488 }
5489 ParsedElement::FootnoteDefinition { .. } => {}
5499 ParsedElement::Table(table) => {
5500 push(TABLE_ANCHOR, &mut out, &mut first);
5507 let columns = table.column_count();
5510 for row in &table.rows {
5511 for column in 0..columns {
5512 let prose = row.get(column).map(cell_prose).unwrap_or_default();
5513 push(&prose, &mut out, &mut first);
5514 }
5515 }
5516 }
5517 }
5518 }
5519 out
5520}
5521
5522#[cfg(test)]
5523mod html_footnote_tests {
5524 use super::*;
5525
5526 #[test]
5530 fn a_literal_reference_in_text_would_be_escaped_into_prose() {
5531 let blocks = parse_html("<p>The ferry.[^1]</p>");
5532 let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
5533 assert!(
5534 text.contains("[^1]"),
5535 "the parser keeps it as characters: {text:?}"
5536 );
5537 assert!(
5538 blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
5539 "text is text — it must not become a reference by accident"
5540 );
5541 }
5542
5543 #[test]
5544 fn an_attributed_element_becomes_a_real_footnote_reference() {
5545 let blocks = parse_html(
5546 r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
5547 );
5548 let refs: Vec<&str> = blocks
5549 .iter()
5550 .flat_map(|b| b.spans.iter())
5551 .filter_map(|s| s.footnote_ref.as_deref())
5552 .collect();
5553 assert_eq!(refs, vec!["3"]);
5554
5555 let marker = blocks[0]
5558 .spans
5559 .iter()
5560 .find(|s| s.footnote_ref.is_some())
5561 .expect("the reference span");
5562 assert!(marker.text.is_empty(), "{marker:?}");
5563 }
5564
5565 #[test]
5568 fn any_element_carrying_the_attribute_works() {
5569 for html in [
5570 r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
5571 r#"<p>a<span data-footnote-ref="1"></span></p>"#,
5572 r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
5573 ] {
5574 let blocks = parse_html(html);
5575 assert_eq!(
5576 blocks
5577 .iter()
5578 .flat_map(|b| b.spans.iter())
5579 .filter_map(|s| s.footnote_ref.as_deref())
5580 .collect::<Vec<_>>(),
5581 vec!["1"],
5582 "failed for {html}"
5583 );
5584 }
5585 }
5586
5587 #[test]
5588 fn an_empty_or_absent_label_is_not_a_reference() {
5589 for html in [
5590 r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
5591 r#"<p>a<sup></sup></p>"#,
5592 ] {
5593 let blocks = parse_html(html);
5594 assert!(
5595 blocks
5596 .iter()
5597 .flat_map(|b| b.spans.iter())
5598 .all(|s| s.footnote_ref.is_none()),
5599 "failed for {html}"
5600 );
5601 }
5602 }
5603}
5604
5605#[cfg(test)]
5606mod html_depth_tests {
5607 use super::*;
5610
5611 fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
5613 blocks
5614 .iter()
5615 .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
5616 .collect()
5617 }
5618
5619 #[test]
5624 fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
5625 use crate::parser_tools::list_depth::MAX_LIST_INDENT;
5626 let depth = 300;
5627 let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
5628 html.push_str(&"</li></ol>".repeat(depth));
5629 let blocks = parse_html(&html);
5630 let items: Vec<(String, u32)> = blocks
5631 .iter()
5632 .filter(|block| block.list_style.is_some())
5633 .map(|block| {
5634 let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
5635 (text, block.list_indent)
5636 })
5637 .collect();
5638 let expected: Vec<(String, u32)> = (0..depth)
5639 .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
5640 .collect();
5641 assert_eq!(items, expected);
5642 }
5643
5644 #[test]
5646 fn text_in_a_deep_html_list_outside_its_items_is_kept() {
5647 let depth = 40;
5648 let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
5649 html.push_str("<ul>loose words<li>last</li></ul>");
5650 html.push_str(&"</li></ul>".repeat(depth));
5651 let texts = block_texts(&parse_html(&html)).join(" ");
5652 assert!(texts.contains("loose words"), "{texts:?}");
5653 assert!(texts.contains("last"), "{texts:?}");
5654 assert!(texts.contains("item 39"), "{texts:?}");
5655 }
5656
5657 #[test]
5660 fn html_nested_past_the_reader_depth_keeps_its_words() {
5661 for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
5662 let html = format!(
5663 "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
5664 open.repeat(300),
5665 close.repeat(300)
5666 );
5667 let texts = block_texts(&parse_html(&html)).join(" ");
5668 let words: Vec<&str> = texts.split_whitespace().collect();
5669 assert_eq!(
5670 words,
5671 ["Top.", "Deep", "words", "here.", "End."],
5672 "for {open}"
5673 );
5674 }
5675 }
5676
5677 #[test]
5679 fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
5680 let html = format!(
5681 "<table><tr><td>{}deep words{}</td></tr></table>",
5682 "<span>".repeat(300),
5683 "</span>".repeat(300)
5684 );
5685 let elements = parse_html_elements(&html);
5686 let Some(ParsedElement::Table(table)) = elements.first() else {
5687 panic!("a table: {elements:?}");
5688 };
5689 let text: String = table.rows[0][0]
5690 .spans
5691 .iter()
5692 .map(|span| span.text.as_str())
5693 .collect();
5694 assert_eq!(text, "deep words");
5695 }
5696
5697 #[test]
5701 fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
5702 for cell in [
5703 "One<br/>Two",
5704 "<p>One</p><p>Two</p>",
5705 "<ul><li>One</li><li>Two</li></ul>",
5706 ] {
5707 let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
5708 let Some(ParsedElement::Table(table)) = elements.first() else {
5709 panic!("a table: {elements:?}");
5710 };
5711 let text: String = table.rows[0][0]
5712 .spans
5713 .iter()
5714 .map(|span| span.text.as_str())
5715 .collect();
5716 assert_eq!(text, "One Two", "for {cell}");
5717 }
5718 }
5719}
5720
5721#[cfg(test)]
5722mod list_delimiter_tests {
5723 use super::*;
5724
5725 fn md(source: &str) -> Vec<ParsedBlock> {
5726 ParsedElement::flatten_to_blocks(parse_markdown(source))
5727 }
5728
5729 fn html(source: &str) -> Vec<ParsedBlock> {
5730 ParsedElement::flatten_to_blocks(parse_html_elements(source))
5731 }
5732
5733 fn suffixes(source: &str) -> Vec<String> {
5735 md(source)
5736 .into_iter()
5737 .filter(|b| b.list_style.is_some())
5738 .map(|b| b.list_suffix)
5739 .collect()
5740 }
5741
5742 #[test]
5743 fn a_period_and_a_paren_are_told_apart() {
5744 assert_eq!(suffixes("1. a\n2. b"), [".", "."]);
5745 assert_eq!(suffixes("1) a\n2) b"), [")", ")"]);
5746 let numbered = md("3) a");
5747 assert_eq!(numbered[0].list_suffix, ")");
5748 assert_eq!(numbered[0].list_start, Some(3));
5749 let numbered = md("3. a");
5750 assert_eq!(numbered[0].list_suffix, ".");
5751 assert_eq!(numbered[0].list_start, Some(3));
5752 }
5753
5754 #[test]
5755 fn a_bullet_list_has_no_suffix() {
5756 assert_eq!(suffixes("- a\n- b"), ["", ""]);
5757 assert_eq!(suffixes("- 1) x"), [")"]);
5758 let nested = md("- x\n 1) y");
5759 assert_eq!(nested[0].list_suffix, "");
5760 assert_eq!(nested[1].list_suffix, ")");
5761 }
5762
5763 #[test]
5767 fn the_range_of_a_list_starts_at_its_marker_wherever_it_stands() {
5768 for source in [
5769 "1) a",
5770 " 1) a\n 2) b",
5771 "> 1) a",
5772 "> > 1) a",
5773 "> - 1) y\n> 2) z",
5774 "text\n1) a",
5775 "1) a\n\n2) b",
5776 "1. a\n 1) b",
5777 "- x\n 1) y",
5778 "- a\n\n 1) b",
5779 "- 1) y",
5780 "1. 2) y",
5781 "x[^n]\n\n[^n]: 1) a",
5782 "[^x] text\n\n1) a",
5783 "1) a\n2) b\n - c\n3) d",
5784 ] {
5785 let mut every_block = Vec::new();
5787 for element in parse_markdown(source) {
5788 match element {
5789 ParsedElement::Block(b) => every_block.push(b),
5790 ParsedElement::FootnoteDefinition { blocks, .. } => every_block.extend(blocks),
5791 ParsedElement::Table(_) => {}
5792 }
5793 }
5794 let found = every_block
5795 .into_iter()
5796 .filter(|b| b.list_style == Some(ListStyle::Decimal))
5797 .map(|b| b.list_suffix)
5798 .collect::<Vec<_>>();
5799 assert!(!found.is_empty(), "no numbered item in {source:?}");
5800 assert!(
5801 found.iter().any(|s| s == ")"),
5802 "{source:?} read {found:?}, the innermost ordered list wears ')'"
5803 );
5804 }
5805 }
5806
5807 #[test]
5808 fn nested_lists_each_wear_their_own_delimiter() {
5809 let blocks = md("1. a\n 1) b\n2. c");
5810 let suffixes: Vec<_> = blocks.iter().map(|b| b.list_suffix.as_str()).collect();
5811 assert_eq!(suffixes, [".", ")", "."]);
5812 }
5813
5814 #[test]
5815 fn a_change_of_delimiter_starts_a_new_list() {
5816 let blocks = md("1. a\n1) b");
5818 let suffixes: Vec<_> = blocks.iter().map(|b| b.list_suffix.as_str()).collect();
5819 assert_eq!(suffixes, [".", ")"]);
5820 }
5821
5822 #[test]
5823 fn the_suffix_does_not_leak_into_a_following_block() {
5824 let blocks = md("1) a\n\nplain");
5825 assert_eq!(blocks[0].list_suffix, ")");
5826 assert_eq!(blocks[1].list_style, None);
5827 assert_eq!(blocks[1].list_suffix, "");
5828 }
5829
5830 #[test]
5831 fn a_loose_list_keeps_the_suffix_on_every_item() {
5832 assert_eq!(suffixes("1) a\n\n2) b\n\n3) c"), [")", ")", ")"]);
5833 }
5834
5835 #[test]
5836 fn the_html_reader_gives_an_ordered_list_a_period() {
5837 let blocks = html("<ol><li>a</li></ol>");
5838 assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
5839 assert_eq!(blocks[0].list_suffix, ".");
5840 let blocks = html("<ol start=\"3\"><li>a</li></ol>");
5841 assert_eq!(blocks[0].list_suffix, ".");
5842 assert_eq!(blocks[0].list_start, Some(3));
5843 let blocks = html("<ul><li>a</li></ul>");
5844 assert_eq!(blocks[0].list_suffix, "");
5845 }
5846
5847 #[test]
5848 fn the_default_suffix_follows_the_style() {
5849 for style in [
5850 ListStyle::Decimal,
5851 ListStyle::LowerAlpha,
5852 ListStyle::UpperAlpha,
5853 ListStyle::LowerRoman,
5854 ListStyle::UpperRoman,
5855 ] {
5856 assert_eq!(default_list_suffix(&style), ".");
5857 }
5858 for style in [ListStyle::Disc, ListStyle::Circle, ListStyle::Square] {
5859 assert_eq!(default_list_suffix(&style), "");
5860 }
5861 }
5862}
5863
5864#[cfg(test)]
5865mod task_list_tests {
5866 use super::*;
5867
5868 fn md(source: &str) -> Vec<ParsedBlock> {
5869 ParsedElement::flatten_to_blocks(parse_markdown(source))
5870 }
5871
5872 fn html(source: &str) -> Vec<ParsedBlock> {
5873 ParsedElement::flatten_to_blocks(parse_html_elements(source))
5874 }
5875
5876 fn text(block: &ParsedBlock) -> String {
5877 block.spans.iter().map(|s| s.text.as_str()).collect()
5878 }
5879
5880 fn markers(blocks: &[ParsedBlock]) -> Vec<Option<MarkerType>> {
5881 blocks.iter().map(|b| b.marker.clone()).collect()
5882 }
5883
5884 #[test]
5885 fn markdown_task_items_keep_their_state_and_lose_the_box_text() {
5886 let blocks = md("- [x] done\n- [ ] todo\n- plain");
5887 assert_eq!(
5888 markers(&blocks),
5889 [Some(MarkerType::Checked), Some(MarkerType::Unchecked), None]
5890 );
5891 assert_eq!(text(&blocks[0]), "done");
5892 assert_eq!(text(&blocks[1]), "todo");
5893 assert_eq!(text(&blocks[2]), "plain");
5894 }
5895
5896 #[test]
5897 fn ordered_and_starred_items_and_a_capital_x() {
5898 assert_eq!(markers(&md("1. [x] a")), [Some(MarkerType::Checked)]);
5899 assert_eq!(markers(&md("* [X] cap")), [Some(MarkerType::Checked)]);
5900 assert_eq!(markers(&md("3) [ ] a")), [Some(MarkerType::Unchecked)]);
5901 }
5902
5903 #[test]
5904 fn a_nested_tight_item_has_its_own_box_and_the_parent_keeps_its_own() {
5905 let blocks = md("- [x] a\n - [ ] b\n- c");
5906 assert_eq!(
5907 markers(&blocks),
5908 [Some(MarkerType::Checked), Some(MarkerType::Unchecked), None]
5909 );
5910 assert_eq!(text(&blocks[0]), "a");
5911 assert_eq!(text(&blocks[1]), "b");
5912 }
5913
5914 #[test]
5915 fn only_the_first_block_of_an_item_wears_the_box() {
5916 let blocks = md("- [ ] a\n\n second para\n- [x] b");
5917 assert_eq!(text(&blocks[0]), "a");
5918 assert_eq!(blocks[0].marker, Some(MarkerType::Unchecked));
5919 assert_eq!(text(&blocks[1]), "second para");
5920 assert_eq!(blocks[1].marker, None);
5921 assert_eq!(text(&blocks[2]), "b");
5922 assert_eq!(blocks[2].marker, Some(MarkerType::Checked));
5923 }
5924
5925 #[test]
5926 fn a_task_item_in_a_quotation_is_read() {
5927 let blocks = md("> - [x] a");
5928 assert_eq!(blocks[0].marker, Some(MarkerType::Checked));
5929 assert_eq!(blocks[0].blockquote_depth, 1);
5930 }
5931
5932 #[test]
5933 fn an_empty_task_item_does_not_leak_into_the_next_one() {
5934 let blocks = md("- [x]");
5937 assert_eq!(markers(&blocks), [None]);
5938 assert_eq!(text(&blocks[0]), "[x]");
5939 let blocks = md("- [x] \n- next");
5940 assert_eq!(text(blocks.last().unwrap()), "next");
5941 assert_eq!(blocks.last().unwrap().marker, None);
5942 }
5943
5944 #[test]
5945 fn a_footnote_with_a_task_list_reads_the_boxes() {
5946 let elements = parse_markdown("x[^n]\n\n[^n]: - [x] a\n");
5947 let note = elements
5948 .into_iter()
5949 .find_map(|e| match e {
5950 ParsedElement::FootnoteDefinition { blocks, .. } => Some(blocks),
5951 _ => None,
5952 })
5953 .expect("a note");
5954 assert!(note.iter().any(|b| b.marker == Some(MarkerType::Checked)));
5955 }
5956
5957 #[test]
5958 fn a_line_break_split_keeps_the_box_on_the_first_line_only() {
5959 let block = ParsedBlock {
5960 spans: vec![ParsedSpan {
5961 text: "one\ntwo".into(),
5962 preserve_whitespace: true,
5963 ..Default::default()
5964 }],
5965 list_style: Some(ListStyle::Disc),
5966 marker: Some(MarkerType::Checked),
5967 ..Default::default()
5968 };
5969 let lines = split_block_at_line_breaks(block);
5970 assert_eq!(
5971 markers(&lines),
5972 [Some(MarkerType::Checked), None],
5973 "the item's box is its first line's"
5974 );
5975 }
5976
5977 #[test]
5978 fn html_checkbox_items_are_read() {
5979 let blocks = html(
5980 r#"<ul><li><input type="checkbox" checked> a</li><li><input type="checkbox" disabled> b</li><li>c</li></ul>"#,
5981 );
5982 assert_eq!(
5983 markers(&blocks),
5984 [Some(MarkerType::Checked), Some(MarkerType::Unchecked), None]
5985 );
5986 assert_eq!(text(&blocks[0]), "a");
5987 assert_eq!(text(&blocks[1]), "b");
5988 }
5989
5990 #[test]
5991 fn html_checkbox_through_a_label_and_in_any_case() {
5992 let blocks = html(r#"<ul><li><label><input type="checkbox"> a</label></li></ul>"#);
5993 assert_eq!(blocks[0].marker, Some(MarkerType::Unchecked));
5994 assert_eq!(text(&blocks[0]), "a");
5995 let blocks = html(r#"<ul><li><input type="CHECKBOX" checked="checked"/> a</li></ul>"#);
5996 assert_eq!(blocks[0].marker, Some(MarkerType::Checked));
5997 }
5998
5999 #[test]
6000 fn html_checkbox_that_is_not_first_or_not_a_checkbox_sets_nothing() {
6001 let blocks = html(r#"<ul><li>a <input type="checkbox" checked></li></ul>"#);
6002 assert_eq!(blocks[0].marker, None);
6003 assert_eq!(text(&blocks[0]), "a");
6004 let blocks = html(r#"<ul><li><input type="text"> a</li></ul>"#);
6005 assert_eq!(blocks[0].marker, None);
6006 }
6007
6008 #[test]
6009 fn html_checkbox_outside_a_list_sets_nothing() {
6010 let blocks = html(r#"<li><input type="checkbox" checked> a</li>"#);
6011 assert!(blocks.iter().all(|b| b.marker.is_none()));
6012 }
6013
6014 #[test]
6015 fn the_checkbox_writers_are_read_back() {
6016 for marker in [MarkerType::Checked, MarkerType::Unchecked] {
6017 let written = task_checkbox_html(Some(&marker));
6018 let blocks = html(&format!("<ul><li>{written}a</li></ul>"));
6019 assert_eq!(blocks[0].marker, Some(marker));
6020 assert_eq!(text(&blocks[0]), "a");
6021 }
6022 assert_eq!(task_checkbox_html(None), "");
6023 assert_eq!(task_box_markdown(Some(&MarkerType::Checked)), "[x] ");
6024 assert_eq!(task_box_markdown(Some(&MarkerType::Unchecked)), "[ ] ");
6025 assert_eq!(task_box_markdown(None), "");
6026 }
6027}
6028
6029#[cfg(test)]
6030mod html_sheet_whitespace_tests {
6031 use super::*;
6032 use proptest::prelude::*;
6033
6034 fn blocks(source: &str) -> Vec<ParsedBlock> {
6035 ParsedElement::flatten_to_blocks(parse_html_elements(source))
6036 }
6037
6038 fn texts(source: &str) -> Vec<String> {
6039 blocks(source)
6040 .iter()
6041 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
6042 .collect()
6043 }
6044
6045 const QT_HEAD: &str = r#"<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"><html><head><meta name="qrichtext" content="1" /><meta charset="utf-8" /><style type="text/css">
6047p, li { white-space: pre-wrap; }
6048hr { height: 1px; border-width: 0; }
6049li.unchecked::marker { content: "\2610"; }
6050li.checked::marker { content: "\2612"; }
6051</style></head>"#;
6052
6053 fn with_sheet(css: &str, body: &str) -> String {
6054 format!("<html><head><style>{css}</style></head><body>{body}</body></html>")
6055 }
6056
6057 #[test]
6058 fn the_qt_sheet_keeps_tabs_and_runs_of_spaces() {
6059 let html = format!(
6060 "{QT_HEAD}<body style=\" font-family:'Sans'; font-size:9pt;\">\n<p style=\" margin-top:0px;\">a\tb c</p></body></html>"
6061 );
6062 assert_eq!(texts(&html), ["a\tb c"]);
6063 let html = format!("{QT_HEAD}<body>\n<ul>\n<li>a\tb c</li></ul></body></html>");
6064 assert_eq!(texts(&html), ["a\tb c"]);
6065 let html = format!(
6066 "{QT_HEAD}<body><table><tr><td><p>a\tb c</p></td></tr></table></body></html>"
6067 );
6068 let elements = parse_html_elements(&html);
6069 let cell = elements
6070 .iter()
6071 .find_map(|e| match e {
6072 ParsedElement::Table(t) => Some(
6073 t.rows[0][0]
6074 .spans
6075 .iter()
6076 .map(|s| s.text.as_str())
6077 .collect::<String>(),
6078 ),
6079 _ => None,
6080 })
6081 .expect("a table");
6082 assert!(cell.contains("a\tb c"), "{cell:?}");
6083 }
6084
6085 #[test]
6086 fn without_a_sheet_the_whitespace_collapses_as_before() {
6087 assert_eq!(texts("<p>a\tb c</p>"), ["a b c"]);
6088 }
6089
6090 #[test]
6093 fn a_sheet_rule_does_not_make_blocks_non_breakable() {
6094 let html = format!("{QT_HEAD}<body><p>a\tb</p></body></html>");
6095 let parsed = blocks(&html);
6096 assert_eq!(parsed.len(), 1);
6097 assert_eq!(parsed[0].non_breakable_lines, None);
6098 assert!(parsed[0].is_inline_only());
6099 }
6100
6101 #[test]
6102 fn only_simple_selectors_count() {
6103 for css in [
6104 "p.x { white-space: pre }",
6105 "#a p { white-space: pre }",
6106 "p:first-child { white-space: pre }",
6107 "p > b { white-space: pre }",
6108 "p[class] { white-space: pre }",
6109 "p, p.x { white-space: pre }",
6110 "@media print { p { white-space: pre } }",
6111 "@font-face { font-family: x; white-space: pre }",
6112 "/* p { white-space: pre } */",
6113 "/* } */ p { color: red }",
6114 "p { color: red }",
6115 "p { white-space: inherit }",
6116 ] {
6117 assert_eq!(texts(&with_sheet(css, "<p>a\tb</p>")), ["a b"], "{css}");
6118 }
6119 }
6120
6121 #[test]
6122 fn a_style_of_another_kind_is_not_read() {
6123 for head in [
6124 r#"<style type="text/less">p { white-space: pre }</style>"#,
6125 r#"<style media="print">p { white-space: pre }</style>"#,
6126 ] {
6127 assert_eq!(
6128 texts(&format!(
6129 "<html><head>{head}</head><body><p>a\tb</p></body></html>"
6130 )),
6131 ["a b"],
6132 "{head}"
6133 );
6134 }
6135 for head in [
6136 r#"<style type="TEXT/CSS" media="screen">p { white-space: pre }</style>"#,
6137 r#"<style media="all">p { white-space: pre }</style>"#,
6138 ] {
6139 assert_eq!(
6140 texts(&format!(
6141 "<html><head>{head}</head><body><p>a\tb</p></body></html>"
6142 )),
6143 ["a\tb"],
6144 "{head}"
6145 );
6146 }
6147 }
6148
6149 #[test]
6150 fn the_later_rule_and_the_more_specific_one_win() {
6151 let case = |css: &str| texts(&with_sheet(css, "<p>a\tb</p>"));
6152 assert_eq!(
6153 case("p { white-space: pre } p { white-space: normal }"),
6154 ["a b"]
6155 );
6156 assert_eq!(
6157 case("p { white-space: normal } p { white-space: pre }"),
6158 ["a\tb"]
6159 );
6160 assert_eq!(
6162 case("* { white-space: pre-wrap } p { white-space: normal }"),
6163 ["a b"]
6164 );
6165 assert_eq!(
6166 case("p { white-space: normal } * { white-space: pre-wrap }"),
6167 ["a b"]
6168 );
6169 assert_eq!(case("p { white-space: pre; white-space: normal }"), ["a b"]);
6170 assert_eq!(case("p { white-space: pre; white-space: bogus }"), ["a\tb"]);
6171 assert_eq!(case("P { white-space: pre }"), ["a\tb"]);
6172 assert_eq!(case("p, b { white-space: pre }"), ["a\tb"]);
6173 assert_eq!(case("p, p.x { white-space: pre }"), ["a b"]);
6175 }
6176
6177 #[test]
6178 fn inline_style_and_importance_decide_between_sheet_and_element() {
6179 let case = |css: &str, style: &str| {
6180 texts(&with_sheet(css, &format!("<p style=\"{style}\">a\tb</p>")))
6181 };
6182 assert_eq!(
6183 case("p { white-space: pre-wrap }", "white-space: normal"),
6184 ["a b"]
6185 );
6186 assert_eq!(
6187 case(
6188 "p { white-space: pre-wrap !important }",
6189 "white-space: normal"
6190 ),
6191 ["a\tb"]
6192 );
6193 assert_eq!(
6194 case(
6195 "p { white-space: pre-wrap !important }",
6196 "white-space: normal !important"
6197 ),
6198 ["a b"]
6199 );
6200 }
6201
6202 #[test]
6203 fn body_html_and_star_start_the_walk_without_making_blocks_of_the_layout() {
6204 for selector in ["body", "html", "*", "body, html"] {
6205 let html = with_sheet(
6206 &format!("{selector} {{ white-space: pre-wrap }}"),
6207 "<p>a\tb</p>\n<p>c</p>\n<ul>\n<li>d</li>\n<li>e<ul>\n<li>f</li>\n</ul></li>\n</ul>\n<table>\n<tr>\n<td>g\th</td>\n</tr>\n</table>\n",
6208 );
6209 let parsed = ParsedElement::flatten_to_blocks(parse_html_elements(&html));
6210 let found: Vec<String> = parsed
6211 .iter()
6212 .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
6213 .collect();
6214 assert_eq!(
6215 found,
6216 ["a\tb", "c", "d", "e", "f", "g\th"],
6217 "{selector}: {found:?}"
6218 );
6219 }
6220 }
6221
6222 #[test]
6225 fn bare_text_starts_from_the_body_as_the_cascade_gives_it() {
6226 let case = |css: &str| texts(&with_sheet(css, "a\tb"));
6227 assert_eq!(
6228 case("body { white-space: normal } * { white-space: pre }"),
6229 ["a b"]
6230 );
6231 assert_eq!(
6232 case("* { white-space: pre } body { white-space: normal }"),
6233 ["a b"]
6234 );
6235 assert_eq!(
6236 case("html { white-space: normal } * { white-space: pre }"),
6237 ["a\tb"]
6238 );
6239 assert_eq!(case("html { white-space: pre }"), ["a\tb"]);
6240 assert_eq!(
6241 case("body { white-space: pre } html { white-space: normal }"),
6242 ["a\tb"]
6243 );
6244 }
6245
6246 #[test]
6247 fn the_style_may_come_after_the_content_or_be_split() {
6248 let html = "<p>a\tb</p><style>p { white-space: pre-wrap }</style>";
6249 assert_eq!(texts(html), ["a\tb"]);
6250 let html =
6251 "<style>p { color: red }</style><style>p { white-space: pre-wrap }</style><p>a\tb</p>";
6252 assert_eq!(texts(html), ["a\tb"]);
6253 }
6254
6255 #[test]
6256 fn a_deep_document_with_a_sheet_is_read_on_its_own_thread() {
6257 let depth = HTML_DEPTH_ON_THE_CALLERS_STACK + 20;
6258 let html = format!(
6259 "<style>p {{ white-space: pre-wrap }}</style>{}<p>a\tb</p>{}",
6260 "<div>".repeat(depth),
6261 "</div>".repeat(depth)
6262 );
6263 assert_eq!(texts(&html), ["a\tb"]);
6264 }
6265
6266 #[test]
6268 fn a_word_sheet_in_comment_markers_is_skipped_without_losing_the_next_rule() {
6269 let css = "\n<!--\n /* Font Definitions */\n @font-face\n\t{font-family:\"Cambria Math\";\n\tpanose-1:2 4 5 3 5 4 6 3 2 4;}\n /* Style Definitions */\n p.MsoNormal, li.MsoNormal, div.MsoNormal\n\t{margin:0cm; font-size:11.0pt;}\n@page WordSection1\n\t{size:612.0pt 792.0pt; margin:72.0pt 72.0pt 72.0pt 72.0pt;}\ndiv.WordSection1\n\t{page:WordSection1;}\n /* List Definitions */\n @list l0:level1\n\t{mso-level-number-format:bullet;}\np { white-space: pre-wrap }\n-->\n";
6270 assert_eq!(texts(&with_sheet(css, "<p>a\tb</p>")), ["a\tb"]);
6271 assert_eq!(
6273 texts(&with_sheet(
6274 "<!-- p { white-space: pre-wrap } -->",
6275 "<p>a\tb</p>"
6276 )),
6277 ["a\tb"]
6278 );
6279 }
6280
6281 #[test]
6282 fn a_blank_paragraph_stays_content_and_blanks_between_blocks_do_not() {
6283 let html = with_sheet(
6284 "p { white-space: pre-wrap }",
6285 "<p>a</p>\n <p> </p>\n<p>b</p>",
6286 );
6287 assert_eq!(texts(&html), ["a", " ", "b"]);
6288 }
6289
6290 #[test]
6291 fn a_hostile_sheet_neither_panics_nor_loops() {
6292 for css in [
6293 "/* unterminated",
6294 "p { white-space: pre",
6295 "p { \"unterminated",
6296 "} } }",
6297 "{ { {",
6298 "p {",
6299 "@media {",
6300 "\u{e9}\u{4e2d}\u{1f600} p { white-space: pre }",
6301 "p\\",
6302 "\\",
6303 "'",
6304 "<!-- -->",
6305 "p { white-space: \u{e9}pre }",
6306 ] {
6307 let _ = texts(&with_sheet(css, "<p>a</p>"));
6308 }
6309 }
6310
6311 #[test]
6316 fn blanks_between_inline_siblings_are_content() {
6317 for html in [
6318 r#"<div style="white-space: pre-wrap"><b>a</b> <i>b</i><p>c</p></div>"#,
6319 r#"<ul style="white-space: pre-wrap"><li><b>a</b> <i>b</i><ul><li>c</li></ul></li></ul>"#,
6320 "<style>li { white-space: pre-wrap }</style><ul><li><b>a</b> <i>b</i><ul><li>c</li></ul></li></ul>",
6321 "<style>* { white-space: pre-wrap }</style><div><b>a</b> <i>b</i><p>c</p></div>",
6322 ] {
6323 let read = texts(html);
6324 assert!(read.iter().any(|t| t == "a b"), "{html}: {read:?}");
6325 assert!(!read.iter().any(|t| t == "ab"), "{html}: {read:?}");
6326 }
6327 let cell = ParsedElement::flatten_to_blocks(parse_html_elements(
6329 r#"<table><tr><td style="white-space: pre-wrap"><b>a</b> <b>b</b><p>c</p></td></tr></table>"#,
6330 ));
6331 let cell: String = cell
6332 .iter()
6333 .flat_map(|b| b.spans.iter().map(|s| s.text.as_str()))
6334 .collect();
6335 assert!(cell.contains("a b"), "{cell:?}");
6336 }
6337
6338 #[test]
6341 fn blanks_between_html5_containers_are_layout() {
6342 let sections = "<section>\n<p>a</p>\n</section>\n<section>\n<p>b</p>\n</section>\n";
6343 for selector in ["body", "html", "*"] {
6344 let html = with_sheet(&format!("{selector} {{ white-space: pre-wrap }}"), sections);
6345 assert_eq!(texts(&html), ["a", "b"], "{selector}");
6346 }
6347 for tag in [
6348 "article", "main", "header", "footer", "nav", "aside", "figure", "dl",
6349 ] {
6350 let html = with_sheet(
6351 "* { white-space: pre-wrap }",
6352 &format!("<{tag}>\n<p>a</p>\n</{tag}>\n<{tag}>\n<p>b</p>\n</{tag}>\n"),
6353 );
6354 assert_eq!(texts(&html), ["a", "b"], "{tag}");
6355 }
6356 }
6357
6358 #[test]
6361 #[ignore = "a timing test: its bound is wide, but wall-clock time depends on the machine; run with --ignored"]
6362 fn a_parent_of_many_children_is_read_in_linear_time() {
6363 let children = "<b>x</b> ".repeat(100_000);
6364 let html =
6365 format!("<style>* {{ white-space: pre-wrap }}</style><div>{children}<p>end</p></div>");
6366 let started = std::time::Instant::now();
6367 let read = texts(&html);
6368 assert!(read.len() >= 2);
6369 assert!(
6370 started.elapsed() < std::time::Duration::from_secs(60),
6371 "100,000 children took {:?}",
6372 started.elapsed()
6373 );
6374 }
6375
6376 #[test]
6379 fn a_cell_takes_the_whitespace_around_the_table_and_no_formatting() {
6380 let parsed = parse_html_elements(
6381 r#"<b><a href="http://x"><table><tr><td>x</td></tr></table></a></b>"#,
6382 );
6383 let cell = parsed
6384 .iter()
6385 .find_map(|e| match e {
6386 ParsedElement::Table(t) => Some(t.rows[0][0].spans.clone()),
6387 _ => None,
6388 })
6389 .expect("a table");
6390 assert!(
6391 cell.iter().all(|s| !s.bold && s.link_href.is_none()),
6392 "{cell:?}"
6393 );
6394 }
6395
6396 #[test]
6398 fn a_cell_of_blocks_under_kept_whitespace_has_no_leading_space() {
6399 let parsed = parse_html_elements(&with_sheet(
6400 "* { white-space: pre-wrap }",
6401 "<table><tr><td><p>x</p></td></tr></table>",
6402 ));
6403 let cell: String = parsed
6404 .iter()
6405 .find_map(|e| match e {
6406 ParsedElement::Table(t) => {
6407 Some(t.rows[0][0].spans.iter().map(|s| s.text.as_str()).collect())
6408 }
6409 _ => None,
6410 })
6411 .expect("a table");
6412 assert_eq!(cell, "x");
6413 let parsed = parse_html_elements(&with_sheet(
6414 "td { white-space: pre-wrap }",
6415 "<table><tr><td>a\tb c</td></tr></table>",
6416 ));
6417 let cell: String = parsed
6418 .iter()
6419 .find_map(|e| match e {
6420 ParsedElement::Table(t) => {
6421 Some(t.rows[0][0].spans.iter().map(|s| s.text.as_str()).collect())
6422 }
6423 _ => None,
6424 })
6425 .expect("a table");
6426 assert_eq!(cell, "a\tb c");
6427 }
6428
6429 #[test]
6431 fn bare_text_with_a_line_break_under_a_kept_whitespace_is_two_blocks() {
6432 let html = "<style>body { white-space: pre-wrap }</style>a\nb";
6433 assert_eq!(texts(html), ["a", "b"]);
6434 }
6435
6436 #[test]
6438 fn a_nested_rule_does_not_leak_into_the_rule_around_it() {
6439 let case = |css: &str| texts(&with_sheet(css, "<p>a\tb</p>"));
6440 assert_eq!(
6441 case("p { span { white-space: normal; white-space: pre; } }"),
6442 ["a b"]
6443 );
6444 assert_eq!(
6445 case("p { span { white-space: pre } white-space: pre-wrap }"),
6446 ["a\tb"]
6447 );
6448 assert_eq!(
6449 case("p { white-space: pre-wrap; & span { white-space: normal } }"),
6450 ["a\tb"]
6451 );
6452 }
6453
6454 #[test]
6457 fn pre_line_nowrap_and_pre_from_a_sheet() {
6458 for keyword in ["pre-line", "nowrap"] {
6459 let html = with_sheet(
6462 &format!("* {{ white-space: pre }} p {{ white-space: {keyword} }}"),
6463 "<p>a\tb c</p>",
6464 );
6465 assert_eq!(texts(&html), ["a b c"], "{keyword}");
6466 }
6467 let html = with_sheet("p { white-space: pre }", "<p>a\tb</p>");
6468 let parsed = blocks(&html);
6469 assert_eq!(parsed[0].non_breakable_lines, None);
6470 assert_eq!(texts(&html), ["a\tb"]);
6471 }
6472
6473 #[test]
6478 fn block_level_elements_the_reader_does_not_split_are_a_line_each() {
6479 let dl = "<dl>\n<dt>a</dt>\n<dd>b</dd>\n</dl>";
6480 assert_eq!(
6481 texts(&format!("<div style=\"white-space: pre-wrap\">{dl}</div>")),
6482 ["a", "b"]
6483 );
6484 assert_eq!(
6485 texts(&with_sheet(
6486 "* { white-space: pre-wrap }",
6487 &format!("<div>{dl}</div>")
6488 )),
6489 ["a", "b"]
6490 );
6491 assert_eq!(
6492 texts(&with_sheet(
6493 "* { white-space: pre-wrap }",
6494 "<section>A</section> <section>B</section>"
6495 )),
6496 ["A", "B"]
6497 );
6498 assert_eq!(
6499 texts(&with_sheet(
6500 "body { white-space: pre-wrap }",
6501 "<header>Title</header>\n<main>Body</main>"
6502 )),
6503 ["Title", "Body"]
6504 );
6505 assert_eq!(
6506 texts(&with_sheet(
6507 "* { white-space: pre-wrap }",
6508 "<div><article>A</article>\n<aside>B</aside></div>"
6509 )),
6510 ["A", "B"]
6511 );
6512 assert_eq!(
6514 texts(&with_sheet(
6515 "* { white-space: pre-wrap }",
6516 "<div><section>A</section> <b>x</b></div>"
6517 )),
6518 ["A", "x"]
6519 );
6520 }
6521
6522 #[test]
6524 fn block_level_elements_the_reader_does_not_split_are_a_word_gap_where_whitespace_collapses() {
6525 let dl = "<dl>\n<dt>a</dt>\n<dd>b</dd>\n</dl>";
6526 assert_eq!(
6527 texts(&with_sheet(
6528 "body { white-space: pre-wrap }",
6529 &format!("<div style=\"white-space: normal\">{dl}</div>")
6530 )),
6531 ["a b"]
6532 );
6533 assert_eq!(texts(&format!("<div>{dl}</div>")), ["a b"]);
6534 assert_eq!(
6536 texts("<div><dl><dt>a </dt>\n<dd> b</dd></dl></div>"),
6537 ["a b"]
6538 );
6539 assert_eq!(
6540 texts(&with_sheet(
6541 "body { white-space: pre-wrap }",
6542 "<div style=\"white-space: normal\"><dl><dt>a </dt>\n<dd> b</dd></dl></div>"
6543 )),
6544 ["a b"]
6545 );
6546 let parsed = parse_html_elements(&with_sheet(
6548 "* { white-space: pre-wrap }",
6549 "<table><tr><td><section>A</section> <section>B</section></td></tr></table>",
6550 ));
6551 let cell: String = parsed
6552 .iter()
6553 .find_map(|e| match e {
6554 ParsedElement::Table(t) => {
6555 Some(t.rows[0][0].spans.iter().map(|s| s.text.as_str()).collect())
6556 }
6557 _ => None,
6558 })
6559 .expect("a table");
6560 assert_eq!(cell, "A B");
6561 let parsed = parse_html_elements(&with_sheet(
6563 "* { white-space: pre-wrap }",
6564 "<table><tr><td><section>A </section> <section>B</section></td></tr></table>",
6565 ));
6566 let cell: String = parsed
6567 .iter()
6568 .find_map(|e| match e {
6569 ParsedElement::Table(t) => {
6570 Some(t.rows[0][0].spans.iter().map(|s| s.text.as_str()).collect())
6571 }
6572 _ => None,
6573 })
6574 .expect("a table");
6575 assert_eq!(cell, "A B");
6576 }
6577
6578 #[test]
6581 fn boundaries_add_no_empty_lines() {
6582 let html = with_sheet(
6583 "* { white-space: pre-wrap }",
6584 "<section>x</section>\n<section><p>p</p></section>\n<section>y</section>",
6585 );
6586 let mut read = texts(&html);
6587 read.sort();
6588 assert_eq!(read, ["p", "x", "y"], "{html}");
6589 let html = with_sheet(
6590 "* { white-space: pre-wrap }",
6591 "<section><p>p</p></section>\n<section><p>q</p></section>",
6592 );
6593 let mut read = texts(&html);
6594 read.sort();
6595 assert_eq!(read, ["p", "q"], "{html}");
6596 }
6597
6598 #[test]
6601 fn a_kept_line_keeps_its_own_blanks_and_gains_none() {
6602 assert_eq!(
6603 texts(&with_sheet(
6604 "* { white-space: pre-wrap }",
6605 "<dl><dt>a </dt>\n<dd> b</dd></dl>"
6606 )),
6607 ["a ", " b"]
6608 );
6609 }
6610
6611 #[test]
6615 fn a_large_sheet_resolves_into_one_declaration_per_tag() {
6616 let mut css = String::new();
6617 for i in 0..4000 {
6618 css.push_str("* { white-space: pre }\n");
6619 css.push_str(&format!("t{i} {{ white-space: pre }}\n"));
6620 }
6621 let sheet = SheetWhitespace::from_css(&css);
6622 assert_eq!(sheet.per_tag.len(), 4000);
6623 assert_eq!(sheet.declaration_for("t17"), Some(("pre", false)));
6624 assert_eq!(sheet.declaration_for("p"), Some(("pre", false)));
6625
6626 let selectors: Vec<String> = (0..20_000).map(|i| format!("t{i}")).collect();
6627 let css = format!("{} {{ white-space: pre }}", selectors.join(","));
6628 let sheet = SheetWhitespace::from_css(&css);
6629 assert_eq!(sheet.per_tag.len(), 20_000);
6630 assert_eq!(sheet.declaration_for("p"), None);
6631 }
6632
6633 proptest! {
6634 #[test]
6635 fn the_rule_scanner_survives_any_text(css in "\\PC{0,200}") {
6636 let _ = parse_whitespace_rules(&css);
6637 }
6638
6639 #[test]
6640 fn the_rule_scanner_survives_css_shaped_noise(
6641 css in "[a-z*.#:>,@{}();!/\"' \\\\\n\u{e9}-]{0,120}"
6642 ) {
6643 let _ = parse_whitespace_rules(&css);
6644 }
6645 }
6646}
6647
6648#[cfg(test)]
6649mod djot_crlf_tests {
6650 use super::*;
6651
6652 fn parse(djot: &str) -> Vec<ParsedElement> {
6653 parse_djot(djot, &DjotImportOptions::default())
6654 }
6655
6656 fn all_text(elements: &[ParsedElement]) -> Vec<String> {
6658 let mut out = Vec::new();
6659 for element in elements {
6660 match element {
6661 ParsedElement::Block(b) => out.extend(b.spans.iter().map(|s| s.text.clone())),
6662 ParsedElement::Table(t) => {
6663 for row in &t.rows {
6664 for cell in row {
6665 out.extend(cell.spans.iter().map(|s| s.text.clone()));
6666 }
6667 }
6668 }
6669 ParsedElement::FootnoteDefinition { blocks, .. } => {
6670 for b in blocks {
6671 out.extend(b.spans.iter().map(|s| s.text.clone()));
6672 }
6673 }
6674 }
6675 }
6676 out
6677 }
6678
6679 #[test]
6681 fn a_crlf_document_reads_as_its_lf_twin() {
6682 for lf in [
6683 "a\nb\n\nc",
6684 "```rust\nlet x = 1;\nlet y = 2;\n```\n",
6685 "`a\nb` and *c\nd*",
6686 "a\\\nb",
6687 "- one\n - nested\n- two\n",
6688 "1. one\n2. two\n",
6689 "| a | b |\n|---|---|\n| c | d |\n",
6690 "> quoted\n> lines\n",
6691 "# Heading\n\nbody\n",
6692 "text[^n]\n\n[^n]: the note\n",
6693 ] {
6694 let crlf = lf.replace('\n', "\r\n");
6695 let from_crlf = parse(&crlf);
6696 let from_lf = parse(lf);
6697 assert_eq!(format!("{from_crlf:?}"), format!("{from_lf:?}"), "{lf:?}");
6698 assert!(
6699 all_text(&from_crlf).iter().all(|t| !t.contains('\r')),
6700 "a carriage return in {lf:?}: {:?}",
6701 all_text(&from_crlf)
6702 );
6703 }
6704 }
6705
6706 #[test]
6708 fn a_soft_break_stays_a_soft_break() {
6709 let blocks = ParsedElement::flatten_to_blocks(parse("a\r\nb\r\n\r\nc"));
6710 assert_eq!(blocks.len(), 2);
6711 let first: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
6712 assert_eq!(first, "a b");
6713 }
6714
6715 #[test]
6718 fn a_too_deep_crlf_document_shows_lines_without_carriage_returns() {
6719 let deep = format!("{}deep\r\nsecond line\r\n", "- ".repeat(700));
6720 assert!(
6721 all_text(&parse(&deep)).iter().all(|t| !t.contains('\r')),
6722 "{:?}",
6723 all_text(&parse(&deep))
6724 );
6725 }
6726
6727 #[test]
6731 fn the_plain_text_round_trip_keeps_every_carriage_return() {
6732 use crate::parser_tools::plain_text_to_djot;
6733 for text in [
6734 "a\rb",
6735 "\rcarriage return\r",
6736 "a\r\nb",
6737 "a\r\n\r\nb",
6738 "x\r",
6739 "a\r\rb",
6740 ] {
6741 let djot = plain_text_to_djot(text);
6742 let back = djot_to_plain_text(&djot, &DjotImportOptions::default());
6743 assert_eq!(back, text, "{text:?} written as {djot:?}");
6744 }
6745 }
6746
6747 #[test]
6749 fn a_lone_carriage_return_is_text() {
6750 let text = all_text(&parse("a\rb")).concat();
6751 assert_eq!(text, "a\rb");
6752 }
6753
6754 #[test]
6757 fn markdown_and_html_already_read_a_crlf_as_a_line_ending() {
6758 let md = ParsedElement::flatten_to_blocks(parse_markdown("a\r\nb\r\n\r\nc"));
6759 let md_text: Vec<String> = md
6760 .iter()
6761 .flat_map(|b| b.spans.iter().map(|s| s.text.clone()))
6762 .collect();
6763 assert!(md_text.iter().all(|t| !t.contains('\r')), "{md_text:?}");
6764 let md = ParsedElement::flatten_to_blocks(parse_markdown("a\rb\n\nc"));
6765 assert!(
6766 md.iter()
6767 .all(|b| b.spans.iter().all(|s| !s.text.contains('\r'))),
6768 "a lone CR in Markdown"
6769 );
6770 let html = ParsedElement::flatten_to_blocks(parse_html_elements("<pre>a\r\nb</pre>"));
6771 let html_text: Vec<String> = html
6772 .iter()
6773 .flat_map(|b| b.spans.iter().map(|s| s.text.clone()))
6774 .collect();
6775 assert!(html_text.iter().all(|t| !t.contains('\r')), "{html_text:?}");
6776 }
6777
6778 #[test]
6779 fn crlf_to_lf_borrows_when_there_is_nothing_to_change() {
6780 use crate::parser_tools::djot_depth::crlf_to_lf;
6781 assert!(matches!(
6782 crlf_to_lf("a\nb\rc"),
6783 std::borrow::Cow::Borrowed(_)
6784 ));
6785 assert_eq!(crlf_to_lf("a\r\nb\r\n\r\n"), "a\nb\n\n");
6786 assert_eq!(crlf_to_lf("\r\r\n\r"), "\r\n\r");
6787 }
6788}