1use crate::document::{
4 ContentLayer, ContentLayers, DoclingDocument, MarkdownExportOptions, Node, Table,
5};
6
7const MISSING_KEY_VALUE_ITEM: &str = "<!-- missing-key-value-item -->";
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
14pub enum ImageMode {
15 #[default]
17 Placeholder,
18 Embedded,
20 Referenced,
23}
24
25struct Ctx {
27 strict: bool,
28 alt_text: bool,
31 compact_tables: bool,
33 images: ImageMode,
34 artifacts_dir: String,
35 artifacts: Vec<(String, Vec<u8>)>,
37 pic_index: usize,
38 in_table_cell: bool,
42 page_break: Option<String>,
46 pending_page_break: bool,
52 emitted_any: bool,
55 layers: ContentLayers,
57 current_layer: Option<ContentLayer>,
62 traverse_pictures: bool,
64 escape_html: bool,
67 escape_underscores: bool,
69 image_placeholder: String,
71 plain: bool,
73}
74
75impl Ctx {
76 fn new(doc: &DoclingDocument, strict: bool, options: &MarkdownExportOptions) -> Self {
79 Ctx {
80 strict,
81 alt_text: strict || doc.picture_alt_text,
82 compact_tables: doc.compact_tables,
83 images: options.image_mode,
84 artifacts_dir: options.artifacts_dir.clone(),
85 artifacts: Vec::new(),
86 pic_index: 0,
87 in_table_cell: false,
88 page_break: doc.page_break_placeholder.clone(),
89 pending_page_break: false,
90 emitted_any: false,
91 layers: options.layers,
92 current_layer: None,
93 traverse_pictures: options.traverse_pictures,
94 escape_html: options.escape_html,
95 escape_underscores: options.escape_underscores,
96 image_placeholder: options.image_placeholder.clone(),
97 plain: options.plain_text,
98 }
99 }
100
101 fn layer_visible(&self) -> bool {
103 self.layers.contains(self.current_layer)
104 }
105
106 fn unescape(&self, text: &str) -> String {
113 let mut out = std::borrow::Cow::Borrowed(text);
114 if !self.escape_html
115 && (out.contains("&") || out.contains("<") || out.contains(">"))
116 {
117 out = std::borrow::Cow::Owned(
118 out.replace("<", "<")
119 .replace(">", ">")
120 .replace("&", "&"),
121 );
122 }
123 if !self.escape_underscores && out.contains("\\_") {
124 out = std::borrow::Cow::Owned(out.replace("\\_", "_"));
125 }
126 out.into_owned()
127 }
128
129 fn text(&self, text: &str) -> String {
132 let text = strict_text(text, self.strict);
133 if self.plain {
134 self.unescape(&plain_inline(&self.drop_image_markers(&text)))
135 } else {
136 self.unescape(&text)
137 }
138 }
139
140 fn caption(&self, text: &str) -> String {
144 if self.plain {
145 self.unescape(&plain_inline(text))
146 } else {
147 self.unescape(text)
148 }
149 }
150
151 fn breaks(&self, text: &str) -> String {
154 if self.plain {
155 text.to_string()
156 } else {
157 md_line_breaks(text)
158 }
159 }
160
161 fn plain_table(&self, table: &Table) -> Table {
168 let mut plain = table.clone();
169 for cell in plain.rows.iter_mut().flatten() {
170 if cell.contains(['*', '~', '`', '[', '\n', '<']) {
171 let text = self.drop_image_markers(&strip_hard_breaks(cell));
172 *cell = plain_inline(&text);
173 }
174 }
175 plain
176 }
177
178 fn drop_image_markers(&self, text: &str) -> String {
185 const MARKER: &str = "<!-- image -->";
186 if !text.contains(MARKER) && !text.contains("```") {
187 return text.to_string();
188 }
189 text.split("\n\n")
193 .filter_map(|block| {
194 let lines: Vec<String> = block
195 .split('\n')
196 .filter_map(|line| {
197 if line.trim() == MARKER || is_fence_line(line.trim()) {
200 return None;
201 }
202 Some(line.replace(MARKER, &self.image_placeholder))
203 })
204 .collect();
205 (!lines.is_empty()).then(|| lines.join("\n"))
206 })
207 .collect::<Vec<_>>()
208 .join("\n\n")
209 }
210}
211
212pub fn to_markdown(doc: &DoclingDocument, strict: bool) -> String {
218 to_markdown_images(doc, strict, ImageMode::Placeholder, "artifacts").0
219}
220
221pub fn to_markdown_images(
225 doc: &DoclingDocument,
226 strict: bool,
227 images: ImageMode,
228 artifacts_dir: &str,
229) -> (String, Vec<(String, Vec<u8>)>) {
230 to_markdown_with_options(
231 doc,
232 strict,
233 &MarkdownExportOptions {
234 image_mode: images,
235 artifacts_dir: artifacts_dir.to_string(),
236 ..MarkdownExportOptions::default()
237 },
238 )
239}
240
241pub fn to_markdown_with_options(
245 doc: &DoclingDocument,
246 strict: bool,
247 options: &MarkdownExportOptions,
248) -> (String, Vec<(String, Vec<u8>)>) {
249 let mut ctx = Ctx::new(doc, strict, options);
250 let mut blocks: Vec<String> = Vec::new();
251 render(&doc.nodes, &mut blocks, &mut ctx);
252 let mut body = blocks.join("\n\n");
253 if strict && !options.plain_text && !doc.links.is_empty() {
257 body = apply_links(&body, &doc.links);
258 }
259 let front = if doc.metadata_output.front_matter() && !options.plain_text {
262 doc.metadata.to_front_matter()
263 } else {
264 String::new()
265 };
266 let md = match (front.is_empty(), body.is_empty()) {
267 (true, true) => String::new(),
268 (true, false) => format!("{body}\n"),
269 (false, true) => front,
270 (false, false) => format!("{front}\n{body}\n"),
271 };
272 (md, ctx.artifacts)
273}
274
275pub fn to_markdown_table_cell(doc: &DoclingDocument, strict: bool) -> String {
284 let mut ctx = Ctx::new(doc, strict, &MarkdownExportOptions::default());
285 ctx.in_table_cell = true;
286 ctx.artifacts_dir = String::new();
287 ctx.page_break = None;
291 let mut blocks: Vec<String> = Vec::new();
292 render(&doc.nodes, &mut blocks, &mut ctx);
293 blocks.join("\n\n")
294}
295
296fn apply_links(body: &str, links: &[(String, String)]) -> String {
304 let mut out = body.to_string();
305 let mut cursor = 0usize;
306 for (anchor, href) in links {
307 let anchor = anchor
308 .replace('&', "&")
309 .replace('<', "<")
310 .replace('>', ">");
311 if anchor.is_empty() {
312 continue;
313 }
314 if let Some(rel) = out[cursor..].find(&anchor) {
315 let at = cursor + rel;
316 let replacement = crate::hyperlink::markdown_link(&anchor, href);
319 out.replace_range(at..at + anchor.len(), &replacement);
320 cursor = at + replacement.len();
321 }
322 }
323 out
324}
325
326fn apply_links_chunk(chunk: &str, queue: &mut Vec<(String, String)>) -> String {
337 let mut out = chunk.to_string();
338 let mut cursor = 0usize;
339 let mut carried: Vec<(String, String)> = Vec::new();
340 for (anchor_raw, href) in std::mem::take(queue) {
341 let anchor = anchor_raw
342 .replace('&', "&")
343 .replace('<', "<")
344 .replace('>', ">");
345 if anchor.is_empty() {
346 continue;
347 }
348 if let Some(rel) = out[cursor..].find(&anchor) {
349 let at = cursor + rel;
350 let replacement = crate::hyperlink::markdown_link(&anchor, &href);
351 out.replace_range(at..at + anchor.len(), &replacement);
352 cursor = at + replacement.len();
353 } else {
354 carried.push((anchor_raw, href));
356 }
357 }
358 *queue = carried;
359 out
360}
361
362pub struct MarkdownStreamer {
380 strict: bool,
381 images: ImageMode,
382 compact_tables: bool,
383 picture_alt_text: bool,
385 front_matter: Option<String>,
389 emitted_any: bool,
392 links: Vec<(String, String)>,
394 artifacts_dir: String,
398 artifacts: Vec<(String, Vec<u8>)>,
399 pic_index: usize,
400 page_break: Option<String>,
404 pending_page_break: bool,
405 items_emitted: bool,
411 held: Option<Node>,
416 layers: ContentLayers,
419 traverse_pictures: bool,
420 escape_html: bool,
421 escape_underscores: bool,
422 image_placeholder: String,
423 plain: bool,
424}
425
426impl MarkdownStreamer {
427 pub fn new(strict: bool, images: ImageMode, compact_tables: bool) -> Self {
430 debug_assert!(
431 images != ImageMode::Referenced,
432 "referenced image mode needs an artifacts dir; use with_artifacts"
433 );
434 Self::with_artifacts(strict, images, compact_tables, "artifacts")
435 }
436
437 pub fn with_artifacts(
444 strict: bool,
445 images: ImageMode,
446 compact_tables: bool,
447 artifacts_dir: &str,
448 ) -> Self {
449 Self {
450 strict,
451 images,
452 compact_tables,
453 picture_alt_text: false,
454 front_matter: None,
455 emitted_any: false,
456 links: Vec::new(),
457 artifacts_dir: artifacts_dir.to_string(),
458 artifacts: Vec::new(),
459 pic_index: 0,
460 page_break: None,
461 pending_page_break: false,
462 items_emitted: false,
463 held: None,
464 layers: ContentLayers::BODY,
465 traverse_pictures: false,
466 escape_html: true,
467 escape_underscores: true,
468 image_placeholder: "<!-- image -->".to_string(),
469 plain: false,
470 }
471 }
472
473 pub fn with_export_options(mut self, options: &MarkdownExportOptions) -> Self {
480 self.images = options.image_mode;
481 self.artifacts_dir = options.artifacts_dir.clone();
482 self.layers = options.layers;
483 self.traverse_pictures = options.traverse_pictures;
484 self.escape_html = options.escape_html;
485 self.escape_underscores = options.escape_underscores;
486 self.image_placeholder = options.image_placeholder.clone();
487 self.plain = options.plain_text;
488 self
489 }
490
491 pub fn with_page_break_placeholder(mut self, placeholder: Option<String>) -> Self {
496 self.page_break = placeholder;
497 self
498 }
499
500 pub fn with_picture_alt_text(mut self, on: bool) -> Self {
504 self.picture_alt_text = on;
505 self
506 }
507
508 pub fn with_metadata(mut self, metadata: &crate::metadata::DocumentMetadata) -> Self {
513 self.front_matter = (!self.plain)
514 .then(|| metadata.to_front_matter())
515 .filter(|f| !f.is_empty());
516 self
517 }
518
519 pub fn take_artifacts(&mut self) -> Vec<(String, Vec<u8>)> {
524 std::mem::take(&mut self.artifacts)
525 }
526
527 pub fn push(&mut self, nodes: &[Node], links: &[(String, String)]) -> String {
532 self.push_holding(nodes, links, true)
533 }
534
535 fn push_holding(&mut self, nodes: &[Node], links: &[(String, String)], hold: bool) -> String {
538 self.links.extend(links.iter().cloned());
539 let images = self.images;
543 let ends_on_picture = |nodes: &[Node]| {
544 matches!(images, ImageMode::Placeholder)
545 && matches!(nodes.last(), Some(Node::Picture { .. }))
546 };
547 let mut owned: Vec<Node> = Vec::new();
548 let mut nodes = nodes;
549 if self.held.is_some() || (hold && ends_on_picture(nodes)) {
550 owned.extend(self.held.take());
551 owned.extend(nodes.iter().cloned());
552 if hold && ends_on_picture(&owned) {
553 self.held = owned.pop();
554 }
555 nodes = &owned;
556 }
557 let mut ctx = Ctx {
558 strict: self.strict,
559 alt_text: self.strict || self.picture_alt_text,
560 compact_tables: self.compact_tables,
561 images: self.images,
562 artifacts_dir: std::mem::take(&mut self.artifacts_dir),
563 artifacts: std::mem::take(&mut self.artifacts),
564 pic_index: self.pic_index,
565 in_table_cell: false,
566 page_break: std::mem::take(&mut self.page_break),
567 pending_page_break: self.pending_page_break,
568 emitted_any: self.items_emitted,
569 layers: self.layers,
570 current_layer: None,
571 traverse_pictures: self.traverse_pictures,
572 escape_html: self.escape_html,
573 escape_underscores: self.escape_underscores,
574 image_placeholder: std::mem::take(&mut self.image_placeholder),
575 plain: self.plain,
576 };
577 let mut blocks: Vec<String> = Vec::new();
578 render(nodes, &mut blocks, &mut ctx);
579 self.image_placeholder = std::mem::take(&mut ctx.image_placeholder);
580 self.artifacts_dir = std::mem::take(&mut ctx.artifacts_dir);
581 self.artifacts = std::mem::take(&mut ctx.artifacts);
582 self.pic_index = ctx.pic_index;
583 self.page_break = std::mem::take(&mut ctx.page_break);
584 self.pending_page_break = ctx.pending_page_break;
585 self.items_emitted = ctx.emitted_any;
586 if blocks.is_empty() {
587 return String::new();
588 }
589 let mut body = blocks.join("\n\n");
590 if self.strict && !self.plain && !self.links.is_empty() {
591 body = apply_links_chunk(&body, &mut self.links);
592 }
593 let chunk = if let Some(front) = self.front_matter.take() {
594 format!("{front}\n{body}")
595 } else if self.emitted_any {
596 format!("\n\n{body}")
597 } else {
598 body
599 };
600 self.emitted_any = true;
601 chunk
602 }
603
604 pub fn finish(mut self) -> String {
607 let mut out = String::new();
608 if self.held.is_some() {
609 out.push_str(&self.push_holding(&[], &[], false));
610 }
611 if let Some(front) = self.front_matter {
612 out.push_str(&front);
613 } else if self.emitted_any {
614 out.push('\n');
615 }
616 out
617 }
618}
619
620fn strict_text(text: &str, strict: bool) -> String {
628 if !strict {
629 return text.to_string();
630 }
631 if let Some(rest) = text.strip_prefix("[ ] ") {
634 return format!("[ ] {}", strict_text(rest, strict));
635 }
636 text.replace("\\_", "_")
637 .replace(" ,", ",")
638 .replace(" .", ".")
639 .replace(" ;", ";")
640 .replace(" )", ")")
641 .replace("( ", "(")
642 .replace(" ]", "]")
643 .replace("[ ", "[")
644}
645
646pub fn longest_backtick_run(text: &str) -> usize {
649 let (mut best, mut run) = (0, 0);
650 for c in text.chars() {
651 if c == '`' {
652 run += 1;
653 best = best.max(run);
654 } else {
655 run = 0;
656 }
657 }
658 best
659}
660
661pub fn code_span(text: &str) -> String {
667 let ticks = "`".repeat(longest_backtick_run(text) + 1);
668 let pad = if text.starts_with('`') || text.ends_with('`') {
669 " "
670 } else {
671 ""
672 };
673 format!("{ticks}{pad}{text}{pad}{ticks}")
674}
675
676pub fn code_fence(language: &str, body: &str) -> String {
681 let fence = "`".repeat(longest_backtick_run(body).max(2) + 1);
682 format!("{fence}{language}\n{body}\n{fence}")
683}
684
685pub(crate) fn code_span_bounds(chars: &[char], i: usize) -> Option<(usize, usize, usize)> {
693 let n = chars[i..].iter().take_while(|&&c| c == '`').count();
694 if n == 0 {
695 return None;
696 }
697 let mut k = i + n;
698 while k < chars.len() {
699 if chars[k] != '`' {
700 k += 1;
701 continue;
702 }
703 let run = chars[k..].iter().take_while(|&&c| c == '`').count();
704 if run == n {
705 let (mut s, mut e) = (i + n, k);
706 if n > 1
707 && e - s >= 2
708 && chars[s] == ' '
709 && chars[e - 1] == ' '
710 && chars[s..e].iter().any(|&c| c != ' ')
711 {
712 s += 1;
713 e -= 1;
714 }
715 return Some((s, e, k + run));
716 }
717 k += run;
718 }
719 None
720}
721
722fn is_fence_line(line: &str) -> bool {
724 line.len() >= 3 && line.bytes().all(|b| b == b'`')
725}
726
727pub(crate) fn plain_inline(text: &str) -> String {
740 if !text.contains(['*', '~', '`', '[']) {
741 return text.to_string();
742 }
743 let chars: Vec<char> = text.chars().collect();
744 let mut out = String::with_capacity(text.len());
745 plain_inline_into(&chars, &mut out);
746 out
747}
748
749fn plain_inline_into(chars: &[char], out: &mut String) {
750 let find = |open: usize, pat: &[char]| -> Option<usize> {
753 chars
754 .get(open..)?
755 .windows(pat.len())
756 .enumerate()
757 .position(|(k, w)| w == pat && (open + k == 0 || chars[open + k - 1] != '\\'))
758 .map(|p| open + p)
759 };
760 let span = |i: usize, pat: &[char]| -> Option<usize> {
763 if !chars[i..].starts_with(pat) {
764 return None;
765 }
766 let open = i + pat.len();
767 let end = find(open, pat)?;
768 (end > open && !chars[open].is_whitespace() && !chars[end - 1].is_whitespace())
769 .then_some(end)
770 };
771 let mut i = 0;
772 while i < chars.len() {
773 if chars[i] == '\\' && chars.get(i + 1) == Some(&'*') {
775 out.push('*');
776 i += 2;
777 continue;
778 }
779 let mut matched = false;
780 for pat in [&['*', '*', '*'][..], &['*', '*'], &['~', '~'], &['*']] {
781 if let Some(end) = span(i, pat) {
782 plain_inline_into(&chars[i + pat.len()..end], out);
783 i = end + pat.len();
784 matched = true;
785 break;
786 }
787 }
788 if matched {
789 continue;
790 }
791 if chars[i] == '`' {
795 if let Some((s, e, after)) = code_span_bounds(chars, i).filter(|&(s, e, _)| {
796 e > s && !chars[s].is_whitespace() && !chars[e - 1].is_whitespace()
797 }) {
798 out.extend(&chars[s..e]);
799 i = after;
800 continue;
801 }
802 }
803 if chars[i] == '$' {
806 if let Some(end) = find(i + 1, &['$']) {
807 out.extend(&chars[i..=end]);
808 i = end + 1;
809 continue;
810 }
811 }
812 if chars[i] == '[' {
815 if let Some(close) = closing_bracket(chars, i) {
816 if chars.get(close + 1) == Some(&'(') {
817 if let Some(endp) = crate::doclang::link_dest_end(chars, close + 2) {
818 plain_inline_into(&chars[i + 1..close], out);
819 i = endp + 1;
820 continue;
821 }
822 }
823 }
824 }
825 let c = chars[i];
828 out.push(c);
829 i += 1;
830 if matches!(c, '*' | '~') {
831 while i < chars.len() && chars[i] == c {
832 out.push(c);
833 i += 1;
834 }
835 }
836 }
837}
838
839fn closing_bracket(chars: &[char], open: usize) -> Option<usize> {
842 let mut depth = 0usize;
843 for (k, c) in chars.iter().enumerate().skip(open + 1) {
844 match c {
845 '[' => depth += 1,
846 ']' if depth == 0 => return Some(k),
847 ']' => depth -= 1,
848 _ => {}
849 }
850 }
851 None
852}
853
854fn md_line_breaks(text: &str) -> String {
862 if !text.contains('\n') {
863 return text.to_string();
864 }
865 text.split("\n\n")
866 .map(|para| para.replace('\n', " \n"))
867 .collect::<Vec<_>>()
868 .join("\n\n")
869}
870
871pub(crate) fn strip_hard_breaks(text: &str) -> String {
876 if text.contains(" \n") {
877 text.replace(" \n", "\n")
878 } else {
879 text.to_string()
880 }
881}
882
883fn heading_line_breaks(text: &str) -> String {
887 text.replace('\n', " ")
888}
889
890fn render(nodes: &[Node], blocks: &mut Vec<String>, ctx: &mut Ctx) {
891 let mut i = 0;
892 while i < nodes.len() {
893 let before = blocks.len();
894 match &nodes[i] {
895 Node::PageBreak | Node::PageInfo { .. } => {
905 if ctx.page_break.is_some() && ctx.emitted_any {
906 ctx.pending_page_break = true;
907 }
908 i += 1;
909 }
910 Node::ListItem { .. } => {
911 let start = i;
912 i += 1;
913 loop {
914 match nodes.get(i) {
915 Some(Node::ListItem { .. } | Node::ListText { .. }) => i += 1,
916 Some(Node::Paragraph { text })
920 if text.is_empty()
921 && matches!(nodes.get(i + 1), Some(Node::ListItem { .. })) =>
922 {
923 i += 1
924 }
925 _ => break,
926 }
927 }
928 render_list_run(&nodes[start..i], blocks, ctx);
929 }
930 other => {
931 let before_item = blocks.len();
932 render_one(other, blocks, ctx);
933 if matches!(other, Node::Picture { .. })
939 && matches!(ctx.images, ImageMode::Placeholder)
940 && blocks.len() > before_item
941 && matches!(
942 nodes.get(i + 1),
943 Some(Node::Code { language: Some(l), .. }) if l == "mermaid"
944 )
945 {
946 blocks.pop();
947 }
948 i += 1;
949 }
950 }
951 if blocks.len() > before {
952 let kept: Vec<String> = blocks.drain(before..).filter(|b| !b.is_empty()).collect();
962 blocks.extend(kept);
963 if ctx.pending_page_break {
964 if let Some(placeholder) = &ctx.page_break {
969 blocks.insert(before, placeholder.clone());
970 }
971 ctx.pending_page_break = false;
972 }
973 ctx.emitted_any = true;
974 }
975 }
976}
977
978fn render_list_run(items: &[Node], blocks: &mut Vec<String>, ctx: &Ctx) {
984 let mut lines: Vec<String> = Vec::new();
985 let mut any_top = false;
988
989 for item in items {
990 if let Node::ListText { level, text, glue } = item {
991 let text = ctx.text(text);
992 match lines.last_mut() {
993 Some(last) if *glue => last.push_str(&text),
994 _ => lines.push(format!("{}{text}", " ".repeat(*level as usize))),
995 }
996 continue;
997 }
998 let Node::ListItem {
999 ordered,
1000 number,
1001 first_in_list,
1002 text,
1003 level,
1004 marker: orig_marker,
1005 location: _,
1006 dclx: _,
1007 split: _,
1008 href: _,
1009 layer,
1010 } = item
1011 else {
1012 continue;
1013 };
1014 if !ctx.layers.contains(layer.or(ctx.current_layer)) {
1018 continue;
1019 }
1020 let level = *level as usize;
1021
1022 if level == 0 {
1031 if any_top && *first_in_list {
1032 lines.push(String::new());
1033 }
1034 any_top = true;
1035 }
1036
1037 let indent = " ".repeat(level);
1038 lines.push(format!(
1039 "{indent}{} {}",
1040 list_item_marker(orig_marker.as_deref().unwrap_or(""), *ordered, *number),
1041 list_item_text(text, ctx)
1042 ));
1043 }
1044
1045 if !lines.is_empty() {
1048 blocks.push(lines.join("\n"));
1049 }
1050}
1051
1052fn list_item_marker(marker: &str, ordered: bool, number: u64) -> String {
1061 let already_valid = matches!(marker, "-" | "*" | "+")
1062 || marker
1063 .strip_suffix('.')
1064 .is_some_and(|d| !d.is_empty() && d.chars().all(char::is_numeric));
1065 if already_valid {
1066 return marker.to_string();
1067 }
1068 let outer = if ordered && marker.is_empty() {
1069 format!("{number}.")
1070 } else {
1071 "-".to_string()
1072 };
1073 if marker.chars().any(char::is_alphanumeric) {
1074 format!("{outer} {marker}")
1075 } else {
1076 outer
1077 }
1078}
1079
1080fn list_item_text(text: &str, ctx: &Ctx) -> String {
1088 let escaped = ctx.text(text);
1089 if ctx.plain {
1090 return escaped;
1091 }
1092 if let Some((own, tail)) = escaped.split_once('\n') {
1093 if is_folded_child_tail(tail) {
1094 return format!("{}\n{tail}", md_line_breaks(own));
1095 }
1096 }
1097 md_line_breaks(&escaped)
1098}
1099
1100fn is_folded_child_tail(tail: &str) -> bool {
1107 const MARKER: &str = "<!-- image -->";
1108 let mut lines = tail.split('\n').peekable();
1109 let mut any = false;
1110 while let Some(line) = lines.next() {
1111 let line = line.trim_start();
1112 if line == MARKER {
1113 any = true;
1114 } else if is_fence_line(line) {
1115 loop {
1118 match lines.next() {
1119 Some(l) if l.trim_start() == line => break,
1120 Some(_) => {}
1121 None => return false,
1122 }
1123 }
1124 any = true;
1125 } else if lines.next().map(str::trim_start) == Some(MARKER) {
1126 any = true; } else {
1128 return false;
1129 }
1130 }
1131 any
1132}
1133
1134fn render_one(node: &Node, blocks: &mut Vec<String>, ctx: &mut Ctx) {
1139 match node {
1140 Node::Group { .. }
1142 | Node::Furniture { .. }
1143 | Node::PageFurniture { .. }
1144 | Node::FurnitureText { .. }
1145 | Node::CommentSection { .. }
1146 | Node::PictureChildren(_)
1147 | Node::Commented { .. }
1148 | Node::ScriptText { .. }
1149 | Node::Located { .. }
1150 | Node::Prov { .. }
1151 | Node::Track { .. }
1152 | Node::PageBreak
1153 | Node::PageInfo { .. }
1154 | Node::ProvContinuation { .. }
1155 | Node::DoclangOnly(_) => {}
1156 _ if !ctx.layer_visible() => return,
1157 _ => {}
1158 }
1159 render_item(node, blocks, ctx)
1160}
1161
1162fn render_item(node: &Node, blocks: &mut Vec<String>, ctx: &mut Ctx) {
1163 match node {
1164 Node::Heading { level, text } => {
1165 if ctx.plain {
1166 blocks.push(ctx.text(text));
1169 return;
1170 }
1171 let text = heading_line_breaks(&ctx.text(text));
1172 if ctx.in_table_cell {
1173 blocks.push(text);
1175 } else {
1176 let hashes = "#".repeat((*level).clamp(1, 6) as usize);
1177 blocks.push(format!("{hashes} {text}"));
1178 }
1179 }
1180 Node::Paragraph { text } if text.is_empty() => {}
1183 Node::Paragraph { text } => blocks.push(ctx.breaks(&ctx.text(text))),
1184 Node::LabeledText { .. } => render_item(&node.labeled_as_paragraph(), blocks, ctx),
1185 Node::Caption { text, .. } if text.is_empty() => {}
1188 Node::Caption { text, href } => {
1189 let body = ctx.breaks(&ctx.text(text));
1190 blocks.push(match href {
1191 Some(url) if !ctx.plain => crate::hyperlink::markdown_link(&body, url),
1192 _ => body,
1193 });
1194 }
1195 Node::CheckboxItem { checked, text } => {
1196 let mark = if *checked { "- [x]" } else { "- [ ]" };
1199 let sep = if text.starts_with(' ') { "" } else { " " };
1200 blocks.push(ctx.breaks(&ctx.text(&format!("{mark}{sep}{text}"))));
1201 }
1202 Node::Code {
1203 language,
1204 text,
1205 pretty,
1206 ..
1207 } => {
1208 let lang = match language {
1213 Some(l) if ctx.strict || l == "mermaid" => l.as_str(),
1214 _ => "",
1215 };
1216 let body = match pretty {
1219 Some(p) if ctx.strict => p.as_str(),
1220 _ => text.as_str(),
1221 };
1222 blocks.push(if ctx.plain {
1225 body.to_string()
1226 } else {
1227 code_fence(lang, body)
1228 });
1229 }
1230 Node::Formula { latex, orig, .. } => {
1236 if !latex.is_empty() {
1237 blocks.push(format!("$${latex}$$"));
1238 } else if !orig.is_empty() {
1239 blocks.push(FORMULA_NOT_DECODED.to_string());
1240 }
1241 }
1242 Node::Table(table) => {
1243 if let Some(cap) = &table.caption {
1246 if !cap.is_empty() {
1247 blocks.push(ctx.text(cap));
1248 }
1249 }
1250 let rendered = if ctx.plain {
1251 render_table(&ctx.plain_table(table), ctx.compact_tables)
1252 } else {
1253 render_table(table, ctx.compact_tables)
1254 };
1255 if !rendered.is_empty() {
1256 blocks.push(rendered);
1257 }
1258 for note in table.footnotes.iter().filter(|n| !n.is_empty()) {
1261 blocks.push(ctx.text(note));
1262 }
1263 }
1264 Node::Picture {
1273 caption,
1274 image,
1275 description,
1276 ..
1277 } => {
1278 if let Some(cap) = caption {
1279 if !cap.is_empty() {
1280 blocks.push(ctx.caption(cap));
1281 }
1282 }
1283 let alt = description
1284 .as_ref()
1285 .filter(|d| d.is_alt_text() && ctx.alt_text && !d.text.is_empty());
1286 let marker = picture_marker(image.as_ref(), ctx, alt.map(|d| d.text.as_str()));
1287 let native = description
1291 .as_ref()
1292 .filter(|d| d.is_docling_meta() && !d.text.is_empty());
1293 if let Some(desc) = description.as_ref().filter(|d| !d.is_docling_meta()) {
1294 let shown = !desc.is_alt_text() || (ctx.alt_text && !marker.starts_with("!["));
1295 if !desc.text.is_empty() && shown {
1296 blocks.push(ctx.breaks(&ctx.text(&desc.text)));
1297 }
1298 }
1299 blocks.push(marker);
1300 if let Some(desc) = native {
1301 blocks.push(ctx.breaks(&ctx.text(&desc.text)));
1302 }
1303 }
1304 Node::Chart {
1308 kind,
1309 table,
1310 caption,
1311 ..
1312 } => {
1313 if let Some(cap) = caption {
1314 if !cap.is_empty() {
1315 blocks.push(ctx.caption(cap));
1316 }
1317 }
1318 blocks.push(picture_marker(None, ctx, None));
1319 blocks.push(humanize_label(kind));
1320 let rendered = if ctx.plain {
1321 render_table(&ctx.plain_table(table), false)
1322 } else {
1323 render_table(table, false)
1324 };
1325 if !rendered.is_empty() {
1326 blocks.push(rendered);
1327 }
1328 }
1329 Node::DoclangOnly(_) => {}
1331 Node::Group {
1336 layer: Some(layer),
1337 children,
1338 ..
1339 } => {
1340 let outer = ctx.current_layer.replace(*layer);
1341 render(children, blocks, ctx);
1342 ctx.current_layer = outer;
1343 }
1344 Node::Group { children, .. } => render(children, blocks, ctx),
1345 Node::FieldRegion { items } => {
1346 for item in items {
1351 for part in [&item.marker, &item.key, &item.value].into_iter().flatten() {
1352 blocks.push(ctx.breaks(&ctx.text(part)));
1353 }
1354 }
1355 }
1356 Node::KeyValueGraph { .. } => blocks.push(MISSING_KEY_VALUE_ITEM.to_string()),
1359 Node::InlineGroup { md_text, .. } => blocks.push(ctx.breaks(&ctx.text(md_text))),
1362 Node::TextDump(text) => {
1364 if !text.is_empty() {
1365 blocks.push(text.clone());
1366 }
1367 }
1368 Node::Furniture { layer, inner } => {
1372 let outer = ctx.current_layer.replace(*layer);
1373 render_one(inner, blocks, ctx);
1374 ctx.current_layer = outer;
1375 }
1376 Node::PageFurniture { text, .. } | Node::FurnitureText { text, .. } => {
1380 if ctx.layers.furniture && !text.is_empty() {
1381 blocks.push(ctx.breaks(&ctx.text(text)));
1382 }
1383 }
1384 Node::PictureChildren(children) => {
1389 if ctx.traverse_pictures {
1390 render(children, blocks, ctx);
1391 }
1392 }
1393 Node::CommentSection { text, .. } => {
1398 if ctx.layers.notes && !text.is_empty() {
1399 blocks.push(ctx.breaks(&ctx.text(text)));
1400 }
1401 }
1402 Node::Commented { inner, .. } => render_one(inner, blocks, ctx),
1403 Node::ScriptText { inner, .. } => render_one(inner, blocks, ctx),
1406 Node::Located { inner, .. } | Node::Prov { inner, .. } | Node::Track { inner, .. } => {
1408 render_one(inner, blocks, ctx)
1409 }
1410 Node::PageBreak => {}
1412 Node::PageInfo { .. } | Node::ProvContinuation { .. } => {}
1414 Node::Anchor { .. } => {}
1416 Node::ListItem { .. } => render_list_run(std::slice::from_ref(node), blocks, ctx),
1421 Node::ListText { text, .. } if text.is_empty() => {}
1423 Node::ListText { text, .. } => blocks.push(ctx.breaks(&ctx.text(text))),
1424 }
1425}
1426
1427fn humanize_label(label: &str) -> String {
1432 let text = label.replace('_', " ");
1433 let mut chars = text.chars();
1434 match chars.next() {
1435 Some(f) => f.to_uppercase().collect::<String>() + chars.as_str(),
1436 None => text,
1437 }
1438}
1439
1440fn image_alt(alt: Option<&str>) -> String {
1443 match alt {
1444 Some(text) => text
1445 .split_whitespace()
1446 .collect::<Vec<_>>()
1447 .join(" ")
1448 .replace('\\', "\\\\")
1449 .replace(']', "\\]"),
1450 None => "Image".to_string(),
1451 }
1452}
1453
1454fn picture_marker(image: Option<&crate::PictureImage>, ctx: &mut Ctx, alt: Option<&str>) -> String {
1455 match (ctx.images, image) {
1456 (ImageMode::Embedded, Some(img)) => {
1458 format!(
1459 "",
1460 image_alt(alt),
1461 crate::pixel_digest::docling_data_uri(img).1
1462 )
1463 }
1464 (ImageMode::Referenced, Some(img)) => {
1465 let path = format!(
1466 "{}/image_{:06}.{}",
1467 ctx.artifacts_dir,
1468 ctx.pic_index,
1469 ext_for(&img.mimetype)
1470 );
1471 ctx.pic_index += 1;
1472 ctx.artifacts.push((path.clone(), img.data.clone()));
1473 format!("", image_alt(alt), escape_uri_path(&path))
1474 }
1475 _ => ctx.image_placeholder.clone(),
1477 }
1478}
1479
1480pub(crate) fn escape_uri_path(value: &str) -> String {
1493 const KEEP: &str = "/%:@+,;=~$!&'*";
1494 let s = value.replace('\\', "/");
1495 if let Some(rest) = s.strip_prefix("//") {
1496 let rest = rest.trim_start_matches('/');
1498 let (host, tail) = rest.split_once('/').unwrap_or((rest, ""));
1499 return format!("file://{host}{}", percent_quote(&format!("/{tail}"), KEEP));
1500 }
1501 let bytes = s.as_bytes();
1502 if bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' && bytes[2] == b'/' {
1503 return format!("file:///{}", percent_quote(&s, KEEP));
1505 }
1506 if let Some((scheme, rest)) = s.split_once(':') {
1510 let valid_scheme = scheme.len() > 1
1511 && scheme.as_bytes()[0].is_ascii_alphabetic()
1512 && scheme
1513 .bytes()
1514 .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.'));
1515 if valid_scheme {
1516 let (authority, rest) = match rest.strip_prefix("//") {
1517 Some(r) => {
1518 let end = r.find(['/', '?', '#']).unwrap_or(r.len());
1519 (Some(&r[..end]), &r[end..])
1520 }
1521 None => (None, rest),
1522 };
1523 let (before_frag, fragment) = rest.split_once('#').unwrap_or((rest, ""));
1524 let (path, query) = before_frag.split_once('?').unwrap_or((before_frag, ""));
1525 let mut out = format!("{scheme}:");
1526 if let Some(a) = authority {
1527 out.push_str("//");
1528 out.push_str(a);
1529 }
1530 out.push_str(&percent_quote(path, KEEP));
1531 if !query.is_empty() {
1532 out.push('?');
1533 out.push_str(&percent_quote(query, KEEP));
1534 }
1535 if !fragment.is_empty() {
1536 out.push('#');
1537 out.push_str(&percent_quote(fragment, KEEP));
1538 }
1539 return out;
1540 }
1541 }
1542 percent_quote(&s, KEEP)
1544}
1545
1546fn percent_quote(s: &str, safe: &str) -> String {
1549 let mut out = String::with_capacity(s.len());
1550 for &b in s.as_bytes() {
1551 let keep = b.is_ascii_alphanumeric()
1552 || matches!(b, b'_' | b'.' | b'-' | b'~')
1553 || (b.is_ascii() && safe.contains(b as char));
1554 if keep {
1555 out.push(b as char);
1556 } else {
1557 out.push_str(&format!("%{b:02X}"));
1558 }
1559 }
1560 out
1561}
1562
1563pub(crate) fn ext_for(mimetype: &str) -> &str {
1564 match mimetype {
1565 "image/jpeg" => "jpg",
1566 "image/gif" => "gif",
1567 "image/webp" => "webp",
1568 "image/bmp" => "bmp",
1569 "image/tiff" => "tif",
1570 "image/svg+xml" => "svg",
1571 _ => "png",
1572 }
1573}
1574
1575fn is_number_cell(t: &str) -> bool {
1594 t.parse::<f64>().is_ok() || is_thousands_number(t)
1595}
1596
1597fn is_thousands_number(t: &str) -> bool {
1603 let b = t.as_bytes();
1604 let mut i = 0;
1605 let start = i;
1606 if i < b.len() && (b[i] == b'+' || b[i] == b'-') {
1607 i += 1;
1608 }
1609 let d0 = i;
1611 while i < b.len() && b[i].is_ascii_digit() && i - d0 < 3 {
1612 i += 1;
1613 }
1614 let has_int = i > d0;
1615 if has_int {
1616 while i + 3 < b.len() + 1
1618 && b.get(i) == Some(&b',')
1619 && b.get(i + 1).is_some_and(u8::is_ascii_digit)
1620 && b.get(i + 2).is_some_and(u8::is_ascii_digit)
1621 && b.get(i + 3).is_some_and(u8::is_ascii_digit)
1622 {
1623 i += 4;
1624 }
1625 } else {
1626 i = start;
1628 }
1629 if i < b.len() && b[i] == b'.' {
1631 i += 1;
1632 let f0 = i;
1633 while i < b.len() && b[i].is_ascii_digit() {
1634 i += 1;
1635 }
1636 if !has_int && i == f0 {
1637 return false; }
1639 } else if !has_int {
1640 return false; }
1642 i == b.len()
1643}
1644
1645fn flatten_header_rows(header_rows: &[Vec<String>], num_cols: usize) -> Vec<String> {
1654 (0..num_cols)
1655 .map(|c| {
1656 let mut parts: Vec<&str> = Vec::new();
1657 for row in header_rows {
1658 let text = row.get(c).map(String::as_str).unwrap_or("");
1659 if !text.is_empty() && parts.last() != Some(&text) {
1660 parts.push(text);
1661 }
1662 }
1663 parts.join(" - ")
1664 })
1665 .collect()
1666}
1667
1668pub(crate) const FORMULA_NOT_DECODED: &str = "<!-- formula-not-decoded -->";
1671
1672pub(crate) fn render_table(table: &Table, compact: bool) -> String {
1673 if table.rows.is_empty() {
1674 return String::new();
1675 }
1676 let num_cols = table.rows.iter().map(Vec::len).max().unwrap_or(0);
1677 if num_cols == 0 {
1678 return String::new();
1679 }
1680
1681 let num_headers = table.header_row_count().min(table.rows.len());
1686 let escaped = |r: usize| -> Vec<String> {
1687 (0..num_cols)
1688 .map(|c| escape_cell(table.rows[r].get(c).map(String::as_str).unwrap_or("")))
1689 .collect()
1690 };
1691 let header_rows: Vec<Vec<String>> = (0..num_headers).map(escaped).collect();
1692 let header = flatten_header_rows(&header_rows, num_cols);
1693 let body: Vec<Vec<String>> = (num_headers..table.rows.len())
1694 .map(|r| {
1695 escaped(r)
1696 .into_iter()
1697 .map(|c| c.trim().to_string())
1698 .collect()
1699 })
1700 .collect();
1701
1702 if compact {
1703 let render_row = |row: &[String]| -> String { format!("| {} |", row.join(" | ")) };
1705 let mut lines = Vec::with_capacity(body.len() + 2);
1706 lines.push(render_row(&header));
1707 let sep: Vec<&str> = (0..num_cols).map(|_| "-").collect();
1708 lines.push(format!("| {} |", sep.join(" | ")));
1709 for row in &body {
1710 lines.push(render_row(row));
1711 }
1712 return lines.join("\n");
1713 }
1714
1715 let dw = |s: &str| s.chars().count();
1717
1718 let right: Vec<bool> = (0..num_cols)
1723 .map(|c| {
1724 let mut any = false;
1725 for row in &body {
1726 let t = row[c].trim();
1727 if t.is_empty() {
1728 continue;
1729 }
1730 if !is_number_cell(t) {
1731 return false;
1732 }
1733 any = true;
1734 }
1735 any
1736 })
1737 .collect();
1738
1739 let width: Vec<usize> = (0..num_cols)
1741 .map(|c| {
1742 let mut w = dw(&header[c]) + 2;
1743 for row in &body {
1744 w = w.max(dw(&row[c]));
1745 }
1746 w
1747 })
1748 .collect();
1749
1750 let fmt_cell = |s: &str, c: usize| -> String {
1751 let pad = " ".repeat(width[c].saturating_sub(dw(s)));
1752 let body = if right[c] {
1753 format!("{pad}{s}")
1754 } else {
1755 format!("{s}{pad}")
1756 };
1757 format!(" {body} ")
1758 };
1759 let render_row = |row: &[String]| -> String {
1760 let cells: Vec<String> = (0..num_cols).map(|c| fmt_cell(&row[c], c)).collect();
1761 format!("|{}|", cells.join("|"))
1762 };
1763
1764 let mut lines = Vec::with_capacity(body.len() + 2);
1765 lines.push(render_row(&header));
1766 let sep: Vec<String> = (0..num_cols).map(|c| "-".repeat(width[c] + 2)).collect();
1767 lines.push(format!("|{}|", sep.join("|")));
1768 for row in &body {
1769 lines.push(render_row(row));
1770 }
1771 lines.join("\n")
1772}
1773
1774fn escape_cell(s: &str) -> String {
1777 s.replace('\n', " ").replace('|', "|")
1778}
1779
1780#[cfg(test)]
1781mod tests {
1782 use super::*;
1783 use crate::{PictureImage, TableCell, TableStructure};
1784
1785 #[test]
1790 fn an_undecoded_formula_prints_the_placeholder() {
1791 let formula = |latex: &str, orig: &str| Node::Formula {
1792 latex: latex.into(),
1793 orig: orig.into(),
1794 location: None,
1795 };
1796 let mut doc = DoclingDocument::new("t");
1797 doc.nodes = vec![
1798 formula("E = mc^2", "E = mc 2"),
1799 formula("", "E = mc 2"),
1800 formula("", ""),
1801 Node::Paragraph {
1802 text: "After.".into(),
1803 },
1804 ];
1805 assert_eq!(
1806 doc.export_to_markdown().trim(),
1807 "$$E = mc^2$$\n\n<!-- formula-not-decoded -->\n\nAfter."
1808 );
1809 let chunks: Vec<String> = crate::chunker::HierarchicalChunker
1810 .chunk(&doc)
1811 .into_iter()
1812 .map(|c| c.text)
1813 .collect();
1814 assert_eq!(
1815 chunks,
1816 ["$$E = mc^2$$", "<!-- formula-not-decoded -->", "After."]
1817 );
1818 }
1819
1820 #[test]
1826 fn list_boundaries_come_from_the_backend_not_the_numbering() {
1827 let item = |ordered: bool, number: u64, first_in_list: bool, text: &str| Node::ListItem {
1828 ordered,
1829 number,
1830 first_in_list,
1831 text: text.into(),
1832 level: 0,
1833 marker: None,
1834 location: None,
1835 split: None,
1836 dclx: None,
1837 href: None,
1838 layer: None,
1839 };
1840 let md = |items: Vec<Node>| {
1841 let mut doc = DoclingDocument::new("t");
1842 for n in items {
1843 doc.push(n);
1844 }
1845 doc.export_to_markdown()
1846 };
1847 assert_eq!(
1849 md(vec![
1850 item(true, 1, true, "one"),
1851 item(true, 5, false, "five")
1852 ]),
1853 "1. one\n5. five\n"
1854 );
1855 assert_eq!(
1857 md(vec![
1858 item(false, 0, true, "bullet"),
1859 item(true, 1, false, "one"),
1860 item(false, 0, false, "bullet two"),
1861 ]),
1862 "- bullet\n1. one\n- bullet two\n"
1863 );
1864 assert_eq!(
1866 md(vec![
1867 item(true, 1, true, "a"),
1868 item(true, 2, false, "b"),
1869 item(true, 3, true, "new list, continuing count"),
1870 ]),
1871 "1. a\n2. b\n\n3. new list, continuing count\n"
1872 );
1873 }
1874
1875 #[test]
1876 fn renders_headings_paragraphs_and_lists() {
1877 let mut doc = DoclingDocument::new("demo");
1878 doc.add_heading(1, "Title");
1879 doc.add_paragraph("Hello world.");
1880 doc.push(Node::ListItem {
1881 ordered: false,
1882 number: 1,
1883 first_in_list: true,
1884 text: "first".into(),
1885 level: 0,
1886 marker: None,
1887 location: None,
1888 split: None,
1889 dclx: None,
1890 href: None,
1891 layer: None,
1892 });
1893 doc.push(Node::ListItem {
1894 ordered: false,
1895 number: 2,
1896 first_in_list: false,
1897 text: "second".into(),
1898 level: 0,
1899 marker: None,
1900 location: None,
1901 split: None,
1902 dclx: None,
1903 href: None,
1904 layer: None,
1905 });
1906 let md = doc.export_to_markdown();
1907 assert_eq!(md, "# Title\n\nHello world.\n\n- first\n- second\n");
1908 }
1909
1910 #[test]
1914 fn single_newlines_become_gfm_hard_line_breaks() {
1915 let mut doc = DoclingDocument::new("t");
1916 doc.push(Node::Heading {
1917 level: 1,
1918 text: "Hello\nWorld".into(),
1919 });
1920 doc.push(Node::Paragraph {
1921 text: "line one\nline two\n\npara two".into(),
1922 });
1923 doc.push(Node::ListItem {
1924 ordered: false,
1925 number: 1,
1926 first_in_list: true,
1927 text: "item\ncontinued".into(),
1928 level: 0,
1929 marker: None,
1930 location: None,
1931 split: None,
1932 dclx: None,
1933 href: None,
1934 layer: None,
1935 });
1936 doc.push(Node::TextDump("A1 B1 \n\n\nC1".into()));
1937 assert_eq!(
1938 doc.export_to_markdown(),
1939 "# Hello World\n\nline one \nline two\n\npara two\n\n- item \ncontinued\n\nA1 B1 \n\n\nC1\n"
1940 );
1941 }
1942
1943 #[test]
1946 fn table_cell_mode_and_field_regions() {
1947 let mut doc = DoclingDocument::new("t");
1948 doc.push(Node::Heading {
1949 level: 2,
1950 text: "A text".into(),
1951 });
1952 doc.push(Node::Paragraph {
1953 text: "body".into(),
1954 });
1955 assert_eq!(to_markdown_table_cell(&doc, false), "A text\n\nbody");
1956 assert_eq!(doc.export_to_markdown(), "## A text\n\nbody\n");
1957
1958 let mut doc = DoclingDocument::new("f");
1959 doc.push(Node::FieldRegion {
1960 items: vec![crate::FieldItem {
1961 marker: None,
1962 key: Some("Name:".into()),
1963 value: Some("John Doe".into()),
1964 value_kind: None,
1965 }],
1966 });
1967 assert_eq!(doc.export_to_markdown(), "Name:\n\nJohn Doe\n");
1968 }
1969
1970 #[test]
1971 fn strict_renders_recovered_links_legacy_does_not() {
1972 let mut doc = DoclingDocument::new("cv");
1973 doc.add_paragraph("Find me on LinkedIn or GitHub.");
1974 doc.links = vec![
1975 ("LinkedIn".into(), "https://www.linkedin.com/in/x/".into()),
1976 ("GitHub".into(), "https://github.com/x/".into()),
1977 ];
1978 assert_eq!(doc.export_to_markdown(), "Find me on LinkedIn or GitHub.\n");
1980 assert_eq!(
1982 doc.export_to_markdown_with(true),
1983 "Find me on [LinkedIn](https://www.linkedin.com/in/x/) or [GitHub](https://github.com/x/).\n"
1984 );
1985 }
1986
1987 #[test]
1988 fn strict_links_match_escaped_anchor_and_consume_in_order() {
1989 let mut doc = DoclingDocument::new("d");
1990 doc.add_paragraph("AI & ML here, and issues here, then issues there.");
1994 doc.links = vec![
1995 ("AI & ML".into(), "https://a/".into()),
1996 ("issues".into(), "https://first/".into()),
1997 ("issues".into(), "https://second/".into()),
1998 ];
1999 assert_eq!(
2000 doc.export_to_markdown_with(true),
2001 "[AI & ML](https://a/) here, and [issues](https://first/) here, then [issues](https://second/) there.\n"
2002 );
2003 }
2004
2005 #[test]
2009 fn referenced_image_destinations_are_escaped() {
2010 let cases = [
2011 (
2012 "doc_artifacts/image_000001_ab12.png",
2013 "doc_artifacts/image_000001_ab12.png",
2014 ),
2015 (
2016 "My Report_artifacts/img.png",
2017 "My%20Report_artifacts/img.png",
2018 ),
2019 ("artifacts/img (1).png", "artifacts/img%20%281%29.png"),
2020 ("100%_scale/a#b?c.png", "100%_scale/a%23b%3Fc.png"),
2021 ("/home/a b/img.png", "/home/a%20b/img.png"),
2022 (
2023 "My Report_artifacts\\img.png",
2024 "My%20Report_artifacts/img.png",
2025 ),
2026 (
2027 "C:/Users/me/My Docs/img.png",
2028 "file:///C:/Users/me/My%20Docs/img.png",
2029 ),
2030 ("C:\\Users\\me\\img.png", "file:///C:/Users/me/img.png"),
2031 (
2032 "//server/share/My Docs/img.png",
2033 "file://server/share/My%20Docs/img.png",
2034 ),
2035 ("\\\\server\\share\\img.png", "file://server/share/img.png"),
2036 ("file:///home/a b/img.png", "file:///home/a%20b/img.png"),
2037 (
2038 "s3://bucket/My Report_artifacts/img.png",
2039 "s3://bucket/My%20Report_artifacts/img.png",
2040 ),
2041 (
2042 "https://example.com:8080/a b.png?w=1&h=2#frag",
2043 "https://example.com:8080/a%20b.png?w=1&h=2#frag",
2044 ),
2045 (
2046 "https://example.com/img (1).png",
2047 "https://example.com/img%20%281%29.png",
2048 ),
2049 ("caf\u{e9}/im\u{e4}ge.png", "caf%C3%A9/im%C3%A4ge.png"),
2050 ];
2051 for (input, expected) in cases {
2052 assert_eq!(escape_uri_path(input), expected, "input {input:?}");
2053 assert_eq!(
2054 escape_uri_path(expected),
2055 expected,
2056 "idempotent {expected:?}"
2057 );
2058 }
2059 let mut doc = DoclingDocument::new("t");
2061 doc.push(Node::Picture {
2062 caption: None,
2063 caption_href: None,
2064 image: Some(PictureImage {
2065 dpi: PictureImage::DEFAULT_DPI,
2066 mimetype: "image/png".into(),
2067 width: 1,
2068 height: 1,
2069 data: b"x".to_vec(),
2070 display_size: None,
2071 }),
2072 classification: None,
2073 description: None,
2074 caption_parent: Default::default(),
2075 caption_location: None,
2076 });
2077 let (md, files) = doc
2078 .export_to_markdown_with_images(ImageMode::Referenced, "My Report (final)_artifacts");
2079 assert!(
2080 md.contains(""),
2081 "got:\n{md}"
2082 );
2083 assert_eq!(files[0].0, "My Report (final)_artifacts/image_000000.png");
2085 }
2086
2087 #[test]
2091 fn folded_list_item_pictures_keep_plain_newlines() {
2092 let doc = DoclingDocument::new("t");
2093 let ctx = Ctx::new(&doc, false, &MarkdownExportOptions::default());
2094 assert_eq!(
2095 list_item_text("Step\n<!-- image -->", &ctx),
2096 "Step\n<!-- image -->"
2097 );
2098 assert_eq!(
2099 list_item_text("Step\nAlt text\n<!-- image -->\n<!-- image -->", &ctx),
2100 "Step\nAlt text\n<!-- image -->\n<!-- image -->"
2101 );
2102 assert_eq!(
2103 list_item_text("line one\nline two", &ctx),
2104 "line one \nline two"
2105 );
2106 }
2107
2108 #[test]
2111 fn stacked_header_rows_flatten_into_one() {
2112 let mut t = Table {
2113 rows: vec![
2114 vec!["".into(), "% of Total".into(), "% of Total".into()],
2115 vec!["class".into(), "Train".into(), "Test".into()],
2116 vec!["Caption".into(), "2.04".into(), "1.77".into()],
2117 ],
2118 ..Default::default()
2119 };
2120 t.structure = Some(TableStructure {
2121 header_row: vec![true, true, false],
2122 col_continuation: vec![
2123 vec![false, false, true],
2124 vec![false, false, false],
2125 vec![false, false, false],
2126 ],
2127 ..Default::default()
2128 });
2129 assert_eq!(t.header_row_count(), 2);
2130 assert_eq!(
2131 render_table(&t, true),
2132 "| class | % of Total - Train | % of Total - Test |\n| - | - | - |\n| Caption | 2.04 | 1.77 |"
2133 );
2134 assert_eq!(
2136 render_table(&t, false),
2137 "| class | % of Total - Train | % of Total - Test |\n\
2138 |---------|----------------------|---------------------|\n\
2139 | Caption | 2.04 | 1.77 |"
2140 );
2141 }
2142
2143 #[test]
2146 fn vertically_spanning_header_does_not_extend_the_block() {
2147 let mut t = Table {
2148 rows: vec![
2149 vec!["Name".into(), "Value".into()],
2150 vec!["Name".into(), "1".into()],
2151 vec!["x".into(), "2".into()],
2152 ],
2153 ..Default::default()
2154 };
2155 t.structure = Some(TableStructure {
2156 col_header: vec![vec![true, true], vec![true, false], vec![false, false]],
2157 row_continuation: vec![vec![false, false], vec![true, false], vec![false, false]],
2158 ..Default::default()
2159 });
2160 assert_eq!(t.header_row_count(), 1);
2161 assert_eq!(
2162 render_table(&t, true),
2163 "| Name | Value |\n| - | - |\n| Name | 1 |\n| x | 2 |"
2164 );
2165 }
2166
2167 #[test]
2170 fn header_flags_not_on_row_zero_keep_all_rows_in_the_body() {
2171 let mut t = Table {
2172 rows: vec![
2173 vec!["1".into(), "2".into()],
2174 vec!["a".into(), "b".into()],
2175 vec!["333".into(), "4".into()],
2176 ],
2177 ..Default::default()
2178 };
2179 t.structure = Some(TableStructure {
2180 header_row: vec![false, true, false],
2181 ..Default::default()
2182 });
2183 assert_eq!(t.header_row_count(), 0);
2184 assert_eq!(
2185 render_table(&t, false),
2186 "| | |\n|-----|----|\n| 1 | 2 |\n| a | b |\n| 333 | 4 |"
2187 );
2188 }
2189
2190 #[test]
2195 fn pivot_row_headers_do_not_extend_the_header() {
2196 let mut t = Table {
2197 rows: vec![
2198 vec!["Year".into(), "Month".into()],
2199 vec!["2025".into(), "January".into()],
2200 vec!["2025".into(), "February".into()],
2201 ],
2202 ..Default::default()
2203 };
2204 t.structure = Some(TableStructure {
2205 col_header: vec![vec![true, true], vec![false, false], vec![false, false]],
2206 row_header: vec![vec![false, false], vec![true, false], vec![true, false]],
2207 row_continuation: vec![vec![false, false], vec![false, false], vec![true, false]],
2208 ..Default::default()
2209 });
2210 assert_eq!(t.header_row_count(), 1);
2211 assert_eq!(
2212 render_table(&t, true),
2213 "| Year | Month |\n| - | - |\n| 2025 | January |\n| 2025 | February |"
2214 );
2215 }
2216
2217 #[test]
2220 fn unflagged_cells_keep_row_zero_as_header() {
2221 let mut t = Table {
2222 rows: vec![vec!["h".into()], vec!["d".into()]],
2223 ..Default::default()
2224 };
2225 t.cells = Some(
2226 [(0usize, "h"), (1, "d")]
2227 .into_iter()
2228 .map(|(r, text)| TableCell {
2229 text: text.into(),
2230 bbox: None,
2231 start_row: r,
2232 start_col: 0,
2233 row_span: 1,
2234 col_span: 1,
2235 column_header: false,
2236 row_header: false,
2237 row_section: false,
2238 })
2239 .collect(),
2240 );
2241 assert_eq!(t.header_row_count(), 1);
2242 assert_eq!(render_table(&t, true), "| h |\n| - |\n| d |");
2243 }
2244
2245 fn flagged_table(rows: &[&[&str]], ched: impl Fn(usize, usize) -> bool) -> Table {
2247 let mut cells = Vec::new();
2248 for (r, row) in rows.iter().enumerate() {
2249 for (c, text) in row.iter().enumerate() {
2250 cells.push(TableCell {
2251 text: (*text).into(),
2252 bbox: None,
2253 start_row: r,
2254 start_col: c,
2255 row_span: 1,
2256 col_span: 1,
2257 column_header: ched(r, c),
2258 row_header: false,
2259 row_section: false,
2260 });
2261 }
2262 }
2263 Table {
2264 rows: rows
2265 .iter()
2266 .map(|row| row.iter().map(|t| (*t).to_string()).collect())
2267 .collect(),
2268 cells: Some(cells),
2269 ..Default::default()
2270 }
2271 }
2272
2273 #[test]
2277 fn rows_with_body_text_stop_the_header_block() {
2278 let t = flagged_table(
2280 &[
2281 &["Item", "Q1", "Q2"],
2282 &["Sales", "10", "20"],
2283 &["Costs", "5", "7"],
2284 ],
2285 |r, c| r == 0 || c == 0,
2286 );
2287 assert_eq!(t.header_row_count(), 1);
2288 assert_eq!(
2289 render_table(&t, false),
2290 "| Item | Q1 | Q2 |\n\
2291 |--------|------|------|\n\
2292 | Sales | 10 | 20 |\n\
2293 | Costs | 5 | 7 |"
2294 );
2295 let t = flagged_table(&[&["Sales", "10", "20"], &["Costs", "5", "7"]], |_, c| {
2298 c == 0
2299 });
2300 assert_eq!(t.header_row_count(), 0);
2301 assert_eq!(
2302 render_table(&t, false),
2303 "| | | |\n\
2304 |-------|----|----|\n\
2305 | Sales | 10 | 20 |\n\
2306 | Costs | 5 | 7 |"
2307 );
2308 let t = flagged_table(
2309 &[&["a", "1", "2"], &["b", "3", "4"], &["c", "5", "6"]],
2310 |r, c| c == 0 && r < 2,
2311 );
2312 assert_eq!(t.header_row_count(), 0);
2313 let t = flagged_table(&[&["a", "1", "2"], &["b", "3", "4"]], |r, c| {
2316 r == 0 && c == 0
2317 });
2318 assert_eq!(t.header_row_count(), 1);
2319 assert_eq!(
2320 render_table(&t, false),
2321 "| a | 1 | 2 |\n\
2322 |-----|-----|-----|\n\
2323 | b | 3 | 4 |"
2324 );
2325 let t = flagged_table(&[&["Item", "", ""], &["x", "1", "2"]], |_, c| c == 0);
2327 assert_eq!(t.header_row_count(), 1);
2328 assert_eq!(
2329 render_table(&t, false),
2330 "| Item | | |\n\
2331 |--------|----|----|\n\
2332 | x | 1 | 2 |"
2333 );
2334 }
2335
2336 #[test]
2337 fn renders_compact_table() {
2338 let mut doc = DoclingDocument::new("t");
2339 doc.compact_tables = true;
2342 doc.push(Node::Table(Table {
2343 footnotes: Vec::new(),
2344 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2345 location: None,
2346 structure: None,
2347 cell_blocks: None,
2348 cells: None,
2349 caption: None,
2350 caption_parent: Default::default(),
2351 caption_location: None,
2352 document_index: false,
2353 }));
2354 let md = doc.export_to_markdown();
2355 assert_eq!(md, "| a | b |\n| - | - |\n| 1 | 2 |\n");
2356 }
2357
2358 #[test]
2359 fn renders_padded_github_table_by_default() {
2360 let mut doc = DoclingDocument::new("t");
2361 doc.push(Node::Table(Table {
2362 footnotes: Vec::new(),
2363 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2364 location: None,
2365 structure: None,
2366 cell_blocks: None,
2367 cells: None,
2368 caption: None,
2369 caption_parent: Default::default(),
2370 caption_location: None,
2371 document_index: false,
2372 }));
2373 let md = doc.export_to_markdown();
2374 assert_eq!(md, "| a | b |\n|-----|-----|\n| 1 | 2 |\n");
2376 }
2377
2378 #[test]
2379 fn strict_unescapes_inline_underscores_legacy_keeps_them() {
2380 let mut doc = DoclingDocument::new("t");
2381 doc.add_heading(1, "a\\_b");
2382 doc.add_paragraph("x\\_y");
2383 doc.push(Node::ListItem {
2384 ordered: false,
2385 number: 1,
2386 first_in_list: true,
2387 text: "i\\_j".into(),
2388 level: 0,
2389 marker: None,
2390 location: None,
2391 split: None,
2392 dclx: None,
2393 href: None,
2394 layer: None,
2395 });
2396 assert_eq!(doc.export_to_markdown(), "# a\\_b\n\nx\\_y\n\n- i\\_j\n");
2398 assert_eq!(doc.export_to_markdown_with(true), "# a_b\n\nx_y\n\n- i_j\n");
2400 }
2401
2402 fn assert_stream_matches(
2405 doc: &DoclingDocument,
2406 strict: bool,
2407 images: ImageMode,
2408 splits: &[usize],
2409 ) {
2410 let (want, want_artifacts) = to_markdown_images(doc, strict, images, "artifacts");
2411 let mut streamer =
2412 MarkdownStreamer::with_artifacts(strict, images, doc.compact_tables, "artifacts")
2413 .with_page_break_placeholder(doc.page_break_placeholder.clone());
2414 let mut got = String::new();
2415 let mut got_artifacts = Vec::new();
2416 let mut start = 0;
2417 for &end in splits {
2418 let links = if start == 0 {
2421 doc.links.as_slice()
2422 } else {
2423 &[]
2424 };
2425 got.push_str(&streamer.push(&doc.nodes[start..end], links));
2426 got_artifacts.extend(streamer.take_artifacts());
2429 start = end;
2430 }
2431 got.push_str(&streamer.push(
2432 &doc.nodes[start..],
2433 if start == 0 {
2434 doc.links.as_slice()
2435 } else {
2436 &[]
2437 },
2438 ));
2439 got_artifacts.extend(streamer.take_artifacts());
2440 got.push_str(&streamer.finish());
2441 assert_eq!(
2442 got, want,
2443 "streamed output diverged (splits={splits:?}, strict={strict})"
2444 );
2445 assert_eq!(
2446 got_artifacts, want_artifacts,
2447 "streamed artifacts diverged (splits={splits:?}, strict={strict})"
2448 );
2449 }
2450
2451 #[test]
2452 fn streaming_is_byte_identical_to_buffered() {
2453 let mut doc = DoclingDocument::new("d");
2454 doc.add_heading(1, "Title");
2455 doc.add_paragraph("First paragraph.");
2456 doc.push(Node::ListItem {
2457 ordered: false,
2458 number: 1,
2459 first_in_list: true,
2460 text: "a".into(),
2461 level: 0,
2462 marker: None,
2463 location: None,
2464 split: None,
2465 dclx: None,
2466 href: None,
2467 layer: None,
2468 });
2469 doc.push(Node::ListItem {
2470 ordered: false,
2471 number: 2,
2472 first_in_list: false,
2473 text: "b".into(),
2474 level: 0,
2475 marker: None,
2476 location: None,
2477 split: None,
2478 dclx: None,
2479 href: None,
2480 layer: None,
2481 });
2482 doc.push(Node::Code {
2483 language: Some("rust".into()),
2484 text: "let x = 1;".into(),
2485 orig: None,
2486 pretty: None,
2487 });
2488 doc.push(Node::Table(Table {
2489 footnotes: Vec::new(),
2490 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2491 location: None,
2492 structure: None,
2493 cell_blocks: None,
2494 cells: None,
2495 caption: None,
2496 caption_parent: Default::default(),
2497 caption_location: None,
2498 document_index: false,
2499 }));
2500 doc.push(Node::Picture {
2501 caption: Some("Fig 1".into()),
2502 caption_href: None,
2503 image: Some(PictureImage {
2504 dpi: PictureImage::DEFAULT_DPI,
2505 mimetype: "image/png".into(),
2506 width: 2,
2507 height: 2,
2508 data: b"png-one".to_vec(),
2509 display_size: None,
2510 }),
2511 classification: None,
2512 description: None,
2513 caption_parent: Default::default(),
2514 caption_location: None,
2515 });
2516 doc.add_paragraph("Last paragraph.");
2517 doc.push(Node::Picture {
2520 caption: None,
2521 caption_href: None,
2522 image: Some(PictureImage {
2523 dpi: PictureImage::DEFAULT_DPI,
2524 mimetype: "image/png".into(),
2525 width: 2,
2526 height: 2,
2527 data: b"png-two".to_vec(),
2528 display_size: None,
2529 }),
2530 classification: None,
2531 description: None,
2532 caption_parent: Default::default(),
2533 caption_location: None,
2534 });
2535
2536 for &strict in &[false, true] {
2539 for &images in &[
2540 ImageMode::Placeholder,
2541 ImageMode::Embedded,
2542 ImageMode::Referenced,
2543 ] {
2544 for splits in [&[][..], &[1][..], &[2][..], &[4][..], &[1, 4, 6, 7][..]] {
2545 assert_stream_matches(&doc, strict, images, splits);
2546 }
2547 }
2548 }
2549 }
2550
2551 #[test]
2552 fn streaming_applies_recovered_links_in_strict_mode() {
2553 let mut doc = DoclingDocument::new("d");
2554 doc.add_paragraph("See LinkedIn for details.");
2555 doc.add_paragraph("And GitHub too.");
2556 doc.links = vec![
2557 ("LinkedIn".into(), "https://lnkd/".into()),
2558 ("GitHub".into(), "https://gh/".into()),
2559 ];
2560 assert_stream_matches(&doc, true, ImageMode::Placeholder, &[1]);
2563 }
2564
2565 fn paged_doc() -> DoclingDocument {
2568 let mut doc = DoclingDocument::new("p");
2569 doc.push(Node::PageInfo {
2570 page_no: 1,
2571 width: 100.0,
2572 height: 100.0,
2573 });
2574 doc.add_heading(1, "Title");
2575 doc.add_paragraph("Page one.");
2576 doc.push(Node::PageBreak);
2578 doc.push(Node::PageInfo {
2579 page_no: 2,
2580 width: 100.0,
2581 height: 100.0,
2582 });
2583 doc.push(Node::PageFurniture {
2584 footer: true,
2585 location: [0, 500, 511, 511],
2586 text: "2".into(),
2587 });
2588 doc.push(Node::PageBreak);
2589 doc.push(Node::PageInfo {
2590 page_no: 3,
2591 width: 100.0,
2592 height: 100.0,
2593 });
2594 doc.add_paragraph("Page three.");
2595 doc.push(Node::PageBreak);
2597 doc
2598 }
2599
2600 #[test]
2601 fn page_break_placeholder_lands_between_pages_only() {
2602 let mut doc = paged_doc();
2603 assert_eq!(
2605 doc.export_to_markdown(),
2606 "# Title\n\nPage one.\n\nPage three.\n"
2607 );
2608 doc.page_break_placeholder = Some("<!-- page break -->".into());
2609 assert_eq!(
2613 doc.export_to_markdown(),
2614 "# Title\n\nPage one.\n\n<!-- page break -->\n\nPage three.\n"
2615 );
2616 doc.page_break_placeholder = Some(String::new());
2619 assert_eq!(
2620 doc.export_to_markdown(),
2621 "# Title\n\nPage one.\n\n\n\nPage three.\n"
2622 );
2623 }
2624
2625 #[test]
2626 fn page_break_placeholder_never_leads_a_single_page() {
2627 let mut doc = DoclingDocument::new("one");
2628 doc.page_break_placeholder = Some("---".into());
2629 doc.push(Node::PageBreak);
2630 doc.push(Node::PageInfo {
2631 page_no: 1,
2632 width: 10.0,
2633 height: 10.0,
2634 });
2635 doc.add_paragraph("Only page.");
2636 assert_eq!(doc.export_to_markdown(), "Only page.\n");
2637 doc.push(Node::PageBreak);
2639 doc.push(Node::PageBreak);
2640 doc.add_paragraph("Next.");
2641 assert_eq!(doc.export_to_markdown(), "Only page.\n\n---\n\nNext.\n");
2642 }
2643
2644 #[test]
2645 fn page_break_placeholder_streams_byte_identical() {
2646 let mut doc = paged_doc();
2647 doc.page_break_placeholder = Some("<!-- page break -->".into());
2648 for splits in [
2652 &[3usize][..],
2653 &[3, 6],
2654 &[3, 6, 8],
2655 &[1, 2, 3, 4, 5, 6, 7, 8, 9],
2656 &[8],
2657 ] {
2658 assert_stream_matches(&doc, false, ImageMode::Placeholder, splits);
2659 assert_stream_matches(&doc, true, ImageMode::Placeholder, splits);
2660 }
2661 }
2662
2663 #[test]
2664 fn strict_tightens_punctuation_spacing_legacy_keeps_it() {
2665 let mut doc = DoclingDocument::new("t");
2666 doc.add_paragraph("see [ 37 , 36 ] and ( x ) .");
2667 assert_eq!(doc.export_to_markdown(), "see [ 37 , 36 ] and ( x ) .\n");
2669 assert_eq!(doc.export_to_markdown_with(true), "see [37, 36] and (x).\n");
2671 }
2672
2673 fn options_doc() -> DoclingDocument {
2678 use crate::document::ContentLayer;
2679 let mut doc = DoclingDocument::new("opts");
2680 doc.push(Node::PageFurniture {
2681 footer: false,
2682 location: [0, 0, 0, 0],
2683 text: "FAX COVER \\_ R&D".into(),
2684 });
2685 doc.add_heading(1, "R&D \\_ report");
2686 doc.add_paragraph("Body a\\_b <tag>");
2687 doc.push(Node::Picture {
2688 caption: Some("Figure 1".into()),
2689 caption_href: None,
2690 image: None,
2691 classification: None,
2692 description: None,
2693 caption_parent: Default::default(),
2694 caption_location: None,
2695 });
2696 doc.push(Node::PictureChildren(vec![
2697 Node::Paragraph {
2698 text: "Field: Name".into(),
2699 },
2700 Node::Paragraph {
2701 text: "Field: Date".into(),
2702 },
2703 ]));
2704 doc.push(Node::CommentSection {
2705 name: "comment-1".into(),
2706 text: "[author: A]: note".into(),
2707 refs_note_text: false,
2708 grouped: true,
2709 });
2710 doc.push(Node::Group {
2711 label: "section".into(),
2712 name: Some("sheet: Hidden".into()),
2713 layer: Some(ContentLayer::Invisible),
2714 children: vec![Node::Paragraph {
2715 text: "hidden cell".into(),
2716 }],
2717 });
2718 doc.push(Node::Furniture {
2719 layer: ContentLayer::Furniture,
2720 inner: Box::new(Node::Paragraph {
2721 text: "Page 1 of 2".into(),
2722 }),
2723 });
2724 doc.push(Node::FurnitureText {
2725 label: "page_footer".into(),
2726 text: "footer line".into(),
2727 });
2728 doc
2729 }
2730
2731 #[test]
2735 fn default_options_are_the_default_export() {
2736 let doc = options_doc();
2737 let plain = doc.export_to_markdown();
2738 assert_eq!(
2739 plain,
2740 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2741 );
2742 let (with_default, artifacts) =
2743 doc.export_to_markdown_with_options(&MarkdownExportOptions::default());
2744 assert_eq!(with_default, plain);
2745 assert!(artifacts.is_empty());
2746 }
2747
2748 #[test]
2749 fn layers_render_furniture_notes_and_hidden_sheets() {
2750 use crate::document::ContentLayer;
2751 let doc = options_doc();
2752 let md = |layers: ContentLayers| {
2753 doc.export_to_markdown_with_options(&MarkdownExportOptions {
2754 layers,
2755 ..MarkdownExportOptions::default()
2756 })
2757 .0
2758 };
2759 assert_eq!(
2760 md(ContentLayers::BODY.with(ContentLayer::Furniture)),
2761 "FAX COVER \\_ R&D\n\n# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nPage 1 of 2\n\nfooter line\n",
2762 "a page header, a furniture-wrapped paragraph and a furniture text item render as paragraphs"
2763 );
2764 assert_eq!(
2765 md(ContentLayers::BODY.with(ContentLayer::Notes)),
2766 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\n[author: A]: note\n"
2767 );
2768 assert_eq!(
2769 md(ContentLayers::BODY.with(ContentLayer::Invisible)),
2770 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nhidden cell\n",
2771 "a hidden sheet's items sit on the sheet's layer"
2772 );
2773 assert_eq!(
2774 md(ContentLayers::NONE.with(ContentLayer::Furniture)),
2775 "FAX COVER \\_ R&D\n\nPage 1 of 2\n\nfooter line\n",
2776 "without the body layer only the furniture prints"
2777 );
2778 assert_eq!(md(ContentLayers::NONE), "");
2779 assert_eq!(
2780 md(ContentLayers::ALL),
2781 "FAX COVER \\_ R&D\n\n# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\n[author: A]: note\n\nhidden cell\n\nPage 1 of 2\n\nfooter line\n"
2782 );
2783 }
2784
2785 #[test]
2788 fn layers_apply_to_list_items() {
2789 use crate::document::ContentLayer;
2790 let mut doc = DoclingDocument::new("lists");
2791 doc.push(Node::ListItem {
2792 ordered: false,
2793 number: 1,
2794 first_in_list: true,
2795 text: "nav link".into(),
2796 level: 0,
2797 marker: None,
2798 location: None,
2799 split: None,
2800 dclx: None,
2801 href: None,
2802 layer: Some(ContentLayer::Furniture),
2803 });
2804 doc.add_paragraph("body");
2805 assert_eq!(doc.export_to_markdown(), "body\n");
2806 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2807 layers: ContentLayers::BODY.with(ContentLayer::Furniture),
2808 ..MarkdownExportOptions::default()
2809 });
2810 assert_eq!(md, "- nav link\n\nbody\n");
2811 }
2812
2813 #[test]
2814 fn traverse_pictures_prints_the_nested_text_after_the_picture() {
2815 let doc = options_doc();
2816 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2817 traverse_pictures: true,
2818 ..MarkdownExportOptions::default()
2819 });
2820 assert_eq!(
2821 md,
2822 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nField: Name\n\nField: Date\n"
2823 );
2824 }
2825
2826 #[test]
2827 fn escaping_can_be_turned_off_per_kind() {
2828 let doc = options_doc();
2829 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2830 escape_html: false,
2831 ..MarkdownExportOptions::default()
2832 });
2833 assert_eq!(
2834 md,
2835 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2836 );
2837 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2838 escape_underscores: false,
2839 ..MarkdownExportOptions::default()
2840 });
2841 assert_eq!(
2842 md,
2843 "# R&D _ report\n\nBody a_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2844 );
2845 let mut doc = DoclingDocument::new("code");
2848 doc.push(Node::Code {
2849 language: None,
2850 text: "a & b \\_ c".into(),
2851 orig: None,
2852 pretty: None,
2853 });
2854 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2855 escape_html: false,
2856 escape_underscores: false,
2857 ..MarkdownExportOptions::default()
2858 });
2859 assert_eq!(md, "```\na & b \\_ c\n```\n");
2860 }
2861
2862 #[test]
2866 fn fences_and_spans_out_length_inner_backticks() {
2867 let code = |text: &str| Node::Code {
2868 language: Some("md".into()),
2869 text: text.into(),
2870 orig: None,
2871 pretty: None,
2872 };
2873 let fence_of = |text: &str| {
2874 let mut doc = DoclingDocument::new("d");
2875 doc.push(code(text));
2876 (doc.export_to_markdown(), doc.export_to_markdown_with(true))
2877 };
2878 assert_eq!(fence_of("x").0, "```\nx\n```\n");
2879 assert_eq!(fence_of("a ` b").0, "```\na ` b\n```\n");
2880 assert_eq!(fence_of("a ``` b").0, "````\na ``` b\n````\n");
2881 assert_eq!(
2882 fence_of("a ``` b ```` c"),
2883 (
2884 "`````\na ``` b ```` c\n`````\n".to_string(),
2885 "`````md\na ``` b ```` c\n`````\n".to_string()
2886 )
2887 );
2888 assert_eq!(code_span("x"), "`x`");
2889 assert_eq!(code_span("a`b"), "``a`b``");
2890 assert_eq!(code_span("a ``` b"), "````a ``` b````");
2891 assert_eq!(code_span("`x"), "`` `x ``");
2892 assert_eq!(code_span("x`"), "`` x` ``");
2893 for (span, text) in [
2894 ("`x`", "x"),
2895 ("``a`b``", "a`b"),
2896 ("`` `x ``", "`x"),
2897 ("````a ``` b````", "a ``` b"),
2898 ] {
2899 assert_eq!(
2900 plain_inline(&format!("see {span} here")),
2901 format!("see {text} here")
2902 );
2903 }
2904 let mut doc = DoclingDocument::new("d");
2907 doc.push(code("a ``` b"));
2908 assert_eq!(doc.export_to_text(), "a ``` b");
2909 let mut doc = DoclingDocument::new("d");
2911 doc.add_paragraph("Before.");
2912 doc.push(code("a ``` b ```` c"));
2913 doc.add_paragraph("After.");
2914 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[1, 2]);
2915 assert_stream_matches(&doc, true, ImageMode::Placeholder, &[1, 2]);
2916 }
2917
2918 #[test]
2923 fn mermaid_code_item_replaces_its_picture_placeholder() {
2924 let picture = || Node::Picture {
2925 caption: None,
2926 caption_href: None,
2927 image: None,
2928 classification: None,
2929 description: None,
2930 caption_parent: Default::default(),
2931 caption_location: None,
2932 };
2933 let mut doc = DoclingDocument::new("d");
2934 doc.add_paragraph("Before.");
2935 doc.push(picture());
2936 doc.push(Node::Code {
2937 language: Some("mermaid".into()),
2938 text: "flowchart TD\n A --> B".into(),
2939 orig: None,
2940 pretty: None,
2941 });
2942 doc.add_paragraph("After.");
2943 doc.push(picture());
2944 let md = doc.export_to_markdown();
2945 assert_eq!(
2946 md,
2947 "Before.\n\n```mermaid\nflowchart TD\n A --> B\n```\n\nAfter.\n\n<!-- image -->\n"
2948 );
2949 assert_eq!(doc.export_to_markdown_with(true), md);
2950 for splits in [&[1][..], &[2], &[1, 2], &[2, 3], &[1, 2, 3, 4]] {
2951 assert_stream_matches(&doc, false, ImageMode::Placeholder, splits);
2952 assert_stream_matches(&doc, true, ImageMode::Placeholder, splits);
2953 }
2954 let mut doc = DoclingDocument::new("d");
2956 doc.add_paragraph("Only.");
2957 doc.push(picture());
2958 assert_eq!(doc.export_to_markdown(), "Only.\n\n<!-- image -->\n");
2959 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[1]);
2960 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[2]);
2961 }
2962
2963 #[test]
2964 fn image_placeholder_is_configurable() {
2965 let doc = options_doc();
2966 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2967 image_placeholder: "[figure]".into(),
2968 ..MarkdownExportOptions::default()
2969 });
2970 assert!(md.contains("Figure 1\n\n[figure]\n"), "{md}");
2971 assert!(!md.contains("<!-- image -->"));
2972 }
2973
2974 #[test]
2980 fn empty_image_placeholder_prints_nothing() {
2981 fn picture(caption: Option<&str>) -> Node {
2982 Node::Picture {
2983 caption: caption.map(Into::into),
2984 caption_href: None,
2985 image: None,
2986 classification: None,
2987 description: None,
2988 caption_parent: Default::default(),
2989 caption_location: None,
2990 }
2991 }
2992 fn page(no: usize) -> Node {
2993 Node::PageInfo {
2994 page_no: no,
2995 width: 100.0,
2996 height: 100.0,
2997 }
2998 }
2999 fn para(text: &str) -> Node {
3000 Node::Paragraph { text: text.into() }
3001 }
3002 let options = MarkdownExportOptions {
3003 image_placeholder: String::new(),
3004 ..MarkdownExportOptions::default()
3005 };
3006 let cases: Vec<(Vec<Node>, Option<&str>, &str)> = vec![
3008 (
3009 vec![page(1), para("a"), picture(None), para("b")],
3010 None,
3011 "a\n\nb\n",
3012 ),
3013 (
3014 vec![page(1), para("a"), picture(Some("Figure 1")), para("b")],
3015 None,
3016 "a\n\nFigure 1\n\nb\n",
3017 ),
3018 (
3019 vec![
3020 page(1),
3021 para("a"),
3022 page(2),
3023 picture(None),
3024 page(3),
3025 para("b"),
3026 ],
3027 Some("<!-- pb -->"),
3028 "a\n\n<!-- pb -->\n\n<!-- pb -->\n\nb\n",
3029 ),
3030 (
3031 vec![
3032 page(1),
3033 para("a"),
3034 page(2),
3035 picture(None),
3036 page(3),
3037 para("b"),
3038 ],
3039 Some(""),
3040 "a\n\n\n\n\n\nb\n",
3041 ),
3042 (
3043 vec![page(1), picture(None), page(2), para("b")],
3044 Some("<!-- pb -->"),
3045 "<!-- pb -->\n\nb\n",
3046 ),
3047 (
3048 vec![page(1), picture(None), page(2), para("b")],
3049 Some(""),
3050 "\n\nb\n",
3051 ),
3052 (
3053 vec![page(1), para("a"), page(2), picture(None)],
3054 Some("<!-- pb -->"),
3055 "a\n\n<!-- pb -->\n",
3056 ),
3057 ];
3058 for (nodes, placeholder, expected) in cases {
3059 let mut doc = DoclingDocument::new("t");
3060 doc.page_break_placeholder = placeholder.map(Into::into);
3061 for node in nodes {
3062 doc.push(node);
3063 }
3064 let (md, _) = doc.export_to_markdown_with_options(&options);
3065 assert_eq!(md, expected, "page break {placeholder:?}");
3066 for size in 1..=doc.nodes.len() {
3069 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3070 .with_export_options(&options)
3071 .with_page_break_placeholder(placeholder.map(Into::into));
3072 let mut out = String::new();
3073 for chunk in doc.nodes.chunks(size) {
3074 out.push_str(&streamer.push(chunk, &[]));
3075 }
3076 out.push_str(&streamer.finish());
3077 assert_eq!(
3078 out, expected,
3079 "chunks of {size}, page break {placeholder:?}"
3080 );
3081 }
3082 }
3083 let mut doc = DoclingDocument::new("t");
3085 doc.push(para("a"));
3086 doc.push(picture(None));
3087 doc.push(para("b"));
3088 assert_eq!(doc.export_to_markdown(), "a\n\n<!-- image -->\n\nb\n");
3089 }
3090
3091 #[test]
3094 fn streamer_matches_buffered_export_with_options() {
3095 let doc = options_doc();
3096 let options = MarkdownExportOptions {
3097 layers: ContentLayers::ALL,
3098 traverse_pictures: true,
3099 escape_html: false,
3100 escape_underscores: false,
3101 image_placeholder: "(img)".into(),
3102 ..MarkdownExportOptions::default()
3103 };
3104 let (buffered, _) = doc.export_to_markdown_with_options(&options);
3105 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3106 .with_export_options(&options);
3107 let mut out = String::new();
3108 for chunk in doc.nodes.chunks(2) {
3109 out.push_str(&streamer.push(chunk, &[]));
3110 }
3111 out.push_str(&streamer.finish());
3112 assert_eq!(out, buffered);
3113 assert!(
3114 buffered.contains("(img)") && buffered.contains("R&D _ report"),
3115 "{buffered}"
3116 );
3117 }
3118
3119 #[test]
3122 fn plain_text_reads_escaped_stars() {
3123 assert_eq!(plain_inline("A\\*\\*\\* GmbH"), "A*** GmbH");
3124 assert_eq!(
3125 plain_inline("**FN \\*0\\*271\\*h** and *h\\*.at*"),
3126 "FN *0*271*h and h*.at"
3127 );
3128 }
3129
3130 #[test]
3133 fn plain_text_drops_the_markdown_decoration() {
3134 let item = |text: &str, ordered: bool, number: u64, level: u8| Node::ListItem {
3135 ordered,
3136 number,
3137 first_in_list: number == 1 && level == 0,
3138 text: text.into(),
3139 level,
3140 marker: None,
3141 location: None,
3142 split: None,
3143 dclx: None,
3144 href: None,
3145 layer: None,
3146 };
3147 let mut doc = DoclingDocument::new("plain");
3148 doc.add_heading(1, "Title with **bold**");
3149 doc.add_heading(2, "Section");
3150 doc.add_paragraph("Some **bold**, *italic*, ~~gone~~ and `code` with a [link](https://x.org).\nNext line.");
3151 doc.push(item("*one*", false, 1, 0));
3152 doc.push(item("[two](https://y.org)", false, 2, 0));
3153 doc.push(item("nested", true, 1, 1));
3154 doc.push(Node::CheckboxItem {
3155 checked: true,
3156 text: "done".into(),
3157 });
3158 doc.push(Node::Code {
3159 language: Some("rust".into()),
3160 text: "let x = **y**;".into(),
3161 orig: None,
3162 pretty: None,
3163 });
3164 doc.push(Node::Picture {
3165 caption: Some("Figure *1*".into()),
3166 caption_href: None,
3167 image: None,
3168 classification: None,
3169 description: None,
3170 caption_parent: Default::default(),
3171 caption_location: None,
3172 });
3173 doc.push(Node::Caption {
3174 text: "Linked caption".into(),
3175 href: Some("https://z.org".into()),
3176 });
3177 doc.push(Node::Table(Table {
3178 rows: vec![
3179 vec!["**n**".into(), "rich".into()],
3180 vec!["**2**".into(), "a \nb".into()],
3181 ],
3182 ..Default::default()
3183 }));
3184 doc.add_paragraph("R&D \\_ x");
3185 let text = doc.export_to_text();
3186 assert_eq!(
3187 text,
3188 "Title with bold\n\n\
3189 Section\n\n\
3190 Some bold, italic, gone and code with a link.\nNext line.\n\n\
3191 - one\n- two\n 1. nested\n\n\
3192 - [x] done\n\n\
3193 let x = **y**;\n\n\
3194 Figure 1\n\n\
3195 Linked caption\n\n\
3196 | n | rich |\n\
3197 |-----|--------|\n\
3198 | 2 | a b |\n\n\
3199 R&D _ x"
3200 );
3201
3202 let options = MarkdownExportOptions::plain_text();
3203 let (buffered, _) = doc.export_to_markdown_with_options(&options);
3204 assert_eq!(buffered, format!("{text}\n"));
3205 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3206 .with_export_options(&options);
3207 let mut out = String::new();
3208 for chunk in doc.nodes.chunks(3) {
3209 out.push_str(&streamer.push(chunk, &[]));
3210 }
3211 out.push_str(&streamer.finish());
3212 assert_eq!(out, buffered);
3213 }
3214
3215 #[test]
3217 fn plain_inline_keeps_literal_markers() {
3218 for (input, want) in [
3219 ("**bold** and *it* and ***both***", "bold and it and both"),
3220 ("**[label](https://a.b/c_(d))** end", "label end"),
3221 (
3222 "grant *USE and *OBJMGT authority",
3223 "grant *USE and *OBJMGT authority",
3224 ),
3225 ("2 * 3 * 4", "2 * 3 * 4"),
3226 ("a ** b", "a ** b"),
3227 ("the `quoted’ and `x` term", "the `quoted’ and x term"),
3228 ("see [12] and [docs](u)", "see [12] and docs"),
3229 ("formula $[a](b)$ stays", "formula $[a](b)$ stays"),
3230 ("price $5 and **bold**", "price $5 and bold"),
3231 ("[outer [inner] text](u)", "outer [inner] text"),
3232 ("unclosed **bold", "unclosed **bold"),
3233 ] {
3234 assert_eq!(plain_inline(input), want, "{input}");
3235 }
3236 }
3237}