1use crate::document::{
4 ContentLayer, ContentLayers, DoclingDocument, MarkdownExportOptions, Node, Table,
5};
6
7const MISSING_KEY_VALUE_ITEM: &str = "<!-- missing-key-value-item -->";
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
14pub enum ImageMode {
15 #[default]
17 Placeholder,
18 Embedded,
20 Referenced,
23}
24
25struct Ctx {
27 strict: bool,
28 alt_text: bool,
31 compact_tables: bool,
33 images: ImageMode,
34 artifacts_dir: String,
35 artifacts: Vec<(String, Vec<u8>)>,
37 pic_index: usize,
38 in_table_cell: bool,
42 page_break: Option<String>,
46 pending_page_break: bool,
52 emitted_any: bool,
55 layers: ContentLayers,
57 current_layer: Option<ContentLayer>,
62 traverse_pictures: bool,
64 escape_html: bool,
67 escape_underscores: bool,
69 image_placeholder: String,
71 plain: bool,
73}
74
75impl Ctx {
76 fn new(doc: &DoclingDocument, strict: bool, options: &MarkdownExportOptions) -> Self {
79 Ctx {
80 strict,
81 alt_text: strict || doc.picture_alt_text,
82 compact_tables: doc.compact_tables,
83 images: options.image_mode,
84 artifacts_dir: options.artifacts_dir.clone(),
85 artifacts: Vec::new(),
86 pic_index: 0,
87 in_table_cell: false,
88 page_break: doc.page_break_placeholder.clone(),
89 pending_page_break: false,
90 emitted_any: false,
91 layers: options.layers,
92 current_layer: None,
93 traverse_pictures: options.traverse_pictures,
94 escape_html: options.escape_html,
95 escape_underscores: options.escape_underscores,
96 image_placeholder: options.image_placeholder.clone(),
97 plain: options.plain_text,
98 }
99 }
100
101 fn layer_visible(&self) -> bool {
103 self.layers.contains(self.current_layer)
104 }
105
106 fn unescape(&self, text: &str) -> String {
113 let mut out = std::borrow::Cow::Borrowed(text);
114 if !self.escape_html
115 && (out.contains("&") || out.contains("<") || out.contains(">"))
116 {
117 out = std::borrow::Cow::Owned(
118 out.replace("<", "<")
119 .replace(">", ">")
120 .replace("&", "&"),
121 );
122 }
123 if !self.escape_underscores && out.contains("\\_") {
124 out = std::borrow::Cow::Owned(out.replace("\\_", "_"));
125 }
126 out.into_owned()
127 }
128
129 fn text(&self, text: &str) -> String {
132 let text = strict_text(text, self.strict);
133 if self.plain {
134 self.unescape(&plain_inline(&self.drop_image_markers(&text)))
135 } else {
136 self.unescape(&text)
137 }
138 }
139
140 fn caption(&self, text: &str) -> String {
144 if self.plain {
145 self.unescape(&plain_inline(text))
146 } else {
147 self.unescape(text)
148 }
149 }
150
151 fn breaks(&self, text: &str) -> String {
154 if self.plain {
155 text.to_string()
156 } else {
157 md_line_breaks(text)
158 }
159 }
160
161 fn plain_table(&self, table: &Table) -> Table {
168 let mut plain = table.clone();
169 for cell in plain.rows.iter_mut().flatten() {
170 if cell.contains(['*', '~', '`', '[', '\n', '<']) {
171 let text = self.drop_image_markers(&strip_hard_breaks(cell));
172 *cell = plain_inline(&text);
173 }
174 }
175 plain
176 }
177
178 fn drop_image_markers(&self, text: &str) -> String {
185 const MARKER: &str = "<!-- image -->";
186 if !text.contains(MARKER) && !text.contains("```") {
187 return text.to_string();
188 }
189 text.split("\n\n")
193 .filter_map(|block| {
194 let lines: Vec<String> = block
195 .split('\n')
196 .filter_map(|line| {
197 if line.trim() == MARKER || is_fence_line(line.trim()) {
200 return None;
201 }
202 Some(line.replace(MARKER, &self.image_placeholder))
203 })
204 .collect();
205 (!lines.is_empty()).then(|| lines.join("\n"))
206 })
207 .collect::<Vec<_>>()
208 .join("\n\n")
209 }
210}
211
212pub fn to_markdown(doc: &DoclingDocument, strict: bool) -> String {
218 to_markdown_images(doc, strict, ImageMode::Placeholder, "artifacts").0
219}
220
221pub fn to_markdown_images(
225 doc: &DoclingDocument,
226 strict: bool,
227 images: ImageMode,
228 artifacts_dir: &str,
229) -> (String, Vec<(String, Vec<u8>)>) {
230 to_markdown_with_options(
231 doc,
232 strict,
233 &MarkdownExportOptions {
234 image_mode: images,
235 artifacts_dir: artifacts_dir.to_string(),
236 ..MarkdownExportOptions::default()
237 },
238 )
239}
240
241pub fn to_markdown_with_options(
245 doc: &DoclingDocument,
246 strict: bool,
247 options: &MarkdownExportOptions,
248) -> (String, Vec<(String, Vec<u8>)>) {
249 let mut ctx = Ctx::new(doc, strict, options);
250 let mut blocks: Vec<String> = Vec::new();
251 render(&doc.nodes, &mut blocks, &mut ctx);
252 let mut body = blocks.join("\n\n");
253 if strict && !options.plain_text && !doc.links.is_empty() {
257 body = apply_links(&body, &doc.links);
258 }
259 let front = if doc.metadata_output.front_matter() && !options.plain_text {
262 doc.metadata.to_front_matter()
263 } else {
264 String::new()
265 };
266 let md = match (front.is_empty(), body.is_empty()) {
267 (true, true) => String::new(),
268 (true, false) => format!("{body}\n"),
269 (false, true) => front,
270 (false, false) => format!("{front}\n{body}\n"),
271 };
272 (md, ctx.artifacts)
273}
274
275pub fn to_markdown_table_cell(doc: &DoclingDocument, strict: bool) -> String {
284 let mut ctx = Ctx::new(doc, strict, &MarkdownExportOptions::default());
285 ctx.in_table_cell = true;
286 ctx.artifacts_dir = String::new();
287 ctx.page_break = None;
291 let mut blocks: Vec<String> = Vec::new();
292 render(&doc.nodes, &mut blocks, &mut ctx);
293 blocks.join("\n\n")
294}
295
296fn apply_links(body: &str, links: &[(String, String)]) -> String {
304 let mut out = body.to_string();
305 let mut cursor = 0usize;
306 for (anchor, href) in links {
307 let anchor = anchor
308 .replace('&', "&")
309 .replace('<', "<")
310 .replace('>', ">");
311 if anchor.is_empty() {
312 continue;
313 }
314 if let Some(rel) = out[cursor..].find(&anchor) {
315 let at = cursor + rel;
316 let replacement = crate::hyperlink::markdown_link(&anchor, href);
319 out.replace_range(at..at + anchor.len(), &replacement);
320 cursor = at + replacement.len();
321 }
322 }
323 out
324}
325
326fn apply_links_chunk(chunk: &str, queue: &mut Vec<(String, String)>) -> String {
337 let mut out = chunk.to_string();
338 let mut cursor = 0usize;
339 let mut carried: Vec<(String, String)> = Vec::new();
340 for (anchor_raw, href) in std::mem::take(queue) {
341 let anchor = anchor_raw
342 .replace('&', "&")
343 .replace('<', "<")
344 .replace('>', ">");
345 if anchor.is_empty() {
346 continue;
347 }
348 if let Some(rel) = out[cursor..].find(&anchor) {
349 let at = cursor + rel;
350 let replacement = crate::hyperlink::markdown_link(&anchor, &href);
351 out.replace_range(at..at + anchor.len(), &replacement);
352 cursor = at + replacement.len();
353 } else {
354 carried.push((anchor_raw, href));
356 }
357 }
358 *queue = carried;
359 out
360}
361
362pub struct MarkdownStreamer {
380 strict: bool,
381 images: ImageMode,
382 compact_tables: bool,
383 picture_alt_text: bool,
385 front_matter: Option<String>,
389 emitted_any: bool,
392 links: Vec<(String, String)>,
394 artifacts_dir: String,
398 artifacts: Vec<(String, Vec<u8>)>,
399 pic_index: usize,
400 page_break: Option<String>,
404 pending_page_break: bool,
405 items_emitted: bool,
411 held: Option<Node>,
416 layers: ContentLayers,
419 traverse_pictures: bool,
420 escape_html: bool,
421 escape_underscores: bool,
422 image_placeholder: String,
423 plain: bool,
424}
425
426impl MarkdownStreamer {
427 pub fn new(strict: bool, images: ImageMode, compact_tables: bool) -> Self {
430 debug_assert!(
431 images != ImageMode::Referenced,
432 "referenced image mode needs an artifacts dir; use with_artifacts"
433 );
434 Self::with_artifacts(strict, images, compact_tables, "artifacts")
435 }
436
437 pub fn with_artifacts(
444 strict: bool,
445 images: ImageMode,
446 compact_tables: bool,
447 artifacts_dir: &str,
448 ) -> Self {
449 Self {
450 strict,
451 images,
452 compact_tables,
453 picture_alt_text: false,
454 front_matter: None,
455 emitted_any: false,
456 links: Vec::new(),
457 artifacts_dir: artifacts_dir.to_string(),
458 artifacts: Vec::new(),
459 pic_index: 0,
460 page_break: None,
461 pending_page_break: false,
462 items_emitted: false,
463 held: None,
464 layers: ContentLayers::BODY,
465 traverse_pictures: false,
466 escape_html: true,
467 escape_underscores: true,
468 image_placeholder: "<!-- image -->".to_string(),
469 plain: false,
470 }
471 }
472
473 pub fn with_export_options(mut self, options: &MarkdownExportOptions) -> Self {
480 self.images = options.image_mode;
481 self.artifacts_dir = options.artifacts_dir.clone();
482 self.layers = options.layers;
483 self.traverse_pictures = options.traverse_pictures;
484 self.escape_html = options.escape_html;
485 self.escape_underscores = options.escape_underscores;
486 self.image_placeholder = options.image_placeholder.clone();
487 self.plain = options.plain_text;
488 self
489 }
490
491 pub fn with_page_break_placeholder(mut self, placeholder: Option<String>) -> Self {
496 self.page_break = placeholder;
497 self
498 }
499
500 pub fn with_picture_alt_text(mut self, on: bool) -> Self {
504 self.picture_alt_text = on;
505 self
506 }
507
508 pub fn with_metadata(mut self, metadata: &crate::metadata::DocumentMetadata) -> Self {
513 self.front_matter = (!self.plain)
514 .then(|| metadata.to_front_matter())
515 .filter(|f| !f.is_empty());
516 self
517 }
518
519 pub fn take_artifacts(&mut self) -> Vec<(String, Vec<u8>)> {
524 std::mem::take(&mut self.artifacts)
525 }
526
527 pub fn push(&mut self, nodes: &[Node], links: &[(String, String)]) -> String {
532 self.push_holding(nodes, links, true)
533 }
534
535 fn push_holding(&mut self, nodes: &[Node], links: &[(String, String)], hold: bool) -> String {
538 self.links.extend(links.iter().cloned());
539 let images = self.images;
543 let ends_on_picture = |nodes: &[Node]| {
544 matches!(images, ImageMode::Placeholder)
545 && matches!(nodes.last(), Some(Node::Picture { .. }))
546 };
547 let mut owned: Vec<Node> = Vec::new();
548 let mut nodes = nodes;
549 if self.held.is_some() || (hold && ends_on_picture(nodes)) {
550 owned.extend(self.held.take());
551 owned.extend(nodes.iter().cloned());
552 if hold && ends_on_picture(&owned) {
553 self.held = owned.pop();
554 }
555 nodes = &owned;
556 }
557 let mut ctx = Ctx {
558 strict: self.strict,
559 alt_text: self.strict || self.picture_alt_text,
560 compact_tables: self.compact_tables,
561 images: self.images,
562 artifacts_dir: std::mem::take(&mut self.artifacts_dir),
563 artifacts: std::mem::take(&mut self.artifacts),
564 pic_index: self.pic_index,
565 in_table_cell: false,
566 page_break: std::mem::take(&mut self.page_break),
567 pending_page_break: self.pending_page_break,
568 emitted_any: self.items_emitted,
569 layers: self.layers,
570 current_layer: None,
571 traverse_pictures: self.traverse_pictures,
572 escape_html: self.escape_html,
573 escape_underscores: self.escape_underscores,
574 image_placeholder: std::mem::take(&mut self.image_placeholder),
575 plain: self.plain,
576 };
577 let mut blocks: Vec<String> = Vec::new();
578 render(nodes, &mut blocks, &mut ctx);
579 self.image_placeholder = std::mem::take(&mut ctx.image_placeholder);
580 self.artifacts_dir = std::mem::take(&mut ctx.artifacts_dir);
581 self.artifacts = std::mem::take(&mut ctx.artifacts);
582 self.pic_index = ctx.pic_index;
583 self.page_break = std::mem::take(&mut ctx.page_break);
584 self.pending_page_break = ctx.pending_page_break;
585 self.items_emitted = ctx.emitted_any;
586 if blocks.is_empty() {
587 return String::new();
588 }
589 let mut body = blocks.join("\n\n");
590 if self.strict && !self.plain && !self.links.is_empty() {
591 body = apply_links_chunk(&body, &mut self.links);
592 }
593 let chunk = if let Some(front) = self.front_matter.take() {
594 format!("{front}\n{body}")
595 } else if self.emitted_any {
596 format!("\n\n{body}")
597 } else {
598 body
599 };
600 self.emitted_any = true;
601 chunk
602 }
603
604 pub fn finish(mut self) -> String {
607 let mut out = String::new();
608 if self.held.is_some() {
609 out.push_str(&self.push_holding(&[], &[], false));
610 }
611 if let Some(front) = self.front_matter {
612 out.push_str(&front);
613 } else if self.emitted_any {
614 out.push('\n');
615 }
616 out
617 }
618}
619
620fn strict_text(text: &str, strict: bool) -> String {
628 if !strict {
629 return text.to_string();
630 }
631 text.replace("\\_", "_")
632 .replace(" ,", ",")
633 .replace(" .", ".")
634 .replace(" ;", ";")
635 .replace(" )", ")")
636 .replace("( ", "(")
637 .replace(" ]", "]")
638 .replace("[ ", "[")
639}
640
641pub fn longest_backtick_run(text: &str) -> usize {
644 let (mut best, mut run) = (0, 0);
645 for c in text.chars() {
646 if c == '`' {
647 run += 1;
648 best = best.max(run);
649 } else {
650 run = 0;
651 }
652 }
653 best
654}
655
656pub fn code_span(text: &str) -> String {
662 let ticks = "`".repeat(longest_backtick_run(text) + 1);
663 let pad = if text.starts_with('`') || text.ends_with('`') {
664 " "
665 } else {
666 ""
667 };
668 format!("{ticks}{pad}{text}{pad}{ticks}")
669}
670
671pub fn code_fence(language: &str, body: &str) -> String {
676 let fence = "`".repeat(longest_backtick_run(body).max(2) + 1);
677 format!("{fence}{language}\n{body}\n{fence}")
678}
679
680pub(crate) fn code_span_bounds(chars: &[char], i: usize) -> Option<(usize, usize, usize)> {
688 let n = chars[i..].iter().take_while(|&&c| c == '`').count();
689 if n == 0 {
690 return None;
691 }
692 let mut k = i + n;
693 while k < chars.len() {
694 if chars[k] != '`' {
695 k += 1;
696 continue;
697 }
698 let run = chars[k..].iter().take_while(|&&c| c == '`').count();
699 if run == n {
700 let (mut s, mut e) = (i + n, k);
701 if n > 1
702 && e - s >= 2
703 && chars[s] == ' '
704 && chars[e - 1] == ' '
705 && chars[s..e].iter().any(|&c| c != ' ')
706 {
707 s += 1;
708 e -= 1;
709 }
710 return Some((s, e, k + run));
711 }
712 k += run;
713 }
714 None
715}
716
717fn is_fence_line(line: &str) -> bool {
719 line.len() >= 3 && line.bytes().all(|b| b == b'`')
720}
721
722pub(crate) fn plain_inline(text: &str) -> String {
735 if !text.contains(['*', '~', '`', '[']) {
736 return text.to_string();
737 }
738 let chars: Vec<char> = text.chars().collect();
739 let mut out = String::with_capacity(text.len());
740 plain_inline_into(&chars, &mut out);
741 out
742}
743
744fn plain_inline_into(chars: &[char], out: &mut String) {
745 let find = |open: usize, pat: &[char]| -> Option<usize> {
748 chars
749 .get(open..)?
750 .windows(pat.len())
751 .enumerate()
752 .position(|(k, w)| w == pat && (open + k == 0 || chars[open + k - 1] != '\\'))
753 .map(|p| open + p)
754 };
755 let span = |i: usize, pat: &[char]| -> Option<usize> {
758 if !chars[i..].starts_with(pat) {
759 return None;
760 }
761 let open = i + pat.len();
762 let end = find(open, pat)?;
763 (end > open && !chars[open].is_whitespace() && !chars[end - 1].is_whitespace())
764 .then_some(end)
765 };
766 let mut i = 0;
767 while i < chars.len() {
768 if chars[i] == '\\' && chars.get(i + 1) == Some(&'*') {
770 out.push('*');
771 i += 2;
772 continue;
773 }
774 let mut matched = false;
775 for pat in [&['*', '*', '*'][..], &['*', '*'], &['~', '~'], &['*']] {
776 if let Some(end) = span(i, pat) {
777 plain_inline_into(&chars[i + pat.len()..end], out);
778 i = end + pat.len();
779 matched = true;
780 break;
781 }
782 }
783 if matched {
784 continue;
785 }
786 if chars[i] == '`' {
790 if let Some((s, e, after)) = code_span_bounds(chars, i).filter(|&(s, e, _)| {
791 e > s && !chars[s].is_whitespace() && !chars[e - 1].is_whitespace()
792 }) {
793 out.extend(&chars[s..e]);
794 i = after;
795 continue;
796 }
797 }
798 if chars[i] == '$' {
801 if let Some(end) = find(i + 1, &['$']) {
802 out.extend(&chars[i..=end]);
803 i = end + 1;
804 continue;
805 }
806 }
807 if chars[i] == '[' {
810 if let Some(close) = closing_bracket(chars, i) {
811 if chars.get(close + 1) == Some(&'(') {
812 if let Some(endp) = crate::doclang::link_dest_end(chars, close + 2) {
813 plain_inline_into(&chars[i + 1..close], out);
814 i = endp + 1;
815 continue;
816 }
817 }
818 }
819 }
820 let c = chars[i];
823 out.push(c);
824 i += 1;
825 if matches!(c, '*' | '~') {
826 while i < chars.len() && chars[i] == c {
827 out.push(c);
828 i += 1;
829 }
830 }
831 }
832}
833
834fn closing_bracket(chars: &[char], open: usize) -> Option<usize> {
837 let mut depth = 0usize;
838 for (k, c) in chars.iter().enumerate().skip(open + 1) {
839 match c {
840 '[' => depth += 1,
841 ']' if depth == 0 => return Some(k),
842 ']' => depth -= 1,
843 _ => {}
844 }
845 }
846 None
847}
848
849fn md_line_breaks(text: &str) -> String {
857 if !text.contains('\n') {
858 return text.to_string();
859 }
860 text.split("\n\n")
861 .map(|para| para.replace('\n', " \n"))
862 .collect::<Vec<_>>()
863 .join("\n\n")
864}
865
866pub(crate) fn strip_hard_breaks(text: &str) -> String {
871 if text.contains(" \n") {
872 text.replace(" \n", "\n")
873 } else {
874 text.to_string()
875 }
876}
877
878fn heading_line_breaks(text: &str) -> String {
882 text.replace('\n', " ")
883}
884
885fn render(nodes: &[Node], blocks: &mut Vec<String>, ctx: &mut Ctx) {
886 let mut i = 0;
887 while i < nodes.len() {
888 let before = blocks.len();
889 match &nodes[i] {
890 Node::PageBreak | Node::PageInfo { .. } => {
900 if ctx.page_break.is_some() && ctx.emitted_any {
901 ctx.pending_page_break = true;
902 }
903 i += 1;
904 }
905 Node::ListItem { .. } => {
906 let start = i;
907 i += 1;
908 loop {
909 match nodes.get(i) {
910 Some(Node::ListItem { .. } | Node::ListText { .. }) => i += 1,
911 Some(Node::Paragraph { text })
915 if text.is_empty()
916 && matches!(nodes.get(i + 1), Some(Node::ListItem { .. })) =>
917 {
918 i += 1
919 }
920 _ => break,
921 }
922 }
923 render_list_run(&nodes[start..i], blocks, ctx);
924 }
925 other => {
926 let before_item = blocks.len();
927 render_one(other, blocks, ctx);
928 if matches!(other, Node::Picture { .. })
934 && matches!(ctx.images, ImageMode::Placeholder)
935 && blocks.len() > before_item
936 && matches!(
937 nodes.get(i + 1),
938 Some(Node::Code { language: Some(l), .. }) if l == "mermaid"
939 )
940 {
941 blocks.pop();
942 }
943 i += 1;
944 }
945 }
946 if blocks.len() > before {
947 let kept: Vec<String> = blocks.drain(before..).filter(|b| !b.is_empty()).collect();
957 blocks.extend(kept);
958 if ctx.pending_page_break {
959 if let Some(placeholder) = &ctx.page_break {
964 blocks.insert(before, placeholder.clone());
965 }
966 ctx.pending_page_break = false;
967 }
968 ctx.emitted_any = true;
969 }
970 }
971}
972
973fn render_list_run(items: &[Node], blocks: &mut Vec<String>, ctx: &Ctx) {
979 let mut lines: Vec<String> = Vec::new();
980 let mut any_top = false;
983
984 for item in items {
985 if let Node::ListText { level, text, glue } = item {
986 let text = ctx.text(text);
987 match lines.last_mut() {
988 Some(last) if *glue => last.push_str(&text),
989 _ => lines.push(format!("{}{text}", " ".repeat(*level as usize))),
990 }
991 continue;
992 }
993 let Node::ListItem {
994 ordered,
995 number,
996 first_in_list,
997 text,
998 level,
999 marker: orig_marker,
1000 location: _,
1001 dclx: _,
1002 href: _,
1003 layer,
1004 } = item
1005 else {
1006 continue;
1007 };
1008 if !ctx.layers.contains(layer.or(ctx.current_layer)) {
1012 continue;
1013 }
1014 let level = *level as usize;
1015
1016 if level == 0 {
1025 if any_top && *first_in_list {
1026 lines.push(String::new());
1027 }
1028 any_top = true;
1029 }
1030
1031 let indent = " ".repeat(level);
1032 lines.push(format!(
1033 "{indent}{} {}",
1034 list_item_marker(orig_marker.as_deref().unwrap_or(""), *ordered, *number),
1035 list_item_text(text, ctx)
1036 ));
1037 }
1038
1039 if !lines.is_empty() {
1042 blocks.push(lines.join("\n"));
1043 }
1044}
1045
1046fn list_item_marker(marker: &str, ordered: bool, number: u64) -> String {
1055 let already_valid = matches!(marker, "-" | "*" | "+")
1056 || marker
1057 .strip_suffix('.')
1058 .is_some_and(|d| !d.is_empty() && d.chars().all(char::is_numeric));
1059 if already_valid {
1060 return marker.to_string();
1061 }
1062 let outer = if ordered && marker.is_empty() {
1063 format!("{number}.")
1064 } else {
1065 "-".to_string()
1066 };
1067 if marker.chars().any(char::is_alphanumeric) {
1068 format!("{outer} {marker}")
1069 } else {
1070 outer
1071 }
1072}
1073
1074fn list_item_text(text: &str, ctx: &Ctx) -> String {
1082 let escaped = ctx.text(text);
1083 if ctx.plain {
1084 return escaped;
1085 }
1086 if let Some((own, tail)) = escaped.split_once('\n') {
1087 if is_folded_child_tail(tail) {
1088 return format!("{}\n{tail}", md_line_breaks(own));
1089 }
1090 }
1091 md_line_breaks(&escaped)
1092}
1093
1094fn is_folded_child_tail(tail: &str) -> bool {
1101 const MARKER: &str = "<!-- image -->";
1102 let mut lines = tail.split('\n').peekable();
1103 let mut any = false;
1104 while let Some(line) = lines.next() {
1105 let line = line.trim_start();
1106 if line == MARKER {
1107 any = true;
1108 } else if is_fence_line(line) {
1109 loop {
1112 match lines.next() {
1113 Some(l) if l.trim_start() == line => break,
1114 Some(_) => {}
1115 None => return false,
1116 }
1117 }
1118 any = true;
1119 } else if lines.next().map(str::trim_start) == Some(MARKER) {
1120 any = true; } else {
1122 return false;
1123 }
1124 }
1125 any
1126}
1127
1128fn render_one(node: &Node, blocks: &mut Vec<String>, ctx: &mut Ctx) {
1133 match node {
1134 Node::Group { .. }
1136 | Node::Furniture { .. }
1137 | Node::PageFurniture { .. }
1138 | Node::FurnitureText { .. }
1139 | Node::CommentSection { .. }
1140 | Node::PictureChildren(_)
1141 | Node::Commented { .. }
1142 | Node::Located { .. }
1143 | Node::Prov { .. }
1144 | Node::Track { .. }
1145 | Node::PageBreak
1146 | Node::PageInfo { .. }
1147 | Node::DoclangOnly(_) => {}
1148 _ if !ctx.layer_visible() => return,
1149 _ => {}
1150 }
1151 render_item(node, blocks, ctx)
1152}
1153
1154fn render_item(node: &Node, blocks: &mut Vec<String>, ctx: &mut Ctx) {
1155 match node {
1156 Node::Heading { level, text } => {
1157 if ctx.plain {
1158 blocks.push(ctx.text(text));
1161 return;
1162 }
1163 let text = heading_line_breaks(&ctx.text(text));
1164 if ctx.in_table_cell {
1165 blocks.push(text);
1167 } else {
1168 let hashes = "#".repeat((*level).clamp(1, 6) as usize);
1169 blocks.push(format!("{hashes} {text}"));
1170 }
1171 }
1172 Node::Paragraph { text } if text.is_empty() => {}
1175 Node::Paragraph { text } => blocks.push(ctx.breaks(&ctx.text(text))),
1176 Node::LabeledText { .. } => render_item(&node.labeled_as_paragraph(), blocks, ctx),
1177 Node::Caption { text, .. } if text.is_empty() => {}
1180 Node::Caption { text, href } => {
1181 let body = ctx.breaks(&ctx.text(text));
1182 blocks.push(match href {
1183 Some(url) if !ctx.plain => crate::hyperlink::markdown_link(&body, url),
1184 _ => body,
1185 });
1186 }
1187 Node::CheckboxItem { checked, text } => {
1188 let mark = if *checked { "- [x] " } else { "- [ ] " };
1189 blocks.push(ctx.breaks(&ctx.text(&format!("{mark}{text}"))));
1190 }
1191 Node::Code {
1192 language,
1193 text,
1194 pretty,
1195 ..
1196 } => {
1197 let lang = match language {
1202 Some(l) if ctx.strict || l == "mermaid" => l.as_str(),
1203 _ => "",
1204 };
1205 let body = match pretty {
1208 Some(p) if ctx.strict => p.as_str(),
1209 _ => text.as_str(),
1210 };
1211 blocks.push(if ctx.plain {
1214 body.to_string()
1215 } else {
1216 code_fence(lang, body)
1217 });
1218 }
1219 Node::Formula { latex, .. } => blocks.push(format!("$${latex}$$")),
1222 Node::Table(table) => {
1223 if let Some(cap) = &table.caption {
1226 if !cap.is_empty() {
1227 blocks.push(ctx.text(cap));
1228 }
1229 }
1230 let rendered = if ctx.plain {
1231 render_table(&ctx.plain_table(table), ctx.compact_tables)
1232 } else {
1233 render_table(table, ctx.compact_tables)
1234 };
1235 if !rendered.is_empty() {
1236 blocks.push(rendered);
1237 }
1238 }
1239 Node::Picture {
1248 caption,
1249 image,
1250 description,
1251 ..
1252 } => {
1253 if let Some(cap) = caption {
1254 if !cap.is_empty() {
1255 blocks.push(ctx.caption(cap));
1256 }
1257 }
1258 let alt = description
1259 .as_ref()
1260 .filter(|d| d.is_alt_text() && ctx.alt_text && !d.text.is_empty());
1261 let marker = picture_marker(image.as_ref(), ctx, alt.map(|d| d.text.as_str()));
1262 if let Some(desc) = description {
1263 let shown = !desc.is_alt_text() || (ctx.alt_text && !marker.starts_with("!["));
1264 if !desc.text.is_empty() && shown {
1265 blocks.push(ctx.breaks(&ctx.text(&desc.text)));
1266 }
1267 }
1268 blocks.push(marker);
1269 }
1270 Node::Chart {
1274 kind,
1275 table,
1276 caption,
1277 ..
1278 } => {
1279 if let Some(cap) = caption {
1280 if !cap.is_empty() {
1281 blocks.push(ctx.caption(cap));
1282 }
1283 }
1284 blocks.push(picture_marker(None, ctx, None));
1285 blocks.push(humanize_label(kind));
1286 let rendered = if ctx.plain {
1287 render_table(&ctx.plain_table(table), false)
1288 } else {
1289 render_table(table, false)
1290 };
1291 if !rendered.is_empty() {
1292 blocks.push(rendered);
1293 }
1294 }
1295 Node::DoclangOnly(_) => {}
1297 Node::Group {
1302 layer: Some(layer),
1303 children,
1304 ..
1305 } => {
1306 let outer = ctx.current_layer.replace(*layer);
1307 render(children, blocks, ctx);
1308 ctx.current_layer = outer;
1309 }
1310 Node::Group { children, .. } => render(children, blocks, ctx),
1311 Node::FieldRegion { items } => {
1312 for item in items {
1317 for part in [&item.marker, &item.key, &item.value].into_iter().flatten() {
1318 blocks.push(ctx.breaks(&ctx.text(part)));
1319 }
1320 }
1321 }
1322 Node::KeyValueGraph { .. } => blocks.push(MISSING_KEY_VALUE_ITEM.to_string()),
1325 Node::InlineGroup { md_text, .. } => blocks.push(ctx.breaks(&ctx.text(md_text))),
1328 Node::TextDump(text) => {
1330 if !text.is_empty() {
1331 blocks.push(text.clone());
1332 }
1333 }
1334 Node::Furniture { layer, inner } => {
1338 let outer = ctx.current_layer.replace(*layer);
1339 render_one(inner, blocks, ctx);
1340 ctx.current_layer = outer;
1341 }
1342 Node::PageFurniture { text, .. } | Node::FurnitureText { text, .. } => {
1346 if ctx.layers.furniture && !text.is_empty() {
1347 blocks.push(ctx.breaks(&ctx.text(text)));
1348 }
1349 }
1350 Node::PictureChildren(children) => {
1355 if ctx.traverse_pictures {
1356 render(children, blocks, ctx);
1357 }
1358 }
1359 Node::CommentSection { text, .. } => {
1364 if ctx.layers.notes && !text.is_empty() {
1365 blocks.push(ctx.breaks(&ctx.text(text)));
1366 }
1367 }
1368 Node::Commented { inner, .. } => render_one(inner, blocks, ctx),
1369 Node::Located { inner, .. } | Node::Prov { inner, .. } | Node::Track { inner, .. } => {
1371 render_one(inner, blocks, ctx)
1372 }
1373 Node::PageBreak => {}
1375 Node::PageInfo { .. } => {}
1377 Node::Anchor { .. } => {}
1379 Node::ListItem { .. } => render_list_run(std::slice::from_ref(node), blocks, ctx),
1384 Node::ListText { text, .. } if text.is_empty() => {}
1386 Node::ListText { text, .. } => blocks.push(ctx.breaks(&ctx.text(text))),
1387 }
1388}
1389
1390fn humanize_label(label: &str) -> String {
1395 let text = label.replace('_', " ");
1396 let mut chars = text.chars();
1397 match chars.next() {
1398 Some(f) => f.to_uppercase().collect::<String>() + chars.as_str(),
1399 None => text,
1400 }
1401}
1402
1403fn image_alt(alt: Option<&str>) -> String {
1406 match alt {
1407 Some(text) => text
1408 .split_whitespace()
1409 .collect::<Vec<_>>()
1410 .join(" ")
1411 .replace('\\', "\\\\")
1412 .replace(']', "\\]"),
1413 None => "Image".to_string(),
1414 }
1415}
1416
1417fn picture_marker(image: Option<&crate::PictureImage>, ctx: &mut Ctx, alt: Option<&str>) -> String {
1418 match (ctx.images, image) {
1419 (ImageMode::Embedded, Some(img)) => {
1421 format!(
1422 "",
1423 image_alt(alt),
1424 crate::pixel_digest::docling_data_uri(img).1
1425 )
1426 }
1427 (ImageMode::Referenced, Some(img)) => {
1428 let path = format!(
1429 "{}/image_{:06}.{}",
1430 ctx.artifacts_dir,
1431 ctx.pic_index,
1432 ext_for(&img.mimetype)
1433 );
1434 ctx.pic_index += 1;
1435 ctx.artifacts.push((path.clone(), img.data.clone()));
1436 format!("", image_alt(alt), escape_uri_path(&path))
1437 }
1438 _ => ctx.image_placeholder.clone(),
1440 }
1441}
1442
1443pub(crate) fn escape_uri_path(value: &str) -> String {
1456 const KEEP: &str = "/%:@+,;=~$!&'*";
1457 let s = value.replace('\\', "/");
1458 if let Some(rest) = s.strip_prefix("//") {
1459 let rest = rest.trim_start_matches('/');
1461 let (host, tail) = rest.split_once('/').unwrap_or((rest, ""));
1462 return format!("file://{host}{}", percent_quote(&format!("/{tail}"), KEEP));
1463 }
1464 let bytes = s.as_bytes();
1465 if bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' && bytes[2] == b'/' {
1466 return format!("file:///{}", percent_quote(&s, KEEP));
1468 }
1469 if let Some((scheme, rest)) = s.split_once(':') {
1473 let valid_scheme = scheme.len() > 1
1474 && scheme.as_bytes()[0].is_ascii_alphabetic()
1475 && scheme
1476 .bytes()
1477 .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.'));
1478 if valid_scheme {
1479 let (authority, rest) = match rest.strip_prefix("//") {
1480 Some(r) => {
1481 let end = r.find(['/', '?', '#']).unwrap_or(r.len());
1482 (Some(&r[..end]), &r[end..])
1483 }
1484 None => (None, rest),
1485 };
1486 let (before_frag, fragment) = rest.split_once('#').unwrap_or((rest, ""));
1487 let (path, query) = before_frag.split_once('?').unwrap_or((before_frag, ""));
1488 let mut out = format!("{scheme}:");
1489 if let Some(a) = authority {
1490 out.push_str("//");
1491 out.push_str(a);
1492 }
1493 out.push_str(&percent_quote(path, KEEP));
1494 if !query.is_empty() {
1495 out.push('?');
1496 out.push_str(&percent_quote(query, KEEP));
1497 }
1498 if !fragment.is_empty() {
1499 out.push('#');
1500 out.push_str(&percent_quote(fragment, KEEP));
1501 }
1502 return out;
1503 }
1504 }
1505 percent_quote(&s, KEEP)
1507}
1508
1509fn percent_quote(s: &str, safe: &str) -> String {
1512 let mut out = String::with_capacity(s.len());
1513 for &b in s.as_bytes() {
1514 let keep = b.is_ascii_alphanumeric()
1515 || matches!(b, b'_' | b'.' | b'-' | b'~')
1516 || (b.is_ascii() && safe.contains(b as char));
1517 if keep {
1518 out.push(b as char);
1519 } else {
1520 out.push_str(&format!("%{b:02X}"));
1521 }
1522 }
1523 out
1524}
1525
1526pub(crate) fn ext_for(mimetype: &str) -> &str {
1527 match mimetype {
1528 "image/jpeg" => "jpg",
1529 "image/gif" => "gif",
1530 "image/webp" => "webp",
1531 "image/bmp" => "bmp",
1532 "image/tiff" => "tif",
1533 "image/svg+xml" => "svg",
1534 _ => "png",
1535 }
1536}
1537
1538fn is_number_cell(t: &str) -> bool {
1557 t.parse::<f64>().is_ok() || is_thousands_number(t)
1558}
1559
1560fn is_thousands_number(t: &str) -> bool {
1566 let b = t.as_bytes();
1567 let mut i = 0;
1568 let start = i;
1569 if i < b.len() && (b[i] == b'+' || b[i] == b'-') {
1570 i += 1;
1571 }
1572 let d0 = i;
1574 while i < b.len() && b[i].is_ascii_digit() && i - d0 < 3 {
1575 i += 1;
1576 }
1577 let has_int = i > d0;
1578 if has_int {
1579 while i + 3 < b.len() + 1
1581 && b.get(i) == Some(&b',')
1582 && b.get(i + 1).is_some_and(u8::is_ascii_digit)
1583 && b.get(i + 2).is_some_and(u8::is_ascii_digit)
1584 && b.get(i + 3).is_some_and(u8::is_ascii_digit)
1585 {
1586 i += 4;
1587 }
1588 } else {
1589 i = start;
1591 }
1592 if i < b.len() && b[i] == b'.' {
1594 i += 1;
1595 let f0 = i;
1596 while i < b.len() && b[i].is_ascii_digit() {
1597 i += 1;
1598 }
1599 if !has_int && i == f0 {
1600 return false; }
1602 } else if !has_int {
1603 return false; }
1605 i == b.len()
1606}
1607
1608fn flatten_header_rows(header_rows: &[Vec<String>], num_cols: usize) -> Vec<String> {
1617 (0..num_cols)
1618 .map(|c| {
1619 let mut parts: Vec<&str> = Vec::new();
1620 for row in header_rows {
1621 let text = row.get(c).map(String::as_str).unwrap_or("");
1622 if !text.is_empty() && parts.last() != Some(&text) {
1623 parts.push(text);
1624 }
1625 }
1626 parts.join(" - ")
1627 })
1628 .collect()
1629}
1630
1631pub(crate) fn render_table(table: &Table, compact: bool) -> String {
1632 if table.rows.is_empty() {
1633 return String::new();
1634 }
1635 let num_cols = table.rows.iter().map(Vec::len).max().unwrap_or(0);
1636 if num_cols == 0 {
1637 return String::new();
1638 }
1639
1640 let num_headers = table.header_row_count().min(table.rows.len());
1645 let escaped = |r: usize| -> Vec<String> {
1646 (0..num_cols)
1647 .map(|c| escape_cell(table.rows[r].get(c).map(String::as_str).unwrap_or("")))
1648 .collect()
1649 };
1650 let header_rows: Vec<Vec<String>> = (0..num_headers).map(escaped).collect();
1651 let header = flatten_header_rows(&header_rows, num_cols);
1652 let body: Vec<Vec<String>> = (num_headers..table.rows.len())
1653 .map(|r| {
1654 escaped(r)
1655 .into_iter()
1656 .map(|c| c.trim().to_string())
1657 .collect()
1658 })
1659 .collect();
1660
1661 if compact {
1662 let render_row = |row: &[String]| -> String { format!("| {} |", row.join(" | ")) };
1664 let mut lines = Vec::with_capacity(body.len() + 2);
1665 lines.push(render_row(&header));
1666 let sep: Vec<&str> = (0..num_cols).map(|_| "-").collect();
1667 lines.push(format!("| {} |", sep.join(" | ")));
1668 for row in &body {
1669 lines.push(render_row(row));
1670 }
1671 return lines.join("\n");
1672 }
1673
1674 let dw = |s: &str| s.chars().count();
1676
1677 let right: Vec<bool> = (0..num_cols)
1682 .map(|c| {
1683 let mut any = false;
1684 for row in &body {
1685 let t = row[c].trim();
1686 if t.is_empty() {
1687 continue;
1688 }
1689 if !is_number_cell(t) {
1690 return false;
1691 }
1692 any = true;
1693 }
1694 any
1695 })
1696 .collect();
1697
1698 let width: Vec<usize> = (0..num_cols)
1700 .map(|c| {
1701 let mut w = dw(&header[c]) + 2;
1702 for row in &body {
1703 w = w.max(dw(&row[c]));
1704 }
1705 w
1706 })
1707 .collect();
1708
1709 let fmt_cell = |s: &str, c: usize| -> String {
1710 let pad = " ".repeat(width[c].saturating_sub(dw(s)));
1711 let body = if right[c] {
1712 format!("{pad}{s}")
1713 } else {
1714 format!("{s}{pad}")
1715 };
1716 format!(" {body} ")
1717 };
1718 let render_row = |row: &[String]| -> String {
1719 let cells: Vec<String> = (0..num_cols).map(|c| fmt_cell(&row[c], c)).collect();
1720 format!("|{}|", cells.join("|"))
1721 };
1722
1723 let mut lines = Vec::with_capacity(body.len() + 2);
1724 lines.push(render_row(&header));
1725 let sep: Vec<String> = (0..num_cols).map(|c| "-".repeat(width[c] + 2)).collect();
1726 lines.push(format!("|{}|", sep.join("|")));
1727 for row in &body {
1728 lines.push(render_row(row));
1729 }
1730 lines.join("\n")
1731}
1732
1733fn escape_cell(s: &str) -> String {
1736 s.replace('\n', " ").replace('|', "|")
1737}
1738
1739#[cfg(test)]
1740mod tests {
1741 use super::*;
1742 use crate::{PictureImage, TableCell, TableStructure};
1743
1744 #[test]
1750 fn list_boundaries_come_from_the_backend_not_the_numbering() {
1751 let item = |ordered: bool, number: u64, first_in_list: bool, text: &str| Node::ListItem {
1752 ordered,
1753 number,
1754 first_in_list,
1755 text: text.into(),
1756 level: 0,
1757 marker: None,
1758 location: None,
1759 dclx: None,
1760 href: None,
1761 layer: None,
1762 };
1763 let md = |items: Vec<Node>| {
1764 let mut doc = DoclingDocument::new("t");
1765 for n in items {
1766 doc.push(n);
1767 }
1768 doc.export_to_markdown()
1769 };
1770 assert_eq!(
1772 md(vec![
1773 item(true, 1, true, "one"),
1774 item(true, 5, false, "five")
1775 ]),
1776 "1. one\n5. five\n"
1777 );
1778 assert_eq!(
1780 md(vec![
1781 item(false, 0, true, "bullet"),
1782 item(true, 1, false, "one"),
1783 item(false, 0, false, "bullet two"),
1784 ]),
1785 "- bullet\n1. one\n- bullet two\n"
1786 );
1787 assert_eq!(
1789 md(vec![
1790 item(true, 1, true, "a"),
1791 item(true, 2, false, "b"),
1792 item(true, 3, true, "new list, continuing count"),
1793 ]),
1794 "1. a\n2. b\n\n3. new list, continuing count\n"
1795 );
1796 }
1797
1798 #[test]
1799 fn renders_headings_paragraphs_and_lists() {
1800 let mut doc = DoclingDocument::new("demo");
1801 doc.add_heading(1, "Title");
1802 doc.add_paragraph("Hello world.");
1803 doc.push(Node::ListItem {
1804 ordered: false,
1805 number: 1,
1806 first_in_list: true,
1807 text: "first".into(),
1808 level: 0,
1809 marker: None,
1810 location: None,
1811 dclx: None,
1812 href: None,
1813 layer: None,
1814 });
1815 doc.push(Node::ListItem {
1816 ordered: false,
1817 number: 2,
1818 first_in_list: false,
1819 text: "second".into(),
1820 level: 0,
1821 marker: None,
1822 location: None,
1823 dclx: None,
1824 href: None,
1825 layer: None,
1826 });
1827 let md = doc.export_to_markdown();
1828 assert_eq!(md, "# Title\n\nHello world.\n\n- first\n- second\n");
1829 }
1830
1831 #[test]
1835 fn single_newlines_become_gfm_hard_line_breaks() {
1836 let mut doc = DoclingDocument::new("t");
1837 doc.push(Node::Heading {
1838 level: 1,
1839 text: "Hello\nWorld".into(),
1840 });
1841 doc.push(Node::Paragraph {
1842 text: "line one\nline two\n\npara two".into(),
1843 });
1844 doc.push(Node::ListItem {
1845 ordered: false,
1846 number: 1,
1847 first_in_list: true,
1848 text: "item\ncontinued".into(),
1849 level: 0,
1850 marker: None,
1851 location: None,
1852 dclx: None,
1853 href: None,
1854 layer: None,
1855 });
1856 doc.push(Node::TextDump("A1 B1 \n\n\nC1".into()));
1857 assert_eq!(
1858 doc.export_to_markdown(),
1859 "# Hello World\n\nline one \nline two\n\npara two\n\n- item \ncontinued\n\nA1 B1 \n\n\nC1\n"
1860 );
1861 }
1862
1863 #[test]
1866 fn table_cell_mode_and_field_regions() {
1867 let mut doc = DoclingDocument::new("t");
1868 doc.push(Node::Heading {
1869 level: 2,
1870 text: "A text".into(),
1871 });
1872 doc.push(Node::Paragraph {
1873 text: "body".into(),
1874 });
1875 assert_eq!(to_markdown_table_cell(&doc, false), "A text\n\nbody");
1876 assert_eq!(doc.export_to_markdown(), "## A text\n\nbody\n");
1877
1878 let mut doc = DoclingDocument::new("f");
1879 doc.push(Node::FieldRegion {
1880 items: vec![crate::FieldItem {
1881 marker: None,
1882 key: Some("Name:".into()),
1883 value: Some("John Doe".into()),
1884 value_kind: None,
1885 }],
1886 });
1887 assert_eq!(doc.export_to_markdown(), "Name:\n\nJohn Doe\n");
1888 }
1889
1890 #[test]
1891 fn strict_renders_recovered_links_legacy_does_not() {
1892 let mut doc = DoclingDocument::new("cv");
1893 doc.add_paragraph("Find me on LinkedIn or GitHub.");
1894 doc.links = vec![
1895 ("LinkedIn".into(), "https://www.linkedin.com/in/x/".into()),
1896 ("GitHub".into(), "https://github.com/x/".into()),
1897 ];
1898 assert_eq!(doc.export_to_markdown(), "Find me on LinkedIn or GitHub.\n");
1900 assert_eq!(
1902 doc.export_to_markdown_with(true),
1903 "Find me on [LinkedIn](https://www.linkedin.com/in/x/) or [GitHub](https://github.com/x/).\n"
1904 );
1905 }
1906
1907 #[test]
1908 fn strict_links_match_escaped_anchor_and_consume_in_order() {
1909 let mut doc = DoclingDocument::new("d");
1910 doc.add_paragraph("AI & ML here, and issues here, then issues there.");
1914 doc.links = vec![
1915 ("AI & ML".into(), "https://a/".into()),
1916 ("issues".into(), "https://first/".into()),
1917 ("issues".into(), "https://second/".into()),
1918 ];
1919 assert_eq!(
1920 doc.export_to_markdown_with(true),
1921 "[AI & ML](https://a/) here, and [issues](https://first/) here, then [issues](https://second/) there.\n"
1922 );
1923 }
1924
1925 #[test]
1929 fn referenced_image_destinations_are_escaped() {
1930 let cases = [
1931 (
1932 "doc_artifacts/image_000001_ab12.png",
1933 "doc_artifacts/image_000001_ab12.png",
1934 ),
1935 (
1936 "My Report_artifacts/img.png",
1937 "My%20Report_artifacts/img.png",
1938 ),
1939 ("artifacts/img (1).png", "artifacts/img%20%281%29.png"),
1940 ("100%_scale/a#b?c.png", "100%_scale/a%23b%3Fc.png"),
1941 ("/home/a b/img.png", "/home/a%20b/img.png"),
1942 (
1943 "My Report_artifacts\\img.png",
1944 "My%20Report_artifacts/img.png",
1945 ),
1946 (
1947 "C:/Users/me/My Docs/img.png",
1948 "file:///C:/Users/me/My%20Docs/img.png",
1949 ),
1950 ("C:\\Users\\me\\img.png", "file:///C:/Users/me/img.png"),
1951 (
1952 "//server/share/My Docs/img.png",
1953 "file://server/share/My%20Docs/img.png",
1954 ),
1955 ("\\\\server\\share\\img.png", "file://server/share/img.png"),
1956 ("file:///home/a b/img.png", "file:///home/a%20b/img.png"),
1957 (
1958 "s3://bucket/My Report_artifacts/img.png",
1959 "s3://bucket/My%20Report_artifacts/img.png",
1960 ),
1961 (
1962 "https://example.com:8080/a b.png?w=1&h=2#frag",
1963 "https://example.com:8080/a%20b.png?w=1&h=2#frag",
1964 ),
1965 (
1966 "https://example.com/img (1).png",
1967 "https://example.com/img%20%281%29.png",
1968 ),
1969 ("caf\u{e9}/im\u{e4}ge.png", "caf%C3%A9/im%C3%A4ge.png"),
1970 ];
1971 for (input, expected) in cases {
1972 assert_eq!(escape_uri_path(input), expected, "input {input:?}");
1973 assert_eq!(
1974 escape_uri_path(expected),
1975 expected,
1976 "idempotent {expected:?}"
1977 );
1978 }
1979 let mut doc = DoclingDocument::new("t");
1981 doc.push(Node::Picture {
1982 caption: None,
1983 caption_href: None,
1984 image: Some(PictureImage {
1985 dpi: PictureImage::DEFAULT_DPI,
1986 mimetype: "image/png".into(),
1987 width: 1,
1988 height: 1,
1989 data: b"x".to_vec(),
1990 display_size: None,
1991 }),
1992 classification: None,
1993 description: None,
1994 caption_parent: Default::default(),
1995 caption_location: None,
1996 });
1997 let (md, files) = doc
1998 .export_to_markdown_with_images(ImageMode::Referenced, "My Report (final)_artifacts");
1999 assert!(
2000 md.contains(""),
2001 "got:\n{md}"
2002 );
2003 assert_eq!(files[0].0, "My Report (final)_artifacts/image_000000.png");
2005 }
2006
2007 #[test]
2011 fn folded_list_item_pictures_keep_plain_newlines() {
2012 let doc = DoclingDocument::new("t");
2013 let ctx = Ctx::new(&doc, false, &MarkdownExportOptions::default());
2014 assert_eq!(
2015 list_item_text("Step\n<!-- image -->", &ctx),
2016 "Step\n<!-- image -->"
2017 );
2018 assert_eq!(
2019 list_item_text("Step\nAlt text\n<!-- image -->\n<!-- image -->", &ctx),
2020 "Step\nAlt text\n<!-- image -->\n<!-- image -->"
2021 );
2022 assert_eq!(
2023 list_item_text("line one\nline two", &ctx),
2024 "line one \nline two"
2025 );
2026 }
2027
2028 #[test]
2031 fn stacked_header_rows_flatten_into_one() {
2032 let mut t = Table {
2033 rows: vec![
2034 vec!["".into(), "% of Total".into(), "% of Total".into()],
2035 vec!["class".into(), "Train".into(), "Test".into()],
2036 vec!["Caption".into(), "2.04".into(), "1.77".into()],
2037 ],
2038 ..Default::default()
2039 };
2040 t.structure = Some(TableStructure {
2041 header_row: vec![true, true, false],
2042 col_continuation: vec![
2043 vec![false, false, true],
2044 vec![false, false, false],
2045 vec![false, false, false],
2046 ],
2047 ..Default::default()
2048 });
2049 assert_eq!(t.header_row_count(), 2);
2050 assert_eq!(
2051 render_table(&t, true),
2052 "| class | % of Total - Train | % of Total - Test |\n| - | - | - |\n| Caption | 2.04 | 1.77 |"
2053 );
2054 assert_eq!(
2056 render_table(&t, false),
2057 "| class | % of Total - Train | % of Total - Test |\n\
2058 |---------|----------------------|---------------------|\n\
2059 | Caption | 2.04 | 1.77 |"
2060 );
2061 }
2062
2063 #[test]
2066 fn vertically_spanning_header_does_not_extend_the_block() {
2067 let mut t = Table {
2068 rows: vec![
2069 vec!["Name".into(), "Value".into()],
2070 vec!["Name".into(), "1".into()],
2071 vec!["x".into(), "2".into()],
2072 ],
2073 ..Default::default()
2074 };
2075 t.structure = Some(TableStructure {
2076 col_header: vec![vec![true, true], vec![true, false], vec![false, false]],
2077 row_continuation: vec![vec![false, false], vec![true, false], vec![false, false]],
2078 ..Default::default()
2079 });
2080 assert_eq!(t.header_row_count(), 1);
2081 assert_eq!(
2082 render_table(&t, true),
2083 "| Name | Value |\n| - | - |\n| Name | 1 |\n| x | 2 |"
2084 );
2085 }
2086
2087 #[test]
2090 fn header_flags_not_on_row_zero_keep_all_rows_in_the_body() {
2091 let mut t = Table {
2092 rows: vec![
2093 vec!["1".into(), "2".into()],
2094 vec!["a".into(), "b".into()],
2095 vec!["333".into(), "4".into()],
2096 ],
2097 ..Default::default()
2098 };
2099 t.structure = Some(TableStructure {
2100 header_row: vec![false, true, false],
2101 ..Default::default()
2102 });
2103 assert_eq!(t.header_row_count(), 0);
2104 assert_eq!(
2105 render_table(&t, false),
2106 "| | |\n|-----|----|\n| 1 | 2 |\n| a | b |\n| 333 | 4 |"
2107 );
2108 }
2109
2110 #[test]
2115 fn pivot_row_headers_do_not_extend_the_header() {
2116 let mut t = Table {
2117 rows: vec![
2118 vec!["Year".into(), "Month".into()],
2119 vec!["2025".into(), "January".into()],
2120 vec!["2025".into(), "February".into()],
2121 ],
2122 ..Default::default()
2123 };
2124 t.structure = Some(TableStructure {
2125 col_header: vec![vec![true, true], vec![false, false], vec![false, false]],
2126 row_header: vec![vec![false, false], vec![true, false], vec![true, false]],
2127 row_continuation: vec![vec![false, false], vec![false, false], vec![true, false]],
2128 ..Default::default()
2129 });
2130 assert_eq!(t.header_row_count(), 1);
2131 assert_eq!(
2132 render_table(&t, true),
2133 "| Year | Month |\n| - | - |\n| 2025 | January |\n| 2025 | February |"
2134 );
2135 }
2136
2137 #[test]
2140 fn unflagged_cells_keep_row_zero_as_header() {
2141 let mut t = Table {
2142 rows: vec![vec!["h".into()], vec!["d".into()]],
2143 ..Default::default()
2144 };
2145 t.cells = Some(
2146 [(0usize, "h"), (1, "d")]
2147 .into_iter()
2148 .map(|(r, text)| TableCell {
2149 text: text.into(),
2150 bbox: None,
2151 start_row: r,
2152 start_col: 0,
2153 row_span: 1,
2154 col_span: 1,
2155 column_header: false,
2156 row_header: false,
2157 row_section: false,
2158 })
2159 .collect(),
2160 );
2161 assert_eq!(t.header_row_count(), 1);
2162 assert_eq!(render_table(&t, true), "| h |\n| - |\n| d |");
2163 }
2164
2165 fn flagged_table(rows: &[&[&str]], ched: impl Fn(usize, usize) -> bool) -> Table {
2167 let mut cells = Vec::new();
2168 for (r, row) in rows.iter().enumerate() {
2169 for (c, text) in row.iter().enumerate() {
2170 cells.push(TableCell {
2171 text: (*text).into(),
2172 bbox: None,
2173 start_row: r,
2174 start_col: c,
2175 row_span: 1,
2176 col_span: 1,
2177 column_header: ched(r, c),
2178 row_header: false,
2179 row_section: false,
2180 });
2181 }
2182 }
2183 Table {
2184 rows: rows
2185 .iter()
2186 .map(|row| row.iter().map(|t| (*t).to_string()).collect())
2187 .collect(),
2188 cells: Some(cells),
2189 ..Default::default()
2190 }
2191 }
2192
2193 #[test]
2197 fn rows_with_body_text_stop_the_header_block() {
2198 let t = flagged_table(
2200 &[
2201 &["Item", "Q1", "Q2"],
2202 &["Sales", "10", "20"],
2203 &["Costs", "5", "7"],
2204 ],
2205 |r, c| r == 0 || c == 0,
2206 );
2207 assert_eq!(t.header_row_count(), 1);
2208 assert_eq!(
2209 render_table(&t, false),
2210 "| Item | Q1 | Q2 |\n\
2211 |--------|------|------|\n\
2212 | Sales | 10 | 20 |\n\
2213 | Costs | 5 | 7 |"
2214 );
2215 let t = flagged_table(&[&["Sales", "10", "20"], &["Costs", "5", "7"]], |_, c| {
2218 c == 0
2219 });
2220 assert_eq!(t.header_row_count(), 0);
2221 assert_eq!(
2222 render_table(&t, false),
2223 "| | | |\n\
2224 |-------|----|----|\n\
2225 | Sales | 10 | 20 |\n\
2226 | Costs | 5 | 7 |"
2227 );
2228 let t = flagged_table(
2229 &[&["a", "1", "2"], &["b", "3", "4"], &["c", "5", "6"]],
2230 |r, c| c == 0 && r < 2,
2231 );
2232 assert_eq!(t.header_row_count(), 0);
2233 let t = flagged_table(&[&["a", "1", "2"], &["b", "3", "4"]], |r, c| {
2236 r == 0 && c == 0
2237 });
2238 assert_eq!(t.header_row_count(), 1);
2239 assert_eq!(
2240 render_table(&t, false),
2241 "| a | 1 | 2 |\n\
2242 |-----|-----|-----|\n\
2243 | b | 3 | 4 |"
2244 );
2245 let t = flagged_table(&[&["Item", "", ""], &["x", "1", "2"]], |_, c| c == 0);
2247 assert_eq!(t.header_row_count(), 1);
2248 assert_eq!(
2249 render_table(&t, false),
2250 "| Item | | |\n\
2251 |--------|----|----|\n\
2252 | x | 1 | 2 |"
2253 );
2254 }
2255
2256 #[test]
2257 fn renders_compact_table() {
2258 let mut doc = DoclingDocument::new("t");
2259 doc.compact_tables = true;
2262 doc.push(Node::Table(Table {
2263 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2264 location: None,
2265 structure: None,
2266 cell_blocks: None,
2267 cells: None,
2268 caption: None,
2269 caption_parent: Default::default(),
2270 caption_location: None,
2271 }));
2272 let md = doc.export_to_markdown();
2273 assert_eq!(md, "| a | b |\n| - | - |\n| 1 | 2 |\n");
2274 }
2275
2276 #[test]
2277 fn renders_padded_github_table_by_default() {
2278 let mut doc = DoclingDocument::new("t");
2279 doc.push(Node::Table(Table {
2280 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2281 location: None,
2282 structure: None,
2283 cell_blocks: None,
2284 cells: None,
2285 caption: None,
2286 caption_parent: Default::default(),
2287 caption_location: None,
2288 }));
2289 let md = doc.export_to_markdown();
2290 assert_eq!(md, "| a | b |\n|-----|-----|\n| 1 | 2 |\n");
2292 }
2293
2294 #[test]
2295 fn strict_unescapes_inline_underscores_legacy_keeps_them() {
2296 let mut doc = DoclingDocument::new("t");
2297 doc.add_heading(1, "a\\_b");
2298 doc.add_paragraph("x\\_y");
2299 doc.push(Node::ListItem {
2300 ordered: false,
2301 number: 1,
2302 first_in_list: true,
2303 text: "i\\_j".into(),
2304 level: 0,
2305 marker: None,
2306 location: None,
2307 dclx: None,
2308 href: None,
2309 layer: None,
2310 });
2311 assert_eq!(doc.export_to_markdown(), "# a\\_b\n\nx\\_y\n\n- i\\_j\n");
2313 assert_eq!(doc.export_to_markdown_with(true), "# a_b\n\nx_y\n\n- i_j\n");
2315 }
2316
2317 fn assert_stream_matches(
2320 doc: &DoclingDocument,
2321 strict: bool,
2322 images: ImageMode,
2323 splits: &[usize],
2324 ) {
2325 let (want, want_artifacts) = to_markdown_images(doc, strict, images, "artifacts");
2326 let mut streamer =
2327 MarkdownStreamer::with_artifacts(strict, images, doc.compact_tables, "artifacts")
2328 .with_page_break_placeholder(doc.page_break_placeholder.clone());
2329 let mut got = String::new();
2330 let mut got_artifacts = Vec::new();
2331 let mut start = 0;
2332 for &end in splits {
2333 let links = if start == 0 {
2336 doc.links.as_slice()
2337 } else {
2338 &[]
2339 };
2340 got.push_str(&streamer.push(&doc.nodes[start..end], links));
2341 got_artifacts.extend(streamer.take_artifacts());
2344 start = end;
2345 }
2346 got.push_str(&streamer.push(
2347 &doc.nodes[start..],
2348 if start == 0 {
2349 doc.links.as_slice()
2350 } else {
2351 &[]
2352 },
2353 ));
2354 got_artifacts.extend(streamer.take_artifacts());
2355 got.push_str(&streamer.finish());
2356 assert_eq!(
2357 got, want,
2358 "streamed output diverged (splits={splits:?}, strict={strict})"
2359 );
2360 assert_eq!(
2361 got_artifacts, want_artifacts,
2362 "streamed artifacts diverged (splits={splits:?}, strict={strict})"
2363 );
2364 }
2365
2366 #[test]
2367 fn streaming_is_byte_identical_to_buffered() {
2368 let mut doc = DoclingDocument::new("d");
2369 doc.add_heading(1, "Title");
2370 doc.add_paragraph("First paragraph.");
2371 doc.push(Node::ListItem {
2372 ordered: false,
2373 number: 1,
2374 first_in_list: true,
2375 text: "a".into(),
2376 level: 0,
2377 marker: None,
2378 location: None,
2379 dclx: None,
2380 href: None,
2381 layer: None,
2382 });
2383 doc.push(Node::ListItem {
2384 ordered: false,
2385 number: 2,
2386 first_in_list: false,
2387 text: "b".into(),
2388 level: 0,
2389 marker: None,
2390 location: None,
2391 dclx: None,
2392 href: None,
2393 layer: None,
2394 });
2395 doc.push(Node::Code {
2396 language: Some("rust".into()),
2397 text: "let x = 1;".into(),
2398 orig: None,
2399 pretty: None,
2400 });
2401 doc.push(Node::Table(Table {
2402 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2403 location: None,
2404 structure: None,
2405 cell_blocks: None,
2406 cells: None,
2407 caption: None,
2408 caption_parent: Default::default(),
2409 caption_location: None,
2410 }));
2411 doc.push(Node::Picture {
2412 caption: Some("Fig 1".into()),
2413 caption_href: None,
2414 image: Some(PictureImage {
2415 dpi: PictureImage::DEFAULT_DPI,
2416 mimetype: "image/png".into(),
2417 width: 2,
2418 height: 2,
2419 data: b"png-one".to_vec(),
2420 display_size: None,
2421 }),
2422 classification: None,
2423 description: None,
2424 caption_parent: Default::default(),
2425 caption_location: None,
2426 });
2427 doc.add_paragraph("Last paragraph.");
2428 doc.push(Node::Picture {
2431 caption: None,
2432 caption_href: None,
2433 image: Some(PictureImage {
2434 dpi: PictureImage::DEFAULT_DPI,
2435 mimetype: "image/png".into(),
2436 width: 2,
2437 height: 2,
2438 data: b"png-two".to_vec(),
2439 display_size: None,
2440 }),
2441 classification: None,
2442 description: None,
2443 caption_parent: Default::default(),
2444 caption_location: None,
2445 });
2446
2447 for &strict in &[false, true] {
2450 for &images in &[
2451 ImageMode::Placeholder,
2452 ImageMode::Embedded,
2453 ImageMode::Referenced,
2454 ] {
2455 for splits in [&[][..], &[1][..], &[2][..], &[4][..], &[1, 4, 6, 7][..]] {
2456 assert_stream_matches(&doc, strict, images, splits);
2457 }
2458 }
2459 }
2460 }
2461
2462 #[test]
2463 fn streaming_applies_recovered_links_in_strict_mode() {
2464 let mut doc = DoclingDocument::new("d");
2465 doc.add_paragraph("See LinkedIn for details.");
2466 doc.add_paragraph("And GitHub too.");
2467 doc.links = vec![
2468 ("LinkedIn".into(), "https://lnkd/".into()),
2469 ("GitHub".into(), "https://gh/".into()),
2470 ];
2471 assert_stream_matches(&doc, true, ImageMode::Placeholder, &[1]);
2474 }
2475
2476 fn paged_doc() -> DoclingDocument {
2479 let mut doc = DoclingDocument::new("p");
2480 doc.push(Node::PageInfo {
2481 page_no: 1,
2482 width: 100.0,
2483 height: 100.0,
2484 });
2485 doc.add_heading(1, "Title");
2486 doc.add_paragraph("Page one.");
2487 doc.push(Node::PageBreak);
2489 doc.push(Node::PageInfo {
2490 page_no: 2,
2491 width: 100.0,
2492 height: 100.0,
2493 });
2494 doc.push(Node::PageFurniture {
2495 footer: true,
2496 location: [0, 500, 511, 511],
2497 text: "2".into(),
2498 });
2499 doc.push(Node::PageBreak);
2500 doc.push(Node::PageInfo {
2501 page_no: 3,
2502 width: 100.0,
2503 height: 100.0,
2504 });
2505 doc.add_paragraph("Page three.");
2506 doc.push(Node::PageBreak);
2508 doc
2509 }
2510
2511 #[test]
2512 fn page_break_placeholder_lands_between_pages_only() {
2513 let mut doc = paged_doc();
2514 assert_eq!(
2516 doc.export_to_markdown(),
2517 "# Title\n\nPage one.\n\nPage three.\n"
2518 );
2519 doc.page_break_placeholder = Some("<!-- page break -->".into());
2520 assert_eq!(
2524 doc.export_to_markdown(),
2525 "# Title\n\nPage one.\n\n<!-- page break -->\n\nPage three.\n"
2526 );
2527 doc.page_break_placeholder = Some(String::new());
2530 assert_eq!(
2531 doc.export_to_markdown(),
2532 "# Title\n\nPage one.\n\n\n\nPage three.\n"
2533 );
2534 }
2535
2536 #[test]
2537 fn page_break_placeholder_never_leads_a_single_page() {
2538 let mut doc = DoclingDocument::new("one");
2539 doc.page_break_placeholder = Some("---".into());
2540 doc.push(Node::PageBreak);
2541 doc.push(Node::PageInfo {
2542 page_no: 1,
2543 width: 10.0,
2544 height: 10.0,
2545 });
2546 doc.add_paragraph("Only page.");
2547 assert_eq!(doc.export_to_markdown(), "Only page.\n");
2548 doc.push(Node::PageBreak);
2550 doc.push(Node::PageBreak);
2551 doc.add_paragraph("Next.");
2552 assert_eq!(doc.export_to_markdown(), "Only page.\n\n---\n\nNext.\n");
2553 }
2554
2555 #[test]
2556 fn page_break_placeholder_streams_byte_identical() {
2557 let mut doc = paged_doc();
2558 doc.page_break_placeholder = Some("<!-- page break -->".into());
2559 for splits in [
2563 &[3usize][..],
2564 &[3, 6],
2565 &[3, 6, 8],
2566 &[1, 2, 3, 4, 5, 6, 7, 8, 9],
2567 &[8],
2568 ] {
2569 assert_stream_matches(&doc, false, ImageMode::Placeholder, splits);
2570 assert_stream_matches(&doc, true, ImageMode::Placeholder, splits);
2571 }
2572 }
2573
2574 #[test]
2575 fn strict_tightens_punctuation_spacing_legacy_keeps_it() {
2576 let mut doc = DoclingDocument::new("t");
2577 doc.add_paragraph("see [ 37 , 36 ] and ( x ) .");
2578 assert_eq!(doc.export_to_markdown(), "see [ 37 , 36 ] and ( x ) .\n");
2580 assert_eq!(doc.export_to_markdown_with(true), "see [37, 36] and (x).\n");
2582 }
2583
2584 fn options_doc() -> DoclingDocument {
2589 use crate::document::ContentLayer;
2590 let mut doc = DoclingDocument::new("opts");
2591 doc.push(Node::PageFurniture {
2592 footer: false,
2593 location: [0, 0, 0, 0],
2594 text: "FAX COVER \\_ R&D".into(),
2595 });
2596 doc.add_heading(1, "R&D \\_ report");
2597 doc.add_paragraph("Body a\\_b <tag>");
2598 doc.push(Node::Picture {
2599 caption: Some("Figure 1".into()),
2600 caption_href: None,
2601 image: None,
2602 classification: None,
2603 description: None,
2604 caption_parent: Default::default(),
2605 caption_location: None,
2606 });
2607 doc.push(Node::PictureChildren(vec![
2608 Node::Paragraph {
2609 text: "Field: Name".into(),
2610 },
2611 Node::Paragraph {
2612 text: "Field: Date".into(),
2613 },
2614 ]));
2615 doc.push(Node::CommentSection {
2616 name: "comment-1".into(),
2617 text: "[author: A]: note".into(),
2618 refs_note_text: false,
2619 grouped: true,
2620 });
2621 doc.push(Node::Group {
2622 label: "section".into(),
2623 name: Some("sheet: Hidden".into()),
2624 layer: Some(ContentLayer::Invisible),
2625 children: vec![Node::Paragraph {
2626 text: "hidden cell".into(),
2627 }],
2628 });
2629 doc.push(Node::Furniture {
2630 layer: ContentLayer::Furniture,
2631 inner: Box::new(Node::Paragraph {
2632 text: "Page 1 of 2".into(),
2633 }),
2634 });
2635 doc.push(Node::FurnitureText {
2636 label: "page_footer".into(),
2637 text: "footer line".into(),
2638 });
2639 doc
2640 }
2641
2642 #[test]
2646 fn default_options_are_the_default_export() {
2647 let doc = options_doc();
2648 let plain = doc.export_to_markdown();
2649 assert_eq!(
2650 plain,
2651 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2652 );
2653 let (with_default, artifacts) =
2654 doc.export_to_markdown_with_options(&MarkdownExportOptions::default());
2655 assert_eq!(with_default, plain);
2656 assert!(artifacts.is_empty());
2657 }
2658
2659 #[test]
2660 fn layers_render_furniture_notes_and_hidden_sheets() {
2661 use crate::document::ContentLayer;
2662 let doc = options_doc();
2663 let md = |layers: ContentLayers| {
2664 doc.export_to_markdown_with_options(&MarkdownExportOptions {
2665 layers,
2666 ..MarkdownExportOptions::default()
2667 })
2668 .0
2669 };
2670 assert_eq!(
2671 md(ContentLayers::BODY.with(ContentLayer::Furniture)),
2672 "FAX COVER \\_ R&D\n\n# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nPage 1 of 2\n\nfooter line\n",
2673 "a page header, a furniture-wrapped paragraph and a furniture text item render as paragraphs"
2674 );
2675 assert_eq!(
2676 md(ContentLayers::BODY.with(ContentLayer::Notes)),
2677 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\n[author: A]: note\n"
2678 );
2679 assert_eq!(
2680 md(ContentLayers::BODY.with(ContentLayer::Invisible)),
2681 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nhidden cell\n",
2682 "a hidden sheet's items sit on the sheet's layer"
2683 );
2684 assert_eq!(
2685 md(ContentLayers::NONE.with(ContentLayer::Furniture)),
2686 "FAX COVER \\_ R&D\n\nPage 1 of 2\n\nfooter line\n",
2687 "without the body layer only the furniture prints"
2688 );
2689 assert_eq!(md(ContentLayers::NONE), "");
2690 assert_eq!(
2691 md(ContentLayers::ALL),
2692 "FAX COVER \\_ R&D\n\n# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\n[author: A]: note\n\nhidden cell\n\nPage 1 of 2\n\nfooter line\n"
2693 );
2694 }
2695
2696 #[test]
2699 fn layers_apply_to_list_items() {
2700 use crate::document::ContentLayer;
2701 let mut doc = DoclingDocument::new("lists");
2702 doc.push(Node::ListItem {
2703 ordered: false,
2704 number: 1,
2705 first_in_list: true,
2706 text: "nav link".into(),
2707 level: 0,
2708 marker: None,
2709 location: None,
2710 dclx: None,
2711 href: None,
2712 layer: Some(ContentLayer::Furniture),
2713 });
2714 doc.add_paragraph("body");
2715 assert_eq!(doc.export_to_markdown(), "body\n");
2716 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2717 layers: ContentLayers::BODY.with(ContentLayer::Furniture),
2718 ..MarkdownExportOptions::default()
2719 });
2720 assert_eq!(md, "- nav link\n\nbody\n");
2721 }
2722
2723 #[test]
2724 fn traverse_pictures_prints_the_nested_text_after_the_picture() {
2725 let doc = options_doc();
2726 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2727 traverse_pictures: true,
2728 ..MarkdownExportOptions::default()
2729 });
2730 assert_eq!(
2731 md,
2732 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nField: Name\n\nField: Date\n"
2733 );
2734 }
2735
2736 #[test]
2737 fn escaping_can_be_turned_off_per_kind() {
2738 let doc = options_doc();
2739 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2740 escape_html: false,
2741 ..MarkdownExportOptions::default()
2742 });
2743 assert_eq!(
2744 md,
2745 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2746 );
2747 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2748 escape_underscores: false,
2749 ..MarkdownExportOptions::default()
2750 });
2751 assert_eq!(
2752 md,
2753 "# R&D _ report\n\nBody a_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2754 );
2755 let mut doc = DoclingDocument::new("code");
2758 doc.push(Node::Code {
2759 language: None,
2760 text: "a & b \\_ c".into(),
2761 orig: None,
2762 pretty: None,
2763 });
2764 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2765 escape_html: false,
2766 escape_underscores: false,
2767 ..MarkdownExportOptions::default()
2768 });
2769 assert_eq!(md, "```\na & b \\_ c\n```\n");
2770 }
2771
2772 #[test]
2776 fn fences_and_spans_out_length_inner_backticks() {
2777 let code = |text: &str| Node::Code {
2778 language: Some("md".into()),
2779 text: text.into(),
2780 orig: None,
2781 pretty: None,
2782 };
2783 let fence_of = |text: &str| {
2784 let mut doc = DoclingDocument::new("d");
2785 doc.push(code(text));
2786 (doc.export_to_markdown(), doc.export_to_markdown_with(true))
2787 };
2788 assert_eq!(fence_of("x").0, "```\nx\n```\n");
2789 assert_eq!(fence_of("a ` b").0, "```\na ` b\n```\n");
2790 assert_eq!(fence_of("a ``` b").0, "````\na ``` b\n````\n");
2791 assert_eq!(
2792 fence_of("a ``` b ```` c"),
2793 (
2794 "`````\na ``` b ```` c\n`````\n".to_string(),
2795 "`````md\na ``` b ```` c\n`````\n".to_string()
2796 )
2797 );
2798 assert_eq!(code_span("x"), "`x`");
2799 assert_eq!(code_span("a`b"), "``a`b``");
2800 assert_eq!(code_span("a ``` b"), "````a ``` b````");
2801 assert_eq!(code_span("`x"), "`` `x ``");
2802 assert_eq!(code_span("x`"), "`` x` ``");
2803 for (span, text) in [
2804 ("`x`", "x"),
2805 ("``a`b``", "a`b"),
2806 ("`` `x ``", "`x"),
2807 ("````a ``` b````", "a ``` b"),
2808 ] {
2809 assert_eq!(
2810 plain_inline(&format!("see {span} here")),
2811 format!("see {text} here")
2812 );
2813 }
2814 let mut doc = DoclingDocument::new("d");
2817 doc.push(code("a ``` b"));
2818 assert_eq!(doc.export_to_text(), "a ``` b");
2819 let mut doc = DoclingDocument::new("d");
2821 doc.add_paragraph("Before.");
2822 doc.push(code("a ``` b ```` c"));
2823 doc.add_paragraph("After.");
2824 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[1, 2]);
2825 assert_stream_matches(&doc, true, ImageMode::Placeholder, &[1, 2]);
2826 }
2827
2828 #[test]
2833 fn mermaid_code_item_replaces_its_picture_placeholder() {
2834 let picture = || Node::Picture {
2835 caption: None,
2836 caption_href: None,
2837 image: None,
2838 classification: None,
2839 description: None,
2840 caption_parent: Default::default(),
2841 caption_location: None,
2842 };
2843 let mut doc = DoclingDocument::new("d");
2844 doc.add_paragraph("Before.");
2845 doc.push(picture());
2846 doc.push(Node::Code {
2847 language: Some("mermaid".into()),
2848 text: "flowchart TD\n A --> B".into(),
2849 orig: None,
2850 pretty: None,
2851 });
2852 doc.add_paragraph("After.");
2853 doc.push(picture());
2854 let md = doc.export_to_markdown();
2855 assert_eq!(
2856 md,
2857 "Before.\n\n```mermaid\nflowchart TD\n A --> B\n```\n\nAfter.\n\n<!-- image -->\n"
2858 );
2859 assert_eq!(doc.export_to_markdown_with(true), md);
2860 for splits in [&[1][..], &[2], &[1, 2], &[2, 3], &[1, 2, 3, 4]] {
2861 assert_stream_matches(&doc, false, ImageMode::Placeholder, splits);
2862 assert_stream_matches(&doc, true, ImageMode::Placeholder, splits);
2863 }
2864 let mut doc = DoclingDocument::new("d");
2866 doc.add_paragraph("Only.");
2867 doc.push(picture());
2868 assert_eq!(doc.export_to_markdown(), "Only.\n\n<!-- image -->\n");
2869 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[1]);
2870 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[2]);
2871 }
2872
2873 #[test]
2874 fn image_placeholder_is_configurable() {
2875 let doc = options_doc();
2876 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2877 image_placeholder: "[figure]".into(),
2878 ..MarkdownExportOptions::default()
2879 });
2880 assert!(md.contains("Figure 1\n\n[figure]\n"), "{md}");
2881 assert!(!md.contains("<!-- image -->"));
2882 }
2883
2884 #[test]
2890 fn empty_image_placeholder_prints_nothing() {
2891 fn picture(caption: Option<&str>) -> Node {
2892 Node::Picture {
2893 caption: caption.map(Into::into),
2894 caption_href: None,
2895 image: None,
2896 classification: None,
2897 description: None,
2898 caption_parent: Default::default(),
2899 caption_location: None,
2900 }
2901 }
2902 fn page(no: usize) -> Node {
2903 Node::PageInfo {
2904 page_no: no,
2905 width: 100.0,
2906 height: 100.0,
2907 }
2908 }
2909 fn para(text: &str) -> Node {
2910 Node::Paragraph { text: text.into() }
2911 }
2912 let options = MarkdownExportOptions {
2913 image_placeholder: String::new(),
2914 ..MarkdownExportOptions::default()
2915 };
2916 let cases: Vec<(Vec<Node>, Option<&str>, &str)> = vec![
2918 (
2919 vec![page(1), para("a"), picture(None), para("b")],
2920 None,
2921 "a\n\nb\n",
2922 ),
2923 (
2924 vec![page(1), para("a"), picture(Some("Figure 1")), para("b")],
2925 None,
2926 "a\n\nFigure 1\n\nb\n",
2927 ),
2928 (
2929 vec![
2930 page(1),
2931 para("a"),
2932 page(2),
2933 picture(None),
2934 page(3),
2935 para("b"),
2936 ],
2937 Some("<!-- pb -->"),
2938 "a\n\n<!-- pb -->\n\n<!-- pb -->\n\nb\n",
2939 ),
2940 (
2941 vec![
2942 page(1),
2943 para("a"),
2944 page(2),
2945 picture(None),
2946 page(3),
2947 para("b"),
2948 ],
2949 Some(""),
2950 "a\n\n\n\n\n\nb\n",
2951 ),
2952 (
2953 vec![page(1), picture(None), page(2), para("b")],
2954 Some("<!-- pb -->"),
2955 "<!-- pb -->\n\nb\n",
2956 ),
2957 (
2958 vec![page(1), picture(None), page(2), para("b")],
2959 Some(""),
2960 "\n\nb\n",
2961 ),
2962 (
2963 vec![page(1), para("a"), page(2), picture(None)],
2964 Some("<!-- pb -->"),
2965 "a\n\n<!-- pb -->\n",
2966 ),
2967 ];
2968 for (nodes, placeholder, expected) in cases {
2969 let mut doc = DoclingDocument::new("t");
2970 doc.page_break_placeholder = placeholder.map(Into::into);
2971 for node in nodes {
2972 doc.push(node);
2973 }
2974 let (md, _) = doc.export_to_markdown_with_options(&options);
2975 assert_eq!(md, expected, "page break {placeholder:?}");
2976 for size in 1..=doc.nodes.len() {
2979 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
2980 .with_export_options(&options)
2981 .with_page_break_placeholder(placeholder.map(Into::into));
2982 let mut out = String::new();
2983 for chunk in doc.nodes.chunks(size) {
2984 out.push_str(&streamer.push(chunk, &[]));
2985 }
2986 out.push_str(&streamer.finish());
2987 assert_eq!(
2988 out, expected,
2989 "chunks of {size}, page break {placeholder:?}"
2990 );
2991 }
2992 }
2993 let mut doc = DoclingDocument::new("t");
2995 doc.push(para("a"));
2996 doc.push(picture(None));
2997 doc.push(para("b"));
2998 assert_eq!(doc.export_to_markdown(), "a\n\n<!-- image -->\n\nb\n");
2999 }
3000
3001 #[test]
3004 fn streamer_matches_buffered_export_with_options() {
3005 let doc = options_doc();
3006 let options = MarkdownExportOptions {
3007 layers: ContentLayers::ALL,
3008 traverse_pictures: true,
3009 escape_html: false,
3010 escape_underscores: false,
3011 image_placeholder: "(img)".into(),
3012 ..MarkdownExportOptions::default()
3013 };
3014 let (buffered, _) = doc.export_to_markdown_with_options(&options);
3015 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3016 .with_export_options(&options);
3017 let mut out = String::new();
3018 for chunk in doc.nodes.chunks(2) {
3019 out.push_str(&streamer.push(chunk, &[]));
3020 }
3021 out.push_str(&streamer.finish());
3022 assert_eq!(out, buffered);
3023 assert!(
3024 buffered.contains("(img)") && buffered.contains("R&D _ report"),
3025 "{buffered}"
3026 );
3027 }
3028
3029 #[test]
3032 fn plain_text_reads_escaped_stars() {
3033 assert_eq!(plain_inline("A\\*\\*\\* GmbH"), "A*** GmbH");
3034 assert_eq!(
3035 plain_inline("**FN \\*0\\*271\\*h** and *h\\*.at*"),
3036 "FN *0*271*h and h*.at"
3037 );
3038 }
3039
3040 #[test]
3043 fn plain_text_drops_the_markdown_decoration() {
3044 let item = |text: &str, ordered: bool, number: u64, level: u8| Node::ListItem {
3045 ordered,
3046 number,
3047 first_in_list: number == 1 && level == 0,
3048 text: text.into(),
3049 level,
3050 marker: None,
3051 location: None,
3052 dclx: None,
3053 href: None,
3054 layer: None,
3055 };
3056 let mut doc = DoclingDocument::new("plain");
3057 doc.add_heading(1, "Title with **bold**");
3058 doc.add_heading(2, "Section");
3059 doc.add_paragraph("Some **bold**, *italic*, ~~gone~~ and `code` with a [link](https://x.org).\nNext line.");
3060 doc.push(item("*one*", false, 1, 0));
3061 doc.push(item("[two](https://y.org)", false, 2, 0));
3062 doc.push(item("nested", true, 1, 1));
3063 doc.push(Node::CheckboxItem {
3064 checked: true,
3065 text: "done".into(),
3066 });
3067 doc.push(Node::Code {
3068 language: Some("rust".into()),
3069 text: "let x = **y**;".into(),
3070 orig: None,
3071 pretty: None,
3072 });
3073 doc.push(Node::Picture {
3074 caption: Some("Figure *1*".into()),
3075 caption_href: None,
3076 image: None,
3077 classification: None,
3078 description: None,
3079 caption_parent: Default::default(),
3080 caption_location: None,
3081 });
3082 doc.push(Node::Caption {
3083 text: "Linked caption".into(),
3084 href: Some("https://z.org".into()),
3085 });
3086 doc.push(Node::Table(Table {
3087 rows: vec![
3088 vec!["**n**".into(), "rich".into()],
3089 vec!["**2**".into(), "a \nb".into()],
3090 ],
3091 ..Default::default()
3092 }));
3093 doc.add_paragraph("R&D \\_ x");
3094 let text = doc.export_to_text();
3095 assert_eq!(
3096 text,
3097 "Title with bold\n\n\
3098 Section\n\n\
3099 Some bold, italic, gone and code with a link.\nNext line.\n\n\
3100 - one\n- two\n 1. nested\n\n\
3101 - [x] done\n\n\
3102 let x = **y**;\n\n\
3103 Figure 1\n\n\
3104 Linked caption\n\n\
3105 | n | rich |\n\
3106 |-----|--------|\n\
3107 | 2 | a b |\n\n\
3108 R&D _ x"
3109 );
3110
3111 let options = MarkdownExportOptions::plain_text();
3112 let (buffered, _) = doc.export_to_markdown_with_options(&options);
3113 assert_eq!(buffered, format!("{text}\n"));
3114 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3115 .with_export_options(&options);
3116 let mut out = String::new();
3117 for chunk in doc.nodes.chunks(3) {
3118 out.push_str(&streamer.push(chunk, &[]));
3119 }
3120 out.push_str(&streamer.finish());
3121 assert_eq!(out, buffered);
3122 }
3123
3124 #[test]
3126 fn plain_inline_keeps_literal_markers() {
3127 for (input, want) in [
3128 ("**bold** and *it* and ***both***", "bold and it and both"),
3129 ("**[label](https://a.b/c_(d))** end", "label end"),
3130 (
3131 "grant *USE and *OBJMGT authority",
3132 "grant *USE and *OBJMGT authority",
3133 ),
3134 ("2 * 3 * 4", "2 * 3 * 4"),
3135 ("a ** b", "a ** b"),
3136 ("the `quoted’ and `x` term", "the `quoted’ and x term"),
3137 ("see [12] and [docs](u)", "see [12] and docs"),
3138 ("formula $[a](b)$ stays", "formula $[a](b)$ stays"),
3139 ("price $5 and **bold**", "price $5 and bold"),
3140 ("[outer [inner] text](u)", "outer [inner] text"),
3141 ("unclosed **bold", "unclosed **bold"),
3142 ] {
3143 assert_eq!(plain_inline(input), want, "{input}");
3144 }
3145 }
3146}