1use crate::document::{
4 ContentLayer, ContentLayers, DoclingDocument, MarkdownExportOptions, Node, Table,
5};
6
7const MISSING_KEY_VALUE_ITEM: &str = "<!-- missing-key-value-item -->";
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
14pub enum ImageMode {
15 #[default]
17 Placeholder,
18 Embedded,
20 Referenced,
23}
24
25struct Ctx {
27 strict: bool,
28 alt_text: bool,
31 compact_tables: bool,
33 images: ImageMode,
34 artifacts_dir: String,
35 artifacts: Vec<(String, Vec<u8>)>,
37 pic_index: usize,
38 in_table_cell: bool,
42 page_break: Option<String>,
46 pending_page_break: bool,
52 emitted_any: bool,
55 layers: ContentLayers,
57 current_layer: Option<ContentLayer>,
62 traverse_pictures: bool,
64 escape_html: bool,
67 escape_underscores: bool,
69 image_placeholder: String,
71 plain: bool,
73}
74
75impl Ctx {
76 fn new(doc: &DoclingDocument, strict: bool, options: &MarkdownExportOptions) -> Self {
79 Ctx {
80 strict,
81 alt_text: strict || doc.picture_alt_text,
82 compact_tables: doc.compact_tables,
83 images: options.image_mode,
84 artifacts_dir: options.artifacts_dir.clone(),
85 artifacts: Vec::new(),
86 pic_index: 0,
87 in_table_cell: false,
88 page_break: doc.page_break_placeholder.clone(),
89 pending_page_break: false,
90 emitted_any: false,
91 layers: options.layers,
92 current_layer: None,
93 traverse_pictures: options.traverse_pictures,
94 escape_html: options.escape_html,
95 escape_underscores: options.escape_underscores,
96 image_placeholder: options.image_placeholder.clone(),
97 plain: options.plain_text,
98 }
99 }
100
101 fn layer_visible(&self) -> bool {
103 self.layers.contains(self.current_layer)
104 }
105
106 fn unescape(&self, text: &str) -> String {
113 let mut out = std::borrow::Cow::Borrowed(text);
114 if !self.escape_html
115 && (out.contains("&") || out.contains("<") || out.contains(">"))
116 {
117 out = std::borrow::Cow::Owned(
118 out.replace("<", "<")
119 .replace(">", ">")
120 .replace("&", "&"),
121 );
122 }
123 if !self.escape_underscores && out.contains("\\_") {
124 out = std::borrow::Cow::Owned(out.replace("\\_", "_"));
125 }
126 out.into_owned()
127 }
128
129 fn text(&self, text: &str) -> String {
132 let text = strict_text(text, self.strict);
133 if self.plain {
134 self.unescape(&plain_inline(&self.drop_image_markers(&text)))
135 } else {
136 self.unescape(&text)
137 }
138 }
139
140 fn caption(&self, text: &str) -> String {
144 if self.plain {
145 self.unescape(&plain_inline(text))
146 } else {
147 self.unescape(text)
148 }
149 }
150
151 fn breaks(&self, text: &str) -> String {
154 if self.plain {
155 text.to_string()
156 } else {
157 md_line_breaks(text)
158 }
159 }
160
161 fn plain_table(&self, table: &Table) -> Table {
168 let mut plain = table.clone();
169 for cell in plain.rows.iter_mut().flatten() {
170 if cell.contains(['*', '~', '`', '[', '\n', '<']) {
171 let text = self.drop_image_markers(&strip_hard_breaks(cell));
172 *cell = plain_inline(&text);
173 }
174 }
175 plain
176 }
177
178 fn drop_image_markers(&self, text: &str) -> String {
185 const MARKER: &str = "<!-- image -->";
186 if !text.contains(MARKER) && !text.contains("```") {
187 return text.to_string();
188 }
189 text.split("\n\n")
193 .filter_map(|block| {
194 let lines: Vec<String> = block
195 .split('\n')
196 .filter_map(|line| {
197 if line.trim() == MARKER || is_fence_line(line.trim()) {
200 return None;
201 }
202 Some(line.replace(MARKER, &self.image_placeholder))
203 })
204 .collect();
205 (!lines.is_empty()).then(|| lines.join("\n"))
206 })
207 .collect::<Vec<_>>()
208 .join("\n\n")
209 }
210}
211
212pub fn to_markdown(doc: &DoclingDocument, strict: bool) -> String {
218 to_markdown_images(doc, strict, ImageMode::Placeholder, "artifacts").0
219}
220
221pub fn to_markdown_images(
225 doc: &DoclingDocument,
226 strict: bool,
227 images: ImageMode,
228 artifacts_dir: &str,
229) -> (String, Vec<(String, Vec<u8>)>) {
230 to_markdown_with_options(
231 doc,
232 strict,
233 &MarkdownExportOptions {
234 image_mode: images,
235 artifacts_dir: artifacts_dir.to_string(),
236 ..MarkdownExportOptions::default()
237 },
238 )
239}
240
241pub fn to_markdown_with_options(
245 doc: &DoclingDocument,
246 strict: bool,
247 options: &MarkdownExportOptions,
248) -> (String, Vec<(String, Vec<u8>)>) {
249 let mut ctx = Ctx::new(doc, strict, options);
250 let mut blocks: Vec<String> = Vec::new();
251 render(&doc.nodes, &mut blocks, &mut ctx);
252 let mut body = blocks.join("\n\n");
253 if strict && !options.plain_text && !doc.links.is_empty() {
257 body = apply_links(&body, &doc.links);
258 }
259 let front = if doc.metadata_output.front_matter() && !options.plain_text {
262 doc.metadata.to_front_matter()
263 } else {
264 String::new()
265 };
266 let md = match (front.is_empty(), body.is_empty()) {
267 (true, true) => String::new(),
268 (true, false) => format!("{body}\n"),
269 (false, true) => front,
270 (false, false) => format!("{front}\n{body}\n"),
271 };
272 (md, ctx.artifacts)
273}
274
275pub fn to_markdown_table_cell(doc: &DoclingDocument, strict: bool) -> String {
284 let mut ctx = Ctx::new(doc, strict, &MarkdownExportOptions::default());
285 ctx.in_table_cell = true;
286 ctx.artifacts_dir = String::new();
287 ctx.page_break = None;
291 let mut blocks: Vec<String> = Vec::new();
292 render(&doc.nodes, &mut blocks, &mut ctx);
293 blocks.join("\n\n")
294}
295
296fn apply_links(body: &str, links: &[(String, String)]) -> String {
304 let mut out = body.to_string();
305 let mut cursor = 0usize;
306 for (anchor, href) in links {
307 let anchor = anchor
308 .replace('&', "&")
309 .replace('<', "<")
310 .replace('>', ">");
311 if anchor.is_empty() {
312 continue;
313 }
314 if let Some(rel) = out[cursor..].find(&anchor) {
315 let at = cursor + rel;
316 let replacement = crate::hyperlink::markdown_link(&anchor, href);
319 out.replace_range(at..at + anchor.len(), &replacement);
320 cursor = at + replacement.len();
321 }
322 }
323 out
324}
325
326fn apply_links_chunk(chunk: &str, queue: &mut Vec<(String, String)>) -> String {
337 let mut out = chunk.to_string();
338 let mut cursor = 0usize;
339 let mut carried: Vec<(String, String)> = Vec::new();
340 for (anchor_raw, href) in std::mem::take(queue) {
341 let anchor = anchor_raw
342 .replace('&', "&")
343 .replace('<', "<")
344 .replace('>', ">");
345 if anchor.is_empty() {
346 continue;
347 }
348 if let Some(rel) = out[cursor..].find(&anchor) {
349 let at = cursor + rel;
350 let replacement = crate::hyperlink::markdown_link(&anchor, &href);
351 out.replace_range(at..at + anchor.len(), &replacement);
352 cursor = at + replacement.len();
353 } else {
354 carried.push((anchor_raw, href));
356 }
357 }
358 *queue = carried;
359 out
360}
361
362pub struct MarkdownStreamer {
380 strict: bool,
381 images: ImageMode,
382 compact_tables: bool,
383 picture_alt_text: bool,
385 front_matter: Option<String>,
389 emitted_any: bool,
392 links: Vec<(String, String)>,
394 artifacts_dir: String,
398 artifacts: Vec<(String, Vec<u8>)>,
399 pic_index: usize,
400 page_break: Option<String>,
404 pending_page_break: bool,
405 items_emitted: bool,
411 held: Option<Node>,
416 layers: ContentLayers,
419 traverse_pictures: bool,
420 escape_html: bool,
421 escape_underscores: bool,
422 image_placeholder: String,
423 plain: bool,
424}
425
426impl MarkdownStreamer {
427 pub fn new(strict: bool, images: ImageMode, compact_tables: bool) -> Self {
430 debug_assert!(
431 images != ImageMode::Referenced,
432 "referenced image mode needs an artifacts dir; use with_artifacts"
433 );
434 Self::with_artifacts(strict, images, compact_tables, "artifacts")
435 }
436
437 pub fn with_artifacts(
444 strict: bool,
445 images: ImageMode,
446 compact_tables: bool,
447 artifacts_dir: &str,
448 ) -> Self {
449 Self {
450 strict,
451 images,
452 compact_tables,
453 picture_alt_text: false,
454 front_matter: None,
455 emitted_any: false,
456 links: Vec::new(),
457 artifacts_dir: artifacts_dir.to_string(),
458 artifacts: Vec::new(),
459 pic_index: 0,
460 page_break: None,
461 pending_page_break: false,
462 items_emitted: false,
463 held: None,
464 layers: ContentLayers::BODY,
465 traverse_pictures: false,
466 escape_html: true,
467 escape_underscores: true,
468 image_placeholder: "<!-- image -->".to_string(),
469 plain: false,
470 }
471 }
472
473 pub fn with_export_options(mut self, options: &MarkdownExportOptions) -> Self {
480 self.images = options.image_mode;
481 self.artifacts_dir = options.artifacts_dir.clone();
482 self.layers = options.layers;
483 self.traverse_pictures = options.traverse_pictures;
484 self.escape_html = options.escape_html;
485 self.escape_underscores = options.escape_underscores;
486 self.image_placeholder = options.image_placeholder.clone();
487 self.plain = options.plain_text;
488 self
489 }
490
491 pub fn with_page_break_placeholder(mut self, placeholder: Option<String>) -> Self {
496 self.page_break = placeholder;
497 self
498 }
499
500 pub fn with_picture_alt_text(mut self, on: bool) -> Self {
504 self.picture_alt_text = on;
505 self
506 }
507
508 pub fn with_metadata(mut self, metadata: &crate::metadata::DocumentMetadata) -> Self {
513 self.front_matter = (!self.plain)
514 .then(|| metadata.to_front_matter())
515 .filter(|f| !f.is_empty());
516 self
517 }
518
519 pub fn take_artifacts(&mut self) -> Vec<(String, Vec<u8>)> {
524 std::mem::take(&mut self.artifacts)
525 }
526
527 pub fn push(&mut self, nodes: &[Node], links: &[(String, String)]) -> String {
532 self.push_holding(nodes, links, true)
533 }
534
535 fn push_holding(&mut self, nodes: &[Node], links: &[(String, String)], hold: bool) -> String {
538 self.links.extend(links.iter().cloned());
539 let images = self.images;
543 let ends_on_picture = |nodes: &[Node]| {
544 matches!(images, ImageMode::Placeholder)
545 && matches!(nodes.last(), Some(Node::Picture { .. }))
546 };
547 let mut owned: Vec<Node> = Vec::new();
548 let mut nodes = nodes;
549 if self.held.is_some() || (hold && ends_on_picture(nodes)) {
550 owned.extend(self.held.take());
551 owned.extend(nodes.iter().cloned());
552 if hold && ends_on_picture(&owned) {
553 self.held = owned.pop();
554 }
555 nodes = &owned;
556 }
557 let mut ctx = Ctx {
558 strict: self.strict,
559 alt_text: self.strict || self.picture_alt_text,
560 compact_tables: self.compact_tables,
561 images: self.images,
562 artifacts_dir: std::mem::take(&mut self.artifacts_dir),
563 artifacts: std::mem::take(&mut self.artifacts),
564 pic_index: self.pic_index,
565 in_table_cell: false,
566 page_break: std::mem::take(&mut self.page_break),
567 pending_page_break: self.pending_page_break,
568 emitted_any: self.items_emitted,
569 layers: self.layers,
570 current_layer: None,
571 traverse_pictures: self.traverse_pictures,
572 escape_html: self.escape_html,
573 escape_underscores: self.escape_underscores,
574 image_placeholder: std::mem::take(&mut self.image_placeholder),
575 plain: self.plain,
576 };
577 let mut blocks: Vec<String> = Vec::new();
578 render(nodes, &mut blocks, &mut ctx);
579 self.image_placeholder = std::mem::take(&mut ctx.image_placeholder);
580 self.artifacts_dir = std::mem::take(&mut ctx.artifacts_dir);
581 self.artifacts = std::mem::take(&mut ctx.artifacts);
582 self.pic_index = ctx.pic_index;
583 self.page_break = std::mem::take(&mut ctx.page_break);
584 self.pending_page_break = ctx.pending_page_break;
585 self.items_emitted = ctx.emitted_any;
586 if blocks.is_empty() {
587 return String::new();
588 }
589 let mut body = blocks.join("\n\n");
590 if self.strict && !self.plain && !self.links.is_empty() {
591 body = apply_links_chunk(&body, &mut self.links);
592 }
593 let chunk = if let Some(front) = self.front_matter.take() {
594 format!("{front}\n{body}")
595 } else if self.emitted_any {
596 format!("\n\n{body}")
597 } else {
598 body
599 };
600 self.emitted_any = true;
601 chunk
602 }
603
604 pub fn finish(mut self) -> String {
607 let mut out = String::new();
608 if self.held.is_some() {
609 out.push_str(&self.push_holding(&[], &[], false));
610 }
611 if let Some(front) = self.front_matter {
612 out.push_str(&front);
613 } else if self.emitted_any {
614 out.push('\n');
615 }
616 out
617 }
618}
619
620fn strict_text(text: &str, strict: bool) -> String {
628 if !strict {
629 return text.to_string();
630 }
631 if let Some(rest) = text.strip_prefix("[ ] ") {
634 return format!("[ ] {}", strict_text(rest, strict));
635 }
636 text.replace("\\_", "_")
637 .replace(" ,", ",")
638 .replace(" .", ".")
639 .replace(" ;", ";")
640 .replace(" )", ")")
641 .replace("( ", "(")
642 .replace(" ]", "]")
643 .replace("[ ", "[")
644}
645
646pub fn longest_backtick_run(text: &str) -> usize {
649 let (mut best, mut run) = (0, 0);
650 for c in text.chars() {
651 if c == '`' {
652 run += 1;
653 best = best.max(run);
654 } else {
655 run = 0;
656 }
657 }
658 best
659}
660
661pub fn code_span(text: &str) -> String {
667 let ticks = "`".repeat(longest_backtick_run(text) + 1);
668 let pad = if text.starts_with('`') || text.ends_with('`') {
669 " "
670 } else {
671 ""
672 };
673 format!("{ticks}{pad}{text}{pad}{ticks}")
674}
675
676pub fn code_fence(language: &str, body: &str) -> String {
681 let fence = "`".repeat(longest_backtick_run(body).max(2) + 1);
682 format!("{fence}{language}\n{body}\n{fence}")
683}
684
685pub(crate) fn code_span_bounds(chars: &[char], i: usize) -> Option<(usize, usize, usize)> {
693 let n = chars[i..].iter().take_while(|&&c| c == '`').count();
694 if n == 0 {
695 return None;
696 }
697 let mut k = i + n;
698 while k < chars.len() {
699 if chars[k] != '`' {
700 k += 1;
701 continue;
702 }
703 let run = chars[k..].iter().take_while(|&&c| c == '`').count();
704 if run == n {
705 let (mut s, mut e) = (i + n, k);
706 if n > 1
707 && e - s >= 2
708 && chars[s] == ' '
709 && chars[e - 1] == ' '
710 && chars[s..e].iter().any(|&c| c != ' ')
711 {
712 s += 1;
713 e -= 1;
714 }
715 return Some((s, e, k + run));
716 }
717 k += run;
718 }
719 None
720}
721
722fn is_fence_line(line: &str) -> bool {
724 line.len() >= 3 && line.bytes().all(|b| b == b'`')
725}
726
727pub(crate) fn plain_inline(text: &str) -> String {
740 if !text.contains(['*', '~', '`', '[']) {
741 return text.to_string();
742 }
743 let chars: Vec<char> = text.chars().collect();
744 let mut out = String::with_capacity(text.len());
745 plain_inline_into(&chars, &mut out);
746 out
747}
748
749fn plain_inline_into(chars: &[char], out: &mut String) {
750 let find = |open: usize, pat: &[char]| -> Option<usize> {
753 chars
754 .get(open..)?
755 .windows(pat.len())
756 .enumerate()
757 .position(|(k, w)| w == pat && (open + k == 0 || chars[open + k - 1] != '\\'))
758 .map(|p| open + p)
759 };
760 let span = |i: usize, pat: &[char]| -> Option<usize> {
763 if !chars[i..].starts_with(pat) {
764 return None;
765 }
766 let open = i + pat.len();
767 let end = find(open, pat)?;
768 (end > open && !chars[open].is_whitespace() && !chars[end - 1].is_whitespace())
769 .then_some(end)
770 };
771 let mut i = 0;
772 while i < chars.len() {
773 if chars[i] == '\\' && chars.get(i + 1) == Some(&'*') {
775 out.push('*');
776 i += 2;
777 continue;
778 }
779 let mut matched = false;
780 for pat in [&['*', '*', '*'][..], &['*', '*'], &['~', '~'], &['*']] {
781 if let Some(end) = span(i, pat) {
782 plain_inline_into(&chars[i + pat.len()..end], out);
783 i = end + pat.len();
784 matched = true;
785 break;
786 }
787 }
788 if matched {
789 continue;
790 }
791 if chars[i] == '`' {
795 if let Some((s, e, after)) = code_span_bounds(chars, i).filter(|&(s, e, _)| {
796 e > s && !chars[s].is_whitespace() && !chars[e - 1].is_whitespace()
797 }) {
798 out.extend(&chars[s..e]);
799 i = after;
800 continue;
801 }
802 }
803 if chars[i] == '$' {
806 if let Some(end) = find(i + 1, &['$']) {
807 out.extend(&chars[i..=end]);
808 i = end + 1;
809 continue;
810 }
811 }
812 if chars[i] == '[' {
815 if let Some(close) = closing_bracket(chars, i) {
816 if chars.get(close + 1) == Some(&'(') {
817 if let Some(endp) = crate::doclang::link_dest_end(chars, close + 2) {
818 plain_inline_into(&chars[i + 1..close], out);
819 i = endp + 1;
820 continue;
821 }
822 }
823 }
824 }
825 let c = chars[i];
828 out.push(c);
829 i += 1;
830 if matches!(c, '*' | '~') {
831 while i < chars.len() && chars[i] == c {
832 out.push(c);
833 i += 1;
834 }
835 }
836 }
837}
838
839fn closing_bracket(chars: &[char], open: usize) -> Option<usize> {
842 let mut depth = 0usize;
843 for (k, c) in chars.iter().enumerate().skip(open + 1) {
844 match c {
845 '[' => depth += 1,
846 ']' if depth == 0 => return Some(k),
847 ']' => depth -= 1,
848 _ => {}
849 }
850 }
851 None
852}
853
854fn md_line_breaks(text: &str) -> String {
862 if !text.contains('\n') {
863 return text.to_string();
864 }
865 text.split("\n\n")
866 .map(|para| para.replace('\n', " \n"))
867 .collect::<Vec<_>>()
868 .join("\n\n")
869}
870
871pub(crate) fn strip_hard_breaks(text: &str) -> String {
876 if text.contains(" \n") {
877 text.replace(" \n", "\n")
878 } else {
879 text.to_string()
880 }
881}
882
883fn heading_line_breaks(text: &str) -> String {
887 text.replace('\n', " ")
888}
889
890fn render(nodes: &[Node], blocks: &mut Vec<String>, ctx: &mut Ctx) {
891 let mut i = 0;
892 while i < nodes.len() {
893 let before = blocks.len();
894 match &nodes[i] {
895 Node::PageBreak | Node::PageInfo { .. } => {
905 if ctx.page_break.is_some() && ctx.emitted_any {
906 ctx.pending_page_break = true;
907 }
908 i += 1;
909 }
910 Node::ListItem { .. } => {
911 let start = i;
912 i += 1;
913 loop {
914 match nodes.get(i) {
915 Some(Node::ListItem { .. } | Node::ListText { .. }) => i += 1,
916 Some(Node::Paragraph { text })
920 if text.is_empty()
921 && matches!(nodes.get(i + 1), Some(Node::ListItem { .. })) =>
922 {
923 i += 1
924 }
925 _ => break,
926 }
927 }
928 render_list_run(&nodes[start..i], blocks, ctx);
929 }
930 other => {
931 let before_item = blocks.len();
932 render_one(other, blocks, ctx);
933 if matches!(other, Node::Picture { .. })
939 && matches!(ctx.images, ImageMode::Placeholder)
940 && blocks.len() > before_item
941 && matches!(
942 nodes.get(i + 1),
943 Some(Node::Code { language: Some(l), .. }) if l == "mermaid"
944 )
945 {
946 blocks.pop();
947 }
948 i += 1;
949 }
950 }
951 if blocks.len() > before {
952 let kept: Vec<String> = blocks.drain(before..).filter(|b| !b.is_empty()).collect();
962 blocks.extend(kept);
963 if ctx.pending_page_break {
964 if let Some(placeholder) = &ctx.page_break {
969 blocks.insert(before, placeholder.clone());
970 }
971 ctx.pending_page_break = false;
972 }
973 ctx.emitted_any = true;
974 }
975 }
976}
977
978fn render_list_run(items: &[Node], blocks: &mut Vec<String>, ctx: &Ctx) {
984 let mut lines: Vec<String> = Vec::new();
985 let mut any_top = false;
988
989 for item in items {
990 if let Node::ListText { level, text, glue } = item {
991 let text = ctx.text(text);
992 match lines.last_mut() {
993 Some(last) if *glue => last.push_str(&text),
994 _ => lines.push(format!("{}{text}", " ".repeat(*level as usize))),
995 }
996 continue;
997 }
998 let Node::ListItem {
999 ordered,
1000 number,
1001 first_in_list,
1002 text,
1003 level,
1004 marker: orig_marker,
1005 location: _,
1006 dclx: _,
1007 split: _,
1008 href: _,
1009 layer,
1010 } = item
1011 else {
1012 continue;
1013 };
1014 if !ctx.layers.contains(layer.or(ctx.current_layer)) {
1018 continue;
1019 }
1020 let level = *level as usize;
1021
1022 if level == 0 {
1031 if any_top && *first_in_list {
1032 lines.push(String::new());
1033 }
1034 any_top = true;
1035 }
1036
1037 let indent = " ".repeat(level);
1038 lines.push(format!(
1039 "{indent}{} {}",
1040 list_item_marker(orig_marker.as_deref().unwrap_or(""), *ordered, *number),
1041 list_item_text(text, ctx)
1042 ));
1043 }
1044
1045 if !lines.is_empty() {
1048 blocks.push(lines.join("\n"));
1049 }
1050}
1051
1052fn list_item_marker(marker: &str, ordered: bool, number: u64) -> String {
1061 let already_valid = matches!(marker, "-" | "*" | "+")
1062 || marker
1063 .strip_suffix('.')
1064 .is_some_and(|d| !d.is_empty() && d.chars().all(char::is_numeric));
1065 if already_valid {
1066 return marker.to_string();
1067 }
1068 let outer = if ordered && marker.is_empty() {
1069 format!("{number}.")
1070 } else {
1071 "-".to_string()
1072 };
1073 if marker.chars().any(char::is_alphanumeric) {
1074 format!("{outer} {marker}")
1075 } else {
1076 outer
1077 }
1078}
1079
1080fn list_item_text(text: &str, ctx: &Ctx) -> String {
1088 let escaped = ctx.text(text);
1089 if ctx.plain {
1090 return escaped;
1091 }
1092 if let Some((own, tail)) = escaped.split_once('\n') {
1093 if is_folded_child_tail(tail) {
1094 return format!("{}\n{tail}", md_line_breaks(own));
1095 }
1096 }
1097 md_line_breaks(&escaped)
1098}
1099
1100fn is_folded_child_tail(tail: &str) -> bool {
1107 const MARKER: &str = "<!-- image -->";
1108 let mut lines = tail.split('\n').peekable();
1109 let mut any = false;
1110 while let Some(line) = lines.next() {
1111 let line = line.trim_start();
1112 if line == MARKER {
1113 any = true;
1114 } else if is_fence_line(line) {
1115 loop {
1118 match lines.next() {
1119 Some(l) if l.trim_start() == line => break,
1120 Some(_) => {}
1121 None => return false,
1122 }
1123 }
1124 any = true;
1125 } else if lines.next().map(str::trim_start) == Some(MARKER) {
1126 any = true; } else {
1128 return false;
1129 }
1130 }
1131 any
1132}
1133
1134fn render_one(node: &Node, blocks: &mut Vec<String>, ctx: &mut Ctx) {
1139 match node {
1140 Node::Group { .. }
1142 | Node::Furniture { .. }
1143 | Node::PageFurniture { .. }
1144 | Node::FurnitureText { .. }
1145 | Node::CommentSection { .. }
1146 | Node::PictureChildren(_)
1147 | Node::Commented { .. }
1148 | Node::ScriptText { .. }
1149 | Node::Located { .. }
1150 | Node::Prov { .. }
1151 | Node::Track { .. }
1152 | Node::PageBreak
1153 | Node::PageInfo { .. }
1154 | Node::DoclangOnly(_) => {}
1155 _ if !ctx.layer_visible() => return,
1156 _ => {}
1157 }
1158 render_item(node, blocks, ctx)
1159}
1160
1161fn render_item(node: &Node, blocks: &mut Vec<String>, ctx: &mut Ctx) {
1162 match node {
1163 Node::Heading { level, text } => {
1164 if ctx.plain {
1165 blocks.push(ctx.text(text));
1168 return;
1169 }
1170 let text = heading_line_breaks(&ctx.text(text));
1171 if ctx.in_table_cell {
1172 blocks.push(text);
1174 } else {
1175 let hashes = "#".repeat((*level).clamp(1, 6) as usize);
1176 blocks.push(format!("{hashes} {text}"));
1177 }
1178 }
1179 Node::Paragraph { text } if text.is_empty() => {}
1182 Node::Paragraph { text } => blocks.push(ctx.breaks(&ctx.text(text))),
1183 Node::LabeledText { .. } => render_item(&node.labeled_as_paragraph(), blocks, ctx),
1184 Node::Caption { text, .. } if text.is_empty() => {}
1187 Node::Caption { text, href } => {
1188 let body = ctx.breaks(&ctx.text(text));
1189 blocks.push(match href {
1190 Some(url) if !ctx.plain => crate::hyperlink::markdown_link(&body, url),
1191 _ => body,
1192 });
1193 }
1194 Node::CheckboxItem { checked, text } => {
1195 let mark = if *checked { "- [x]" } else { "- [ ]" };
1198 let sep = if text.starts_with(' ') { "" } else { " " };
1199 blocks.push(ctx.breaks(&ctx.text(&format!("{mark}{sep}{text}"))));
1200 }
1201 Node::Code {
1202 language,
1203 text,
1204 pretty,
1205 ..
1206 } => {
1207 let lang = match language {
1212 Some(l) if ctx.strict || l == "mermaid" => l.as_str(),
1213 _ => "",
1214 };
1215 let body = match pretty {
1218 Some(p) if ctx.strict => p.as_str(),
1219 _ => text.as_str(),
1220 };
1221 blocks.push(if ctx.plain {
1224 body.to_string()
1225 } else {
1226 code_fence(lang, body)
1227 });
1228 }
1229 Node::Formula { latex, orig, .. } => {
1235 if !latex.is_empty() {
1236 blocks.push(format!("$${latex}$$"));
1237 } else if !orig.is_empty() {
1238 blocks.push(FORMULA_NOT_DECODED.to_string());
1239 }
1240 }
1241 Node::Table(table) => {
1242 if let Some(cap) = &table.caption {
1245 if !cap.is_empty() {
1246 blocks.push(ctx.text(cap));
1247 }
1248 }
1249 let rendered = if ctx.plain {
1250 render_table(&ctx.plain_table(table), ctx.compact_tables)
1251 } else {
1252 render_table(table, ctx.compact_tables)
1253 };
1254 if !rendered.is_empty() {
1255 blocks.push(rendered);
1256 }
1257 for note in table.footnotes.iter().filter(|n| !n.is_empty()) {
1260 blocks.push(ctx.text(note));
1261 }
1262 }
1263 Node::Picture {
1272 caption,
1273 image,
1274 description,
1275 ..
1276 } => {
1277 if let Some(cap) = caption {
1278 if !cap.is_empty() {
1279 blocks.push(ctx.caption(cap));
1280 }
1281 }
1282 let alt = description
1283 .as_ref()
1284 .filter(|d| d.is_alt_text() && ctx.alt_text && !d.text.is_empty());
1285 let marker = picture_marker(image.as_ref(), ctx, alt.map(|d| d.text.as_str()));
1286 let native = description
1290 .as_ref()
1291 .filter(|d| d.is_docling_meta() && !d.text.is_empty());
1292 if let Some(desc) = description.as_ref().filter(|d| !d.is_docling_meta()) {
1293 let shown = !desc.is_alt_text() || (ctx.alt_text && !marker.starts_with("!["));
1294 if !desc.text.is_empty() && shown {
1295 blocks.push(ctx.breaks(&ctx.text(&desc.text)));
1296 }
1297 }
1298 blocks.push(marker);
1299 if let Some(desc) = native {
1300 blocks.push(ctx.breaks(&ctx.text(&desc.text)));
1301 }
1302 }
1303 Node::Chart {
1307 kind,
1308 table,
1309 caption,
1310 ..
1311 } => {
1312 if let Some(cap) = caption {
1313 if !cap.is_empty() {
1314 blocks.push(ctx.caption(cap));
1315 }
1316 }
1317 blocks.push(picture_marker(None, ctx, None));
1318 blocks.push(humanize_label(kind));
1319 let rendered = if ctx.plain {
1320 render_table(&ctx.plain_table(table), false)
1321 } else {
1322 render_table(table, false)
1323 };
1324 if !rendered.is_empty() {
1325 blocks.push(rendered);
1326 }
1327 }
1328 Node::DoclangOnly(_) => {}
1330 Node::Group {
1335 layer: Some(layer),
1336 children,
1337 ..
1338 } => {
1339 let outer = ctx.current_layer.replace(*layer);
1340 render(children, blocks, ctx);
1341 ctx.current_layer = outer;
1342 }
1343 Node::Group { children, .. } => render(children, blocks, ctx),
1344 Node::FieldRegion { items } => {
1345 for item in items {
1350 for part in [&item.marker, &item.key, &item.value].into_iter().flatten() {
1351 blocks.push(ctx.breaks(&ctx.text(part)));
1352 }
1353 }
1354 }
1355 Node::KeyValueGraph { .. } => blocks.push(MISSING_KEY_VALUE_ITEM.to_string()),
1358 Node::InlineGroup { md_text, .. } => blocks.push(ctx.breaks(&ctx.text(md_text))),
1361 Node::TextDump(text) => {
1363 if !text.is_empty() {
1364 blocks.push(text.clone());
1365 }
1366 }
1367 Node::Furniture { layer, inner } => {
1371 let outer = ctx.current_layer.replace(*layer);
1372 render_one(inner, blocks, ctx);
1373 ctx.current_layer = outer;
1374 }
1375 Node::PageFurniture { text, .. } | Node::FurnitureText { text, .. } => {
1379 if ctx.layers.furniture && !text.is_empty() {
1380 blocks.push(ctx.breaks(&ctx.text(text)));
1381 }
1382 }
1383 Node::PictureChildren(children) => {
1388 if ctx.traverse_pictures {
1389 render(children, blocks, ctx);
1390 }
1391 }
1392 Node::CommentSection { text, .. } => {
1397 if ctx.layers.notes && !text.is_empty() {
1398 blocks.push(ctx.breaks(&ctx.text(text)));
1399 }
1400 }
1401 Node::Commented { inner, .. } => render_one(inner, blocks, ctx),
1402 Node::ScriptText { inner, .. } => render_one(inner, blocks, ctx),
1405 Node::Located { inner, .. } | Node::Prov { inner, .. } | Node::Track { inner, .. } => {
1407 render_one(inner, blocks, ctx)
1408 }
1409 Node::PageBreak => {}
1411 Node::PageInfo { .. } => {}
1413 Node::Anchor { .. } => {}
1415 Node::ListItem { .. } => render_list_run(std::slice::from_ref(node), blocks, ctx),
1420 Node::ListText { text, .. } if text.is_empty() => {}
1422 Node::ListText { text, .. } => blocks.push(ctx.breaks(&ctx.text(text))),
1423 }
1424}
1425
1426fn humanize_label(label: &str) -> String {
1431 let text = label.replace('_', " ");
1432 let mut chars = text.chars();
1433 match chars.next() {
1434 Some(f) => f.to_uppercase().collect::<String>() + chars.as_str(),
1435 None => text,
1436 }
1437}
1438
1439fn image_alt(alt: Option<&str>) -> String {
1442 match alt {
1443 Some(text) => text
1444 .split_whitespace()
1445 .collect::<Vec<_>>()
1446 .join(" ")
1447 .replace('\\', "\\\\")
1448 .replace(']', "\\]"),
1449 None => "Image".to_string(),
1450 }
1451}
1452
1453fn picture_marker(image: Option<&crate::PictureImage>, ctx: &mut Ctx, alt: Option<&str>) -> String {
1454 match (ctx.images, image) {
1455 (ImageMode::Embedded, Some(img)) => {
1457 format!(
1458 "",
1459 image_alt(alt),
1460 crate::pixel_digest::docling_data_uri(img).1
1461 )
1462 }
1463 (ImageMode::Referenced, Some(img)) => {
1464 let path = format!(
1465 "{}/image_{:06}.{}",
1466 ctx.artifacts_dir,
1467 ctx.pic_index,
1468 ext_for(&img.mimetype)
1469 );
1470 ctx.pic_index += 1;
1471 ctx.artifacts.push((path.clone(), img.data.clone()));
1472 format!("", image_alt(alt), escape_uri_path(&path))
1473 }
1474 _ => ctx.image_placeholder.clone(),
1476 }
1477}
1478
1479pub(crate) fn escape_uri_path(value: &str) -> String {
1492 const KEEP: &str = "/%:@+,;=~$!&'*";
1493 let s = value.replace('\\', "/");
1494 if let Some(rest) = s.strip_prefix("//") {
1495 let rest = rest.trim_start_matches('/');
1497 let (host, tail) = rest.split_once('/').unwrap_or((rest, ""));
1498 return format!("file://{host}{}", percent_quote(&format!("/{tail}"), KEEP));
1499 }
1500 let bytes = s.as_bytes();
1501 if bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' && bytes[2] == b'/' {
1502 return format!("file:///{}", percent_quote(&s, KEEP));
1504 }
1505 if let Some((scheme, rest)) = s.split_once(':') {
1509 let valid_scheme = scheme.len() > 1
1510 && scheme.as_bytes()[0].is_ascii_alphabetic()
1511 && scheme
1512 .bytes()
1513 .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.'));
1514 if valid_scheme {
1515 let (authority, rest) = match rest.strip_prefix("//") {
1516 Some(r) => {
1517 let end = r.find(['/', '?', '#']).unwrap_or(r.len());
1518 (Some(&r[..end]), &r[end..])
1519 }
1520 None => (None, rest),
1521 };
1522 let (before_frag, fragment) = rest.split_once('#').unwrap_or((rest, ""));
1523 let (path, query) = before_frag.split_once('?').unwrap_or((before_frag, ""));
1524 let mut out = format!("{scheme}:");
1525 if let Some(a) = authority {
1526 out.push_str("//");
1527 out.push_str(a);
1528 }
1529 out.push_str(&percent_quote(path, KEEP));
1530 if !query.is_empty() {
1531 out.push('?');
1532 out.push_str(&percent_quote(query, KEEP));
1533 }
1534 if !fragment.is_empty() {
1535 out.push('#');
1536 out.push_str(&percent_quote(fragment, KEEP));
1537 }
1538 return out;
1539 }
1540 }
1541 percent_quote(&s, KEEP)
1543}
1544
1545fn percent_quote(s: &str, safe: &str) -> String {
1548 let mut out = String::with_capacity(s.len());
1549 for &b in s.as_bytes() {
1550 let keep = b.is_ascii_alphanumeric()
1551 || matches!(b, b'_' | b'.' | b'-' | b'~')
1552 || (b.is_ascii() && safe.contains(b as char));
1553 if keep {
1554 out.push(b as char);
1555 } else {
1556 out.push_str(&format!("%{b:02X}"));
1557 }
1558 }
1559 out
1560}
1561
1562pub(crate) fn ext_for(mimetype: &str) -> &str {
1563 match mimetype {
1564 "image/jpeg" => "jpg",
1565 "image/gif" => "gif",
1566 "image/webp" => "webp",
1567 "image/bmp" => "bmp",
1568 "image/tiff" => "tif",
1569 "image/svg+xml" => "svg",
1570 _ => "png",
1571 }
1572}
1573
1574fn is_number_cell(t: &str) -> bool {
1593 t.parse::<f64>().is_ok() || is_thousands_number(t)
1594}
1595
1596fn is_thousands_number(t: &str) -> bool {
1602 let b = t.as_bytes();
1603 let mut i = 0;
1604 let start = i;
1605 if i < b.len() && (b[i] == b'+' || b[i] == b'-') {
1606 i += 1;
1607 }
1608 let d0 = i;
1610 while i < b.len() && b[i].is_ascii_digit() && i - d0 < 3 {
1611 i += 1;
1612 }
1613 let has_int = i > d0;
1614 if has_int {
1615 while i + 3 < b.len() + 1
1617 && b.get(i) == Some(&b',')
1618 && b.get(i + 1).is_some_and(u8::is_ascii_digit)
1619 && b.get(i + 2).is_some_and(u8::is_ascii_digit)
1620 && b.get(i + 3).is_some_and(u8::is_ascii_digit)
1621 {
1622 i += 4;
1623 }
1624 } else {
1625 i = start;
1627 }
1628 if i < b.len() && b[i] == b'.' {
1630 i += 1;
1631 let f0 = i;
1632 while i < b.len() && b[i].is_ascii_digit() {
1633 i += 1;
1634 }
1635 if !has_int && i == f0 {
1636 return false; }
1638 } else if !has_int {
1639 return false; }
1641 i == b.len()
1642}
1643
1644fn flatten_header_rows(header_rows: &[Vec<String>], num_cols: usize) -> Vec<String> {
1653 (0..num_cols)
1654 .map(|c| {
1655 let mut parts: Vec<&str> = Vec::new();
1656 for row in header_rows {
1657 let text = row.get(c).map(String::as_str).unwrap_or("");
1658 if !text.is_empty() && parts.last() != Some(&text) {
1659 parts.push(text);
1660 }
1661 }
1662 parts.join(" - ")
1663 })
1664 .collect()
1665}
1666
1667pub(crate) const FORMULA_NOT_DECODED: &str = "<!-- formula-not-decoded -->";
1670
1671pub(crate) fn render_table(table: &Table, compact: bool) -> String {
1672 if table.rows.is_empty() {
1673 return String::new();
1674 }
1675 let num_cols = table.rows.iter().map(Vec::len).max().unwrap_or(0);
1676 if num_cols == 0 {
1677 return String::new();
1678 }
1679
1680 let num_headers = table.header_row_count().min(table.rows.len());
1685 let escaped = |r: usize| -> Vec<String> {
1686 (0..num_cols)
1687 .map(|c| escape_cell(table.rows[r].get(c).map(String::as_str).unwrap_or("")))
1688 .collect()
1689 };
1690 let header_rows: Vec<Vec<String>> = (0..num_headers).map(escaped).collect();
1691 let header = flatten_header_rows(&header_rows, num_cols);
1692 let body: Vec<Vec<String>> = (num_headers..table.rows.len())
1693 .map(|r| {
1694 escaped(r)
1695 .into_iter()
1696 .map(|c| c.trim().to_string())
1697 .collect()
1698 })
1699 .collect();
1700
1701 if compact {
1702 let render_row = |row: &[String]| -> String { format!("| {} |", row.join(" | ")) };
1704 let mut lines = Vec::with_capacity(body.len() + 2);
1705 lines.push(render_row(&header));
1706 let sep: Vec<&str> = (0..num_cols).map(|_| "-").collect();
1707 lines.push(format!("| {} |", sep.join(" | ")));
1708 for row in &body {
1709 lines.push(render_row(row));
1710 }
1711 return lines.join("\n");
1712 }
1713
1714 let dw = |s: &str| s.chars().count();
1716
1717 let right: Vec<bool> = (0..num_cols)
1722 .map(|c| {
1723 let mut any = false;
1724 for row in &body {
1725 let t = row[c].trim();
1726 if t.is_empty() {
1727 continue;
1728 }
1729 if !is_number_cell(t) {
1730 return false;
1731 }
1732 any = true;
1733 }
1734 any
1735 })
1736 .collect();
1737
1738 let width: Vec<usize> = (0..num_cols)
1740 .map(|c| {
1741 let mut w = dw(&header[c]) + 2;
1742 for row in &body {
1743 w = w.max(dw(&row[c]));
1744 }
1745 w
1746 })
1747 .collect();
1748
1749 let fmt_cell = |s: &str, c: usize| -> String {
1750 let pad = " ".repeat(width[c].saturating_sub(dw(s)));
1751 let body = if right[c] {
1752 format!("{pad}{s}")
1753 } else {
1754 format!("{s}{pad}")
1755 };
1756 format!(" {body} ")
1757 };
1758 let render_row = |row: &[String]| -> String {
1759 let cells: Vec<String> = (0..num_cols).map(|c| fmt_cell(&row[c], c)).collect();
1760 format!("|{}|", cells.join("|"))
1761 };
1762
1763 let mut lines = Vec::with_capacity(body.len() + 2);
1764 lines.push(render_row(&header));
1765 let sep: Vec<String> = (0..num_cols).map(|c| "-".repeat(width[c] + 2)).collect();
1766 lines.push(format!("|{}|", sep.join("|")));
1767 for row in &body {
1768 lines.push(render_row(row));
1769 }
1770 lines.join("\n")
1771}
1772
1773fn escape_cell(s: &str) -> String {
1776 s.replace('\n', " ").replace('|', "|")
1777}
1778
1779#[cfg(test)]
1780mod tests {
1781 use super::*;
1782 use crate::{PictureImage, TableCell, TableStructure};
1783
1784 #[test]
1789 fn an_undecoded_formula_prints_the_placeholder() {
1790 let formula = |latex: &str, orig: &str| Node::Formula {
1791 latex: latex.into(),
1792 orig: orig.into(),
1793 location: None,
1794 };
1795 let mut doc = DoclingDocument::new("t");
1796 doc.nodes = vec![
1797 formula("E = mc^2", "E = mc 2"),
1798 formula("", "E = mc 2"),
1799 formula("", ""),
1800 Node::Paragraph {
1801 text: "After.".into(),
1802 },
1803 ];
1804 assert_eq!(
1805 doc.export_to_markdown().trim(),
1806 "$$E = mc^2$$\n\n<!-- formula-not-decoded -->\n\nAfter."
1807 );
1808 let chunks: Vec<String> = crate::chunker::HierarchicalChunker
1809 .chunk(&doc)
1810 .into_iter()
1811 .map(|c| c.text)
1812 .collect();
1813 assert_eq!(
1814 chunks,
1815 ["$$E = mc^2$$", "<!-- formula-not-decoded -->", "After."]
1816 );
1817 }
1818
1819 #[test]
1825 fn list_boundaries_come_from_the_backend_not_the_numbering() {
1826 let item = |ordered: bool, number: u64, first_in_list: bool, text: &str| Node::ListItem {
1827 ordered,
1828 number,
1829 first_in_list,
1830 text: text.into(),
1831 level: 0,
1832 marker: None,
1833 location: None,
1834 split: None,
1835 dclx: None,
1836 href: None,
1837 layer: None,
1838 };
1839 let md = |items: Vec<Node>| {
1840 let mut doc = DoclingDocument::new("t");
1841 for n in items {
1842 doc.push(n);
1843 }
1844 doc.export_to_markdown()
1845 };
1846 assert_eq!(
1848 md(vec![
1849 item(true, 1, true, "one"),
1850 item(true, 5, false, "five")
1851 ]),
1852 "1. one\n5. five\n"
1853 );
1854 assert_eq!(
1856 md(vec![
1857 item(false, 0, true, "bullet"),
1858 item(true, 1, false, "one"),
1859 item(false, 0, false, "bullet two"),
1860 ]),
1861 "- bullet\n1. one\n- bullet two\n"
1862 );
1863 assert_eq!(
1865 md(vec![
1866 item(true, 1, true, "a"),
1867 item(true, 2, false, "b"),
1868 item(true, 3, true, "new list, continuing count"),
1869 ]),
1870 "1. a\n2. b\n\n3. new list, continuing count\n"
1871 );
1872 }
1873
1874 #[test]
1875 fn renders_headings_paragraphs_and_lists() {
1876 let mut doc = DoclingDocument::new("demo");
1877 doc.add_heading(1, "Title");
1878 doc.add_paragraph("Hello world.");
1879 doc.push(Node::ListItem {
1880 ordered: false,
1881 number: 1,
1882 first_in_list: true,
1883 text: "first".into(),
1884 level: 0,
1885 marker: None,
1886 location: None,
1887 split: None,
1888 dclx: None,
1889 href: None,
1890 layer: None,
1891 });
1892 doc.push(Node::ListItem {
1893 ordered: false,
1894 number: 2,
1895 first_in_list: false,
1896 text: "second".into(),
1897 level: 0,
1898 marker: None,
1899 location: None,
1900 split: None,
1901 dclx: None,
1902 href: None,
1903 layer: None,
1904 });
1905 let md = doc.export_to_markdown();
1906 assert_eq!(md, "# Title\n\nHello world.\n\n- first\n- second\n");
1907 }
1908
1909 #[test]
1913 fn single_newlines_become_gfm_hard_line_breaks() {
1914 let mut doc = DoclingDocument::new("t");
1915 doc.push(Node::Heading {
1916 level: 1,
1917 text: "Hello\nWorld".into(),
1918 });
1919 doc.push(Node::Paragraph {
1920 text: "line one\nline two\n\npara two".into(),
1921 });
1922 doc.push(Node::ListItem {
1923 ordered: false,
1924 number: 1,
1925 first_in_list: true,
1926 text: "item\ncontinued".into(),
1927 level: 0,
1928 marker: None,
1929 location: None,
1930 split: None,
1931 dclx: None,
1932 href: None,
1933 layer: None,
1934 });
1935 doc.push(Node::TextDump("A1 B1 \n\n\nC1".into()));
1936 assert_eq!(
1937 doc.export_to_markdown(),
1938 "# Hello World\n\nline one \nline two\n\npara two\n\n- item \ncontinued\n\nA1 B1 \n\n\nC1\n"
1939 );
1940 }
1941
1942 #[test]
1945 fn table_cell_mode_and_field_regions() {
1946 let mut doc = DoclingDocument::new("t");
1947 doc.push(Node::Heading {
1948 level: 2,
1949 text: "A text".into(),
1950 });
1951 doc.push(Node::Paragraph {
1952 text: "body".into(),
1953 });
1954 assert_eq!(to_markdown_table_cell(&doc, false), "A text\n\nbody");
1955 assert_eq!(doc.export_to_markdown(), "## A text\n\nbody\n");
1956
1957 let mut doc = DoclingDocument::new("f");
1958 doc.push(Node::FieldRegion {
1959 items: vec![crate::FieldItem {
1960 marker: None,
1961 key: Some("Name:".into()),
1962 value: Some("John Doe".into()),
1963 value_kind: None,
1964 }],
1965 });
1966 assert_eq!(doc.export_to_markdown(), "Name:\n\nJohn Doe\n");
1967 }
1968
1969 #[test]
1970 fn strict_renders_recovered_links_legacy_does_not() {
1971 let mut doc = DoclingDocument::new("cv");
1972 doc.add_paragraph("Find me on LinkedIn or GitHub.");
1973 doc.links = vec![
1974 ("LinkedIn".into(), "https://www.linkedin.com/in/x/".into()),
1975 ("GitHub".into(), "https://github.com/x/".into()),
1976 ];
1977 assert_eq!(doc.export_to_markdown(), "Find me on LinkedIn or GitHub.\n");
1979 assert_eq!(
1981 doc.export_to_markdown_with(true),
1982 "Find me on [LinkedIn](https://www.linkedin.com/in/x/) or [GitHub](https://github.com/x/).\n"
1983 );
1984 }
1985
1986 #[test]
1987 fn strict_links_match_escaped_anchor_and_consume_in_order() {
1988 let mut doc = DoclingDocument::new("d");
1989 doc.add_paragraph("AI & ML here, and issues here, then issues there.");
1993 doc.links = vec![
1994 ("AI & ML".into(), "https://a/".into()),
1995 ("issues".into(), "https://first/".into()),
1996 ("issues".into(), "https://second/".into()),
1997 ];
1998 assert_eq!(
1999 doc.export_to_markdown_with(true),
2000 "[AI & ML](https://a/) here, and [issues](https://first/) here, then [issues](https://second/) there.\n"
2001 );
2002 }
2003
2004 #[test]
2008 fn referenced_image_destinations_are_escaped() {
2009 let cases = [
2010 (
2011 "doc_artifacts/image_000001_ab12.png",
2012 "doc_artifacts/image_000001_ab12.png",
2013 ),
2014 (
2015 "My Report_artifacts/img.png",
2016 "My%20Report_artifacts/img.png",
2017 ),
2018 ("artifacts/img (1).png", "artifacts/img%20%281%29.png"),
2019 ("100%_scale/a#b?c.png", "100%_scale/a%23b%3Fc.png"),
2020 ("/home/a b/img.png", "/home/a%20b/img.png"),
2021 (
2022 "My Report_artifacts\\img.png",
2023 "My%20Report_artifacts/img.png",
2024 ),
2025 (
2026 "C:/Users/me/My Docs/img.png",
2027 "file:///C:/Users/me/My%20Docs/img.png",
2028 ),
2029 ("C:\\Users\\me\\img.png", "file:///C:/Users/me/img.png"),
2030 (
2031 "//server/share/My Docs/img.png",
2032 "file://server/share/My%20Docs/img.png",
2033 ),
2034 ("\\\\server\\share\\img.png", "file://server/share/img.png"),
2035 ("file:///home/a b/img.png", "file:///home/a%20b/img.png"),
2036 (
2037 "s3://bucket/My Report_artifacts/img.png",
2038 "s3://bucket/My%20Report_artifacts/img.png",
2039 ),
2040 (
2041 "https://example.com:8080/a b.png?w=1&h=2#frag",
2042 "https://example.com:8080/a%20b.png?w=1&h=2#frag",
2043 ),
2044 (
2045 "https://example.com/img (1).png",
2046 "https://example.com/img%20%281%29.png",
2047 ),
2048 ("caf\u{e9}/im\u{e4}ge.png", "caf%C3%A9/im%C3%A4ge.png"),
2049 ];
2050 for (input, expected) in cases {
2051 assert_eq!(escape_uri_path(input), expected, "input {input:?}");
2052 assert_eq!(
2053 escape_uri_path(expected),
2054 expected,
2055 "idempotent {expected:?}"
2056 );
2057 }
2058 let mut doc = DoclingDocument::new("t");
2060 doc.push(Node::Picture {
2061 caption: None,
2062 caption_href: None,
2063 image: Some(PictureImage {
2064 dpi: PictureImage::DEFAULT_DPI,
2065 mimetype: "image/png".into(),
2066 width: 1,
2067 height: 1,
2068 data: b"x".to_vec(),
2069 display_size: None,
2070 }),
2071 classification: None,
2072 description: None,
2073 caption_parent: Default::default(),
2074 caption_location: None,
2075 });
2076 let (md, files) = doc
2077 .export_to_markdown_with_images(ImageMode::Referenced, "My Report (final)_artifacts");
2078 assert!(
2079 md.contains(""),
2080 "got:\n{md}"
2081 );
2082 assert_eq!(files[0].0, "My Report (final)_artifacts/image_000000.png");
2084 }
2085
2086 #[test]
2090 fn folded_list_item_pictures_keep_plain_newlines() {
2091 let doc = DoclingDocument::new("t");
2092 let ctx = Ctx::new(&doc, false, &MarkdownExportOptions::default());
2093 assert_eq!(
2094 list_item_text("Step\n<!-- image -->", &ctx),
2095 "Step\n<!-- image -->"
2096 );
2097 assert_eq!(
2098 list_item_text("Step\nAlt text\n<!-- image -->\n<!-- image -->", &ctx),
2099 "Step\nAlt text\n<!-- image -->\n<!-- image -->"
2100 );
2101 assert_eq!(
2102 list_item_text("line one\nline two", &ctx),
2103 "line one \nline two"
2104 );
2105 }
2106
2107 #[test]
2110 fn stacked_header_rows_flatten_into_one() {
2111 let mut t = Table {
2112 rows: vec![
2113 vec!["".into(), "% of Total".into(), "% of Total".into()],
2114 vec!["class".into(), "Train".into(), "Test".into()],
2115 vec!["Caption".into(), "2.04".into(), "1.77".into()],
2116 ],
2117 ..Default::default()
2118 };
2119 t.structure = Some(TableStructure {
2120 header_row: vec![true, true, false],
2121 col_continuation: vec![
2122 vec![false, false, true],
2123 vec![false, false, false],
2124 vec![false, false, false],
2125 ],
2126 ..Default::default()
2127 });
2128 assert_eq!(t.header_row_count(), 2);
2129 assert_eq!(
2130 render_table(&t, true),
2131 "| class | % of Total - Train | % of Total - Test |\n| - | - | - |\n| Caption | 2.04 | 1.77 |"
2132 );
2133 assert_eq!(
2135 render_table(&t, false),
2136 "| class | % of Total - Train | % of Total - Test |\n\
2137 |---------|----------------------|---------------------|\n\
2138 | Caption | 2.04 | 1.77 |"
2139 );
2140 }
2141
2142 #[test]
2145 fn vertically_spanning_header_does_not_extend_the_block() {
2146 let mut t = Table {
2147 rows: vec![
2148 vec!["Name".into(), "Value".into()],
2149 vec!["Name".into(), "1".into()],
2150 vec!["x".into(), "2".into()],
2151 ],
2152 ..Default::default()
2153 };
2154 t.structure = Some(TableStructure {
2155 col_header: vec![vec![true, true], vec![true, false], vec![false, false]],
2156 row_continuation: vec![vec![false, false], vec![true, false], vec![false, false]],
2157 ..Default::default()
2158 });
2159 assert_eq!(t.header_row_count(), 1);
2160 assert_eq!(
2161 render_table(&t, true),
2162 "| Name | Value |\n| - | - |\n| Name | 1 |\n| x | 2 |"
2163 );
2164 }
2165
2166 #[test]
2169 fn header_flags_not_on_row_zero_keep_all_rows_in_the_body() {
2170 let mut t = Table {
2171 rows: vec![
2172 vec!["1".into(), "2".into()],
2173 vec!["a".into(), "b".into()],
2174 vec!["333".into(), "4".into()],
2175 ],
2176 ..Default::default()
2177 };
2178 t.structure = Some(TableStructure {
2179 header_row: vec![false, true, false],
2180 ..Default::default()
2181 });
2182 assert_eq!(t.header_row_count(), 0);
2183 assert_eq!(
2184 render_table(&t, false),
2185 "| | |\n|-----|----|\n| 1 | 2 |\n| a | b |\n| 333 | 4 |"
2186 );
2187 }
2188
2189 #[test]
2194 fn pivot_row_headers_do_not_extend_the_header() {
2195 let mut t = Table {
2196 rows: vec![
2197 vec!["Year".into(), "Month".into()],
2198 vec!["2025".into(), "January".into()],
2199 vec!["2025".into(), "February".into()],
2200 ],
2201 ..Default::default()
2202 };
2203 t.structure = Some(TableStructure {
2204 col_header: vec![vec![true, true], vec![false, false], vec![false, false]],
2205 row_header: vec![vec![false, false], vec![true, false], vec![true, false]],
2206 row_continuation: vec![vec![false, false], vec![false, false], vec![true, false]],
2207 ..Default::default()
2208 });
2209 assert_eq!(t.header_row_count(), 1);
2210 assert_eq!(
2211 render_table(&t, true),
2212 "| Year | Month |\n| - | - |\n| 2025 | January |\n| 2025 | February |"
2213 );
2214 }
2215
2216 #[test]
2219 fn unflagged_cells_keep_row_zero_as_header() {
2220 let mut t = Table {
2221 rows: vec![vec!["h".into()], vec!["d".into()]],
2222 ..Default::default()
2223 };
2224 t.cells = Some(
2225 [(0usize, "h"), (1, "d")]
2226 .into_iter()
2227 .map(|(r, text)| TableCell {
2228 text: text.into(),
2229 bbox: None,
2230 start_row: r,
2231 start_col: 0,
2232 row_span: 1,
2233 col_span: 1,
2234 column_header: false,
2235 row_header: false,
2236 row_section: false,
2237 })
2238 .collect(),
2239 );
2240 assert_eq!(t.header_row_count(), 1);
2241 assert_eq!(render_table(&t, true), "| h |\n| - |\n| d |");
2242 }
2243
2244 fn flagged_table(rows: &[&[&str]], ched: impl Fn(usize, usize) -> bool) -> Table {
2246 let mut cells = Vec::new();
2247 for (r, row) in rows.iter().enumerate() {
2248 for (c, text) in row.iter().enumerate() {
2249 cells.push(TableCell {
2250 text: (*text).into(),
2251 bbox: None,
2252 start_row: r,
2253 start_col: c,
2254 row_span: 1,
2255 col_span: 1,
2256 column_header: ched(r, c),
2257 row_header: false,
2258 row_section: false,
2259 });
2260 }
2261 }
2262 Table {
2263 rows: rows
2264 .iter()
2265 .map(|row| row.iter().map(|t| (*t).to_string()).collect())
2266 .collect(),
2267 cells: Some(cells),
2268 ..Default::default()
2269 }
2270 }
2271
2272 #[test]
2276 fn rows_with_body_text_stop_the_header_block() {
2277 let t = flagged_table(
2279 &[
2280 &["Item", "Q1", "Q2"],
2281 &["Sales", "10", "20"],
2282 &["Costs", "5", "7"],
2283 ],
2284 |r, c| r == 0 || c == 0,
2285 );
2286 assert_eq!(t.header_row_count(), 1);
2287 assert_eq!(
2288 render_table(&t, false),
2289 "| Item | Q1 | Q2 |\n\
2290 |--------|------|------|\n\
2291 | Sales | 10 | 20 |\n\
2292 | Costs | 5 | 7 |"
2293 );
2294 let t = flagged_table(&[&["Sales", "10", "20"], &["Costs", "5", "7"]], |_, c| {
2297 c == 0
2298 });
2299 assert_eq!(t.header_row_count(), 0);
2300 assert_eq!(
2301 render_table(&t, false),
2302 "| | | |\n\
2303 |-------|----|----|\n\
2304 | Sales | 10 | 20 |\n\
2305 | Costs | 5 | 7 |"
2306 );
2307 let t = flagged_table(
2308 &[&["a", "1", "2"], &["b", "3", "4"], &["c", "5", "6"]],
2309 |r, c| c == 0 && r < 2,
2310 );
2311 assert_eq!(t.header_row_count(), 0);
2312 let t = flagged_table(&[&["a", "1", "2"], &["b", "3", "4"]], |r, c| {
2315 r == 0 && c == 0
2316 });
2317 assert_eq!(t.header_row_count(), 1);
2318 assert_eq!(
2319 render_table(&t, false),
2320 "| a | 1 | 2 |\n\
2321 |-----|-----|-----|\n\
2322 | b | 3 | 4 |"
2323 );
2324 let t = flagged_table(&[&["Item", "", ""], &["x", "1", "2"]], |_, c| c == 0);
2326 assert_eq!(t.header_row_count(), 1);
2327 assert_eq!(
2328 render_table(&t, false),
2329 "| Item | | |\n\
2330 |--------|----|----|\n\
2331 | x | 1 | 2 |"
2332 );
2333 }
2334
2335 #[test]
2336 fn renders_compact_table() {
2337 let mut doc = DoclingDocument::new("t");
2338 doc.compact_tables = true;
2341 doc.push(Node::Table(Table {
2342 footnotes: Vec::new(),
2343 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2344 location: None,
2345 structure: None,
2346 cell_blocks: None,
2347 cells: None,
2348 caption: None,
2349 caption_parent: Default::default(),
2350 caption_location: None,
2351 document_index: false,
2352 }));
2353 let md = doc.export_to_markdown();
2354 assert_eq!(md, "| a | b |\n| - | - |\n| 1 | 2 |\n");
2355 }
2356
2357 #[test]
2358 fn renders_padded_github_table_by_default() {
2359 let mut doc = DoclingDocument::new("t");
2360 doc.push(Node::Table(Table {
2361 footnotes: Vec::new(),
2362 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2363 location: None,
2364 structure: None,
2365 cell_blocks: None,
2366 cells: None,
2367 caption: None,
2368 caption_parent: Default::default(),
2369 caption_location: None,
2370 document_index: false,
2371 }));
2372 let md = doc.export_to_markdown();
2373 assert_eq!(md, "| a | b |\n|-----|-----|\n| 1 | 2 |\n");
2375 }
2376
2377 #[test]
2378 fn strict_unescapes_inline_underscores_legacy_keeps_them() {
2379 let mut doc = DoclingDocument::new("t");
2380 doc.add_heading(1, "a\\_b");
2381 doc.add_paragraph("x\\_y");
2382 doc.push(Node::ListItem {
2383 ordered: false,
2384 number: 1,
2385 first_in_list: true,
2386 text: "i\\_j".into(),
2387 level: 0,
2388 marker: None,
2389 location: None,
2390 split: None,
2391 dclx: None,
2392 href: None,
2393 layer: None,
2394 });
2395 assert_eq!(doc.export_to_markdown(), "# a\\_b\n\nx\\_y\n\n- i\\_j\n");
2397 assert_eq!(doc.export_to_markdown_with(true), "# a_b\n\nx_y\n\n- i_j\n");
2399 }
2400
2401 fn assert_stream_matches(
2404 doc: &DoclingDocument,
2405 strict: bool,
2406 images: ImageMode,
2407 splits: &[usize],
2408 ) {
2409 let (want, want_artifacts) = to_markdown_images(doc, strict, images, "artifacts");
2410 let mut streamer =
2411 MarkdownStreamer::with_artifacts(strict, images, doc.compact_tables, "artifacts")
2412 .with_page_break_placeholder(doc.page_break_placeholder.clone());
2413 let mut got = String::new();
2414 let mut got_artifacts = Vec::new();
2415 let mut start = 0;
2416 for &end in splits {
2417 let links = if start == 0 {
2420 doc.links.as_slice()
2421 } else {
2422 &[]
2423 };
2424 got.push_str(&streamer.push(&doc.nodes[start..end], links));
2425 got_artifacts.extend(streamer.take_artifacts());
2428 start = end;
2429 }
2430 got.push_str(&streamer.push(
2431 &doc.nodes[start..],
2432 if start == 0 {
2433 doc.links.as_slice()
2434 } else {
2435 &[]
2436 },
2437 ));
2438 got_artifacts.extend(streamer.take_artifacts());
2439 got.push_str(&streamer.finish());
2440 assert_eq!(
2441 got, want,
2442 "streamed output diverged (splits={splits:?}, strict={strict})"
2443 );
2444 assert_eq!(
2445 got_artifacts, want_artifacts,
2446 "streamed artifacts diverged (splits={splits:?}, strict={strict})"
2447 );
2448 }
2449
2450 #[test]
2451 fn streaming_is_byte_identical_to_buffered() {
2452 let mut doc = DoclingDocument::new("d");
2453 doc.add_heading(1, "Title");
2454 doc.add_paragraph("First paragraph.");
2455 doc.push(Node::ListItem {
2456 ordered: false,
2457 number: 1,
2458 first_in_list: true,
2459 text: "a".into(),
2460 level: 0,
2461 marker: None,
2462 location: None,
2463 split: None,
2464 dclx: None,
2465 href: None,
2466 layer: None,
2467 });
2468 doc.push(Node::ListItem {
2469 ordered: false,
2470 number: 2,
2471 first_in_list: false,
2472 text: "b".into(),
2473 level: 0,
2474 marker: None,
2475 location: None,
2476 split: None,
2477 dclx: None,
2478 href: None,
2479 layer: None,
2480 });
2481 doc.push(Node::Code {
2482 language: Some("rust".into()),
2483 text: "let x = 1;".into(),
2484 orig: None,
2485 pretty: None,
2486 });
2487 doc.push(Node::Table(Table {
2488 footnotes: Vec::new(),
2489 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2490 location: None,
2491 structure: None,
2492 cell_blocks: None,
2493 cells: None,
2494 caption: None,
2495 caption_parent: Default::default(),
2496 caption_location: None,
2497 document_index: false,
2498 }));
2499 doc.push(Node::Picture {
2500 caption: Some("Fig 1".into()),
2501 caption_href: None,
2502 image: Some(PictureImage {
2503 dpi: PictureImage::DEFAULT_DPI,
2504 mimetype: "image/png".into(),
2505 width: 2,
2506 height: 2,
2507 data: b"png-one".to_vec(),
2508 display_size: None,
2509 }),
2510 classification: None,
2511 description: None,
2512 caption_parent: Default::default(),
2513 caption_location: None,
2514 });
2515 doc.add_paragraph("Last paragraph.");
2516 doc.push(Node::Picture {
2519 caption: None,
2520 caption_href: None,
2521 image: Some(PictureImage {
2522 dpi: PictureImage::DEFAULT_DPI,
2523 mimetype: "image/png".into(),
2524 width: 2,
2525 height: 2,
2526 data: b"png-two".to_vec(),
2527 display_size: None,
2528 }),
2529 classification: None,
2530 description: None,
2531 caption_parent: Default::default(),
2532 caption_location: None,
2533 });
2534
2535 for &strict in &[false, true] {
2538 for &images in &[
2539 ImageMode::Placeholder,
2540 ImageMode::Embedded,
2541 ImageMode::Referenced,
2542 ] {
2543 for splits in [&[][..], &[1][..], &[2][..], &[4][..], &[1, 4, 6, 7][..]] {
2544 assert_stream_matches(&doc, strict, images, splits);
2545 }
2546 }
2547 }
2548 }
2549
2550 #[test]
2551 fn streaming_applies_recovered_links_in_strict_mode() {
2552 let mut doc = DoclingDocument::new("d");
2553 doc.add_paragraph("See LinkedIn for details.");
2554 doc.add_paragraph("And GitHub too.");
2555 doc.links = vec![
2556 ("LinkedIn".into(), "https://lnkd/".into()),
2557 ("GitHub".into(), "https://gh/".into()),
2558 ];
2559 assert_stream_matches(&doc, true, ImageMode::Placeholder, &[1]);
2562 }
2563
2564 fn paged_doc() -> DoclingDocument {
2567 let mut doc = DoclingDocument::new("p");
2568 doc.push(Node::PageInfo {
2569 page_no: 1,
2570 width: 100.0,
2571 height: 100.0,
2572 });
2573 doc.add_heading(1, "Title");
2574 doc.add_paragraph("Page one.");
2575 doc.push(Node::PageBreak);
2577 doc.push(Node::PageInfo {
2578 page_no: 2,
2579 width: 100.0,
2580 height: 100.0,
2581 });
2582 doc.push(Node::PageFurniture {
2583 footer: true,
2584 location: [0, 500, 511, 511],
2585 text: "2".into(),
2586 });
2587 doc.push(Node::PageBreak);
2588 doc.push(Node::PageInfo {
2589 page_no: 3,
2590 width: 100.0,
2591 height: 100.0,
2592 });
2593 doc.add_paragraph("Page three.");
2594 doc.push(Node::PageBreak);
2596 doc
2597 }
2598
2599 #[test]
2600 fn page_break_placeholder_lands_between_pages_only() {
2601 let mut doc = paged_doc();
2602 assert_eq!(
2604 doc.export_to_markdown(),
2605 "# Title\n\nPage one.\n\nPage three.\n"
2606 );
2607 doc.page_break_placeholder = Some("<!-- page break -->".into());
2608 assert_eq!(
2612 doc.export_to_markdown(),
2613 "# Title\n\nPage one.\n\n<!-- page break -->\n\nPage three.\n"
2614 );
2615 doc.page_break_placeholder = Some(String::new());
2618 assert_eq!(
2619 doc.export_to_markdown(),
2620 "# Title\n\nPage one.\n\n\n\nPage three.\n"
2621 );
2622 }
2623
2624 #[test]
2625 fn page_break_placeholder_never_leads_a_single_page() {
2626 let mut doc = DoclingDocument::new("one");
2627 doc.page_break_placeholder = Some("---".into());
2628 doc.push(Node::PageBreak);
2629 doc.push(Node::PageInfo {
2630 page_no: 1,
2631 width: 10.0,
2632 height: 10.0,
2633 });
2634 doc.add_paragraph("Only page.");
2635 assert_eq!(doc.export_to_markdown(), "Only page.\n");
2636 doc.push(Node::PageBreak);
2638 doc.push(Node::PageBreak);
2639 doc.add_paragraph("Next.");
2640 assert_eq!(doc.export_to_markdown(), "Only page.\n\n---\n\nNext.\n");
2641 }
2642
2643 #[test]
2644 fn page_break_placeholder_streams_byte_identical() {
2645 let mut doc = paged_doc();
2646 doc.page_break_placeholder = Some("<!-- page break -->".into());
2647 for splits in [
2651 &[3usize][..],
2652 &[3, 6],
2653 &[3, 6, 8],
2654 &[1, 2, 3, 4, 5, 6, 7, 8, 9],
2655 &[8],
2656 ] {
2657 assert_stream_matches(&doc, false, ImageMode::Placeholder, splits);
2658 assert_stream_matches(&doc, true, ImageMode::Placeholder, splits);
2659 }
2660 }
2661
2662 #[test]
2663 fn strict_tightens_punctuation_spacing_legacy_keeps_it() {
2664 let mut doc = DoclingDocument::new("t");
2665 doc.add_paragraph("see [ 37 , 36 ] and ( x ) .");
2666 assert_eq!(doc.export_to_markdown(), "see [ 37 , 36 ] and ( x ) .\n");
2668 assert_eq!(doc.export_to_markdown_with(true), "see [37, 36] and (x).\n");
2670 }
2671
2672 fn options_doc() -> DoclingDocument {
2677 use crate::document::ContentLayer;
2678 let mut doc = DoclingDocument::new("opts");
2679 doc.push(Node::PageFurniture {
2680 footer: false,
2681 location: [0, 0, 0, 0],
2682 text: "FAX COVER \\_ R&D".into(),
2683 });
2684 doc.add_heading(1, "R&D \\_ report");
2685 doc.add_paragraph("Body a\\_b <tag>");
2686 doc.push(Node::Picture {
2687 caption: Some("Figure 1".into()),
2688 caption_href: None,
2689 image: None,
2690 classification: None,
2691 description: None,
2692 caption_parent: Default::default(),
2693 caption_location: None,
2694 });
2695 doc.push(Node::PictureChildren(vec![
2696 Node::Paragraph {
2697 text: "Field: Name".into(),
2698 },
2699 Node::Paragraph {
2700 text: "Field: Date".into(),
2701 },
2702 ]));
2703 doc.push(Node::CommentSection {
2704 name: "comment-1".into(),
2705 text: "[author: A]: note".into(),
2706 refs_note_text: false,
2707 grouped: true,
2708 });
2709 doc.push(Node::Group {
2710 label: "section".into(),
2711 name: Some("sheet: Hidden".into()),
2712 layer: Some(ContentLayer::Invisible),
2713 children: vec![Node::Paragraph {
2714 text: "hidden cell".into(),
2715 }],
2716 });
2717 doc.push(Node::Furniture {
2718 layer: ContentLayer::Furniture,
2719 inner: Box::new(Node::Paragraph {
2720 text: "Page 1 of 2".into(),
2721 }),
2722 });
2723 doc.push(Node::FurnitureText {
2724 label: "page_footer".into(),
2725 text: "footer line".into(),
2726 });
2727 doc
2728 }
2729
2730 #[test]
2734 fn default_options_are_the_default_export() {
2735 let doc = options_doc();
2736 let plain = doc.export_to_markdown();
2737 assert_eq!(
2738 plain,
2739 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2740 );
2741 let (with_default, artifacts) =
2742 doc.export_to_markdown_with_options(&MarkdownExportOptions::default());
2743 assert_eq!(with_default, plain);
2744 assert!(artifacts.is_empty());
2745 }
2746
2747 #[test]
2748 fn layers_render_furniture_notes_and_hidden_sheets() {
2749 use crate::document::ContentLayer;
2750 let doc = options_doc();
2751 let md = |layers: ContentLayers| {
2752 doc.export_to_markdown_with_options(&MarkdownExportOptions {
2753 layers,
2754 ..MarkdownExportOptions::default()
2755 })
2756 .0
2757 };
2758 assert_eq!(
2759 md(ContentLayers::BODY.with(ContentLayer::Furniture)),
2760 "FAX COVER \\_ R&D\n\n# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nPage 1 of 2\n\nfooter line\n",
2761 "a page header, a furniture-wrapped paragraph and a furniture text item render as paragraphs"
2762 );
2763 assert_eq!(
2764 md(ContentLayers::BODY.with(ContentLayer::Notes)),
2765 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\n[author: A]: note\n"
2766 );
2767 assert_eq!(
2768 md(ContentLayers::BODY.with(ContentLayer::Invisible)),
2769 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nhidden cell\n",
2770 "a hidden sheet's items sit on the sheet's layer"
2771 );
2772 assert_eq!(
2773 md(ContentLayers::NONE.with(ContentLayer::Furniture)),
2774 "FAX COVER \\_ R&D\n\nPage 1 of 2\n\nfooter line\n",
2775 "without the body layer only the furniture prints"
2776 );
2777 assert_eq!(md(ContentLayers::NONE), "");
2778 assert_eq!(
2779 md(ContentLayers::ALL),
2780 "FAX COVER \\_ R&D\n\n# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\n[author: A]: note\n\nhidden cell\n\nPage 1 of 2\n\nfooter line\n"
2781 );
2782 }
2783
2784 #[test]
2787 fn layers_apply_to_list_items() {
2788 use crate::document::ContentLayer;
2789 let mut doc = DoclingDocument::new("lists");
2790 doc.push(Node::ListItem {
2791 ordered: false,
2792 number: 1,
2793 first_in_list: true,
2794 text: "nav link".into(),
2795 level: 0,
2796 marker: None,
2797 location: None,
2798 split: None,
2799 dclx: None,
2800 href: None,
2801 layer: Some(ContentLayer::Furniture),
2802 });
2803 doc.add_paragraph("body");
2804 assert_eq!(doc.export_to_markdown(), "body\n");
2805 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2806 layers: ContentLayers::BODY.with(ContentLayer::Furniture),
2807 ..MarkdownExportOptions::default()
2808 });
2809 assert_eq!(md, "- nav link\n\nbody\n");
2810 }
2811
2812 #[test]
2813 fn traverse_pictures_prints_the_nested_text_after_the_picture() {
2814 let doc = options_doc();
2815 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2816 traverse_pictures: true,
2817 ..MarkdownExportOptions::default()
2818 });
2819 assert_eq!(
2820 md,
2821 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nField: Name\n\nField: Date\n"
2822 );
2823 }
2824
2825 #[test]
2826 fn escaping_can_be_turned_off_per_kind() {
2827 let doc = options_doc();
2828 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2829 escape_html: false,
2830 ..MarkdownExportOptions::default()
2831 });
2832 assert_eq!(
2833 md,
2834 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2835 );
2836 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2837 escape_underscores: false,
2838 ..MarkdownExportOptions::default()
2839 });
2840 assert_eq!(
2841 md,
2842 "# R&D _ report\n\nBody a_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2843 );
2844 let mut doc = DoclingDocument::new("code");
2847 doc.push(Node::Code {
2848 language: None,
2849 text: "a & b \\_ c".into(),
2850 orig: None,
2851 pretty: None,
2852 });
2853 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2854 escape_html: false,
2855 escape_underscores: false,
2856 ..MarkdownExportOptions::default()
2857 });
2858 assert_eq!(md, "```\na & b \\_ c\n```\n");
2859 }
2860
2861 #[test]
2865 fn fences_and_spans_out_length_inner_backticks() {
2866 let code = |text: &str| Node::Code {
2867 language: Some("md".into()),
2868 text: text.into(),
2869 orig: None,
2870 pretty: None,
2871 };
2872 let fence_of = |text: &str| {
2873 let mut doc = DoclingDocument::new("d");
2874 doc.push(code(text));
2875 (doc.export_to_markdown(), doc.export_to_markdown_with(true))
2876 };
2877 assert_eq!(fence_of("x").0, "```\nx\n```\n");
2878 assert_eq!(fence_of("a ` b").0, "```\na ` b\n```\n");
2879 assert_eq!(fence_of("a ``` b").0, "````\na ``` b\n````\n");
2880 assert_eq!(
2881 fence_of("a ``` b ```` c"),
2882 (
2883 "`````\na ``` b ```` c\n`````\n".to_string(),
2884 "`````md\na ``` b ```` c\n`````\n".to_string()
2885 )
2886 );
2887 assert_eq!(code_span("x"), "`x`");
2888 assert_eq!(code_span("a`b"), "``a`b``");
2889 assert_eq!(code_span("a ``` b"), "````a ``` b````");
2890 assert_eq!(code_span("`x"), "`` `x ``");
2891 assert_eq!(code_span("x`"), "`` x` ``");
2892 for (span, text) in [
2893 ("`x`", "x"),
2894 ("``a`b``", "a`b"),
2895 ("`` `x ``", "`x"),
2896 ("````a ``` b````", "a ``` b"),
2897 ] {
2898 assert_eq!(
2899 plain_inline(&format!("see {span} here")),
2900 format!("see {text} here")
2901 );
2902 }
2903 let mut doc = DoclingDocument::new("d");
2906 doc.push(code("a ``` b"));
2907 assert_eq!(doc.export_to_text(), "a ``` b");
2908 let mut doc = DoclingDocument::new("d");
2910 doc.add_paragraph("Before.");
2911 doc.push(code("a ``` b ```` c"));
2912 doc.add_paragraph("After.");
2913 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[1, 2]);
2914 assert_stream_matches(&doc, true, ImageMode::Placeholder, &[1, 2]);
2915 }
2916
2917 #[test]
2922 fn mermaid_code_item_replaces_its_picture_placeholder() {
2923 let picture = || Node::Picture {
2924 caption: None,
2925 caption_href: None,
2926 image: None,
2927 classification: None,
2928 description: None,
2929 caption_parent: Default::default(),
2930 caption_location: None,
2931 };
2932 let mut doc = DoclingDocument::new("d");
2933 doc.add_paragraph("Before.");
2934 doc.push(picture());
2935 doc.push(Node::Code {
2936 language: Some("mermaid".into()),
2937 text: "flowchart TD\n A --> B".into(),
2938 orig: None,
2939 pretty: None,
2940 });
2941 doc.add_paragraph("After.");
2942 doc.push(picture());
2943 let md = doc.export_to_markdown();
2944 assert_eq!(
2945 md,
2946 "Before.\n\n```mermaid\nflowchart TD\n A --> B\n```\n\nAfter.\n\n<!-- image -->\n"
2947 );
2948 assert_eq!(doc.export_to_markdown_with(true), md);
2949 for splits in [&[1][..], &[2], &[1, 2], &[2, 3], &[1, 2, 3, 4]] {
2950 assert_stream_matches(&doc, false, ImageMode::Placeholder, splits);
2951 assert_stream_matches(&doc, true, ImageMode::Placeholder, splits);
2952 }
2953 let mut doc = DoclingDocument::new("d");
2955 doc.add_paragraph("Only.");
2956 doc.push(picture());
2957 assert_eq!(doc.export_to_markdown(), "Only.\n\n<!-- image -->\n");
2958 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[1]);
2959 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[2]);
2960 }
2961
2962 #[test]
2963 fn image_placeholder_is_configurable() {
2964 let doc = options_doc();
2965 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2966 image_placeholder: "[figure]".into(),
2967 ..MarkdownExportOptions::default()
2968 });
2969 assert!(md.contains("Figure 1\n\n[figure]\n"), "{md}");
2970 assert!(!md.contains("<!-- image -->"));
2971 }
2972
2973 #[test]
2979 fn empty_image_placeholder_prints_nothing() {
2980 fn picture(caption: Option<&str>) -> Node {
2981 Node::Picture {
2982 caption: caption.map(Into::into),
2983 caption_href: None,
2984 image: None,
2985 classification: None,
2986 description: None,
2987 caption_parent: Default::default(),
2988 caption_location: None,
2989 }
2990 }
2991 fn page(no: usize) -> Node {
2992 Node::PageInfo {
2993 page_no: no,
2994 width: 100.0,
2995 height: 100.0,
2996 }
2997 }
2998 fn para(text: &str) -> Node {
2999 Node::Paragraph { text: text.into() }
3000 }
3001 let options = MarkdownExportOptions {
3002 image_placeholder: String::new(),
3003 ..MarkdownExportOptions::default()
3004 };
3005 let cases: Vec<(Vec<Node>, Option<&str>, &str)> = vec![
3007 (
3008 vec![page(1), para("a"), picture(None), para("b")],
3009 None,
3010 "a\n\nb\n",
3011 ),
3012 (
3013 vec![page(1), para("a"), picture(Some("Figure 1")), para("b")],
3014 None,
3015 "a\n\nFigure 1\n\nb\n",
3016 ),
3017 (
3018 vec![
3019 page(1),
3020 para("a"),
3021 page(2),
3022 picture(None),
3023 page(3),
3024 para("b"),
3025 ],
3026 Some("<!-- pb -->"),
3027 "a\n\n<!-- pb -->\n\n<!-- pb -->\n\nb\n",
3028 ),
3029 (
3030 vec![
3031 page(1),
3032 para("a"),
3033 page(2),
3034 picture(None),
3035 page(3),
3036 para("b"),
3037 ],
3038 Some(""),
3039 "a\n\n\n\n\n\nb\n",
3040 ),
3041 (
3042 vec![page(1), picture(None), page(2), para("b")],
3043 Some("<!-- pb -->"),
3044 "<!-- pb -->\n\nb\n",
3045 ),
3046 (
3047 vec![page(1), picture(None), page(2), para("b")],
3048 Some(""),
3049 "\n\nb\n",
3050 ),
3051 (
3052 vec![page(1), para("a"), page(2), picture(None)],
3053 Some("<!-- pb -->"),
3054 "a\n\n<!-- pb -->\n",
3055 ),
3056 ];
3057 for (nodes, placeholder, expected) in cases {
3058 let mut doc = DoclingDocument::new("t");
3059 doc.page_break_placeholder = placeholder.map(Into::into);
3060 for node in nodes {
3061 doc.push(node);
3062 }
3063 let (md, _) = doc.export_to_markdown_with_options(&options);
3064 assert_eq!(md, expected, "page break {placeholder:?}");
3065 for size in 1..=doc.nodes.len() {
3068 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3069 .with_export_options(&options)
3070 .with_page_break_placeholder(placeholder.map(Into::into));
3071 let mut out = String::new();
3072 for chunk in doc.nodes.chunks(size) {
3073 out.push_str(&streamer.push(chunk, &[]));
3074 }
3075 out.push_str(&streamer.finish());
3076 assert_eq!(
3077 out, expected,
3078 "chunks of {size}, page break {placeholder:?}"
3079 );
3080 }
3081 }
3082 let mut doc = DoclingDocument::new("t");
3084 doc.push(para("a"));
3085 doc.push(picture(None));
3086 doc.push(para("b"));
3087 assert_eq!(doc.export_to_markdown(), "a\n\n<!-- image -->\n\nb\n");
3088 }
3089
3090 #[test]
3093 fn streamer_matches_buffered_export_with_options() {
3094 let doc = options_doc();
3095 let options = MarkdownExportOptions {
3096 layers: ContentLayers::ALL,
3097 traverse_pictures: true,
3098 escape_html: false,
3099 escape_underscores: false,
3100 image_placeholder: "(img)".into(),
3101 ..MarkdownExportOptions::default()
3102 };
3103 let (buffered, _) = doc.export_to_markdown_with_options(&options);
3104 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3105 .with_export_options(&options);
3106 let mut out = String::new();
3107 for chunk in doc.nodes.chunks(2) {
3108 out.push_str(&streamer.push(chunk, &[]));
3109 }
3110 out.push_str(&streamer.finish());
3111 assert_eq!(out, buffered);
3112 assert!(
3113 buffered.contains("(img)") && buffered.contains("R&D _ report"),
3114 "{buffered}"
3115 );
3116 }
3117
3118 #[test]
3121 fn plain_text_reads_escaped_stars() {
3122 assert_eq!(plain_inline("A\\*\\*\\* GmbH"), "A*** GmbH");
3123 assert_eq!(
3124 plain_inline("**FN \\*0\\*271\\*h** and *h\\*.at*"),
3125 "FN *0*271*h and h*.at"
3126 );
3127 }
3128
3129 #[test]
3132 fn plain_text_drops_the_markdown_decoration() {
3133 let item = |text: &str, ordered: bool, number: u64, level: u8| Node::ListItem {
3134 ordered,
3135 number,
3136 first_in_list: number == 1 && level == 0,
3137 text: text.into(),
3138 level,
3139 marker: None,
3140 location: None,
3141 split: None,
3142 dclx: None,
3143 href: None,
3144 layer: None,
3145 };
3146 let mut doc = DoclingDocument::new("plain");
3147 doc.add_heading(1, "Title with **bold**");
3148 doc.add_heading(2, "Section");
3149 doc.add_paragraph("Some **bold**, *italic*, ~~gone~~ and `code` with a [link](https://x.org).\nNext line.");
3150 doc.push(item("*one*", false, 1, 0));
3151 doc.push(item("[two](https://y.org)", false, 2, 0));
3152 doc.push(item("nested", true, 1, 1));
3153 doc.push(Node::CheckboxItem {
3154 checked: true,
3155 text: "done".into(),
3156 });
3157 doc.push(Node::Code {
3158 language: Some("rust".into()),
3159 text: "let x = **y**;".into(),
3160 orig: None,
3161 pretty: None,
3162 });
3163 doc.push(Node::Picture {
3164 caption: Some("Figure *1*".into()),
3165 caption_href: None,
3166 image: None,
3167 classification: None,
3168 description: None,
3169 caption_parent: Default::default(),
3170 caption_location: None,
3171 });
3172 doc.push(Node::Caption {
3173 text: "Linked caption".into(),
3174 href: Some("https://z.org".into()),
3175 });
3176 doc.push(Node::Table(Table {
3177 rows: vec![
3178 vec!["**n**".into(), "rich".into()],
3179 vec!["**2**".into(), "a \nb".into()],
3180 ],
3181 ..Default::default()
3182 }));
3183 doc.add_paragraph("R&D \\_ x");
3184 let text = doc.export_to_text();
3185 assert_eq!(
3186 text,
3187 "Title with bold\n\n\
3188 Section\n\n\
3189 Some bold, italic, gone and code with a link.\nNext line.\n\n\
3190 - one\n- two\n 1. nested\n\n\
3191 - [x] done\n\n\
3192 let x = **y**;\n\n\
3193 Figure 1\n\n\
3194 Linked caption\n\n\
3195 | n | rich |\n\
3196 |-----|--------|\n\
3197 | 2 | a b |\n\n\
3198 R&D _ x"
3199 );
3200
3201 let options = MarkdownExportOptions::plain_text();
3202 let (buffered, _) = doc.export_to_markdown_with_options(&options);
3203 assert_eq!(buffered, format!("{text}\n"));
3204 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3205 .with_export_options(&options);
3206 let mut out = String::new();
3207 for chunk in doc.nodes.chunks(3) {
3208 out.push_str(&streamer.push(chunk, &[]));
3209 }
3210 out.push_str(&streamer.finish());
3211 assert_eq!(out, buffered);
3212 }
3213
3214 #[test]
3216 fn plain_inline_keeps_literal_markers() {
3217 for (input, want) in [
3218 ("**bold** and *it* and ***both***", "bold and it and both"),
3219 ("**[label](https://a.b/c_(d))** end", "label end"),
3220 (
3221 "grant *USE and *OBJMGT authority",
3222 "grant *USE and *OBJMGT authority",
3223 ),
3224 ("2 * 3 * 4", "2 * 3 * 4"),
3225 ("a ** b", "a ** b"),
3226 ("the `quoted’ and `x` term", "the `quoted’ and x term"),
3227 ("see [12] and [docs](u)", "see [12] and docs"),
3228 ("formula $[a](b)$ stays", "formula $[a](b)$ stays"),
3229 ("price $5 and **bold**", "price $5 and bold"),
3230 ("[outer [inner] text](u)", "outer [inner] text"),
3231 ("unclosed **bold", "unclosed **bold"),
3232 ] {
3233 assert_eq!(plain_inline(input), want, "{input}");
3234 }
3235 }
3236}