1use crate::document::{
4 ContentLayer, ContentLayers, DoclingDocument, MarkdownExportOptions, Node, Table,
5};
6
7const MISSING_KEY_VALUE_ITEM: &str = "<!-- missing-key-value-item -->";
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
14pub enum ImageMode {
15 #[default]
17 Placeholder,
18 Embedded,
20 Referenced,
23}
24
25struct Ctx {
27 strict: bool,
28 alt_text: bool,
31 compact_tables: bool,
33 images: ImageMode,
34 artifacts_dir: String,
35 artifacts: Vec<(String, Vec<u8>)>,
37 pic_index: usize,
38 in_table_cell: bool,
42 page_break: Option<String>,
46 pending_page_break: bool,
52 emitted_any: bool,
55 layers: ContentLayers,
57 current_layer: Option<ContentLayer>,
62 traverse_pictures: bool,
64 escape_html: bool,
67 escape_underscores: bool,
69 image_placeholder: String,
71 plain: bool,
73}
74
75impl Ctx {
76 fn new(doc: &DoclingDocument, strict: bool, options: &MarkdownExportOptions) -> Self {
79 Ctx {
80 strict,
81 alt_text: strict || doc.picture_alt_text,
82 compact_tables: doc.compact_tables,
83 images: options.image_mode,
84 artifacts_dir: options.artifacts_dir.clone(),
85 artifacts: Vec::new(),
86 pic_index: 0,
87 in_table_cell: false,
88 page_break: doc.page_break_placeholder.clone(),
89 pending_page_break: false,
90 emitted_any: false,
91 layers: options.layers,
92 current_layer: None,
93 traverse_pictures: options.traverse_pictures,
94 escape_html: options.escape_html,
95 escape_underscores: options.escape_underscores,
96 image_placeholder: options.image_placeholder.clone(),
97 plain: options.plain_text,
98 }
99 }
100
101 fn layer_visible(&self) -> bool {
103 self.layers.contains(self.current_layer)
104 }
105
106 fn unescape(&self, text: &str) -> String {
113 let mut out = std::borrow::Cow::Borrowed(text);
114 if !self.escape_html
115 && (out.contains("&") || out.contains("<") || out.contains(">"))
116 {
117 out = std::borrow::Cow::Owned(
118 out.replace("<", "<")
119 .replace(">", ">")
120 .replace("&", "&"),
121 );
122 }
123 if !self.escape_underscores && out.contains("\\_") {
124 out = std::borrow::Cow::Owned(out.replace("\\_", "_"));
125 }
126 out.into_owned()
127 }
128
129 fn text(&self, text: &str) -> String {
132 let text = strict_text(text, self.strict);
133 if self.plain {
134 self.unescape(&plain_inline(&self.drop_image_markers(&text)))
135 } else {
136 self.unescape(&text)
137 }
138 }
139
140 fn caption(&self, text: &str) -> String {
144 if self.plain {
145 self.unescape(&plain_inline(text))
146 } else {
147 self.unescape(text)
148 }
149 }
150
151 fn breaks(&self, text: &str) -> String {
154 if self.plain {
155 text.to_string()
156 } else {
157 md_line_breaks(text)
158 }
159 }
160
161 fn plain_table(&self, table: &Table) -> Table {
168 let mut plain = table.clone();
169 for cell in plain.rows.iter_mut().flatten() {
170 if cell.contains(['*', '~', '`', '[', '\n', '<']) {
171 let text = self.drop_image_markers(&strip_hard_breaks(cell));
172 *cell = plain_inline(&text);
173 }
174 }
175 plain
176 }
177
178 fn drop_image_markers(&self, text: &str) -> String {
185 const MARKER: &str = "<!-- image -->";
186 if !text.contains(MARKER) && !text.contains("```") {
187 return text.to_string();
188 }
189 text.split("\n\n")
193 .filter_map(|block| {
194 let lines: Vec<String> = block
195 .split('\n')
196 .filter_map(|line| {
197 if line.trim() == MARKER || is_fence_line(line.trim()) {
200 return None;
201 }
202 Some(line.replace(MARKER, &self.image_placeholder))
203 })
204 .collect();
205 (!lines.is_empty()).then(|| lines.join("\n"))
206 })
207 .collect::<Vec<_>>()
208 .join("\n\n")
209 }
210}
211
212pub fn to_markdown(doc: &DoclingDocument, strict: bool) -> String {
218 to_markdown_images(doc, strict, ImageMode::Placeholder, "artifacts").0
219}
220
221pub fn to_markdown_images(
225 doc: &DoclingDocument,
226 strict: bool,
227 images: ImageMode,
228 artifacts_dir: &str,
229) -> (String, Vec<(String, Vec<u8>)>) {
230 to_markdown_with_options(
231 doc,
232 strict,
233 &MarkdownExportOptions {
234 image_mode: images,
235 artifacts_dir: artifacts_dir.to_string(),
236 ..MarkdownExportOptions::default()
237 },
238 )
239}
240
241pub fn to_markdown_with_options(
245 doc: &DoclingDocument,
246 strict: bool,
247 options: &MarkdownExportOptions,
248) -> (String, Vec<(String, Vec<u8>)>) {
249 let mut ctx = Ctx::new(doc, strict, options);
250 let mut blocks: Vec<String> = Vec::new();
251 render(&doc.nodes, &mut blocks, &mut ctx);
252 let mut body = blocks.join("\n\n");
253 if strict && !options.plain_text && !doc.links.is_empty() {
257 body = apply_links(&body, &doc.links);
258 }
259 let front = if doc.metadata_output.front_matter() && !options.plain_text {
262 doc.metadata.to_front_matter()
263 } else {
264 String::new()
265 };
266 let md = match (front.is_empty(), body.is_empty()) {
267 (true, true) => String::new(),
268 (true, false) => format!("{body}\n"),
269 (false, true) => front,
270 (false, false) => format!("{front}\n{body}\n"),
271 };
272 (md, ctx.artifacts)
273}
274
275pub fn to_markdown_table_cell(doc: &DoclingDocument, strict: bool) -> String {
284 let mut ctx = Ctx::new(doc, strict, &MarkdownExportOptions::default());
285 ctx.in_table_cell = true;
286 ctx.artifacts_dir = String::new();
287 ctx.page_break = None;
291 let mut blocks: Vec<String> = Vec::new();
292 render(&doc.nodes, &mut blocks, &mut ctx);
293 blocks.join("\n\n")
294}
295
296fn apply_links(body: &str, links: &[(String, String)]) -> String {
304 let mut out = body.to_string();
305 let mut cursor = 0usize;
306 for (anchor, href) in links {
307 let anchor = anchor
308 .replace('&', "&")
309 .replace('<', "<")
310 .replace('>', ">");
311 if anchor.is_empty() {
312 continue;
313 }
314 if let Some(rel) = out[cursor..].find(&anchor) {
315 let at = cursor + rel;
316 let replacement = crate::hyperlink::markdown_link(&anchor, href);
319 out.replace_range(at..at + anchor.len(), &replacement);
320 cursor = at + replacement.len();
321 }
322 }
323 out
324}
325
326fn apply_links_chunk(chunk: &str, queue: &mut Vec<(String, String)>) -> String {
337 let mut out = chunk.to_string();
338 let mut cursor = 0usize;
339 let mut carried: Vec<(String, String)> = Vec::new();
340 for (anchor_raw, href) in std::mem::take(queue) {
341 let anchor = anchor_raw
342 .replace('&', "&")
343 .replace('<', "<")
344 .replace('>', ">");
345 if anchor.is_empty() {
346 continue;
347 }
348 if let Some(rel) = out[cursor..].find(&anchor) {
349 let at = cursor + rel;
350 let replacement = crate::hyperlink::markdown_link(&anchor, &href);
351 out.replace_range(at..at + anchor.len(), &replacement);
352 cursor = at + replacement.len();
353 } else {
354 carried.push((anchor_raw, href));
356 }
357 }
358 *queue = carried;
359 out
360}
361
362pub struct MarkdownStreamer {
380 strict: bool,
381 images: ImageMode,
382 compact_tables: bool,
383 picture_alt_text: bool,
385 front_matter: Option<String>,
389 emitted_any: bool,
392 links: Vec<(String, String)>,
394 artifacts_dir: String,
398 artifacts: Vec<(String, Vec<u8>)>,
399 pic_index: usize,
400 page_break: Option<String>,
404 pending_page_break: bool,
405 items_emitted: bool,
411 held: Option<Node>,
416 layers: ContentLayers,
419 traverse_pictures: bool,
420 escape_html: bool,
421 escape_underscores: bool,
422 image_placeholder: String,
423 plain: bool,
424}
425
426impl MarkdownStreamer {
427 pub fn new(strict: bool, images: ImageMode, compact_tables: bool) -> Self {
430 debug_assert!(
431 images != ImageMode::Referenced,
432 "referenced image mode needs an artifacts dir; use with_artifacts"
433 );
434 Self::with_artifacts(strict, images, compact_tables, "artifacts")
435 }
436
437 pub fn with_artifacts(
444 strict: bool,
445 images: ImageMode,
446 compact_tables: bool,
447 artifacts_dir: &str,
448 ) -> Self {
449 Self {
450 strict,
451 images,
452 compact_tables,
453 picture_alt_text: false,
454 front_matter: None,
455 emitted_any: false,
456 links: Vec::new(),
457 artifacts_dir: artifacts_dir.to_string(),
458 artifacts: Vec::new(),
459 pic_index: 0,
460 page_break: None,
461 pending_page_break: false,
462 items_emitted: false,
463 held: None,
464 layers: ContentLayers::BODY,
465 traverse_pictures: false,
466 escape_html: true,
467 escape_underscores: true,
468 image_placeholder: "<!-- image -->".to_string(),
469 plain: false,
470 }
471 }
472
473 pub fn with_export_options(mut self, options: &MarkdownExportOptions) -> Self {
480 self.images = options.image_mode;
481 self.artifacts_dir = options.artifacts_dir.clone();
482 self.layers = options.layers;
483 self.traverse_pictures = options.traverse_pictures;
484 self.escape_html = options.escape_html;
485 self.escape_underscores = options.escape_underscores;
486 self.image_placeholder = options.image_placeholder.clone();
487 self.plain = options.plain_text;
488 self
489 }
490
491 pub fn with_page_break_placeholder(mut self, placeholder: Option<String>) -> Self {
496 self.page_break = placeholder;
497 self
498 }
499
500 pub fn with_picture_alt_text(mut self, on: bool) -> Self {
504 self.picture_alt_text = on;
505 self
506 }
507
508 pub fn with_metadata(mut self, metadata: &crate::metadata::DocumentMetadata) -> Self {
513 self.front_matter = (!self.plain)
514 .then(|| metadata.to_front_matter())
515 .filter(|f| !f.is_empty());
516 self
517 }
518
519 pub fn take_artifacts(&mut self) -> Vec<(String, Vec<u8>)> {
524 std::mem::take(&mut self.artifacts)
525 }
526
527 pub fn push(&mut self, nodes: &[Node], links: &[(String, String)]) -> String {
532 self.push_holding(nodes, links, true)
533 }
534
535 fn push_holding(&mut self, nodes: &[Node], links: &[(String, String)], hold: bool) -> String {
538 self.links.extend(links.iter().cloned());
539 let images = self.images;
543 let ends_on_picture = |nodes: &[Node]| {
544 matches!(images, ImageMode::Placeholder)
545 && matches!(nodes.last(), Some(Node::Picture { .. }))
546 };
547 let mut owned: Vec<Node> = Vec::new();
548 let mut nodes = nodes;
549 if self.held.is_some() || (hold && ends_on_picture(nodes)) {
550 owned.extend(self.held.take());
551 owned.extend(nodes.iter().cloned());
552 if hold && ends_on_picture(&owned) {
553 self.held = owned.pop();
554 }
555 nodes = &owned;
556 }
557 let mut ctx = Ctx {
558 strict: self.strict,
559 alt_text: self.strict || self.picture_alt_text,
560 compact_tables: self.compact_tables,
561 images: self.images,
562 artifacts_dir: std::mem::take(&mut self.artifacts_dir),
563 artifacts: std::mem::take(&mut self.artifacts),
564 pic_index: self.pic_index,
565 in_table_cell: false,
566 page_break: std::mem::take(&mut self.page_break),
567 pending_page_break: self.pending_page_break,
568 emitted_any: self.items_emitted,
569 layers: self.layers,
570 current_layer: None,
571 traverse_pictures: self.traverse_pictures,
572 escape_html: self.escape_html,
573 escape_underscores: self.escape_underscores,
574 image_placeholder: std::mem::take(&mut self.image_placeholder),
575 plain: self.plain,
576 };
577 let mut blocks: Vec<String> = Vec::new();
578 render(nodes, &mut blocks, &mut ctx);
579 self.image_placeholder = std::mem::take(&mut ctx.image_placeholder);
580 self.artifacts_dir = std::mem::take(&mut ctx.artifacts_dir);
581 self.artifacts = std::mem::take(&mut ctx.artifacts);
582 self.pic_index = ctx.pic_index;
583 self.page_break = std::mem::take(&mut ctx.page_break);
584 self.pending_page_break = ctx.pending_page_break;
585 self.items_emitted = ctx.emitted_any;
586 if blocks.is_empty() {
587 return String::new();
588 }
589 let mut body = blocks.join("\n\n");
590 if self.strict && !self.plain && !self.links.is_empty() {
591 body = apply_links_chunk(&body, &mut self.links);
592 }
593 let chunk = if let Some(front) = self.front_matter.take() {
594 format!("{front}\n{body}")
595 } else if self.emitted_any {
596 format!("\n\n{body}")
597 } else {
598 body
599 };
600 self.emitted_any = true;
601 chunk
602 }
603
604 pub fn finish(mut self) -> String {
607 let mut out = String::new();
608 if self.held.is_some() {
609 out.push_str(&self.push_holding(&[], &[], false));
610 }
611 if let Some(front) = self.front_matter {
612 out.push_str(&front);
613 } else if self.emitted_any {
614 out.push('\n');
615 }
616 out
617 }
618}
619
620fn strict_text(text: &str, strict: bool) -> String {
628 if !strict {
629 return text.to_string();
630 }
631 if let Some(rest) = text.strip_prefix("[ ] ") {
634 return format!("[ ] {}", strict_text(rest, strict));
635 }
636 text.replace("\\_", "_")
637 .replace(" ,", ",")
638 .replace(" .", ".")
639 .replace(" ;", ";")
640 .replace(" )", ")")
641 .replace("( ", "(")
642 .replace(" ]", "]")
643 .replace("[ ", "[")
644}
645
646pub fn longest_backtick_run(text: &str) -> usize {
649 let (mut best, mut run) = (0, 0);
650 for c in text.chars() {
651 if c == '`' {
652 run += 1;
653 best = best.max(run);
654 } else {
655 run = 0;
656 }
657 }
658 best
659}
660
661pub fn code_span(text: &str) -> String {
667 let ticks = "`".repeat(longest_backtick_run(text) + 1);
668 let pad = if text.starts_with('`') || text.ends_with('`') {
669 " "
670 } else {
671 ""
672 };
673 format!("{ticks}{pad}{text}{pad}{ticks}")
674}
675
676pub fn code_fence(language: &str, body: &str) -> String {
681 let fence = "`".repeat(longest_backtick_run(body).max(2) + 1);
682 format!("{fence}{language}\n{body}\n{fence}")
683}
684
685pub(crate) fn code_span_bounds(chars: &[char], i: usize) -> Option<(usize, usize, usize)> {
693 let n = chars[i..].iter().take_while(|&&c| c == '`').count();
694 if n == 0 {
695 return None;
696 }
697 let mut k = i + n;
698 while k < chars.len() {
699 if chars[k] != '`' {
700 k += 1;
701 continue;
702 }
703 let run = chars[k..].iter().take_while(|&&c| c == '`').count();
704 if run == n {
705 let (mut s, mut e) = (i + n, k);
706 if n > 1
707 && e - s >= 2
708 && chars[s] == ' '
709 && chars[e - 1] == ' '
710 && chars[s..e].iter().any(|&c| c != ' ')
711 {
712 s += 1;
713 e -= 1;
714 }
715 return Some((s, e, k + run));
716 }
717 k += run;
718 }
719 None
720}
721
722fn is_fence_line(line: &str) -> bool {
724 line.len() >= 3 && line.bytes().all(|b| b == b'`')
725}
726
727pub(crate) fn plain_inline(text: &str) -> String {
740 if !text.contains(['*', '~', '`', '[']) {
741 return text.to_string();
742 }
743 let chars: Vec<char> = text.chars().collect();
744 let mut out = String::with_capacity(text.len());
745 plain_inline_into(&chars, &mut out);
746 out
747}
748
749fn plain_inline_into(chars: &[char], out: &mut String) {
750 let find = |open: usize, pat: &[char]| -> Option<usize> {
753 chars
754 .get(open..)?
755 .windows(pat.len())
756 .enumerate()
757 .position(|(k, w)| w == pat && (open + k == 0 || chars[open + k - 1] != '\\'))
758 .map(|p| open + p)
759 };
760 let span = |i: usize, pat: &[char]| -> Option<usize> {
763 if !chars[i..].starts_with(pat) {
764 return None;
765 }
766 let open = i + pat.len();
767 let end = find(open, pat)?;
768 (end > open && !chars[open].is_whitespace() && !chars[end - 1].is_whitespace())
769 .then_some(end)
770 };
771 let mut i = 0;
772 while i < chars.len() {
773 if chars[i] == '\\' && chars.get(i + 1) == Some(&'*') {
775 out.push('*');
776 i += 2;
777 continue;
778 }
779 let mut matched = false;
780 for pat in [&['*', '*', '*'][..], &['*', '*'], &['~', '~'], &['*']] {
781 if let Some(end) = span(i, pat) {
782 plain_inline_into(&chars[i + pat.len()..end], out);
783 i = end + pat.len();
784 matched = true;
785 break;
786 }
787 }
788 if matched {
789 continue;
790 }
791 if chars[i] == '`' {
795 if let Some((s, e, after)) = code_span_bounds(chars, i).filter(|&(s, e, _)| {
796 e > s && !chars[s].is_whitespace() && !chars[e - 1].is_whitespace()
797 }) {
798 out.extend(&chars[s..e]);
799 i = after;
800 continue;
801 }
802 }
803 if chars[i] == '$' {
806 if let Some(end) = find(i + 1, &['$']) {
807 out.extend(&chars[i..=end]);
808 i = end + 1;
809 continue;
810 }
811 }
812 if chars[i] == '[' {
815 if let Some(close) = closing_bracket(chars, i) {
816 if chars.get(close + 1) == Some(&'(') {
817 if let Some(endp) = crate::doclang::link_dest_end(chars, close + 2) {
818 plain_inline_into(&chars[i + 1..close], out);
819 i = endp + 1;
820 continue;
821 }
822 }
823 }
824 }
825 let c = chars[i];
828 out.push(c);
829 i += 1;
830 if matches!(c, '*' | '~') {
831 while i < chars.len() && chars[i] == c {
832 out.push(c);
833 i += 1;
834 }
835 }
836 }
837}
838
839fn closing_bracket(chars: &[char], open: usize) -> Option<usize> {
842 let mut depth = 0usize;
843 for (k, c) in chars.iter().enumerate().skip(open + 1) {
844 match c {
845 '[' => depth += 1,
846 ']' if depth == 0 => return Some(k),
847 ']' => depth -= 1,
848 _ => {}
849 }
850 }
851 None
852}
853
854fn md_line_breaks(text: &str) -> String {
862 if !text.contains('\n') {
863 return text.to_string();
864 }
865 text.split("\n\n")
866 .map(|para| para.replace('\n', " \n"))
867 .collect::<Vec<_>>()
868 .join("\n\n")
869}
870
871pub(crate) fn strip_hard_breaks(text: &str) -> String {
876 if text.contains(" \n") {
877 text.replace(" \n", "\n")
878 } else {
879 text.to_string()
880 }
881}
882
883fn heading_line_breaks(text: &str) -> String {
887 text.replace('\n', " ")
888}
889
890fn render(nodes: &[Node], blocks: &mut Vec<String>, ctx: &mut Ctx) {
891 let mut i = 0;
892 while i < nodes.len() {
893 let before = blocks.len();
894 match &nodes[i] {
895 Node::PageBreak | Node::PageInfo { .. } => {
905 if ctx.page_break.is_some() && ctx.emitted_any {
906 ctx.pending_page_break = true;
907 }
908 i += 1;
909 }
910 Node::ListItem { .. } => {
911 let start = i;
912 i += 1;
913 loop {
914 match nodes.get(i) {
915 Some(Node::ListItem { .. } | Node::ListText { .. }) => i += 1,
916 Some(Node::Paragraph { text })
920 if text.is_empty()
921 && matches!(nodes.get(i + 1), Some(Node::ListItem { .. })) =>
922 {
923 i += 1
924 }
925 _ => break,
926 }
927 }
928 render_list_run(&nodes[start..i], blocks, ctx);
929 }
930 other => {
931 let before_item = blocks.len();
932 render_one(other, blocks, ctx);
933 if matches!(other, Node::Picture { .. })
939 && matches!(ctx.images, ImageMode::Placeholder)
940 && blocks.len() > before_item
941 && matches!(
942 nodes.get(i + 1),
943 Some(Node::Code { language: Some(l), .. }) if l == "mermaid"
944 )
945 {
946 blocks.pop();
947 }
948 i += 1;
949 }
950 }
951 if blocks.len() > before {
952 let kept: Vec<String> = blocks.drain(before..).filter(|b| !b.is_empty()).collect();
962 blocks.extend(kept);
963 if ctx.pending_page_break {
964 if let Some(placeholder) = &ctx.page_break {
969 blocks.insert(before, placeholder.clone());
970 }
971 ctx.pending_page_break = false;
972 }
973 ctx.emitted_any = true;
974 }
975 }
976}
977
978fn render_list_run(items: &[Node], blocks: &mut Vec<String>, ctx: &Ctx) {
984 let mut lines: Vec<String> = Vec::new();
985 let mut any_top = false;
988
989 for item in items {
990 if let Node::ListText { level, text, glue } = item {
991 let text = ctx.text(text);
992 match lines.last_mut() {
993 Some(last) if *glue => last.push_str(&text),
994 _ => lines.push(format!("{}{text}", " ".repeat(*level as usize))),
995 }
996 continue;
997 }
998 let Node::ListItem {
999 ordered,
1000 number,
1001 first_in_list,
1002 text,
1003 level,
1004 marker: orig_marker,
1005 location: _,
1006 dclx: _,
1007 split: _,
1008 href: _,
1009 layer,
1010 } = item
1011 else {
1012 continue;
1013 };
1014 if !ctx.layers.contains(layer.or(ctx.current_layer)) {
1018 continue;
1019 }
1020 let level = *level as usize;
1021
1022 if level == 0 {
1031 if any_top && *first_in_list {
1032 lines.push(String::new());
1033 }
1034 any_top = true;
1035 }
1036
1037 let indent = " ".repeat(level);
1038 lines.push(format!(
1039 "{indent}{} {}",
1040 list_item_marker(orig_marker.as_deref().unwrap_or(""), *ordered, *number),
1041 list_item_text(text, ctx)
1042 ));
1043 }
1044
1045 if !lines.is_empty() {
1048 blocks.push(lines.join("\n"));
1049 }
1050}
1051
1052fn list_item_marker(marker: &str, ordered: bool, number: u64) -> String {
1061 let already_valid = matches!(marker, "-" | "*" | "+")
1062 || marker
1063 .strip_suffix('.')
1064 .is_some_and(|d| !d.is_empty() && d.chars().all(char::is_numeric));
1065 if already_valid {
1066 return marker.to_string();
1067 }
1068 let outer = if ordered && marker.is_empty() {
1069 format!("{number}.")
1070 } else {
1071 "-".to_string()
1072 };
1073 if marker.chars().any(char::is_alphanumeric) {
1074 format!("{outer} {marker}")
1075 } else {
1076 outer
1077 }
1078}
1079
1080fn list_item_text(text: &str, ctx: &Ctx) -> String {
1088 let escaped = ctx.text(text);
1089 if ctx.plain {
1090 return escaped;
1091 }
1092 if let Some((own, tail)) = escaped.split_once('\n') {
1093 if is_folded_child_tail(tail) {
1094 return format!("{}\n{tail}", md_line_breaks(own));
1095 }
1096 }
1097 md_line_breaks(&escaped)
1098}
1099
1100fn is_folded_child_tail(tail: &str) -> bool {
1107 const MARKER: &str = "<!-- image -->";
1108 let mut lines = tail.split('\n').peekable();
1109 let mut any = false;
1110 while let Some(line) = lines.next() {
1111 let line = line.trim_start();
1112 if line == MARKER {
1113 any = true;
1114 } else if is_fence_line(line) {
1115 loop {
1118 match lines.next() {
1119 Some(l) if l.trim_start() == line => break,
1120 Some(_) => {}
1121 None => return false,
1122 }
1123 }
1124 any = true;
1125 } else if lines.next().map(str::trim_start) == Some(MARKER) {
1126 any = true; } else {
1128 return false;
1129 }
1130 }
1131 any
1132}
1133
1134fn render_one(node: &Node, blocks: &mut Vec<String>, ctx: &mut Ctx) {
1139 match node {
1140 Node::Group { .. }
1142 | Node::Furniture { .. }
1143 | Node::PageFurniture { .. }
1144 | Node::FurnitureText { .. }
1145 | Node::CommentSection { .. }
1146 | Node::PictureChildren(_)
1147 | Node::Commented { .. }
1148 | Node::ScriptText { .. }
1149 | Node::Located { .. }
1150 | Node::Prov { .. }
1151 | Node::Track { .. }
1152 | Node::PageBreak
1153 | Node::PageInfo { .. }
1154 | Node::DoclangOnly(_) => {}
1155 _ if !ctx.layer_visible() => return,
1156 _ => {}
1157 }
1158 render_item(node, blocks, ctx)
1159}
1160
1161fn render_item(node: &Node, blocks: &mut Vec<String>, ctx: &mut Ctx) {
1162 match node {
1163 Node::Heading { level, text } => {
1164 if ctx.plain {
1165 blocks.push(ctx.text(text));
1168 return;
1169 }
1170 let text = heading_line_breaks(&ctx.text(text));
1171 if ctx.in_table_cell {
1172 blocks.push(text);
1174 } else {
1175 let hashes = "#".repeat((*level).clamp(1, 6) as usize);
1176 blocks.push(format!("{hashes} {text}"));
1177 }
1178 }
1179 Node::Paragraph { text } if text.is_empty() => {}
1182 Node::Paragraph { text } => blocks.push(ctx.breaks(&ctx.text(text))),
1183 Node::LabeledText { .. } => render_item(&node.labeled_as_paragraph(), blocks, ctx),
1184 Node::Caption { text, .. } if text.is_empty() => {}
1187 Node::Caption { text, href } => {
1188 let body = ctx.breaks(&ctx.text(text));
1189 blocks.push(match href {
1190 Some(url) if !ctx.plain => crate::hyperlink::markdown_link(&body, url),
1191 _ => body,
1192 });
1193 }
1194 Node::CheckboxItem { checked, text } => {
1195 let mark = if *checked { "- [x]" } else { "- [ ]" };
1198 let sep = if text.starts_with(' ') { "" } else { " " };
1199 blocks.push(ctx.breaks(&ctx.text(&format!("{mark}{sep}{text}"))));
1200 }
1201 Node::Code {
1202 language,
1203 text,
1204 pretty,
1205 ..
1206 } => {
1207 let lang = match language {
1212 Some(l) if ctx.strict || l == "mermaid" => l.as_str(),
1213 _ => "",
1214 };
1215 let body = match pretty {
1218 Some(p) if ctx.strict => p.as_str(),
1219 _ => text.as_str(),
1220 };
1221 blocks.push(if ctx.plain {
1224 body.to_string()
1225 } else {
1226 code_fence(lang, body)
1227 });
1228 }
1229 Node::Formula { latex, .. } => blocks.push(format!("$${latex}$$")),
1232 Node::Table(table) => {
1233 if let Some(cap) = &table.caption {
1236 if !cap.is_empty() {
1237 blocks.push(ctx.text(cap));
1238 }
1239 }
1240 let rendered = if ctx.plain {
1241 render_table(&ctx.plain_table(table), ctx.compact_tables)
1242 } else {
1243 render_table(table, ctx.compact_tables)
1244 };
1245 if !rendered.is_empty() {
1246 blocks.push(rendered);
1247 }
1248 for note in table.footnotes.iter().filter(|n| !n.is_empty()) {
1251 blocks.push(ctx.text(note));
1252 }
1253 }
1254 Node::Picture {
1263 caption,
1264 image,
1265 description,
1266 ..
1267 } => {
1268 if let Some(cap) = caption {
1269 if !cap.is_empty() {
1270 blocks.push(ctx.caption(cap));
1271 }
1272 }
1273 let alt = description
1274 .as_ref()
1275 .filter(|d| d.is_alt_text() && ctx.alt_text && !d.text.is_empty());
1276 let marker = picture_marker(image.as_ref(), ctx, alt.map(|d| d.text.as_str()));
1277 let native = description
1281 .as_ref()
1282 .filter(|d| d.is_docling_meta() && !d.text.is_empty());
1283 if let Some(desc) = description.as_ref().filter(|d| !d.is_docling_meta()) {
1284 let shown = !desc.is_alt_text() || (ctx.alt_text && !marker.starts_with("!["));
1285 if !desc.text.is_empty() && shown {
1286 blocks.push(ctx.breaks(&ctx.text(&desc.text)));
1287 }
1288 }
1289 blocks.push(marker);
1290 if let Some(desc) = native {
1291 blocks.push(ctx.breaks(&ctx.text(&desc.text)));
1292 }
1293 }
1294 Node::Chart {
1298 kind,
1299 table,
1300 caption,
1301 ..
1302 } => {
1303 if let Some(cap) = caption {
1304 if !cap.is_empty() {
1305 blocks.push(ctx.caption(cap));
1306 }
1307 }
1308 blocks.push(picture_marker(None, ctx, None));
1309 blocks.push(humanize_label(kind));
1310 let rendered = if ctx.plain {
1311 render_table(&ctx.plain_table(table), false)
1312 } else {
1313 render_table(table, false)
1314 };
1315 if !rendered.is_empty() {
1316 blocks.push(rendered);
1317 }
1318 }
1319 Node::DoclangOnly(_) => {}
1321 Node::Group {
1326 layer: Some(layer),
1327 children,
1328 ..
1329 } => {
1330 let outer = ctx.current_layer.replace(*layer);
1331 render(children, blocks, ctx);
1332 ctx.current_layer = outer;
1333 }
1334 Node::Group { children, .. } => render(children, blocks, ctx),
1335 Node::FieldRegion { items } => {
1336 for item in items {
1341 for part in [&item.marker, &item.key, &item.value].into_iter().flatten() {
1342 blocks.push(ctx.breaks(&ctx.text(part)));
1343 }
1344 }
1345 }
1346 Node::KeyValueGraph { .. } => blocks.push(MISSING_KEY_VALUE_ITEM.to_string()),
1349 Node::InlineGroup { md_text, .. } => blocks.push(ctx.breaks(&ctx.text(md_text))),
1352 Node::TextDump(text) => {
1354 if !text.is_empty() {
1355 blocks.push(text.clone());
1356 }
1357 }
1358 Node::Furniture { layer, inner } => {
1362 let outer = ctx.current_layer.replace(*layer);
1363 render_one(inner, blocks, ctx);
1364 ctx.current_layer = outer;
1365 }
1366 Node::PageFurniture { text, .. } | Node::FurnitureText { text, .. } => {
1370 if ctx.layers.furniture && !text.is_empty() {
1371 blocks.push(ctx.breaks(&ctx.text(text)));
1372 }
1373 }
1374 Node::PictureChildren(children) => {
1379 if ctx.traverse_pictures {
1380 render(children, blocks, ctx);
1381 }
1382 }
1383 Node::CommentSection { text, .. } => {
1388 if ctx.layers.notes && !text.is_empty() {
1389 blocks.push(ctx.breaks(&ctx.text(text)));
1390 }
1391 }
1392 Node::Commented { inner, .. } => render_one(inner, blocks, ctx),
1393 Node::ScriptText { inner, .. } => render_one(inner, blocks, ctx),
1396 Node::Located { inner, .. } | Node::Prov { inner, .. } | Node::Track { inner, .. } => {
1398 render_one(inner, blocks, ctx)
1399 }
1400 Node::PageBreak => {}
1402 Node::PageInfo { .. } => {}
1404 Node::Anchor { .. } => {}
1406 Node::ListItem { .. } => render_list_run(std::slice::from_ref(node), blocks, ctx),
1411 Node::ListText { text, .. } if text.is_empty() => {}
1413 Node::ListText { text, .. } => blocks.push(ctx.breaks(&ctx.text(text))),
1414 }
1415}
1416
1417fn humanize_label(label: &str) -> String {
1422 let text = label.replace('_', " ");
1423 let mut chars = text.chars();
1424 match chars.next() {
1425 Some(f) => f.to_uppercase().collect::<String>() + chars.as_str(),
1426 None => text,
1427 }
1428}
1429
1430fn image_alt(alt: Option<&str>) -> String {
1433 match alt {
1434 Some(text) => text
1435 .split_whitespace()
1436 .collect::<Vec<_>>()
1437 .join(" ")
1438 .replace('\\', "\\\\")
1439 .replace(']', "\\]"),
1440 None => "Image".to_string(),
1441 }
1442}
1443
1444fn picture_marker(image: Option<&crate::PictureImage>, ctx: &mut Ctx, alt: Option<&str>) -> String {
1445 match (ctx.images, image) {
1446 (ImageMode::Embedded, Some(img)) => {
1448 format!(
1449 "",
1450 image_alt(alt),
1451 crate::pixel_digest::docling_data_uri(img).1
1452 )
1453 }
1454 (ImageMode::Referenced, Some(img)) => {
1455 let path = format!(
1456 "{}/image_{:06}.{}",
1457 ctx.artifacts_dir,
1458 ctx.pic_index,
1459 ext_for(&img.mimetype)
1460 );
1461 ctx.pic_index += 1;
1462 ctx.artifacts.push((path.clone(), img.data.clone()));
1463 format!("", image_alt(alt), escape_uri_path(&path))
1464 }
1465 _ => ctx.image_placeholder.clone(),
1467 }
1468}
1469
1470pub(crate) fn escape_uri_path(value: &str) -> String {
1483 const KEEP: &str = "/%:@+,;=~$!&'*";
1484 let s = value.replace('\\', "/");
1485 if let Some(rest) = s.strip_prefix("//") {
1486 let rest = rest.trim_start_matches('/');
1488 let (host, tail) = rest.split_once('/').unwrap_or((rest, ""));
1489 return format!("file://{host}{}", percent_quote(&format!("/{tail}"), KEEP));
1490 }
1491 let bytes = s.as_bytes();
1492 if bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' && bytes[2] == b'/' {
1493 return format!("file:///{}", percent_quote(&s, KEEP));
1495 }
1496 if let Some((scheme, rest)) = s.split_once(':') {
1500 let valid_scheme = scheme.len() > 1
1501 && scheme.as_bytes()[0].is_ascii_alphabetic()
1502 && scheme
1503 .bytes()
1504 .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.'));
1505 if valid_scheme {
1506 let (authority, rest) = match rest.strip_prefix("//") {
1507 Some(r) => {
1508 let end = r.find(['/', '?', '#']).unwrap_or(r.len());
1509 (Some(&r[..end]), &r[end..])
1510 }
1511 None => (None, rest),
1512 };
1513 let (before_frag, fragment) = rest.split_once('#').unwrap_or((rest, ""));
1514 let (path, query) = before_frag.split_once('?').unwrap_or((before_frag, ""));
1515 let mut out = format!("{scheme}:");
1516 if let Some(a) = authority {
1517 out.push_str("//");
1518 out.push_str(a);
1519 }
1520 out.push_str(&percent_quote(path, KEEP));
1521 if !query.is_empty() {
1522 out.push('?');
1523 out.push_str(&percent_quote(query, KEEP));
1524 }
1525 if !fragment.is_empty() {
1526 out.push('#');
1527 out.push_str(&percent_quote(fragment, KEEP));
1528 }
1529 return out;
1530 }
1531 }
1532 percent_quote(&s, KEEP)
1534}
1535
1536fn percent_quote(s: &str, safe: &str) -> String {
1539 let mut out = String::with_capacity(s.len());
1540 for &b in s.as_bytes() {
1541 let keep = b.is_ascii_alphanumeric()
1542 || matches!(b, b'_' | b'.' | b'-' | b'~')
1543 || (b.is_ascii() && safe.contains(b as char));
1544 if keep {
1545 out.push(b as char);
1546 } else {
1547 out.push_str(&format!("%{b:02X}"));
1548 }
1549 }
1550 out
1551}
1552
1553pub(crate) fn ext_for(mimetype: &str) -> &str {
1554 match mimetype {
1555 "image/jpeg" => "jpg",
1556 "image/gif" => "gif",
1557 "image/webp" => "webp",
1558 "image/bmp" => "bmp",
1559 "image/tiff" => "tif",
1560 "image/svg+xml" => "svg",
1561 _ => "png",
1562 }
1563}
1564
1565fn is_number_cell(t: &str) -> bool {
1584 t.parse::<f64>().is_ok() || is_thousands_number(t)
1585}
1586
1587fn is_thousands_number(t: &str) -> bool {
1593 let b = t.as_bytes();
1594 let mut i = 0;
1595 let start = i;
1596 if i < b.len() && (b[i] == b'+' || b[i] == b'-') {
1597 i += 1;
1598 }
1599 let d0 = i;
1601 while i < b.len() && b[i].is_ascii_digit() && i - d0 < 3 {
1602 i += 1;
1603 }
1604 let has_int = i > d0;
1605 if has_int {
1606 while i + 3 < b.len() + 1
1608 && b.get(i) == Some(&b',')
1609 && b.get(i + 1).is_some_and(u8::is_ascii_digit)
1610 && b.get(i + 2).is_some_and(u8::is_ascii_digit)
1611 && b.get(i + 3).is_some_and(u8::is_ascii_digit)
1612 {
1613 i += 4;
1614 }
1615 } else {
1616 i = start;
1618 }
1619 if i < b.len() && b[i] == b'.' {
1621 i += 1;
1622 let f0 = i;
1623 while i < b.len() && b[i].is_ascii_digit() {
1624 i += 1;
1625 }
1626 if !has_int && i == f0 {
1627 return false; }
1629 } else if !has_int {
1630 return false; }
1632 i == b.len()
1633}
1634
1635fn flatten_header_rows(header_rows: &[Vec<String>], num_cols: usize) -> Vec<String> {
1644 (0..num_cols)
1645 .map(|c| {
1646 let mut parts: Vec<&str> = Vec::new();
1647 for row in header_rows {
1648 let text = row.get(c).map(String::as_str).unwrap_or("");
1649 if !text.is_empty() && parts.last() != Some(&text) {
1650 parts.push(text);
1651 }
1652 }
1653 parts.join(" - ")
1654 })
1655 .collect()
1656}
1657
1658pub(crate) fn render_table(table: &Table, compact: bool) -> String {
1659 if table.rows.is_empty() {
1660 return String::new();
1661 }
1662 let num_cols = table.rows.iter().map(Vec::len).max().unwrap_or(0);
1663 if num_cols == 0 {
1664 return String::new();
1665 }
1666
1667 let num_headers = table.header_row_count().min(table.rows.len());
1672 let escaped = |r: usize| -> Vec<String> {
1673 (0..num_cols)
1674 .map(|c| escape_cell(table.rows[r].get(c).map(String::as_str).unwrap_or("")))
1675 .collect()
1676 };
1677 let header_rows: Vec<Vec<String>> = (0..num_headers).map(escaped).collect();
1678 let header = flatten_header_rows(&header_rows, num_cols);
1679 let body: Vec<Vec<String>> = (num_headers..table.rows.len())
1680 .map(|r| {
1681 escaped(r)
1682 .into_iter()
1683 .map(|c| c.trim().to_string())
1684 .collect()
1685 })
1686 .collect();
1687
1688 if compact {
1689 let render_row = |row: &[String]| -> String { format!("| {} |", row.join(" | ")) };
1691 let mut lines = Vec::with_capacity(body.len() + 2);
1692 lines.push(render_row(&header));
1693 let sep: Vec<&str> = (0..num_cols).map(|_| "-").collect();
1694 lines.push(format!("| {} |", sep.join(" | ")));
1695 for row in &body {
1696 lines.push(render_row(row));
1697 }
1698 return lines.join("\n");
1699 }
1700
1701 let dw = |s: &str| s.chars().count();
1703
1704 let right: Vec<bool> = (0..num_cols)
1709 .map(|c| {
1710 let mut any = false;
1711 for row in &body {
1712 let t = row[c].trim();
1713 if t.is_empty() {
1714 continue;
1715 }
1716 if !is_number_cell(t) {
1717 return false;
1718 }
1719 any = true;
1720 }
1721 any
1722 })
1723 .collect();
1724
1725 let width: Vec<usize> = (0..num_cols)
1727 .map(|c| {
1728 let mut w = dw(&header[c]) + 2;
1729 for row in &body {
1730 w = w.max(dw(&row[c]));
1731 }
1732 w
1733 })
1734 .collect();
1735
1736 let fmt_cell = |s: &str, c: usize| -> String {
1737 let pad = " ".repeat(width[c].saturating_sub(dw(s)));
1738 let body = if right[c] {
1739 format!("{pad}{s}")
1740 } else {
1741 format!("{s}{pad}")
1742 };
1743 format!(" {body} ")
1744 };
1745 let render_row = |row: &[String]| -> String {
1746 let cells: Vec<String> = (0..num_cols).map(|c| fmt_cell(&row[c], c)).collect();
1747 format!("|{}|", cells.join("|"))
1748 };
1749
1750 let mut lines = Vec::with_capacity(body.len() + 2);
1751 lines.push(render_row(&header));
1752 let sep: Vec<String> = (0..num_cols).map(|c| "-".repeat(width[c] + 2)).collect();
1753 lines.push(format!("|{}|", sep.join("|")));
1754 for row in &body {
1755 lines.push(render_row(row));
1756 }
1757 lines.join("\n")
1758}
1759
1760fn escape_cell(s: &str) -> String {
1763 s.replace('\n', " ").replace('|', "|")
1764}
1765
1766#[cfg(test)]
1767mod tests {
1768 use super::*;
1769 use crate::{PictureImage, TableCell, TableStructure};
1770
1771 #[test]
1777 fn list_boundaries_come_from_the_backend_not_the_numbering() {
1778 let item = |ordered: bool, number: u64, first_in_list: bool, text: &str| Node::ListItem {
1779 ordered,
1780 number,
1781 first_in_list,
1782 text: text.into(),
1783 level: 0,
1784 marker: None,
1785 location: None,
1786 split: None,
1787 dclx: None,
1788 href: None,
1789 layer: None,
1790 };
1791 let md = |items: Vec<Node>| {
1792 let mut doc = DoclingDocument::new("t");
1793 for n in items {
1794 doc.push(n);
1795 }
1796 doc.export_to_markdown()
1797 };
1798 assert_eq!(
1800 md(vec![
1801 item(true, 1, true, "one"),
1802 item(true, 5, false, "five")
1803 ]),
1804 "1. one\n5. five\n"
1805 );
1806 assert_eq!(
1808 md(vec![
1809 item(false, 0, true, "bullet"),
1810 item(true, 1, false, "one"),
1811 item(false, 0, false, "bullet two"),
1812 ]),
1813 "- bullet\n1. one\n- bullet two\n"
1814 );
1815 assert_eq!(
1817 md(vec![
1818 item(true, 1, true, "a"),
1819 item(true, 2, false, "b"),
1820 item(true, 3, true, "new list, continuing count"),
1821 ]),
1822 "1. a\n2. b\n\n3. new list, continuing count\n"
1823 );
1824 }
1825
1826 #[test]
1827 fn renders_headings_paragraphs_and_lists() {
1828 let mut doc = DoclingDocument::new("demo");
1829 doc.add_heading(1, "Title");
1830 doc.add_paragraph("Hello world.");
1831 doc.push(Node::ListItem {
1832 ordered: false,
1833 number: 1,
1834 first_in_list: true,
1835 text: "first".into(),
1836 level: 0,
1837 marker: None,
1838 location: None,
1839 split: None,
1840 dclx: None,
1841 href: None,
1842 layer: None,
1843 });
1844 doc.push(Node::ListItem {
1845 ordered: false,
1846 number: 2,
1847 first_in_list: false,
1848 text: "second".into(),
1849 level: 0,
1850 marker: None,
1851 location: None,
1852 split: None,
1853 dclx: None,
1854 href: None,
1855 layer: None,
1856 });
1857 let md = doc.export_to_markdown();
1858 assert_eq!(md, "# Title\n\nHello world.\n\n- first\n- second\n");
1859 }
1860
1861 #[test]
1865 fn single_newlines_become_gfm_hard_line_breaks() {
1866 let mut doc = DoclingDocument::new("t");
1867 doc.push(Node::Heading {
1868 level: 1,
1869 text: "Hello\nWorld".into(),
1870 });
1871 doc.push(Node::Paragraph {
1872 text: "line one\nline two\n\npara two".into(),
1873 });
1874 doc.push(Node::ListItem {
1875 ordered: false,
1876 number: 1,
1877 first_in_list: true,
1878 text: "item\ncontinued".into(),
1879 level: 0,
1880 marker: None,
1881 location: None,
1882 split: None,
1883 dclx: None,
1884 href: None,
1885 layer: None,
1886 });
1887 doc.push(Node::TextDump("A1 B1 \n\n\nC1".into()));
1888 assert_eq!(
1889 doc.export_to_markdown(),
1890 "# Hello World\n\nline one \nline two\n\npara two\n\n- item \ncontinued\n\nA1 B1 \n\n\nC1\n"
1891 );
1892 }
1893
1894 #[test]
1897 fn table_cell_mode_and_field_regions() {
1898 let mut doc = DoclingDocument::new("t");
1899 doc.push(Node::Heading {
1900 level: 2,
1901 text: "A text".into(),
1902 });
1903 doc.push(Node::Paragraph {
1904 text: "body".into(),
1905 });
1906 assert_eq!(to_markdown_table_cell(&doc, false), "A text\n\nbody");
1907 assert_eq!(doc.export_to_markdown(), "## A text\n\nbody\n");
1908
1909 let mut doc = DoclingDocument::new("f");
1910 doc.push(Node::FieldRegion {
1911 items: vec![crate::FieldItem {
1912 marker: None,
1913 key: Some("Name:".into()),
1914 value: Some("John Doe".into()),
1915 value_kind: None,
1916 }],
1917 });
1918 assert_eq!(doc.export_to_markdown(), "Name:\n\nJohn Doe\n");
1919 }
1920
1921 #[test]
1922 fn strict_renders_recovered_links_legacy_does_not() {
1923 let mut doc = DoclingDocument::new("cv");
1924 doc.add_paragraph("Find me on LinkedIn or GitHub.");
1925 doc.links = vec![
1926 ("LinkedIn".into(), "https://www.linkedin.com/in/x/".into()),
1927 ("GitHub".into(), "https://github.com/x/".into()),
1928 ];
1929 assert_eq!(doc.export_to_markdown(), "Find me on LinkedIn or GitHub.\n");
1931 assert_eq!(
1933 doc.export_to_markdown_with(true),
1934 "Find me on [LinkedIn](https://www.linkedin.com/in/x/) or [GitHub](https://github.com/x/).\n"
1935 );
1936 }
1937
1938 #[test]
1939 fn strict_links_match_escaped_anchor_and_consume_in_order() {
1940 let mut doc = DoclingDocument::new("d");
1941 doc.add_paragraph("AI & ML here, and issues here, then issues there.");
1945 doc.links = vec![
1946 ("AI & ML".into(), "https://a/".into()),
1947 ("issues".into(), "https://first/".into()),
1948 ("issues".into(), "https://second/".into()),
1949 ];
1950 assert_eq!(
1951 doc.export_to_markdown_with(true),
1952 "[AI & ML](https://a/) here, and [issues](https://first/) here, then [issues](https://second/) there.\n"
1953 );
1954 }
1955
1956 #[test]
1960 fn referenced_image_destinations_are_escaped() {
1961 let cases = [
1962 (
1963 "doc_artifacts/image_000001_ab12.png",
1964 "doc_artifacts/image_000001_ab12.png",
1965 ),
1966 (
1967 "My Report_artifacts/img.png",
1968 "My%20Report_artifacts/img.png",
1969 ),
1970 ("artifacts/img (1).png", "artifacts/img%20%281%29.png"),
1971 ("100%_scale/a#b?c.png", "100%_scale/a%23b%3Fc.png"),
1972 ("/home/a b/img.png", "/home/a%20b/img.png"),
1973 (
1974 "My Report_artifacts\\img.png",
1975 "My%20Report_artifacts/img.png",
1976 ),
1977 (
1978 "C:/Users/me/My Docs/img.png",
1979 "file:///C:/Users/me/My%20Docs/img.png",
1980 ),
1981 ("C:\\Users\\me\\img.png", "file:///C:/Users/me/img.png"),
1982 (
1983 "//server/share/My Docs/img.png",
1984 "file://server/share/My%20Docs/img.png",
1985 ),
1986 ("\\\\server\\share\\img.png", "file://server/share/img.png"),
1987 ("file:///home/a b/img.png", "file:///home/a%20b/img.png"),
1988 (
1989 "s3://bucket/My Report_artifacts/img.png",
1990 "s3://bucket/My%20Report_artifacts/img.png",
1991 ),
1992 (
1993 "https://example.com:8080/a b.png?w=1&h=2#frag",
1994 "https://example.com:8080/a%20b.png?w=1&h=2#frag",
1995 ),
1996 (
1997 "https://example.com/img (1).png",
1998 "https://example.com/img%20%281%29.png",
1999 ),
2000 ("caf\u{e9}/im\u{e4}ge.png", "caf%C3%A9/im%C3%A4ge.png"),
2001 ];
2002 for (input, expected) in cases {
2003 assert_eq!(escape_uri_path(input), expected, "input {input:?}");
2004 assert_eq!(
2005 escape_uri_path(expected),
2006 expected,
2007 "idempotent {expected:?}"
2008 );
2009 }
2010 let mut doc = DoclingDocument::new("t");
2012 doc.push(Node::Picture {
2013 caption: None,
2014 caption_href: None,
2015 image: Some(PictureImage {
2016 dpi: PictureImage::DEFAULT_DPI,
2017 mimetype: "image/png".into(),
2018 width: 1,
2019 height: 1,
2020 data: b"x".to_vec(),
2021 display_size: None,
2022 }),
2023 classification: None,
2024 description: None,
2025 caption_parent: Default::default(),
2026 caption_location: None,
2027 });
2028 let (md, files) = doc
2029 .export_to_markdown_with_images(ImageMode::Referenced, "My Report (final)_artifacts");
2030 assert!(
2031 md.contains(""),
2032 "got:\n{md}"
2033 );
2034 assert_eq!(files[0].0, "My Report (final)_artifacts/image_000000.png");
2036 }
2037
2038 #[test]
2042 fn folded_list_item_pictures_keep_plain_newlines() {
2043 let doc = DoclingDocument::new("t");
2044 let ctx = Ctx::new(&doc, false, &MarkdownExportOptions::default());
2045 assert_eq!(
2046 list_item_text("Step\n<!-- image -->", &ctx),
2047 "Step\n<!-- image -->"
2048 );
2049 assert_eq!(
2050 list_item_text("Step\nAlt text\n<!-- image -->\n<!-- image -->", &ctx),
2051 "Step\nAlt text\n<!-- image -->\n<!-- image -->"
2052 );
2053 assert_eq!(
2054 list_item_text("line one\nline two", &ctx),
2055 "line one \nline two"
2056 );
2057 }
2058
2059 #[test]
2062 fn stacked_header_rows_flatten_into_one() {
2063 let mut t = Table {
2064 rows: vec![
2065 vec!["".into(), "% of Total".into(), "% of Total".into()],
2066 vec!["class".into(), "Train".into(), "Test".into()],
2067 vec!["Caption".into(), "2.04".into(), "1.77".into()],
2068 ],
2069 ..Default::default()
2070 };
2071 t.structure = Some(TableStructure {
2072 header_row: vec![true, true, false],
2073 col_continuation: vec![
2074 vec![false, false, true],
2075 vec![false, false, false],
2076 vec![false, false, false],
2077 ],
2078 ..Default::default()
2079 });
2080 assert_eq!(t.header_row_count(), 2);
2081 assert_eq!(
2082 render_table(&t, true),
2083 "| class | % of Total - Train | % of Total - Test |\n| - | - | - |\n| Caption | 2.04 | 1.77 |"
2084 );
2085 assert_eq!(
2087 render_table(&t, false),
2088 "| class | % of Total - Train | % of Total - Test |\n\
2089 |---------|----------------------|---------------------|\n\
2090 | Caption | 2.04 | 1.77 |"
2091 );
2092 }
2093
2094 #[test]
2097 fn vertically_spanning_header_does_not_extend_the_block() {
2098 let mut t = Table {
2099 rows: vec![
2100 vec!["Name".into(), "Value".into()],
2101 vec!["Name".into(), "1".into()],
2102 vec!["x".into(), "2".into()],
2103 ],
2104 ..Default::default()
2105 };
2106 t.structure = Some(TableStructure {
2107 col_header: vec![vec![true, true], vec![true, false], vec![false, false]],
2108 row_continuation: vec![vec![false, false], vec![true, false], vec![false, false]],
2109 ..Default::default()
2110 });
2111 assert_eq!(t.header_row_count(), 1);
2112 assert_eq!(
2113 render_table(&t, true),
2114 "| Name | Value |\n| - | - |\n| Name | 1 |\n| x | 2 |"
2115 );
2116 }
2117
2118 #[test]
2121 fn header_flags_not_on_row_zero_keep_all_rows_in_the_body() {
2122 let mut t = Table {
2123 rows: vec![
2124 vec!["1".into(), "2".into()],
2125 vec!["a".into(), "b".into()],
2126 vec!["333".into(), "4".into()],
2127 ],
2128 ..Default::default()
2129 };
2130 t.structure = Some(TableStructure {
2131 header_row: vec![false, true, false],
2132 ..Default::default()
2133 });
2134 assert_eq!(t.header_row_count(), 0);
2135 assert_eq!(
2136 render_table(&t, false),
2137 "| | |\n|-----|----|\n| 1 | 2 |\n| a | b |\n| 333 | 4 |"
2138 );
2139 }
2140
2141 #[test]
2146 fn pivot_row_headers_do_not_extend_the_header() {
2147 let mut t = Table {
2148 rows: vec![
2149 vec!["Year".into(), "Month".into()],
2150 vec!["2025".into(), "January".into()],
2151 vec!["2025".into(), "February".into()],
2152 ],
2153 ..Default::default()
2154 };
2155 t.structure = Some(TableStructure {
2156 col_header: vec![vec![true, true], vec![false, false], vec![false, false]],
2157 row_header: vec![vec![false, false], vec![true, false], vec![true, false]],
2158 row_continuation: vec![vec![false, false], vec![false, false], vec![true, false]],
2159 ..Default::default()
2160 });
2161 assert_eq!(t.header_row_count(), 1);
2162 assert_eq!(
2163 render_table(&t, true),
2164 "| Year | Month |\n| - | - |\n| 2025 | January |\n| 2025 | February |"
2165 );
2166 }
2167
2168 #[test]
2171 fn unflagged_cells_keep_row_zero_as_header() {
2172 let mut t = Table {
2173 rows: vec![vec!["h".into()], vec!["d".into()]],
2174 ..Default::default()
2175 };
2176 t.cells = Some(
2177 [(0usize, "h"), (1, "d")]
2178 .into_iter()
2179 .map(|(r, text)| TableCell {
2180 text: text.into(),
2181 bbox: None,
2182 start_row: r,
2183 start_col: 0,
2184 row_span: 1,
2185 col_span: 1,
2186 column_header: false,
2187 row_header: false,
2188 row_section: false,
2189 })
2190 .collect(),
2191 );
2192 assert_eq!(t.header_row_count(), 1);
2193 assert_eq!(render_table(&t, true), "| h |\n| - |\n| d |");
2194 }
2195
2196 fn flagged_table(rows: &[&[&str]], ched: impl Fn(usize, usize) -> bool) -> Table {
2198 let mut cells = Vec::new();
2199 for (r, row) in rows.iter().enumerate() {
2200 for (c, text) in row.iter().enumerate() {
2201 cells.push(TableCell {
2202 text: (*text).into(),
2203 bbox: None,
2204 start_row: r,
2205 start_col: c,
2206 row_span: 1,
2207 col_span: 1,
2208 column_header: ched(r, c),
2209 row_header: false,
2210 row_section: false,
2211 });
2212 }
2213 }
2214 Table {
2215 rows: rows
2216 .iter()
2217 .map(|row| row.iter().map(|t| (*t).to_string()).collect())
2218 .collect(),
2219 cells: Some(cells),
2220 ..Default::default()
2221 }
2222 }
2223
2224 #[test]
2228 fn rows_with_body_text_stop_the_header_block() {
2229 let t = flagged_table(
2231 &[
2232 &["Item", "Q1", "Q2"],
2233 &["Sales", "10", "20"],
2234 &["Costs", "5", "7"],
2235 ],
2236 |r, c| r == 0 || c == 0,
2237 );
2238 assert_eq!(t.header_row_count(), 1);
2239 assert_eq!(
2240 render_table(&t, false),
2241 "| Item | Q1 | Q2 |\n\
2242 |--------|------|------|\n\
2243 | Sales | 10 | 20 |\n\
2244 | Costs | 5 | 7 |"
2245 );
2246 let t = flagged_table(&[&["Sales", "10", "20"], &["Costs", "5", "7"]], |_, c| {
2249 c == 0
2250 });
2251 assert_eq!(t.header_row_count(), 0);
2252 assert_eq!(
2253 render_table(&t, false),
2254 "| | | |\n\
2255 |-------|----|----|\n\
2256 | Sales | 10 | 20 |\n\
2257 | Costs | 5 | 7 |"
2258 );
2259 let t = flagged_table(
2260 &[&["a", "1", "2"], &["b", "3", "4"], &["c", "5", "6"]],
2261 |r, c| c == 0 && r < 2,
2262 );
2263 assert_eq!(t.header_row_count(), 0);
2264 let t = flagged_table(&[&["a", "1", "2"], &["b", "3", "4"]], |r, c| {
2267 r == 0 && c == 0
2268 });
2269 assert_eq!(t.header_row_count(), 1);
2270 assert_eq!(
2271 render_table(&t, false),
2272 "| a | 1 | 2 |\n\
2273 |-----|-----|-----|\n\
2274 | b | 3 | 4 |"
2275 );
2276 let t = flagged_table(&[&["Item", "", ""], &["x", "1", "2"]], |_, c| c == 0);
2278 assert_eq!(t.header_row_count(), 1);
2279 assert_eq!(
2280 render_table(&t, false),
2281 "| Item | | |\n\
2282 |--------|----|----|\n\
2283 | x | 1 | 2 |"
2284 );
2285 }
2286
2287 #[test]
2288 fn renders_compact_table() {
2289 let mut doc = DoclingDocument::new("t");
2290 doc.compact_tables = true;
2293 doc.push(Node::Table(Table {
2294 footnotes: Vec::new(),
2295 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2296 location: None,
2297 structure: None,
2298 cell_blocks: None,
2299 cells: None,
2300 caption: None,
2301 caption_parent: Default::default(),
2302 caption_location: None,
2303 document_index: false,
2304 }));
2305 let md = doc.export_to_markdown();
2306 assert_eq!(md, "| a | b |\n| - | - |\n| 1 | 2 |\n");
2307 }
2308
2309 #[test]
2310 fn renders_padded_github_table_by_default() {
2311 let mut doc = DoclingDocument::new("t");
2312 doc.push(Node::Table(Table {
2313 footnotes: Vec::new(),
2314 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2315 location: None,
2316 structure: None,
2317 cell_blocks: None,
2318 cells: None,
2319 caption: None,
2320 caption_parent: Default::default(),
2321 caption_location: None,
2322 document_index: false,
2323 }));
2324 let md = doc.export_to_markdown();
2325 assert_eq!(md, "| a | b |\n|-----|-----|\n| 1 | 2 |\n");
2327 }
2328
2329 #[test]
2330 fn strict_unescapes_inline_underscores_legacy_keeps_them() {
2331 let mut doc = DoclingDocument::new("t");
2332 doc.add_heading(1, "a\\_b");
2333 doc.add_paragraph("x\\_y");
2334 doc.push(Node::ListItem {
2335 ordered: false,
2336 number: 1,
2337 first_in_list: true,
2338 text: "i\\_j".into(),
2339 level: 0,
2340 marker: None,
2341 location: None,
2342 split: None,
2343 dclx: None,
2344 href: None,
2345 layer: None,
2346 });
2347 assert_eq!(doc.export_to_markdown(), "# a\\_b\n\nx\\_y\n\n- i\\_j\n");
2349 assert_eq!(doc.export_to_markdown_with(true), "# a_b\n\nx_y\n\n- i_j\n");
2351 }
2352
2353 fn assert_stream_matches(
2356 doc: &DoclingDocument,
2357 strict: bool,
2358 images: ImageMode,
2359 splits: &[usize],
2360 ) {
2361 let (want, want_artifacts) = to_markdown_images(doc, strict, images, "artifacts");
2362 let mut streamer =
2363 MarkdownStreamer::with_artifacts(strict, images, doc.compact_tables, "artifacts")
2364 .with_page_break_placeholder(doc.page_break_placeholder.clone());
2365 let mut got = String::new();
2366 let mut got_artifacts = Vec::new();
2367 let mut start = 0;
2368 for &end in splits {
2369 let links = if start == 0 {
2372 doc.links.as_slice()
2373 } else {
2374 &[]
2375 };
2376 got.push_str(&streamer.push(&doc.nodes[start..end], links));
2377 got_artifacts.extend(streamer.take_artifacts());
2380 start = end;
2381 }
2382 got.push_str(&streamer.push(
2383 &doc.nodes[start..],
2384 if start == 0 {
2385 doc.links.as_slice()
2386 } else {
2387 &[]
2388 },
2389 ));
2390 got_artifacts.extend(streamer.take_artifacts());
2391 got.push_str(&streamer.finish());
2392 assert_eq!(
2393 got, want,
2394 "streamed output diverged (splits={splits:?}, strict={strict})"
2395 );
2396 assert_eq!(
2397 got_artifacts, want_artifacts,
2398 "streamed artifacts diverged (splits={splits:?}, strict={strict})"
2399 );
2400 }
2401
2402 #[test]
2403 fn streaming_is_byte_identical_to_buffered() {
2404 let mut doc = DoclingDocument::new("d");
2405 doc.add_heading(1, "Title");
2406 doc.add_paragraph("First paragraph.");
2407 doc.push(Node::ListItem {
2408 ordered: false,
2409 number: 1,
2410 first_in_list: true,
2411 text: "a".into(),
2412 level: 0,
2413 marker: None,
2414 location: None,
2415 split: None,
2416 dclx: None,
2417 href: None,
2418 layer: None,
2419 });
2420 doc.push(Node::ListItem {
2421 ordered: false,
2422 number: 2,
2423 first_in_list: false,
2424 text: "b".into(),
2425 level: 0,
2426 marker: None,
2427 location: None,
2428 split: None,
2429 dclx: None,
2430 href: None,
2431 layer: None,
2432 });
2433 doc.push(Node::Code {
2434 language: Some("rust".into()),
2435 text: "let x = 1;".into(),
2436 orig: None,
2437 pretty: None,
2438 });
2439 doc.push(Node::Table(Table {
2440 footnotes: Vec::new(),
2441 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2442 location: None,
2443 structure: None,
2444 cell_blocks: None,
2445 cells: None,
2446 caption: None,
2447 caption_parent: Default::default(),
2448 caption_location: None,
2449 document_index: false,
2450 }));
2451 doc.push(Node::Picture {
2452 caption: Some("Fig 1".into()),
2453 caption_href: None,
2454 image: Some(PictureImage {
2455 dpi: PictureImage::DEFAULT_DPI,
2456 mimetype: "image/png".into(),
2457 width: 2,
2458 height: 2,
2459 data: b"png-one".to_vec(),
2460 display_size: None,
2461 }),
2462 classification: None,
2463 description: None,
2464 caption_parent: Default::default(),
2465 caption_location: None,
2466 });
2467 doc.add_paragraph("Last paragraph.");
2468 doc.push(Node::Picture {
2471 caption: None,
2472 caption_href: None,
2473 image: Some(PictureImage {
2474 dpi: PictureImage::DEFAULT_DPI,
2475 mimetype: "image/png".into(),
2476 width: 2,
2477 height: 2,
2478 data: b"png-two".to_vec(),
2479 display_size: None,
2480 }),
2481 classification: None,
2482 description: None,
2483 caption_parent: Default::default(),
2484 caption_location: None,
2485 });
2486
2487 for &strict in &[false, true] {
2490 for &images in &[
2491 ImageMode::Placeholder,
2492 ImageMode::Embedded,
2493 ImageMode::Referenced,
2494 ] {
2495 for splits in [&[][..], &[1][..], &[2][..], &[4][..], &[1, 4, 6, 7][..]] {
2496 assert_stream_matches(&doc, strict, images, splits);
2497 }
2498 }
2499 }
2500 }
2501
2502 #[test]
2503 fn streaming_applies_recovered_links_in_strict_mode() {
2504 let mut doc = DoclingDocument::new("d");
2505 doc.add_paragraph("See LinkedIn for details.");
2506 doc.add_paragraph("And GitHub too.");
2507 doc.links = vec![
2508 ("LinkedIn".into(), "https://lnkd/".into()),
2509 ("GitHub".into(), "https://gh/".into()),
2510 ];
2511 assert_stream_matches(&doc, true, ImageMode::Placeholder, &[1]);
2514 }
2515
2516 fn paged_doc() -> DoclingDocument {
2519 let mut doc = DoclingDocument::new("p");
2520 doc.push(Node::PageInfo {
2521 page_no: 1,
2522 width: 100.0,
2523 height: 100.0,
2524 });
2525 doc.add_heading(1, "Title");
2526 doc.add_paragraph("Page one.");
2527 doc.push(Node::PageBreak);
2529 doc.push(Node::PageInfo {
2530 page_no: 2,
2531 width: 100.0,
2532 height: 100.0,
2533 });
2534 doc.push(Node::PageFurniture {
2535 footer: true,
2536 location: [0, 500, 511, 511],
2537 text: "2".into(),
2538 });
2539 doc.push(Node::PageBreak);
2540 doc.push(Node::PageInfo {
2541 page_no: 3,
2542 width: 100.0,
2543 height: 100.0,
2544 });
2545 doc.add_paragraph("Page three.");
2546 doc.push(Node::PageBreak);
2548 doc
2549 }
2550
2551 #[test]
2552 fn page_break_placeholder_lands_between_pages_only() {
2553 let mut doc = paged_doc();
2554 assert_eq!(
2556 doc.export_to_markdown(),
2557 "# Title\n\nPage one.\n\nPage three.\n"
2558 );
2559 doc.page_break_placeholder = Some("<!-- page break -->".into());
2560 assert_eq!(
2564 doc.export_to_markdown(),
2565 "# Title\n\nPage one.\n\n<!-- page break -->\n\nPage three.\n"
2566 );
2567 doc.page_break_placeholder = Some(String::new());
2570 assert_eq!(
2571 doc.export_to_markdown(),
2572 "# Title\n\nPage one.\n\n\n\nPage three.\n"
2573 );
2574 }
2575
2576 #[test]
2577 fn page_break_placeholder_never_leads_a_single_page() {
2578 let mut doc = DoclingDocument::new("one");
2579 doc.page_break_placeholder = Some("---".into());
2580 doc.push(Node::PageBreak);
2581 doc.push(Node::PageInfo {
2582 page_no: 1,
2583 width: 10.0,
2584 height: 10.0,
2585 });
2586 doc.add_paragraph("Only page.");
2587 assert_eq!(doc.export_to_markdown(), "Only page.\n");
2588 doc.push(Node::PageBreak);
2590 doc.push(Node::PageBreak);
2591 doc.add_paragraph("Next.");
2592 assert_eq!(doc.export_to_markdown(), "Only page.\n\n---\n\nNext.\n");
2593 }
2594
2595 #[test]
2596 fn page_break_placeholder_streams_byte_identical() {
2597 let mut doc = paged_doc();
2598 doc.page_break_placeholder = Some("<!-- page break -->".into());
2599 for splits in [
2603 &[3usize][..],
2604 &[3, 6],
2605 &[3, 6, 8],
2606 &[1, 2, 3, 4, 5, 6, 7, 8, 9],
2607 &[8],
2608 ] {
2609 assert_stream_matches(&doc, false, ImageMode::Placeholder, splits);
2610 assert_stream_matches(&doc, true, ImageMode::Placeholder, splits);
2611 }
2612 }
2613
2614 #[test]
2615 fn strict_tightens_punctuation_spacing_legacy_keeps_it() {
2616 let mut doc = DoclingDocument::new("t");
2617 doc.add_paragraph("see [ 37 , 36 ] and ( x ) .");
2618 assert_eq!(doc.export_to_markdown(), "see [ 37 , 36 ] and ( x ) .\n");
2620 assert_eq!(doc.export_to_markdown_with(true), "see [37, 36] and (x).\n");
2622 }
2623
2624 fn options_doc() -> DoclingDocument {
2629 use crate::document::ContentLayer;
2630 let mut doc = DoclingDocument::new("opts");
2631 doc.push(Node::PageFurniture {
2632 footer: false,
2633 location: [0, 0, 0, 0],
2634 text: "FAX COVER \\_ R&D".into(),
2635 });
2636 doc.add_heading(1, "R&D \\_ report");
2637 doc.add_paragraph("Body a\\_b <tag>");
2638 doc.push(Node::Picture {
2639 caption: Some("Figure 1".into()),
2640 caption_href: None,
2641 image: None,
2642 classification: None,
2643 description: None,
2644 caption_parent: Default::default(),
2645 caption_location: None,
2646 });
2647 doc.push(Node::PictureChildren(vec![
2648 Node::Paragraph {
2649 text: "Field: Name".into(),
2650 },
2651 Node::Paragraph {
2652 text: "Field: Date".into(),
2653 },
2654 ]));
2655 doc.push(Node::CommentSection {
2656 name: "comment-1".into(),
2657 text: "[author: A]: note".into(),
2658 refs_note_text: false,
2659 grouped: true,
2660 });
2661 doc.push(Node::Group {
2662 label: "section".into(),
2663 name: Some("sheet: Hidden".into()),
2664 layer: Some(ContentLayer::Invisible),
2665 children: vec![Node::Paragraph {
2666 text: "hidden cell".into(),
2667 }],
2668 });
2669 doc.push(Node::Furniture {
2670 layer: ContentLayer::Furniture,
2671 inner: Box::new(Node::Paragraph {
2672 text: "Page 1 of 2".into(),
2673 }),
2674 });
2675 doc.push(Node::FurnitureText {
2676 label: "page_footer".into(),
2677 text: "footer line".into(),
2678 });
2679 doc
2680 }
2681
2682 #[test]
2686 fn default_options_are_the_default_export() {
2687 let doc = options_doc();
2688 let plain = doc.export_to_markdown();
2689 assert_eq!(
2690 plain,
2691 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2692 );
2693 let (with_default, artifacts) =
2694 doc.export_to_markdown_with_options(&MarkdownExportOptions::default());
2695 assert_eq!(with_default, plain);
2696 assert!(artifacts.is_empty());
2697 }
2698
2699 #[test]
2700 fn layers_render_furniture_notes_and_hidden_sheets() {
2701 use crate::document::ContentLayer;
2702 let doc = options_doc();
2703 let md = |layers: ContentLayers| {
2704 doc.export_to_markdown_with_options(&MarkdownExportOptions {
2705 layers,
2706 ..MarkdownExportOptions::default()
2707 })
2708 .0
2709 };
2710 assert_eq!(
2711 md(ContentLayers::BODY.with(ContentLayer::Furniture)),
2712 "FAX COVER \\_ R&D\n\n# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nPage 1 of 2\n\nfooter line\n",
2713 "a page header, a furniture-wrapped paragraph and a furniture text item render as paragraphs"
2714 );
2715 assert_eq!(
2716 md(ContentLayers::BODY.with(ContentLayer::Notes)),
2717 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\n[author: A]: note\n"
2718 );
2719 assert_eq!(
2720 md(ContentLayers::BODY.with(ContentLayer::Invisible)),
2721 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nhidden cell\n",
2722 "a hidden sheet's items sit on the sheet's layer"
2723 );
2724 assert_eq!(
2725 md(ContentLayers::NONE.with(ContentLayer::Furniture)),
2726 "FAX COVER \\_ R&D\n\nPage 1 of 2\n\nfooter line\n",
2727 "without the body layer only the furniture prints"
2728 );
2729 assert_eq!(md(ContentLayers::NONE), "");
2730 assert_eq!(
2731 md(ContentLayers::ALL),
2732 "FAX COVER \\_ R&D\n\n# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\n[author: A]: note\n\nhidden cell\n\nPage 1 of 2\n\nfooter line\n"
2733 );
2734 }
2735
2736 #[test]
2739 fn layers_apply_to_list_items() {
2740 use crate::document::ContentLayer;
2741 let mut doc = DoclingDocument::new("lists");
2742 doc.push(Node::ListItem {
2743 ordered: false,
2744 number: 1,
2745 first_in_list: true,
2746 text: "nav link".into(),
2747 level: 0,
2748 marker: None,
2749 location: None,
2750 split: None,
2751 dclx: None,
2752 href: None,
2753 layer: Some(ContentLayer::Furniture),
2754 });
2755 doc.add_paragraph("body");
2756 assert_eq!(doc.export_to_markdown(), "body\n");
2757 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2758 layers: ContentLayers::BODY.with(ContentLayer::Furniture),
2759 ..MarkdownExportOptions::default()
2760 });
2761 assert_eq!(md, "- nav link\n\nbody\n");
2762 }
2763
2764 #[test]
2765 fn traverse_pictures_prints_the_nested_text_after_the_picture() {
2766 let doc = options_doc();
2767 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2768 traverse_pictures: true,
2769 ..MarkdownExportOptions::default()
2770 });
2771 assert_eq!(
2772 md,
2773 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nField: Name\n\nField: Date\n"
2774 );
2775 }
2776
2777 #[test]
2778 fn escaping_can_be_turned_off_per_kind() {
2779 let doc = options_doc();
2780 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2781 escape_html: false,
2782 ..MarkdownExportOptions::default()
2783 });
2784 assert_eq!(
2785 md,
2786 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2787 );
2788 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2789 escape_underscores: false,
2790 ..MarkdownExportOptions::default()
2791 });
2792 assert_eq!(
2793 md,
2794 "# R&D _ report\n\nBody a_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2795 );
2796 let mut doc = DoclingDocument::new("code");
2799 doc.push(Node::Code {
2800 language: None,
2801 text: "a & b \\_ c".into(),
2802 orig: None,
2803 pretty: None,
2804 });
2805 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2806 escape_html: false,
2807 escape_underscores: false,
2808 ..MarkdownExportOptions::default()
2809 });
2810 assert_eq!(md, "```\na & b \\_ c\n```\n");
2811 }
2812
2813 #[test]
2817 fn fences_and_spans_out_length_inner_backticks() {
2818 let code = |text: &str| Node::Code {
2819 language: Some("md".into()),
2820 text: text.into(),
2821 orig: None,
2822 pretty: None,
2823 };
2824 let fence_of = |text: &str| {
2825 let mut doc = DoclingDocument::new("d");
2826 doc.push(code(text));
2827 (doc.export_to_markdown(), doc.export_to_markdown_with(true))
2828 };
2829 assert_eq!(fence_of("x").0, "```\nx\n```\n");
2830 assert_eq!(fence_of("a ` b").0, "```\na ` b\n```\n");
2831 assert_eq!(fence_of("a ``` b").0, "````\na ``` b\n````\n");
2832 assert_eq!(
2833 fence_of("a ``` b ```` c"),
2834 (
2835 "`````\na ``` b ```` c\n`````\n".to_string(),
2836 "`````md\na ``` b ```` c\n`````\n".to_string()
2837 )
2838 );
2839 assert_eq!(code_span("x"), "`x`");
2840 assert_eq!(code_span("a`b"), "``a`b``");
2841 assert_eq!(code_span("a ``` b"), "````a ``` b````");
2842 assert_eq!(code_span("`x"), "`` `x ``");
2843 assert_eq!(code_span("x`"), "`` x` ``");
2844 for (span, text) in [
2845 ("`x`", "x"),
2846 ("``a`b``", "a`b"),
2847 ("`` `x ``", "`x"),
2848 ("````a ``` b````", "a ``` b"),
2849 ] {
2850 assert_eq!(
2851 plain_inline(&format!("see {span} here")),
2852 format!("see {text} here")
2853 );
2854 }
2855 let mut doc = DoclingDocument::new("d");
2858 doc.push(code("a ``` b"));
2859 assert_eq!(doc.export_to_text(), "a ``` b");
2860 let mut doc = DoclingDocument::new("d");
2862 doc.add_paragraph("Before.");
2863 doc.push(code("a ``` b ```` c"));
2864 doc.add_paragraph("After.");
2865 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[1, 2]);
2866 assert_stream_matches(&doc, true, ImageMode::Placeholder, &[1, 2]);
2867 }
2868
2869 #[test]
2874 fn mermaid_code_item_replaces_its_picture_placeholder() {
2875 let picture = || Node::Picture {
2876 caption: None,
2877 caption_href: None,
2878 image: None,
2879 classification: None,
2880 description: None,
2881 caption_parent: Default::default(),
2882 caption_location: None,
2883 };
2884 let mut doc = DoclingDocument::new("d");
2885 doc.add_paragraph("Before.");
2886 doc.push(picture());
2887 doc.push(Node::Code {
2888 language: Some("mermaid".into()),
2889 text: "flowchart TD\n A --> B".into(),
2890 orig: None,
2891 pretty: None,
2892 });
2893 doc.add_paragraph("After.");
2894 doc.push(picture());
2895 let md = doc.export_to_markdown();
2896 assert_eq!(
2897 md,
2898 "Before.\n\n```mermaid\nflowchart TD\n A --> B\n```\n\nAfter.\n\n<!-- image -->\n"
2899 );
2900 assert_eq!(doc.export_to_markdown_with(true), md);
2901 for splits in [&[1][..], &[2], &[1, 2], &[2, 3], &[1, 2, 3, 4]] {
2902 assert_stream_matches(&doc, false, ImageMode::Placeholder, splits);
2903 assert_stream_matches(&doc, true, ImageMode::Placeholder, splits);
2904 }
2905 let mut doc = DoclingDocument::new("d");
2907 doc.add_paragraph("Only.");
2908 doc.push(picture());
2909 assert_eq!(doc.export_to_markdown(), "Only.\n\n<!-- image -->\n");
2910 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[1]);
2911 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[2]);
2912 }
2913
2914 #[test]
2915 fn image_placeholder_is_configurable() {
2916 let doc = options_doc();
2917 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2918 image_placeholder: "[figure]".into(),
2919 ..MarkdownExportOptions::default()
2920 });
2921 assert!(md.contains("Figure 1\n\n[figure]\n"), "{md}");
2922 assert!(!md.contains("<!-- image -->"));
2923 }
2924
2925 #[test]
2931 fn empty_image_placeholder_prints_nothing() {
2932 fn picture(caption: Option<&str>) -> Node {
2933 Node::Picture {
2934 caption: caption.map(Into::into),
2935 caption_href: None,
2936 image: None,
2937 classification: None,
2938 description: None,
2939 caption_parent: Default::default(),
2940 caption_location: None,
2941 }
2942 }
2943 fn page(no: usize) -> Node {
2944 Node::PageInfo {
2945 page_no: no,
2946 width: 100.0,
2947 height: 100.0,
2948 }
2949 }
2950 fn para(text: &str) -> Node {
2951 Node::Paragraph { text: text.into() }
2952 }
2953 let options = MarkdownExportOptions {
2954 image_placeholder: String::new(),
2955 ..MarkdownExportOptions::default()
2956 };
2957 let cases: Vec<(Vec<Node>, Option<&str>, &str)> = vec![
2959 (
2960 vec![page(1), para("a"), picture(None), para("b")],
2961 None,
2962 "a\n\nb\n",
2963 ),
2964 (
2965 vec![page(1), para("a"), picture(Some("Figure 1")), para("b")],
2966 None,
2967 "a\n\nFigure 1\n\nb\n",
2968 ),
2969 (
2970 vec![
2971 page(1),
2972 para("a"),
2973 page(2),
2974 picture(None),
2975 page(3),
2976 para("b"),
2977 ],
2978 Some("<!-- pb -->"),
2979 "a\n\n<!-- pb -->\n\n<!-- pb -->\n\nb\n",
2980 ),
2981 (
2982 vec![
2983 page(1),
2984 para("a"),
2985 page(2),
2986 picture(None),
2987 page(3),
2988 para("b"),
2989 ],
2990 Some(""),
2991 "a\n\n\n\n\n\nb\n",
2992 ),
2993 (
2994 vec![page(1), picture(None), page(2), para("b")],
2995 Some("<!-- pb -->"),
2996 "<!-- pb -->\n\nb\n",
2997 ),
2998 (
2999 vec![page(1), picture(None), page(2), para("b")],
3000 Some(""),
3001 "\n\nb\n",
3002 ),
3003 (
3004 vec![page(1), para("a"), page(2), picture(None)],
3005 Some("<!-- pb -->"),
3006 "a\n\n<!-- pb -->\n",
3007 ),
3008 ];
3009 for (nodes, placeholder, expected) in cases {
3010 let mut doc = DoclingDocument::new("t");
3011 doc.page_break_placeholder = placeholder.map(Into::into);
3012 for node in nodes {
3013 doc.push(node);
3014 }
3015 let (md, _) = doc.export_to_markdown_with_options(&options);
3016 assert_eq!(md, expected, "page break {placeholder:?}");
3017 for size in 1..=doc.nodes.len() {
3020 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3021 .with_export_options(&options)
3022 .with_page_break_placeholder(placeholder.map(Into::into));
3023 let mut out = String::new();
3024 for chunk in doc.nodes.chunks(size) {
3025 out.push_str(&streamer.push(chunk, &[]));
3026 }
3027 out.push_str(&streamer.finish());
3028 assert_eq!(
3029 out, expected,
3030 "chunks of {size}, page break {placeholder:?}"
3031 );
3032 }
3033 }
3034 let mut doc = DoclingDocument::new("t");
3036 doc.push(para("a"));
3037 doc.push(picture(None));
3038 doc.push(para("b"));
3039 assert_eq!(doc.export_to_markdown(), "a\n\n<!-- image -->\n\nb\n");
3040 }
3041
3042 #[test]
3045 fn streamer_matches_buffered_export_with_options() {
3046 let doc = options_doc();
3047 let options = MarkdownExportOptions {
3048 layers: ContentLayers::ALL,
3049 traverse_pictures: true,
3050 escape_html: false,
3051 escape_underscores: false,
3052 image_placeholder: "(img)".into(),
3053 ..MarkdownExportOptions::default()
3054 };
3055 let (buffered, _) = doc.export_to_markdown_with_options(&options);
3056 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3057 .with_export_options(&options);
3058 let mut out = String::new();
3059 for chunk in doc.nodes.chunks(2) {
3060 out.push_str(&streamer.push(chunk, &[]));
3061 }
3062 out.push_str(&streamer.finish());
3063 assert_eq!(out, buffered);
3064 assert!(
3065 buffered.contains("(img)") && buffered.contains("R&D _ report"),
3066 "{buffered}"
3067 );
3068 }
3069
3070 #[test]
3073 fn plain_text_reads_escaped_stars() {
3074 assert_eq!(plain_inline("A\\*\\*\\* GmbH"), "A*** GmbH");
3075 assert_eq!(
3076 plain_inline("**FN \\*0\\*271\\*h** and *h\\*.at*"),
3077 "FN *0*271*h and h*.at"
3078 );
3079 }
3080
3081 #[test]
3084 fn plain_text_drops_the_markdown_decoration() {
3085 let item = |text: &str, ordered: bool, number: u64, level: u8| Node::ListItem {
3086 ordered,
3087 number,
3088 first_in_list: number == 1 && level == 0,
3089 text: text.into(),
3090 level,
3091 marker: None,
3092 location: None,
3093 split: None,
3094 dclx: None,
3095 href: None,
3096 layer: None,
3097 };
3098 let mut doc = DoclingDocument::new("plain");
3099 doc.add_heading(1, "Title with **bold**");
3100 doc.add_heading(2, "Section");
3101 doc.add_paragraph("Some **bold**, *italic*, ~~gone~~ and `code` with a [link](https://x.org).\nNext line.");
3102 doc.push(item("*one*", false, 1, 0));
3103 doc.push(item("[two](https://y.org)", false, 2, 0));
3104 doc.push(item("nested", true, 1, 1));
3105 doc.push(Node::CheckboxItem {
3106 checked: true,
3107 text: "done".into(),
3108 });
3109 doc.push(Node::Code {
3110 language: Some("rust".into()),
3111 text: "let x = **y**;".into(),
3112 orig: None,
3113 pretty: None,
3114 });
3115 doc.push(Node::Picture {
3116 caption: Some("Figure *1*".into()),
3117 caption_href: None,
3118 image: None,
3119 classification: None,
3120 description: None,
3121 caption_parent: Default::default(),
3122 caption_location: None,
3123 });
3124 doc.push(Node::Caption {
3125 text: "Linked caption".into(),
3126 href: Some("https://z.org".into()),
3127 });
3128 doc.push(Node::Table(Table {
3129 rows: vec![
3130 vec!["**n**".into(), "rich".into()],
3131 vec!["**2**".into(), "a \nb".into()],
3132 ],
3133 ..Default::default()
3134 }));
3135 doc.add_paragraph("R&D \\_ x");
3136 let text = doc.export_to_text();
3137 assert_eq!(
3138 text,
3139 "Title with bold\n\n\
3140 Section\n\n\
3141 Some bold, italic, gone and code with a link.\nNext line.\n\n\
3142 - one\n- two\n 1. nested\n\n\
3143 - [x] done\n\n\
3144 let x = **y**;\n\n\
3145 Figure 1\n\n\
3146 Linked caption\n\n\
3147 | n | rich |\n\
3148 |-----|--------|\n\
3149 | 2 | a b |\n\n\
3150 R&D _ x"
3151 );
3152
3153 let options = MarkdownExportOptions::plain_text();
3154 let (buffered, _) = doc.export_to_markdown_with_options(&options);
3155 assert_eq!(buffered, format!("{text}\n"));
3156 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3157 .with_export_options(&options);
3158 let mut out = String::new();
3159 for chunk in doc.nodes.chunks(3) {
3160 out.push_str(&streamer.push(chunk, &[]));
3161 }
3162 out.push_str(&streamer.finish());
3163 assert_eq!(out, buffered);
3164 }
3165
3166 #[test]
3168 fn plain_inline_keeps_literal_markers() {
3169 for (input, want) in [
3170 ("**bold** and *it* and ***both***", "bold and it and both"),
3171 ("**[label](https://a.b/c_(d))** end", "label end"),
3172 (
3173 "grant *USE and *OBJMGT authority",
3174 "grant *USE and *OBJMGT authority",
3175 ),
3176 ("2 * 3 * 4", "2 * 3 * 4"),
3177 ("a ** b", "a ** b"),
3178 ("the `quoted’ and `x` term", "the `quoted’ and x term"),
3179 ("see [12] and [docs](u)", "see [12] and docs"),
3180 ("formula $[a](b)$ stays", "formula $[a](b)$ stays"),
3181 ("price $5 and **bold**", "price $5 and bold"),
3182 ("[outer [inner] text](u)", "outer [inner] text"),
3183 ("unclosed **bold", "unclosed **bold"),
3184 ] {
3185 assert_eq!(plain_inline(input), want, "{input}");
3186 }
3187 }
3188}