1use crate::document::{
4 ContentLayer, ContentLayers, DoclingDocument, MarkdownExportOptions, Node, Table,
5};
6
7const MISSING_KEY_VALUE_ITEM: &str = "<!-- missing-key-value-item -->";
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
14pub enum ImageMode {
15 #[default]
17 Placeholder,
18 Embedded,
20 Referenced,
23}
24
25struct Ctx {
27 strict: bool,
28 alt_text: bool,
31 compact_tables: bool,
33 images: ImageMode,
34 artifacts_dir: String,
35 artifacts: Vec<(String, Vec<u8>)>,
37 pic_index: usize,
38 in_table_cell: bool,
42 page_break: Option<String>,
46 pending_page_break: bool,
52 emitted_any: bool,
55 layers: ContentLayers,
57 current_layer: Option<ContentLayer>,
62 traverse_pictures: bool,
64 escape_html: bool,
67 escape_underscores: bool,
69 image_placeholder: String,
71 plain: bool,
73}
74
75impl Ctx {
76 fn new(doc: &DoclingDocument, strict: bool, options: &MarkdownExportOptions) -> Self {
79 Ctx {
80 strict,
81 alt_text: strict || doc.picture_alt_text,
82 compact_tables: doc.compact_tables,
83 images: options.image_mode,
84 artifacts_dir: options.artifacts_dir.clone(),
85 artifacts: Vec::new(),
86 pic_index: 0,
87 in_table_cell: false,
88 page_break: doc.page_break_placeholder.clone(),
89 pending_page_break: false,
90 emitted_any: false,
91 layers: options.layers,
92 current_layer: None,
93 traverse_pictures: options.traverse_pictures,
94 escape_html: options.escape_html,
95 escape_underscores: options.escape_underscores,
96 image_placeholder: options.image_placeholder.clone(),
97 plain: options.plain_text,
98 }
99 }
100
101 fn layer_visible(&self) -> bool {
103 self.layers.contains(self.current_layer)
104 }
105
106 fn unescape(&self, text: &str) -> String {
113 let mut out = std::borrow::Cow::Borrowed(text);
114 if !self.escape_html
115 && (out.contains("&") || out.contains("<") || out.contains(">"))
116 {
117 out = std::borrow::Cow::Owned(
118 out.replace("<", "<")
119 .replace(">", ">")
120 .replace("&", "&"),
121 );
122 }
123 if !self.escape_underscores && out.contains("\\_") {
124 out = std::borrow::Cow::Owned(out.replace("\\_", "_"));
125 }
126 out.into_owned()
127 }
128
129 fn text(&self, text: &str) -> String {
132 let text = strict_text(text, self.strict);
133 if self.plain {
134 self.unescape(&plain_inline(&self.drop_image_markers(&text)))
135 } else {
136 self.unescape(&text)
137 }
138 }
139
140 fn caption(&self, text: &str) -> String {
144 if self.plain {
145 self.unescape(&plain_inline(text))
146 } else {
147 self.unescape(text)
148 }
149 }
150
151 fn breaks(&self, text: &str) -> String {
154 if self.plain {
155 text.to_string()
156 } else {
157 md_line_breaks(text)
158 }
159 }
160
161 fn plain_table(&self, table: &Table) -> Table {
168 let mut plain = table.clone();
169 for cell in plain.rows.iter_mut().flatten() {
170 if cell.contains(['*', '~', '`', '[', '\n', '<']) {
171 let text = self.drop_image_markers(&strip_hard_breaks(cell));
172 *cell = plain_inline(&text);
173 }
174 }
175 plain
176 }
177
178 fn drop_image_markers(&self, text: &str) -> String {
185 const MARKER: &str = "<!-- image -->";
186 if !text.contains(MARKER) && !text.contains("```") {
187 return text.to_string();
188 }
189 text.split("\n\n")
193 .filter_map(|block| {
194 let lines: Vec<String> = block
195 .split('\n')
196 .filter_map(|line| {
197 if line.trim() == MARKER || is_fence_line(line.trim()) {
200 return None;
201 }
202 Some(line.replace(MARKER, &self.image_placeholder))
203 })
204 .collect();
205 (!lines.is_empty()).then(|| lines.join("\n"))
206 })
207 .collect::<Vec<_>>()
208 .join("\n\n")
209 }
210}
211
212pub fn to_markdown(doc: &DoclingDocument, strict: bool) -> String {
218 to_markdown_images(doc, strict, ImageMode::Placeholder, "artifacts").0
219}
220
221pub fn to_markdown_images(
225 doc: &DoclingDocument,
226 strict: bool,
227 images: ImageMode,
228 artifacts_dir: &str,
229) -> (String, Vec<(String, Vec<u8>)>) {
230 to_markdown_with_options(
231 doc,
232 strict,
233 &MarkdownExportOptions {
234 image_mode: images,
235 artifacts_dir: artifacts_dir.to_string(),
236 ..MarkdownExportOptions::default()
237 },
238 )
239}
240
241pub fn to_markdown_with_options(
245 doc: &DoclingDocument,
246 strict: bool,
247 options: &MarkdownExportOptions,
248) -> (String, Vec<(String, Vec<u8>)>) {
249 let mut ctx = Ctx::new(doc, strict, options);
250 let mut blocks: Vec<String> = Vec::new();
251 render(&doc.nodes, &mut blocks, &mut ctx);
252 let mut body = blocks.join("\n\n");
253 if strict && !options.plain_text && !doc.links.is_empty() {
257 body = apply_links(&body, &doc.links);
258 }
259 let front = if doc.metadata_output.front_matter() && !options.plain_text {
262 doc.metadata.to_front_matter()
263 } else {
264 String::new()
265 };
266 let md = match (front.is_empty(), body.is_empty()) {
267 (true, true) => String::new(),
268 (true, false) => format!("{body}\n"),
269 (false, true) => front,
270 (false, false) => format!("{front}\n{body}\n"),
271 };
272 (md, ctx.artifacts)
273}
274
275pub fn to_markdown_table_cell(doc: &DoclingDocument, strict: bool) -> String {
284 let mut ctx = Ctx::new(doc, strict, &MarkdownExportOptions::default());
285 ctx.in_table_cell = true;
286 ctx.artifacts_dir = String::new();
287 ctx.page_break = None;
291 let mut blocks: Vec<String> = Vec::new();
292 render(&doc.nodes, &mut blocks, &mut ctx);
293 blocks.join("\n\n")
294}
295
296fn apply_links(body: &str, links: &[(String, String)]) -> String {
304 let mut out = body.to_string();
305 let mut cursor = 0usize;
306 for (anchor, href) in links {
307 let anchor = anchor
308 .replace('&', "&")
309 .replace('<', "<")
310 .replace('>', ">");
311 if anchor.is_empty() {
312 continue;
313 }
314 if let Some(rel) = out[cursor..].find(&anchor) {
315 let at = cursor + rel;
316 let replacement = crate::hyperlink::markdown_link(&anchor, href);
319 out.replace_range(at..at + anchor.len(), &replacement);
320 cursor = at + replacement.len();
321 }
322 }
323 out
324}
325
326fn apply_links_chunk(chunk: &str, queue: &mut Vec<(String, String)>) -> String {
337 let mut out = chunk.to_string();
338 let mut cursor = 0usize;
339 let mut carried: Vec<(String, String)> = Vec::new();
340 for (anchor_raw, href) in std::mem::take(queue) {
341 let anchor = anchor_raw
342 .replace('&', "&")
343 .replace('<', "<")
344 .replace('>', ">");
345 if anchor.is_empty() {
346 continue;
347 }
348 if let Some(rel) = out[cursor..].find(&anchor) {
349 let at = cursor + rel;
350 let replacement = crate::hyperlink::markdown_link(&anchor, &href);
351 out.replace_range(at..at + anchor.len(), &replacement);
352 cursor = at + replacement.len();
353 } else {
354 carried.push((anchor_raw, href));
356 }
357 }
358 *queue = carried;
359 out
360}
361
362pub struct MarkdownStreamer {
380 strict: bool,
381 images: ImageMode,
382 compact_tables: bool,
383 picture_alt_text: bool,
385 front_matter: Option<String>,
389 emitted_any: bool,
392 links: Vec<(String, String)>,
394 artifacts_dir: String,
398 artifacts: Vec<(String, Vec<u8>)>,
399 pic_index: usize,
400 page_break: Option<String>,
404 pending_page_break: bool,
405 items_emitted: bool,
411 held: Option<Node>,
416 layers: ContentLayers,
419 traverse_pictures: bool,
420 escape_html: bool,
421 escape_underscores: bool,
422 image_placeholder: String,
423 plain: bool,
424}
425
426impl MarkdownStreamer {
427 pub fn new(strict: bool, images: ImageMode, compact_tables: bool) -> Self {
430 debug_assert!(
431 images != ImageMode::Referenced,
432 "referenced image mode needs an artifacts dir; use with_artifacts"
433 );
434 Self::with_artifacts(strict, images, compact_tables, "artifacts")
435 }
436
437 pub fn with_artifacts(
444 strict: bool,
445 images: ImageMode,
446 compact_tables: bool,
447 artifacts_dir: &str,
448 ) -> Self {
449 Self {
450 strict,
451 images,
452 compact_tables,
453 picture_alt_text: false,
454 front_matter: None,
455 emitted_any: false,
456 links: Vec::new(),
457 artifacts_dir: artifacts_dir.to_string(),
458 artifacts: Vec::new(),
459 pic_index: 0,
460 page_break: None,
461 pending_page_break: false,
462 items_emitted: false,
463 held: None,
464 layers: ContentLayers::BODY,
465 traverse_pictures: false,
466 escape_html: true,
467 escape_underscores: true,
468 image_placeholder: "<!-- image -->".to_string(),
469 plain: false,
470 }
471 }
472
473 pub fn with_export_options(mut self, options: &MarkdownExportOptions) -> Self {
480 self.images = options.image_mode;
481 self.artifacts_dir = options.artifacts_dir.clone();
482 self.layers = options.layers;
483 self.traverse_pictures = options.traverse_pictures;
484 self.escape_html = options.escape_html;
485 self.escape_underscores = options.escape_underscores;
486 self.image_placeholder = options.image_placeholder.clone();
487 self.plain = options.plain_text;
488 self
489 }
490
491 pub fn with_page_break_placeholder(mut self, placeholder: Option<String>) -> Self {
496 self.page_break = placeholder;
497 self
498 }
499
500 pub fn with_picture_alt_text(mut self, on: bool) -> Self {
504 self.picture_alt_text = on;
505 self
506 }
507
508 pub fn with_metadata(mut self, metadata: &crate::metadata::DocumentMetadata) -> Self {
513 self.front_matter = (!self.plain)
514 .then(|| metadata.to_front_matter())
515 .filter(|f| !f.is_empty());
516 self
517 }
518
519 pub fn take_artifacts(&mut self) -> Vec<(String, Vec<u8>)> {
524 std::mem::take(&mut self.artifacts)
525 }
526
527 pub fn push(&mut self, nodes: &[Node], links: &[(String, String)]) -> String {
532 self.push_holding(nodes, links, true)
533 }
534
535 fn push_holding(&mut self, nodes: &[Node], links: &[(String, String)], hold: bool) -> String {
538 self.links.extend(links.iter().cloned());
539 let images = self.images;
543 let ends_on_picture = |nodes: &[Node]| {
544 matches!(images, ImageMode::Placeholder)
545 && matches!(nodes.last(), Some(Node::Picture { .. }))
546 };
547 let mut owned: Vec<Node> = Vec::new();
548 let mut nodes = nodes;
549 if self.held.is_some() || (hold && ends_on_picture(nodes)) {
550 owned.extend(self.held.take());
551 owned.extend(nodes.iter().cloned());
552 if hold && ends_on_picture(&owned) {
553 self.held = owned.pop();
554 }
555 nodes = &owned;
556 }
557 let mut ctx = Ctx {
558 strict: self.strict,
559 alt_text: self.strict || self.picture_alt_text,
560 compact_tables: self.compact_tables,
561 images: self.images,
562 artifacts_dir: std::mem::take(&mut self.artifacts_dir),
563 artifacts: std::mem::take(&mut self.artifacts),
564 pic_index: self.pic_index,
565 in_table_cell: false,
566 page_break: std::mem::take(&mut self.page_break),
567 pending_page_break: self.pending_page_break,
568 emitted_any: self.items_emitted,
569 layers: self.layers,
570 current_layer: None,
571 traverse_pictures: self.traverse_pictures,
572 escape_html: self.escape_html,
573 escape_underscores: self.escape_underscores,
574 image_placeholder: std::mem::take(&mut self.image_placeholder),
575 plain: self.plain,
576 };
577 let mut blocks: Vec<String> = Vec::new();
578 render(nodes, &mut blocks, &mut ctx);
579 self.image_placeholder = std::mem::take(&mut ctx.image_placeholder);
580 self.artifacts_dir = std::mem::take(&mut ctx.artifacts_dir);
581 self.artifacts = std::mem::take(&mut ctx.artifacts);
582 self.pic_index = ctx.pic_index;
583 self.page_break = std::mem::take(&mut ctx.page_break);
584 self.pending_page_break = ctx.pending_page_break;
585 self.items_emitted = ctx.emitted_any;
586 if blocks.is_empty() {
587 return String::new();
588 }
589 let mut body = blocks.join("\n\n");
590 if self.strict && !self.plain && !self.links.is_empty() {
591 body = apply_links_chunk(&body, &mut self.links);
592 }
593 let chunk = if let Some(front) = self.front_matter.take() {
594 format!("{front}\n{body}")
595 } else if self.emitted_any {
596 format!("\n\n{body}")
597 } else {
598 body
599 };
600 self.emitted_any = true;
601 chunk
602 }
603
604 pub fn finish(mut self) -> String {
607 let mut out = String::new();
608 if self.held.is_some() {
609 out.push_str(&self.push_holding(&[], &[], false));
610 }
611 if let Some(front) = self.front_matter {
612 out.push_str(&front);
613 } else if self.emitted_any {
614 out.push('\n');
615 }
616 out
617 }
618}
619
620fn strict_text(text: &str, strict: bool) -> String {
628 if !strict {
629 return text.to_string();
630 }
631 if let Some(rest) = text.strip_prefix("[ ] ") {
634 return format!("[ ] {}", strict_text(rest, strict));
635 }
636 text.replace("\\_", "_")
637 .replace(" ,", ",")
638 .replace(" .", ".")
639 .replace(" ;", ";")
640 .replace(" )", ")")
641 .replace("( ", "(")
642 .replace(" ]", "]")
643 .replace("[ ", "[")
644}
645
646pub fn longest_backtick_run(text: &str) -> usize {
649 let (mut best, mut run) = (0, 0);
650 for c in text.chars() {
651 if c == '`' {
652 run += 1;
653 best = best.max(run);
654 } else {
655 run = 0;
656 }
657 }
658 best
659}
660
661pub fn code_span(text: &str) -> String {
667 let ticks = "`".repeat(longest_backtick_run(text) + 1);
668 let pad = if text.starts_with('`') || text.ends_with('`') {
669 " "
670 } else {
671 ""
672 };
673 format!("{ticks}{pad}{text}{pad}{ticks}")
674}
675
676pub fn code_fence(language: &str, body: &str) -> String {
681 let fence = "`".repeat(longest_backtick_run(body).max(2) + 1);
682 format!("{fence}{language}\n{body}\n{fence}")
683}
684
685pub(crate) fn code_span_bounds(chars: &[char], i: usize) -> Option<(usize, usize, usize)> {
693 let n = chars[i..].iter().take_while(|&&c| c == '`').count();
694 if n == 0 {
695 return None;
696 }
697 let mut k = i + n;
698 while k < chars.len() {
699 if chars[k] != '`' {
700 k += 1;
701 continue;
702 }
703 let run = chars[k..].iter().take_while(|&&c| c == '`').count();
704 if run == n {
705 let (mut s, mut e) = (i + n, k);
706 if n > 1
707 && e - s >= 2
708 && chars[s] == ' '
709 && chars[e - 1] == ' '
710 && chars[s..e].iter().any(|&c| c != ' ')
711 {
712 s += 1;
713 e -= 1;
714 }
715 return Some((s, e, k + run));
716 }
717 k += run;
718 }
719 None
720}
721
722fn is_fence_line(line: &str) -> bool {
724 line.len() >= 3 && line.bytes().all(|b| b == b'`')
725}
726
727pub(crate) fn plain_inline(text: &str) -> String {
740 if !text.contains(['*', '~', '`', '[']) {
741 return text.to_string();
742 }
743 let chars: Vec<char> = text.chars().collect();
744 let mut out = String::with_capacity(text.len());
745 plain_inline_into(&chars, &mut out);
746 out
747}
748
749fn plain_inline_into(chars: &[char], out: &mut String) {
750 let find = |open: usize, pat: &[char]| -> Option<usize> {
753 chars
754 .get(open..)?
755 .windows(pat.len())
756 .enumerate()
757 .position(|(k, w)| w == pat && (open + k == 0 || chars[open + k - 1] != '\\'))
758 .map(|p| open + p)
759 };
760 let span = |i: usize, pat: &[char]| -> Option<usize> {
763 if !chars[i..].starts_with(pat) {
764 return None;
765 }
766 let open = i + pat.len();
767 let end = find(open, pat)?;
768 (end > open && !chars[open].is_whitespace() && !chars[end - 1].is_whitespace())
769 .then_some(end)
770 };
771 let mut i = 0;
772 while i < chars.len() {
773 if chars[i] == '\\' && chars.get(i + 1) == Some(&'*') {
775 out.push('*');
776 i += 2;
777 continue;
778 }
779 let mut matched = false;
780 for pat in [&['*', '*', '*'][..], &['*', '*'], &['~', '~'], &['*']] {
781 if let Some(end) = span(i, pat) {
782 plain_inline_into(&chars[i + pat.len()..end], out);
783 i = end + pat.len();
784 matched = true;
785 break;
786 }
787 }
788 if matched {
789 continue;
790 }
791 if chars[i] == '`' {
795 if let Some((s, e, after)) = code_span_bounds(chars, i).filter(|&(s, e, _)| {
796 e > s && !chars[s].is_whitespace() && !chars[e - 1].is_whitespace()
797 }) {
798 out.extend(&chars[s..e]);
799 i = after;
800 continue;
801 }
802 }
803 if chars[i] == '$' {
806 if let Some(end) = find(i + 1, &['$']) {
807 out.extend(&chars[i..=end]);
808 i = end + 1;
809 continue;
810 }
811 }
812 if chars[i] == '[' {
815 if let Some(close) = closing_bracket(chars, i) {
816 if chars.get(close + 1) == Some(&'(') {
817 if let Some(endp) = crate::doclang::link_dest_end(chars, close + 2) {
818 plain_inline_into(&chars[i + 1..close], out);
819 i = endp + 1;
820 continue;
821 }
822 }
823 }
824 }
825 let c = chars[i];
828 out.push(c);
829 i += 1;
830 if matches!(c, '*' | '~') {
831 while i < chars.len() && chars[i] == c {
832 out.push(c);
833 i += 1;
834 }
835 }
836 }
837}
838
839fn closing_bracket(chars: &[char], open: usize) -> Option<usize> {
842 let mut depth = 0usize;
843 for (k, c) in chars.iter().enumerate().skip(open + 1) {
844 match c {
845 '[' => depth += 1,
846 ']' if depth == 0 => return Some(k),
847 ']' => depth -= 1,
848 _ => {}
849 }
850 }
851 None
852}
853
854fn md_line_breaks(text: &str) -> String {
862 if !text.contains('\n') {
863 return text.to_string();
864 }
865 text.split("\n\n")
866 .map(|para| para.replace('\n', " \n"))
867 .collect::<Vec<_>>()
868 .join("\n\n")
869}
870
871pub(crate) fn strip_hard_breaks(text: &str) -> String {
876 if text.contains(" \n") {
877 text.replace(" \n", "\n")
878 } else {
879 text.to_string()
880 }
881}
882
883fn heading_line_breaks(text: &str) -> String {
887 text.replace('\n', " ")
888}
889
890fn render(nodes: &[Node], blocks: &mut Vec<String>, ctx: &mut Ctx) {
891 let mut i = 0;
892 while i < nodes.len() {
893 let before = blocks.len();
894 match &nodes[i] {
895 Node::PageBreak | Node::PageInfo { .. } => {
905 if ctx.page_break.is_some() && ctx.emitted_any {
906 ctx.pending_page_break = true;
907 }
908 i += 1;
909 }
910 Node::ListItem { .. } => {
911 let start = i;
912 i += 1;
913 loop {
914 match nodes.get(i) {
915 Some(Node::ListItem { .. } | Node::ListText { .. }) => i += 1,
916 Some(Node::Paragraph { text })
920 if text.is_empty()
921 && matches!(nodes.get(i + 1), Some(Node::ListItem { .. })) =>
922 {
923 i += 1
924 }
925 _ => break,
926 }
927 }
928 render_list_run(&nodes[start..i], blocks, ctx);
929 }
930 other => {
931 let before_item = blocks.len();
932 render_one(other, blocks, ctx);
933 if matches!(other, Node::Picture { .. })
939 && matches!(ctx.images, ImageMode::Placeholder)
940 && blocks.len() > before_item
941 && matches!(
942 nodes.get(i + 1),
943 Some(Node::Code { language: Some(l), .. }) if l == "mermaid"
944 )
945 {
946 blocks.pop();
947 }
948 i += 1;
949 }
950 }
951 if blocks.len() > before {
952 let kept: Vec<String> = blocks.drain(before..).filter(|b| !b.is_empty()).collect();
962 blocks.extend(kept);
963 if ctx.pending_page_break {
964 if let Some(placeholder) = &ctx.page_break {
969 blocks.insert(before, placeholder.clone());
970 }
971 ctx.pending_page_break = false;
972 }
973 ctx.emitted_any = true;
974 }
975 }
976}
977
978fn render_list_run(items: &[Node], blocks: &mut Vec<String>, ctx: &Ctx) {
984 let mut lines: Vec<String> = Vec::new();
985 let mut any_top = false;
988
989 for item in items {
990 if let Node::ListText { level, text, glue } = item {
991 let text = ctx.text(text);
992 match lines.last_mut() {
993 Some(last) if *glue => last.push_str(&text),
994 _ => lines.push(format!("{}{text}", " ".repeat(*level as usize))),
995 }
996 continue;
997 }
998 let Node::ListItem {
999 ordered,
1000 number,
1001 first_in_list,
1002 text,
1003 level,
1004 marker: orig_marker,
1005 location: _,
1006 dclx: _,
1007 href: _,
1008 layer,
1009 } = item
1010 else {
1011 continue;
1012 };
1013 if !ctx.layers.contains(layer.or(ctx.current_layer)) {
1017 continue;
1018 }
1019 let level = *level as usize;
1020
1021 if level == 0 {
1030 if any_top && *first_in_list {
1031 lines.push(String::new());
1032 }
1033 any_top = true;
1034 }
1035
1036 let indent = " ".repeat(level);
1037 lines.push(format!(
1038 "{indent}{} {}",
1039 list_item_marker(orig_marker.as_deref().unwrap_or(""), *ordered, *number),
1040 list_item_text(text, ctx)
1041 ));
1042 }
1043
1044 if !lines.is_empty() {
1047 blocks.push(lines.join("\n"));
1048 }
1049}
1050
1051fn list_item_marker(marker: &str, ordered: bool, number: u64) -> String {
1060 let already_valid = matches!(marker, "-" | "*" | "+")
1061 || marker
1062 .strip_suffix('.')
1063 .is_some_and(|d| !d.is_empty() && d.chars().all(char::is_numeric));
1064 if already_valid {
1065 return marker.to_string();
1066 }
1067 let outer = if ordered && marker.is_empty() {
1068 format!("{number}.")
1069 } else {
1070 "-".to_string()
1071 };
1072 if marker.chars().any(char::is_alphanumeric) {
1073 format!("{outer} {marker}")
1074 } else {
1075 outer
1076 }
1077}
1078
1079fn list_item_text(text: &str, ctx: &Ctx) -> String {
1087 let escaped = ctx.text(text);
1088 if ctx.plain {
1089 return escaped;
1090 }
1091 if let Some((own, tail)) = escaped.split_once('\n') {
1092 if is_folded_child_tail(tail) {
1093 return format!("{}\n{tail}", md_line_breaks(own));
1094 }
1095 }
1096 md_line_breaks(&escaped)
1097}
1098
1099fn is_folded_child_tail(tail: &str) -> bool {
1106 const MARKER: &str = "<!-- image -->";
1107 let mut lines = tail.split('\n').peekable();
1108 let mut any = false;
1109 while let Some(line) = lines.next() {
1110 let line = line.trim_start();
1111 if line == MARKER {
1112 any = true;
1113 } else if is_fence_line(line) {
1114 loop {
1117 match lines.next() {
1118 Some(l) if l.trim_start() == line => break,
1119 Some(_) => {}
1120 None => return false,
1121 }
1122 }
1123 any = true;
1124 } else if lines.next().map(str::trim_start) == Some(MARKER) {
1125 any = true; } else {
1127 return false;
1128 }
1129 }
1130 any
1131}
1132
1133fn render_one(node: &Node, blocks: &mut Vec<String>, ctx: &mut Ctx) {
1138 match node {
1139 Node::Group { .. }
1141 | Node::Furniture { .. }
1142 | Node::PageFurniture { .. }
1143 | Node::FurnitureText { .. }
1144 | Node::CommentSection { .. }
1145 | Node::PictureChildren(_)
1146 | Node::Commented { .. }
1147 | Node::ScriptText { .. }
1148 | Node::Located { .. }
1149 | Node::Prov { .. }
1150 | Node::Track { .. }
1151 | Node::PageBreak
1152 | Node::PageInfo { .. }
1153 | Node::DoclangOnly(_) => {}
1154 _ if !ctx.layer_visible() => return,
1155 _ => {}
1156 }
1157 render_item(node, blocks, ctx)
1158}
1159
1160fn render_item(node: &Node, blocks: &mut Vec<String>, ctx: &mut Ctx) {
1161 match node {
1162 Node::Heading { level, text } => {
1163 if ctx.plain {
1164 blocks.push(ctx.text(text));
1167 return;
1168 }
1169 let text = heading_line_breaks(&ctx.text(text));
1170 if ctx.in_table_cell {
1171 blocks.push(text);
1173 } else {
1174 let hashes = "#".repeat((*level).clamp(1, 6) as usize);
1175 blocks.push(format!("{hashes} {text}"));
1176 }
1177 }
1178 Node::Paragraph { text } if text.is_empty() => {}
1181 Node::Paragraph { text } => blocks.push(ctx.breaks(&ctx.text(text))),
1182 Node::LabeledText { .. } => render_item(&node.labeled_as_paragraph(), blocks, ctx),
1183 Node::Caption { text, .. } if text.is_empty() => {}
1186 Node::Caption { text, href } => {
1187 let body = ctx.breaks(&ctx.text(text));
1188 blocks.push(match href {
1189 Some(url) if !ctx.plain => crate::hyperlink::markdown_link(&body, url),
1190 _ => body,
1191 });
1192 }
1193 Node::CheckboxItem { checked, text } => {
1194 let mark = if *checked { "- [x]" } else { "- [ ]" };
1197 let sep = if text.starts_with(' ') { "" } else { " " };
1198 blocks.push(ctx.breaks(&ctx.text(&format!("{mark}{sep}{text}"))));
1199 }
1200 Node::Code {
1201 language,
1202 text,
1203 pretty,
1204 ..
1205 } => {
1206 let lang = match language {
1211 Some(l) if ctx.strict || l == "mermaid" => l.as_str(),
1212 _ => "",
1213 };
1214 let body = match pretty {
1217 Some(p) if ctx.strict => p.as_str(),
1218 _ => text.as_str(),
1219 };
1220 blocks.push(if ctx.plain {
1223 body.to_string()
1224 } else {
1225 code_fence(lang, body)
1226 });
1227 }
1228 Node::Formula { latex, .. } => blocks.push(format!("$${latex}$$")),
1231 Node::Table(table) => {
1232 if let Some(cap) = &table.caption {
1235 if !cap.is_empty() {
1236 blocks.push(ctx.text(cap));
1237 }
1238 }
1239 let rendered = if ctx.plain {
1240 render_table(&ctx.plain_table(table), ctx.compact_tables)
1241 } else {
1242 render_table(table, ctx.compact_tables)
1243 };
1244 if !rendered.is_empty() {
1245 blocks.push(rendered);
1246 }
1247 for note in table.footnotes.iter().filter(|n| !n.is_empty()) {
1250 blocks.push(ctx.text(note));
1251 }
1252 }
1253 Node::Picture {
1262 caption,
1263 image,
1264 description,
1265 ..
1266 } => {
1267 if let Some(cap) = caption {
1268 if !cap.is_empty() {
1269 blocks.push(ctx.caption(cap));
1270 }
1271 }
1272 let alt = description
1273 .as_ref()
1274 .filter(|d| d.is_alt_text() && ctx.alt_text && !d.text.is_empty());
1275 let marker = picture_marker(image.as_ref(), ctx, alt.map(|d| d.text.as_str()));
1276 let native = description
1280 .as_ref()
1281 .filter(|d| d.is_docling_meta() && !d.text.is_empty());
1282 if let Some(desc) = description.as_ref().filter(|d| !d.is_docling_meta()) {
1283 let shown = !desc.is_alt_text() || (ctx.alt_text && !marker.starts_with("!["));
1284 if !desc.text.is_empty() && shown {
1285 blocks.push(ctx.breaks(&ctx.text(&desc.text)));
1286 }
1287 }
1288 blocks.push(marker);
1289 if let Some(desc) = native {
1290 blocks.push(ctx.breaks(&ctx.text(&desc.text)));
1291 }
1292 }
1293 Node::Chart {
1297 kind,
1298 table,
1299 caption,
1300 ..
1301 } => {
1302 if let Some(cap) = caption {
1303 if !cap.is_empty() {
1304 blocks.push(ctx.caption(cap));
1305 }
1306 }
1307 blocks.push(picture_marker(None, ctx, None));
1308 blocks.push(humanize_label(kind));
1309 let rendered = if ctx.plain {
1310 render_table(&ctx.plain_table(table), false)
1311 } else {
1312 render_table(table, false)
1313 };
1314 if !rendered.is_empty() {
1315 blocks.push(rendered);
1316 }
1317 }
1318 Node::DoclangOnly(_) => {}
1320 Node::Group {
1325 layer: Some(layer),
1326 children,
1327 ..
1328 } => {
1329 let outer = ctx.current_layer.replace(*layer);
1330 render(children, blocks, ctx);
1331 ctx.current_layer = outer;
1332 }
1333 Node::Group { children, .. } => render(children, blocks, ctx),
1334 Node::FieldRegion { items } => {
1335 for item in items {
1340 for part in [&item.marker, &item.key, &item.value].into_iter().flatten() {
1341 blocks.push(ctx.breaks(&ctx.text(part)));
1342 }
1343 }
1344 }
1345 Node::KeyValueGraph { .. } => blocks.push(MISSING_KEY_VALUE_ITEM.to_string()),
1348 Node::InlineGroup { md_text, .. } => blocks.push(ctx.breaks(&ctx.text(md_text))),
1351 Node::TextDump(text) => {
1353 if !text.is_empty() {
1354 blocks.push(text.clone());
1355 }
1356 }
1357 Node::Furniture { layer, inner } => {
1361 let outer = ctx.current_layer.replace(*layer);
1362 render_one(inner, blocks, ctx);
1363 ctx.current_layer = outer;
1364 }
1365 Node::PageFurniture { text, .. } | Node::FurnitureText { text, .. } => {
1369 if ctx.layers.furniture && !text.is_empty() {
1370 blocks.push(ctx.breaks(&ctx.text(text)));
1371 }
1372 }
1373 Node::PictureChildren(children) => {
1378 if ctx.traverse_pictures {
1379 render(children, blocks, ctx);
1380 }
1381 }
1382 Node::CommentSection { text, .. } => {
1387 if ctx.layers.notes && !text.is_empty() {
1388 blocks.push(ctx.breaks(&ctx.text(text)));
1389 }
1390 }
1391 Node::Commented { inner, .. } => render_one(inner, blocks, ctx),
1392 Node::ScriptText { inner, .. } => render_one(inner, blocks, ctx),
1395 Node::Located { inner, .. } | Node::Prov { inner, .. } | Node::Track { inner, .. } => {
1397 render_one(inner, blocks, ctx)
1398 }
1399 Node::PageBreak => {}
1401 Node::PageInfo { .. } => {}
1403 Node::Anchor { .. } => {}
1405 Node::ListItem { .. } => render_list_run(std::slice::from_ref(node), blocks, ctx),
1410 Node::ListText { text, .. } if text.is_empty() => {}
1412 Node::ListText { text, .. } => blocks.push(ctx.breaks(&ctx.text(text))),
1413 }
1414}
1415
1416fn humanize_label(label: &str) -> String {
1421 let text = label.replace('_', " ");
1422 let mut chars = text.chars();
1423 match chars.next() {
1424 Some(f) => f.to_uppercase().collect::<String>() + chars.as_str(),
1425 None => text,
1426 }
1427}
1428
1429fn image_alt(alt: Option<&str>) -> String {
1432 match alt {
1433 Some(text) => text
1434 .split_whitespace()
1435 .collect::<Vec<_>>()
1436 .join(" ")
1437 .replace('\\', "\\\\")
1438 .replace(']', "\\]"),
1439 None => "Image".to_string(),
1440 }
1441}
1442
1443fn picture_marker(image: Option<&crate::PictureImage>, ctx: &mut Ctx, alt: Option<&str>) -> String {
1444 match (ctx.images, image) {
1445 (ImageMode::Embedded, Some(img)) => {
1447 format!(
1448 "",
1449 image_alt(alt),
1450 crate::pixel_digest::docling_data_uri(img).1
1451 )
1452 }
1453 (ImageMode::Referenced, Some(img)) => {
1454 let path = format!(
1455 "{}/image_{:06}.{}",
1456 ctx.artifacts_dir,
1457 ctx.pic_index,
1458 ext_for(&img.mimetype)
1459 );
1460 ctx.pic_index += 1;
1461 ctx.artifacts.push((path.clone(), img.data.clone()));
1462 format!("", image_alt(alt), escape_uri_path(&path))
1463 }
1464 _ => ctx.image_placeholder.clone(),
1466 }
1467}
1468
1469pub(crate) fn escape_uri_path(value: &str) -> String {
1482 const KEEP: &str = "/%:@+,;=~$!&'*";
1483 let s = value.replace('\\', "/");
1484 if let Some(rest) = s.strip_prefix("//") {
1485 let rest = rest.trim_start_matches('/');
1487 let (host, tail) = rest.split_once('/').unwrap_or((rest, ""));
1488 return format!("file://{host}{}", percent_quote(&format!("/{tail}"), KEEP));
1489 }
1490 let bytes = s.as_bytes();
1491 if bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' && bytes[2] == b'/' {
1492 return format!("file:///{}", percent_quote(&s, KEEP));
1494 }
1495 if let Some((scheme, rest)) = s.split_once(':') {
1499 let valid_scheme = scheme.len() > 1
1500 && scheme.as_bytes()[0].is_ascii_alphabetic()
1501 && scheme
1502 .bytes()
1503 .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.'));
1504 if valid_scheme {
1505 let (authority, rest) = match rest.strip_prefix("//") {
1506 Some(r) => {
1507 let end = r.find(['/', '?', '#']).unwrap_or(r.len());
1508 (Some(&r[..end]), &r[end..])
1509 }
1510 None => (None, rest),
1511 };
1512 let (before_frag, fragment) = rest.split_once('#').unwrap_or((rest, ""));
1513 let (path, query) = before_frag.split_once('?').unwrap_or((before_frag, ""));
1514 let mut out = format!("{scheme}:");
1515 if let Some(a) = authority {
1516 out.push_str("//");
1517 out.push_str(a);
1518 }
1519 out.push_str(&percent_quote(path, KEEP));
1520 if !query.is_empty() {
1521 out.push('?');
1522 out.push_str(&percent_quote(query, KEEP));
1523 }
1524 if !fragment.is_empty() {
1525 out.push('#');
1526 out.push_str(&percent_quote(fragment, KEEP));
1527 }
1528 return out;
1529 }
1530 }
1531 percent_quote(&s, KEEP)
1533}
1534
1535fn percent_quote(s: &str, safe: &str) -> String {
1538 let mut out = String::with_capacity(s.len());
1539 for &b in s.as_bytes() {
1540 let keep = b.is_ascii_alphanumeric()
1541 || matches!(b, b'_' | b'.' | b'-' | b'~')
1542 || (b.is_ascii() && safe.contains(b as char));
1543 if keep {
1544 out.push(b as char);
1545 } else {
1546 out.push_str(&format!("%{b:02X}"));
1547 }
1548 }
1549 out
1550}
1551
1552pub(crate) fn ext_for(mimetype: &str) -> &str {
1553 match mimetype {
1554 "image/jpeg" => "jpg",
1555 "image/gif" => "gif",
1556 "image/webp" => "webp",
1557 "image/bmp" => "bmp",
1558 "image/tiff" => "tif",
1559 "image/svg+xml" => "svg",
1560 _ => "png",
1561 }
1562}
1563
1564fn is_number_cell(t: &str) -> bool {
1583 t.parse::<f64>().is_ok() || is_thousands_number(t)
1584}
1585
1586fn is_thousands_number(t: &str) -> bool {
1592 let b = t.as_bytes();
1593 let mut i = 0;
1594 let start = i;
1595 if i < b.len() && (b[i] == b'+' || b[i] == b'-') {
1596 i += 1;
1597 }
1598 let d0 = i;
1600 while i < b.len() && b[i].is_ascii_digit() && i - d0 < 3 {
1601 i += 1;
1602 }
1603 let has_int = i > d0;
1604 if has_int {
1605 while i + 3 < b.len() + 1
1607 && b.get(i) == Some(&b',')
1608 && b.get(i + 1).is_some_and(u8::is_ascii_digit)
1609 && b.get(i + 2).is_some_and(u8::is_ascii_digit)
1610 && b.get(i + 3).is_some_and(u8::is_ascii_digit)
1611 {
1612 i += 4;
1613 }
1614 } else {
1615 i = start;
1617 }
1618 if i < b.len() && b[i] == b'.' {
1620 i += 1;
1621 let f0 = i;
1622 while i < b.len() && b[i].is_ascii_digit() {
1623 i += 1;
1624 }
1625 if !has_int && i == f0 {
1626 return false; }
1628 } else if !has_int {
1629 return false; }
1631 i == b.len()
1632}
1633
1634fn flatten_header_rows(header_rows: &[Vec<String>], num_cols: usize) -> Vec<String> {
1643 (0..num_cols)
1644 .map(|c| {
1645 let mut parts: Vec<&str> = Vec::new();
1646 for row in header_rows {
1647 let text = row.get(c).map(String::as_str).unwrap_or("");
1648 if !text.is_empty() && parts.last() != Some(&text) {
1649 parts.push(text);
1650 }
1651 }
1652 parts.join(" - ")
1653 })
1654 .collect()
1655}
1656
1657pub(crate) fn render_table(table: &Table, compact: bool) -> String {
1658 if table.rows.is_empty() {
1659 return String::new();
1660 }
1661 let num_cols = table.rows.iter().map(Vec::len).max().unwrap_or(0);
1662 if num_cols == 0 {
1663 return String::new();
1664 }
1665
1666 let num_headers = table.header_row_count().min(table.rows.len());
1671 let escaped = |r: usize| -> Vec<String> {
1672 (0..num_cols)
1673 .map(|c| escape_cell(table.rows[r].get(c).map(String::as_str).unwrap_or("")))
1674 .collect()
1675 };
1676 let header_rows: Vec<Vec<String>> = (0..num_headers).map(escaped).collect();
1677 let header = flatten_header_rows(&header_rows, num_cols);
1678 let body: Vec<Vec<String>> = (num_headers..table.rows.len())
1679 .map(|r| {
1680 escaped(r)
1681 .into_iter()
1682 .map(|c| c.trim().to_string())
1683 .collect()
1684 })
1685 .collect();
1686
1687 if compact {
1688 let render_row = |row: &[String]| -> String { format!("| {} |", row.join(" | ")) };
1690 let mut lines = Vec::with_capacity(body.len() + 2);
1691 lines.push(render_row(&header));
1692 let sep: Vec<&str> = (0..num_cols).map(|_| "-").collect();
1693 lines.push(format!("| {} |", sep.join(" | ")));
1694 for row in &body {
1695 lines.push(render_row(row));
1696 }
1697 return lines.join("\n");
1698 }
1699
1700 let dw = |s: &str| s.chars().count();
1702
1703 let right: Vec<bool> = (0..num_cols)
1708 .map(|c| {
1709 let mut any = false;
1710 for row in &body {
1711 let t = row[c].trim();
1712 if t.is_empty() {
1713 continue;
1714 }
1715 if !is_number_cell(t) {
1716 return false;
1717 }
1718 any = true;
1719 }
1720 any
1721 })
1722 .collect();
1723
1724 let width: Vec<usize> = (0..num_cols)
1726 .map(|c| {
1727 let mut w = dw(&header[c]) + 2;
1728 for row in &body {
1729 w = w.max(dw(&row[c]));
1730 }
1731 w
1732 })
1733 .collect();
1734
1735 let fmt_cell = |s: &str, c: usize| -> String {
1736 let pad = " ".repeat(width[c].saturating_sub(dw(s)));
1737 let body = if right[c] {
1738 format!("{pad}{s}")
1739 } else {
1740 format!("{s}{pad}")
1741 };
1742 format!(" {body} ")
1743 };
1744 let render_row = |row: &[String]| -> String {
1745 let cells: Vec<String> = (0..num_cols).map(|c| fmt_cell(&row[c], c)).collect();
1746 format!("|{}|", cells.join("|"))
1747 };
1748
1749 let mut lines = Vec::with_capacity(body.len() + 2);
1750 lines.push(render_row(&header));
1751 let sep: Vec<String> = (0..num_cols).map(|c| "-".repeat(width[c] + 2)).collect();
1752 lines.push(format!("|{}|", sep.join("|")));
1753 for row in &body {
1754 lines.push(render_row(row));
1755 }
1756 lines.join("\n")
1757}
1758
1759fn escape_cell(s: &str) -> String {
1762 s.replace('\n', " ").replace('|', "|")
1763}
1764
1765#[cfg(test)]
1766mod tests {
1767 use super::*;
1768 use crate::{PictureImage, TableCell, TableStructure};
1769
1770 #[test]
1776 fn list_boundaries_come_from_the_backend_not_the_numbering() {
1777 let item = |ordered: bool, number: u64, first_in_list: bool, text: &str| Node::ListItem {
1778 ordered,
1779 number,
1780 first_in_list,
1781 text: text.into(),
1782 level: 0,
1783 marker: None,
1784 location: None,
1785 dclx: None,
1786 href: None,
1787 layer: None,
1788 };
1789 let md = |items: Vec<Node>| {
1790 let mut doc = DoclingDocument::new("t");
1791 for n in items {
1792 doc.push(n);
1793 }
1794 doc.export_to_markdown()
1795 };
1796 assert_eq!(
1798 md(vec![
1799 item(true, 1, true, "one"),
1800 item(true, 5, false, "five")
1801 ]),
1802 "1. one\n5. five\n"
1803 );
1804 assert_eq!(
1806 md(vec![
1807 item(false, 0, true, "bullet"),
1808 item(true, 1, false, "one"),
1809 item(false, 0, false, "bullet two"),
1810 ]),
1811 "- bullet\n1. one\n- bullet two\n"
1812 );
1813 assert_eq!(
1815 md(vec![
1816 item(true, 1, true, "a"),
1817 item(true, 2, false, "b"),
1818 item(true, 3, true, "new list, continuing count"),
1819 ]),
1820 "1. a\n2. b\n\n3. new list, continuing count\n"
1821 );
1822 }
1823
1824 #[test]
1825 fn renders_headings_paragraphs_and_lists() {
1826 let mut doc = DoclingDocument::new("demo");
1827 doc.add_heading(1, "Title");
1828 doc.add_paragraph("Hello world.");
1829 doc.push(Node::ListItem {
1830 ordered: false,
1831 number: 1,
1832 first_in_list: true,
1833 text: "first".into(),
1834 level: 0,
1835 marker: None,
1836 location: None,
1837 dclx: None,
1838 href: None,
1839 layer: None,
1840 });
1841 doc.push(Node::ListItem {
1842 ordered: false,
1843 number: 2,
1844 first_in_list: false,
1845 text: "second".into(),
1846 level: 0,
1847 marker: None,
1848 location: None,
1849 dclx: None,
1850 href: None,
1851 layer: None,
1852 });
1853 let md = doc.export_to_markdown();
1854 assert_eq!(md, "# Title\n\nHello world.\n\n- first\n- second\n");
1855 }
1856
1857 #[test]
1861 fn single_newlines_become_gfm_hard_line_breaks() {
1862 let mut doc = DoclingDocument::new("t");
1863 doc.push(Node::Heading {
1864 level: 1,
1865 text: "Hello\nWorld".into(),
1866 });
1867 doc.push(Node::Paragraph {
1868 text: "line one\nline two\n\npara two".into(),
1869 });
1870 doc.push(Node::ListItem {
1871 ordered: false,
1872 number: 1,
1873 first_in_list: true,
1874 text: "item\ncontinued".into(),
1875 level: 0,
1876 marker: None,
1877 location: None,
1878 dclx: None,
1879 href: None,
1880 layer: None,
1881 });
1882 doc.push(Node::TextDump("A1 B1 \n\n\nC1".into()));
1883 assert_eq!(
1884 doc.export_to_markdown(),
1885 "# Hello World\n\nline one \nline two\n\npara two\n\n- item \ncontinued\n\nA1 B1 \n\n\nC1\n"
1886 );
1887 }
1888
1889 #[test]
1892 fn table_cell_mode_and_field_regions() {
1893 let mut doc = DoclingDocument::new("t");
1894 doc.push(Node::Heading {
1895 level: 2,
1896 text: "A text".into(),
1897 });
1898 doc.push(Node::Paragraph {
1899 text: "body".into(),
1900 });
1901 assert_eq!(to_markdown_table_cell(&doc, false), "A text\n\nbody");
1902 assert_eq!(doc.export_to_markdown(), "## A text\n\nbody\n");
1903
1904 let mut doc = DoclingDocument::new("f");
1905 doc.push(Node::FieldRegion {
1906 items: vec![crate::FieldItem {
1907 marker: None,
1908 key: Some("Name:".into()),
1909 value: Some("John Doe".into()),
1910 value_kind: None,
1911 }],
1912 });
1913 assert_eq!(doc.export_to_markdown(), "Name:\n\nJohn Doe\n");
1914 }
1915
1916 #[test]
1917 fn strict_renders_recovered_links_legacy_does_not() {
1918 let mut doc = DoclingDocument::new("cv");
1919 doc.add_paragraph("Find me on LinkedIn or GitHub.");
1920 doc.links = vec![
1921 ("LinkedIn".into(), "https://www.linkedin.com/in/x/".into()),
1922 ("GitHub".into(), "https://github.com/x/".into()),
1923 ];
1924 assert_eq!(doc.export_to_markdown(), "Find me on LinkedIn or GitHub.\n");
1926 assert_eq!(
1928 doc.export_to_markdown_with(true),
1929 "Find me on [LinkedIn](https://www.linkedin.com/in/x/) or [GitHub](https://github.com/x/).\n"
1930 );
1931 }
1932
1933 #[test]
1934 fn strict_links_match_escaped_anchor_and_consume_in_order() {
1935 let mut doc = DoclingDocument::new("d");
1936 doc.add_paragraph("AI & ML here, and issues here, then issues there.");
1940 doc.links = vec![
1941 ("AI & ML".into(), "https://a/".into()),
1942 ("issues".into(), "https://first/".into()),
1943 ("issues".into(), "https://second/".into()),
1944 ];
1945 assert_eq!(
1946 doc.export_to_markdown_with(true),
1947 "[AI & ML](https://a/) here, and [issues](https://first/) here, then [issues](https://second/) there.\n"
1948 );
1949 }
1950
1951 #[test]
1955 fn referenced_image_destinations_are_escaped() {
1956 let cases = [
1957 (
1958 "doc_artifacts/image_000001_ab12.png",
1959 "doc_artifacts/image_000001_ab12.png",
1960 ),
1961 (
1962 "My Report_artifacts/img.png",
1963 "My%20Report_artifacts/img.png",
1964 ),
1965 ("artifacts/img (1).png", "artifacts/img%20%281%29.png"),
1966 ("100%_scale/a#b?c.png", "100%_scale/a%23b%3Fc.png"),
1967 ("/home/a b/img.png", "/home/a%20b/img.png"),
1968 (
1969 "My Report_artifacts\\img.png",
1970 "My%20Report_artifacts/img.png",
1971 ),
1972 (
1973 "C:/Users/me/My Docs/img.png",
1974 "file:///C:/Users/me/My%20Docs/img.png",
1975 ),
1976 ("C:\\Users\\me\\img.png", "file:///C:/Users/me/img.png"),
1977 (
1978 "//server/share/My Docs/img.png",
1979 "file://server/share/My%20Docs/img.png",
1980 ),
1981 ("\\\\server\\share\\img.png", "file://server/share/img.png"),
1982 ("file:///home/a b/img.png", "file:///home/a%20b/img.png"),
1983 (
1984 "s3://bucket/My Report_artifacts/img.png",
1985 "s3://bucket/My%20Report_artifacts/img.png",
1986 ),
1987 (
1988 "https://example.com:8080/a b.png?w=1&h=2#frag",
1989 "https://example.com:8080/a%20b.png?w=1&h=2#frag",
1990 ),
1991 (
1992 "https://example.com/img (1).png",
1993 "https://example.com/img%20%281%29.png",
1994 ),
1995 ("caf\u{e9}/im\u{e4}ge.png", "caf%C3%A9/im%C3%A4ge.png"),
1996 ];
1997 for (input, expected) in cases {
1998 assert_eq!(escape_uri_path(input), expected, "input {input:?}");
1999 assert_eq!(
2000 escape_uri_path(expected),
2001 expected,
2002 "idempotent {expected:?}"
2003 );
2004 }
2005 let mut doc = DoclingDocument::new("t");
2007 doc.push(Node::Picture {
2008 caption: None,
2009 caption_href: None,
2010 image: Some(PictureImage {
2011 dpi: PictureImage::DEFAULT_DPI,
2012 mimetype: "image/png".into(),
2013 width: 1,
2014 height: 1,
2015 data: b"x".to_vec(),
2016 display_size: None,
2017 }),
2018 classification: None,
2019 description: None,
2020 caption_parent: Default::default(),
2021 caption_location: None,
2022 });
2023 let (md, files) = doc
2024 .export_to_markdown_with_images(ImageMode::Referenced, "My Report (final)_artifacts");
2025 assert!(
2026 md.contains(""),
2027 "got:\n{md}"
2028 );
2029 assert_eq!(files[0].0, "My Report (final)_artifacts/image_000000.png");
2031 }
2032
2033 #[test]
2037 fn folded_list_item_pictures_keep_plain_newlines() {
2038 let doc = DoclingDocument::new("t");
2039 let ctx = Ctx::new(&doc, false, &MarkdownExportOptions::default());
2040 assert_eq!(
2041 list_item_text("Step\n<!-- image -->", &ctx),
2042 "Step\n<!-- image -->"
2043 );
2044 assert_eq!(
2045 list_item_text("Step\nAlt text\n<!-- image -->\n<!-- image -->", &ctx),
2046 "Step\nAlt text\n<!-- image -->\n<!-- image -->"
2047 );
2048 assert_eq!(
2049 list_item_text("line one\nline two", &ctx),
2050 "line one \nline two"
2051 );
2052 }
2053
2054 #[test]
2057 fn stacked_header_rows_flatten_into_one() {
2058 let mut t = Table {
2059 rows: vec![
2060 vec!["".into(), "% of Total".into(), "% of Total".into()],
2061 vec!["class".into(), "Train".into(), "Test".into()],
2062 vec!["Caption".into(), "2.04".into(), "1.77".into()],
2063 ],
2064 ..Default::default()
2065 };
2066 t.structure = Some(TableStructure {
2067 header_row: vec![true, true, false],
2068 col_continuation: vec![
2069 vec![false, false, true],
2070 vec![false, false, false],
2071 vec![false, false, false],
2072 ],
2073 ..Default::default()
2074 });
2075 assert_eq!(t.header_row_count(), 2);
2076 assert_eq!(
2077 render_table(&t, true),
2078 "| class | % of Total - Train | % of Total - Test |\n| - | - | - |\n| Caption | 2.04 | 1.77 |"
2079 );
2080 assert_eq!(
2082 render_table(&t, false),
2083 "| class | % of Total - Train | % of Total - Test |\n\
2084 |---------|----------------------|---------------------|\n\
2085 | Caption | 2.04 | 1.77 |"
2086 );
2087 }
2088
2089 #[test]
2092 fn vertically_spanning_header_does_not_extend_the_block() {
2093 let mut t = Table {
2094 rows: vec![
2095 vec!["Name".into(), "Value".into()],
2096 vec!["Name".into(), "1".into()],
2097 vec!["x".into(), "2".into()],
2098 ],
2099 ..Default::default()
2100 };
2101 t.structure = Some(TableStructure {
2102 col_header: vec![vec![true, true], vec![true, false], vec![false, false]],
2103 row_continuation: vec![vec![false, false], vec![true, false], vec![false, false]],
2104 ..Default::default()
2105 });
2106 assert_eq!(t.header_row_count(), 1);
2107 assert_eq!(
2108 render_table(&t, true),
2109 "| Name | Value |\n| - | - |\n| Name | 1 |\n| x | 2 |"
2110 );
2111 }
2112
2113 #[test]
2116 fn header_flags_not_on_row_zero_keep_all_rows_in_the_body() {
2117 let mut t = Table {
2118 rows: vec![
2119 vec!["1".into(), "2".into()],
2120 vec!["a".into(), "b".into()],
2121 vec!["333".into(), "4".into()],
2122 ],
2123 ..Default::default()
2124 };
2125 t.structure = Some(TableStructure {
2126 header_row: vec![false, true, false],
2127 ..Default::default()
2128 });
2129 assert_eq!(t.header_row_count(), 0);
2130 assert_eq!(
2131 render_table(&t, false),
2132 "| | |\n|-----|----|\n| 1 | 2 |\n| a | b |\n| 333 | 4 |"
2133 );
2134 }
2135
2136 #[test]
2141 fn pivot_row_headers_do_not_extend_the_header() {
2142 let mut t = Table {
2143 rows: vec![
2144 vec!["Year".into(), "Month".into()],
2145 vec!["2025".into(), "January".into()],
2146 vec!["2025".into(), "February".into()],
2147 ],
2148 ..Default::default()
2149 };
2150 t.structure = Some(TableStructure {
2151 col_header: vec![vec![true, true], vec![false, false], vec![false, false]],
2152 row_header: vec![vec![false, false], vec![true, false], vec![true, false]],
2153 row_continuation: vec![vec![false, false], vec![false, false], vec![true, false]],
2154 ..Default::default()
2155 });
2156 assert_eq!(t.header_row_count(), 1);
2157 assert_eq!(
2158 render_table(&t, true),
2159 "| Year | Month |\n| - | - |\n| 2025 | January |\n| 2025 | February |"
2160 );
2161 }
2162
2163 #[test]
2166 fn unflagged_cells_keep_row_zero_as_header() {
2167 let mut t = Table {
2168 rows: vec![vec!["h".into()], vec!["d".into()]],
2169 ..Default::default()
2170 };
2171 t.cells = Some(
2172 [(0usize, "h"), (1, "d")]
2173 .into_iter()
2174 .map(|(r, text)| TableCell {
2175 text: text.into(),
2176 bbox: None,
2177 start_row: r,
2178 start_col: 0,
2179 row_span: 1,
2180 col_span: 1,
2181 column_header: false,
2182 row_header: false,
2183 row_section: false,
2184 })
2185 .collect(),
2186 );
2187 assert_eq!(t.header_row_count(), 1);
2188 assert_eq!(render_table(&t, true), "| h |\n| - |\n| d |");
2189 }
2190
2191 fn flagged_table(rows: &[&[&str]], ched: impl Fn(usize, usize) -> bool) -> Table {
2193 let mut cells = Vec::new();
2194 for (r, row) in rows.iter().enumerate() {
2195 for (c, text) in row.iter().enumerate() {
2196 cells.push(TableCell {
2197 text: (*text).into(),
2198 bbox: None,
2199 start_row: r,
2200 start_col: c,
2201 row_span: 1,
2202 col_span: 1,
2203 column_header: ched(r, c),
2204 row_header: false,
2205 row_section: false,
2206 });
2207 }
2208 }
2209 Table {
2210 rows: rows
2211 .iter()
2212 .map(|row| row.iter().map(|t| (*t).to_string()).collect())
2213 .collect(),
2214 cells: Some(cells),
2215 ..Default::default()
2216 }
2217 }
2218
2219 #[test]
2223 fn rows_with_body_text_stop_the_header_block() {
2224 let t = flagged_table(
2226 &[
2227 &["Item", "Q1", "Q2"],
2228 &["Sales", "10", "20"],
2229 &["Costs", "5", "7"],
2230 ],
2231 |r, c| r == 0 || c == 0,
2232 );
2233 assert_eq!(t.header_row_count(), 1);
2234 assert_eq!(
2235 render_table(&t, false),
2236 "| Item | Q1 | Q2 |\n\
2237 |--------|------|------|\n\
2238 | Sales | 10 | 20 |\n\
2239 | Costs | 5 | 7 |"
2240 );
2241 let t = flagged_table(&[&["Sales", "10", "20"], &["Costs", "5", "7"]], |_, c| {
2244 c == 0
2245 });
2246 assert_eq!(t.header_row_count(), 0);
2247 assert_eq!(
2248 render_table(&t, false),
2249 "| | | |\n\
2250 |-------|----|----|\n\
2251 | Sales | 10 | 20 |\n\
2252 | Costs | 5 | 7 |"
2253 );
2254 let t = flagged_table(
2255 &[&["a", "1", "2"], &["b", "3", "4"], &["c", "5", "6"]],
2256 |r, c| c == 0 && r < 2,
2257 );
2258 assert_eq!(t.header_row_count(), 0);
2259 let t = flagged_table(&[&["a", "1", "2"], &["b", "3", "4"]], |r, c| {
2262 r == 0 && c == 0
2263 });
2264 assert_eq!(t.header_row_count(), 1);
2265 assert_eq!(
2266 render_table(&t, false),
2267 "| a | 1 | 2 |\n\
2268 |-----|-----|-----|\n\
2269 | b | 3 | 4 |"
2270 );
2271 let t = flagged_table(&[&["Item", "", ""], &["x", "1", "2"]], |_, c| c == 0);
2273 assert_eq!(t.header_row_count(), 1);
2274 assert_eq!(
2275 render_table(&t, false),
2276 "| Item | | |\n\
2277 |--------|----|----|\n\
2278 | x | 1 | 2 |"
2279 );
2280 }
2281
2282 #[test]
2283 fn renders_compact_table() {
2284 let mut doc = DoclingDocument::new("t");
2285 doc.compact_tables = true;
2288 doc.push(Node::Table(Table {
2289 footnotes: Vec::new(),
2290 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2291 location: None,
2292 structure: None,
2293 cell_blocks: None,
2294 cells: None,
2295 caption: None,
2296 caption_parent: Default::default(),
2297 caption_location: None,
2298 document_index: false,
2299 }));
2300 let md = doc.export_to_markdown();
2301 assert_eq!(md, "| a | b |\n| - | - |\n| 1 | 2 |\n");
2302 }
2303
2304 #[test]
2305 fn renders_padded_github_table_by_default() {
2306 let mut doc = DoclingDocument::new("t");
2307 doc.push(Node::Table(Table {
2308 footnotes: Vec::new(),
2309 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2310 location: None,
2311 structure: None,
2312 cell_blocks: None,
2313 cells: None,
2314 caption: None,
2315 caption_parent: Default::default(),
2316 caption_location: None,
2317 document_index: false,
2318 }));
2319 let md = doc.export_to_markdown();
2320 assert_eq!(md, "| a | b |\n|-----|-----|\n| 1 | 2 |\n");
2322 }
2323
2324 #[test]
2325 fn strict_unescapes_inline_underscores_legacy_keeps_them() {
2326 let mut doc = DoclingDocument::new("t");
2327 doc.add_heading(1, "a\\_b");
2328 doc.add_paragraph("x\\_y");
2329 doc.push(Node::ListItem {
2330 ordered: false,
2331 number: 1,
2332 first_in_list: true,
2333 text: "i\\_j".into(),
2334 level: 0,
2335 marker: None,
2336 location: None,
2337 dclx: None,
2338 href: None,
2339 layer: None,
2340 });
2341 assert_eq!(doc.export_to_markdown(), "# a\\_b\n\nx\\_y\n\n- i\\_j\n");
2343 assert_eq!(doc.export_to_markdown_with(true), "# a_b\n\nx_y\n\n- i_j\n");
2345 }
2346
2347 fn assert_stream_matches(
2350 doc: &DoclingDocument,
2351 strict: bool,
2352 images: ImageMode,
2353 splits: &[usize],
2354 ) {
2355 let (want, want_artifacts) = to_markdown_images(doc, strict, images, "artifacts");
2356 let mut streamer =
2357 MarkdownStreamer::with_artifacts(strict, images, doc.compact_tables, "artifacts")
2358 .with_page_break_placeholder(doc.page_break_placeholder.clone());
2359 let mut got = String::new();
2360 let mut got_artifacts = Vec::new();
2361 let mut start = 0;
2362 for &end in splits {
2363 let links = if start == 0 {
2366 doc.links.as_slice()
2367 } else {
2368 &[]
2369 };
2370 got.push_str(&streamer.push(&doc.nodes[start..end], links));
2371 got_artifacts.extend(streamer.take_artifacts());
2374 start = end;
2375 }
2376 got.push_str(&streamer.push(
2377 &doc.nodes[start..],
2378 if start == 0 {
2379 doc.links.as_slice()
2380 } else {
2381 &[]
2382 },
2383 ));
2384 got_artifacts.extend(streamer.take_artifacts());
2385 got.push_str(&streamer.finish());
2386 assert_eq!(
2387 got, want,
2388 "streamed output diverged (splits={splits:?}, strict={strict})"
2389 );
2390 assert_eq!(
2391 got_artifacts, want_artifacts,
2392 "streamed artifacts diverged (splits={splits:?}, strict={strict})"
2393 );
2394 }
2395
2396 #[test]
2397 fn streaming_is_byte_identical_to_buffered() {
2398 let mut doc = DoclingDocument::new("d");
2399 doc.add_heading(1, "Title");
2400 doc.add_paragraph("First paragraph.");
2401 doc.push(Node::ListItem {
2402 ordered: false,
2403 number: 1,
2404 first_in_list: true,
2405 text: "a".into(),
2406 level: 0,
2407 marker: None,
2408 location: None,
2409 dclx: None,
2410 href: None,
2411 layer: None,
2412 });
2413 doc.push(Node::ListItem {
2414 ordered: false,
2415 number: 2,
2416 first_in_list: false,
2417 text: "b".into(),
2418 level: 0,
2419 marker: None,
2420 location: None,
2421 dclx: None,
2422 href: None,
2423 layer: None,
2424 });
2425 doc.push(Node::Code {
2426 language: Some("rust".into()),
2427 text: "let x = 1;".into(),
2428 orig: None,
2429 pretty: None,
2430 });
2431 doc.push(Node::Table(Table {
2432 footnotes: Vec::new(),
2433 rows: vec![vec!["a".into(), "b".into()], vec!["1".into(), "2".into()]],
2434 location: None,
2435 structure: None,
2436 cell_blocks: None,
2437 cells: None,
2438 caption: None,
2439 caption_parent: Default::default(),
2440 caption_location: None,
2441 document_index: false,
2442 }));
2443 doc.push(Node::Picture {
2444 caption: Some("Fig 1".into()),
2445 caption_href: None,
2446 image: Some(PictureImage {
2447 dpi: PictureImage::DEFAULT_DPI,
2448 mimetype: "image/png".into(),
2449 width: 2,
2450 height: 2,
2451 data: b"png-one".to_vec(),
2452 display_size: None,
2453 }),
2454 classification: None,
2455 description: None,
2456 caption_parent: Default::default(),
2457 caption_location: None,
2458 });
2459 doc.add_paragraph("Last paragraph.");
2460 doc.push(Node::Picture {
2463 caption: None,
2464 caption_href: None,
2465 image: Some(PictureImage {
2466 dpi: PictureImage::DEFAULT_DPI,
2467 mimetype: "image/png".into(),
2468 width: 2,
2469 height: 2,
2470 data: b"png-two".to_vec(),
2471 display_size: None,
2472 }),
2473 classification: None,
2474 description: None,
2475 caption_parent: Default::default(),
2476 caption_location: None,
2477 });
2478
2479 for &strict in &[false, true] {
2482 for &images in &[
2483 ImageMode::Placeholder,
2484 ImageMode::Embedded,
2485 ImageMode::Referenced,
2486 ] {
2487 for splits in [&[][..], &[1][..], &[2][..], &[4][..], &[1, 4, 6, 7][..]] {
2488 assert_stream_matches(&doc, strict, images, splits);
2489 }
2490 }
2491 }
2492 }
2493
2494 #[test]
2495 fn streaming_applies_recovered_links_in_strict_mode() {
2496 let mut doc = DoclingDocument::new("d");
2497 doc.add_paragraph("See LinkedIn for details.");
2498 doc.add_paragraph("And GitHub too.");
2499 doc.links = vec![
2500 ("LinkedIn".into(), "https://lnkd/".into()),
2501 ("GitHub".into(), "https://gh/".into()),
2502 ];
2503 assert_stream_matches(&doc, true, ImageMode::Placeholder, &[1]);
2506 }
2507
2508 fn paged_doc() -> DoclingDocument {
2511 let mut doc = DoclingDocument::new("p");
2512 doc.push(Node::PageInfo {
2513 page_no: 1,
2514 width: 100.0,
2515 height: 100.0,
2516 });
2517 doc.add_heading(1, "Title");
2518 doc.add_paragraph("Page one.");
2519 doc.push(Node::PageBreak);
2521 doc.push(Node::PageInfo {
2522 page_no: 2,
2523 width: 100.0,
2524 height: 100.0,
2525 });
2526 doc.push(Node::PageFurniture {
2527 footer: true,
2528 location: [0, 500, 511, 511],
2529 text: "2".into(),
2530 });
2531 doc.push(Node::PageBreak);
2532 doc.push(Node::PageInfo {
2533 page_no: 3,
2534 width: 100.0,
2535 height: 100.0,
2536 });
2537 doc.add_paragraph("Page three.");
2538 doc.push(Node::PageBreak);
2540 doc
2541 }
2542
2543 #[test]
2544 fn page_break_placeholder_lands_between_pages_only() {
2545 let mut doc = paged_doc();
2546 assert_eq!(
2548 doc.export_to_markdown(),
2549 "# Title\n\nPage one.\n\nPage three.\n"
2550 );
2551 doc.page_break_placeholder = Some("<!-- page break -->".into());
2552 assert_eq!(
2556 doc.export_to_markdown(),
2557 "# Title\n\nPage one.\n\n<!-- page break -->\n\nPage three.\n"
2558 );
2559 doc.page_break_placeholder = Some(String::new());
2562 assert_eq!(
2563 doc.export_to_markdown(),
2564 "# Title\n\nPage one.\n\n\n\nPage three.\n"
2565 );
2566 }
2567
2568 #[test]
2569 fn page_break_placeholder_never_leads_a_single_page() {
2570 let mut doc = DoclingDocument::new("one");
2571 doc.page_break_placeholder = Some("---".into());
2572 doc.push(Node::PageBreak);
2573 doc.push(Node::PageInfo {
2574 page_no: 1,
2575 width: 10.0,
2576 height: 10.0,
2577 });
2578 doc.add_paragraph("Only page.");
2579 assert_eq!(doc.export_to_markdown(), "Only page.\n");
2580 doc.push(Node::PageBreak);
2582 doc.push(Node::PageBreak);
2583 doc.add_paragraph("Next.");
2584 assert_eq!(doc.export_to_markdown(), "Only page.\n\n---\n\nNext.\n");
2585 }
2586
2587 #[test]
2588 fn page_break_placeholder_streams_byte_identical() {
2589 let mut doc = paged_doc();
2590 doc.page_break_placeholder = Some("<!-- page break -->".into());
2591 for splits in [
2595 &[3usize][..],
2596 &[3, 6],
2597 &[3, 6, 8],
2598 &[1, 2, 3, 4, 5, 6, 7, 8, 9],
2599 &[8],
2600 ] {
2601 assert_stream_matches(&doc, false, ImageMode::Placeholder, splits);
2602 assert_stream_matches(&doc, true, ImageMode::Placeholder, splits);
2603 }
2604 }
2605
2606 #[test]
2607 fn strict_tightens_punctuation_spacing_legacy_keeps_it() {
2608 let mut doc = DoclingDocument::new("t");
2609 doc.add_paragraph("see [ 37 , 36 ] and ( x ) .");
2610 assert_eq!(doc.export_to_markdown(), "see [ 37 , 36 ] and ( x ) .\n");
2612 assert_eq!(doc.export_to_markdown_with(true), "see [37, 36] and (x).\n");
2614 }
2615
2616 fn options_doc() -> DoclingDocument {
2621 use crate::document::ContentLayer;
2622 let mut doc = DoclingDocument::new("opts");
2623 doc.push(Node::PageFurniture {
2624 footer: false,
2625 location: [0, 0, 0, 0],
2626 text: "FAX COVER \\_ R&D".into(),
2627 });
2628 doc.add_heading(1, "R&D \\_ report");
2629 doc.add_paragraph("Body a\\_b <tag>");
2630 doc.push(Node::Picture {
2631 caption: Some("Figure 1".into()),
2632 caption_href: None,
2633 image: None,
2634 classification: None,
2635 description: None,
2636 caption_parent: Default::default(),
2637 caption_location: None,
2638 });
2639 doc.push(Node::PictureChildren(vec![
2640 Node::Paragraph {
2641 text: "Field: Name".into(),
2642 },
2643 Node::Paragraph {
2644 text: "Field: Date".into(),
2645 },
2646 ]));
2647 doc.push(Node::CommentSection {
2648 name: "comment-1".into(),
2649 text: "[author: A]: note".into(),
2650 refs_note_text: false,
2651 grouped: true,
2652 });
2653 doc.push(Node::Group {
2654 label: "section".into(),
2655 name: Some("sheet: Hidden".into()),
2656 layer: Some(ContentLayer::Invisible),
2657 children: vec![Node::Paragraph {
2658 text: "hidden cell".into(),
2659 }],
2660 });
2661 doc.push(Node::Furniture {
2662 layer: ContentLayer::Furniture,
2663 inner: Box::new(Node::Paragraph {
2664 text: "Page 1 of 2".into(),
2665 }),
2666 });
2667 doc.push(Node::FurnitureText {
2668 label: "page_footer".into(),
2669 text: "footer line".into(),
2670 });
2671 doc
2672 }
2673
2674 #[test]
2678 fn default_options_are_the_default_export() {
2679 let doc = options_doc();
2680 let plain = doc.export_to_markdown();
2681 assert_eq!(
2682 plain,
2683 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2684 );
2685 let (with_default, artifacts) =
2686 doc.export_to_markdown_with_options(&MarkdownExportOptions::default());
2687 assert_eq!(with_default, plain);
2688 assert!(artifacts.is_empty());
2689 }
2690
2691 #[test]
2692 fn layers_render_furniture_notes_and_hidden_sheets() {
2693 use crate::document::ContentLayer;
2694 let doc = options_doc();
2695 let md = |layers: ContentLayers| {
2696 doc.export_to_markdown_with_options(&MarkdownExportOptions {
2697 layers,
2698 ..MarkdownExportOptions::default()
2699 })
2700 .0
2701 };
2702 assert_eq!(
2703 md(ContentLayers::BODY.with(ContentLayer::Furniture)),
2704 "FAX COVER \\_ R&D\n\n# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nPage 1 of 2\n\nfooter line\n",
2705 "a page header, a furniture-wrapped paragraph and a furniture text item render as paragraphs"
2706 );
2707 assert_eq!(
2708 md(ContentLayers::BODY.with(ContentLayer::Notes)),
2709 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\n[author: A]: note\n"
2710 );
2711 assert_eq!(
2712 md(ContentLayers::BODY.with(ContentLayer::Invisible)),
2713 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nhidden cell\n",
2714 "a hidden sheet's items sit on the sheet's layer"
2715 );
2716 assert_eq!(
2717 md(ContentLayers::NONE.with(ContentLayer::Furniture)),
2718 "FAX COVER \\_ R&D\n\nPage 1 of 2\n\nfooter line\n",
2719 "without the body layer only the furniture prints"
2720 );
2721 assert_eq!(md(ContentLayers::NONE), "");
2722 assert_eq!(
2723 md(ContentLayers::ALL),
2724 "FAX COVER \\_ R&D\n\n# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\n[author: A]: note\n\nhidden cell\n\nPage 1 of 2\n\nfooter line\n"
2725 );
2726 }
2727
2728 #[test]
2731 fn layers_apply_to_list_items() {
2732 use crate::document::ContentLayer;
2733 let mut doc = DoclingDocument::new("lists");
2734 doc.push(Node::ListItem {
2735 ordered: false,
2736 number: 1,
2737 first_in_list: true,
2738 text: "nav link".into(),
2739 level: 0,
2740 marker: None,
2741 location: None,
2742 dclx: None,
2743 href: None,
2744 layer: Some(ContentLayer::Furniture),
2745 });
2746 doc.add_paragraph("body");
2747 assert_eq!(doc.export_to_markdown(), "body\n");
2748 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2749 layers: ContentLayers::BODY.with(ContentLayer::Furniture),
2750 ..MarkdownExportOptions::default()
2751 });
2752 assert_eq!(md, "- nav link\n\nbody\n");
2753 }
2754
2755 #[test]
2756 fn traverse_pictures_prints_the_nested_text_after_the_picture() {
2757 let doc = options_doc();
2758 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2759 traverse_pictures: true,
2760 ..MarkdownExportOptions::default()
2761 });
2762 assert_eq!(
2763 md,
2764 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n\nField: Name\n\nField: Date\n"
2765 );
2766 }
2767
2768 #[test]
2769 fn escaping_can_be_turned_off_per_kind() {
2770 let doc = options_doc();
2771 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2772 escape_html: false,
2773 ..MarkdownExportOptions::default()
2774 });
2775 assert_eq!(
2776 md,
2777 "# R&D \\_ report\n\nBody a\\_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2778 );
2779 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2780 escape_underscores: false,
2781 ..MarkdownExportOptions::default()
2782 });
2783 assert_eq!(
2784 md,
2785 "# R&D _ report\n\nBody a_b <tag>\n\nFigure 1\n\n<!-- image -->\n"
2786 );
2787 let mut doc = DoclingDocument::new("code");
2790 doc.push(Node::Code {
2791 language: None,
2792 text: "a & b \\_ c".into(),
2793 orig: None,
2794 pretty: None,
2795 });
2796 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2797 escape_html: false,
2798 escape_underscores: false,
2799 ..MarkdownExportOptions::default()
2800 });
2801 assert_eq!(md, "```\na & b \\_ c\n```\n");
2802 }
2803
2804 #[test]
2808 fn fences_and_spans_out_length_inner_backticks() {
2809 let code = |text: &str| Node::Code {
2810 language: Some("md".into()),
2811 text: text.into(),
2812 orig: None,
2813 pretty: None,
2814 };
2815 let fence_of = |text: &str| {
2816 let mut doc = DoclingDocument::new("d");
2817 doc.push(code(text));
2818 (doc.export_to_markdown(), doc.export_to_markdown_with(true))
2819 };
2820 assert_eq!(fence_of("x").0, "```\nx\n```\n");
2821 assert_eq!(fence_of("a ` b").0, "```\na ` b\n```\n");
2822 assert_eq!(fence_of("a ``` b").0, "````\na ``` b\n````\n");
2823 assert_eq!(
2824 fence_of("a ``` b ```` c"),
2825 (
2826 "`````\na ``` b ```` c\n`````\n".to_string(),
2827 "`````md\na ``` b ```` c\n`````\n".to_string()
2828 )
2829 );
2830 assert_eq!(code_span("x"), "`x`");
2831 assert_eq!(code_span("a`b"), "``a`b``");
2832 assert_eq!(code_span("a ``` b"), "````a ``` b````");
2833 assert_eq!(code_span("`x"), "`` `x ``");
2834 assert_eq!(code_span("x`"), "`` x` ``");
2835 for (span, text) in [
2836 ("`x`", "x"),
2837 ("``a`b``", "a`b"),
2838 ("`` `x ``", "`x"),
2839 ("````a ``` b````", "a ``` b"),
2840 ] {
2841 assert_eq!(
2842 plain_inline(&format!("see {span} here")),
2843 format!("see {text} here")
2844 );
2845 }
2846 let mut doc = DoclingDocument::new("d");
2849 doc.push(code("a ``` b"));
2850 assert_eq!(doc.export_to_text(), "a ``` b");
2851 let mut doc = DoclingDocument::new("d");
2853 doc.add_paragraph("Before.");
2854 doc.push(code("a ``` b ```` c"));
2855 doc.add_paragraph("After.");
2856 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[1, 2]);
2857 assert_stream_matches(&doc, true, ImageMode::Placeholder, &[1, 2]);
2858 }
2859
2860 #[test]
2865 fn mermaid_code_item_replaces_its_picture_placeholder() {
2866 let picture = || Node::Picture {
2867 caption: None,
2868 caption_href: None,
2869 image: None,
2870 classification: None,
2871 description: None,
2872 caption_parent: Default::default(),
2873 caption_location: None,
2874 };
2875 let mut doc = DoclingDocument::new("d");
2876 doc.add_paragraph("Before.");
2877 doc.push(picture());
2878 doc.push(Node::Code {
2879 language: Some("mermaid".into()),
2880 text: "flowchart TD\n A --> B".into(),
2881 orig: None,
2882 pretty: None,
2883 });
2884 doc.add_paragraph("After.");
2885 doc.push(picture());
2886 let md = doc.export_to_markdown();
2887 assert_eq!(
2888 md,
2889 "Before.\n\n```mermaid\nflowchart TD\n A --> B\n```\n\nAfter.\n\n<!-- image -->\n"
2890 );
2891 assert_eq!(doc.export_to_markdown_with(true), md);
2892 for splits in [&[1][..], &[2], &[1, 2], &[2, 3], &[1, 2, 3, 4]] {
2893 assert_stream_matches(&doc, false, ImageMode::Placeholder, splits);
2894 assert_stream_matches(&doc, true, ImageMode::Placeholder, splits);
2895 }
2896 let mut doc = DoclingDocument::new("d");
2898 doc.add_paragraph("Only.");
2899 doc.push(picture());
2900 assert_eq!(doc.export_to_markdown(), "Only.\n\n<!-- image -->\n");
2901 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[1]);
2902 assert_stream_matches(&doc, false, ImageMode::Placeholder, &[2]);
2903 }
2904
2905 #[test]
2906 fn image_placeholder_is_configurable() {
2907 let doc = options_doc();
2908 let (md, _) = doc.export_to_markdown_with_options(&MarkdownExportOptions {
2909 image_placeholder: "[figure]".into(),
2910 ..MarkdownExportOptions::default()
2911 });
2912 assert!(md.contains("Figure 1\n\n[figure]\n"), "{md}");
2913 assert!(!md.contains("<!-- image -->"));
2914 }
2915
2916 #[test]
2922 fn empty_image_placeholder_prints_nothing() {
2923 fn picture(caption: Option<&str>) -> Node {
2924 Node::Picture {
2925 caption: caption.map(Into::into),
2926 caption_href: None,
2927 image: None,
2928 classification: None,
2929 description: None,
2930 caption_parent: Default::default(),
2931 caption_location: None,
2932 }
2933 }
2934 fn page(no: usize) -> Node {
2935 Node::PageInfo {
2936 page_no: no,
2937 width: 100.0,
2938 height: 100.0,
2939 }
2940 }
2941 fn para(text: &str) -> Node {
2942 Node::Paragraph { text: text.into() }
2943 }
2944 let options = MarkdownExportOptions {
2945 image_placeholder: String::new(),
2946 ..MarkdownExportOptions::default()
2947 };
2948 let cases: Vec<(Vec<Node>, Option<&str>, &str)> = vec![
2950 (
2951 vec![page(1), para("a"), picture(None), para("b")],
2952 None,
2953 "a\n\nb\n",
2954 ),
2955 (
2956 vec![page(1), para("a"), picture(Some("Figure 1")), para("b")],
2957 None,
2958 "a\n\nFigure 1\n\nb\n",
2959 ),
2960 (
2961 vec![
2962 page(1),
2963 para("a"),
2964 page(2),
2965 picture(None),
2966 page(3),
2967 para("b"),
2968 ],
2969 Some("<!-- pb -->"),
2970 "a\n\n<!-- pb -->\n\n<!-- pb -->\n\nb\n",
2971 ),
2972 (
2973 vec![
2974 page(1),
2975 para("a"),
2976 page(2),
2977 picture(None),
2978 page(3),
2979 para("b"),
2980 ],
2981 Some(""),
2982 "a\n\n\n\n\n\nb\n",
2983 ),
2984 (
2985 vec![page(1), picture(None), page(2), para("b")],
2986 Some("<!-- pb -->"),
2987 "<!-- pb -->\n\nb\n",
2988 ),
2989 (
2990 vec![page(1), picture(None), page(2), para("b")],
2991 Some(""),
2992 "\n\nb\n",
2993 ),
2994 (
2995 vec![page(1), para("a"), page(2), picture(None)],
2996 Some("<!-- pb -->"),
2997 "a\n\n<!-- pb -->\n",
2998 ),
2999 ];
3000 for (nodes, placeholder, expected) in cases {
3001 let mut doc = DoclingDocument::new("t");
3002 doc.page_break_placeholder = placeholder.map(Into::into);
3003 for node in nodes {
3004 doc.push(node);
3005 }
3006 let (md, _) = doc.export_to_markdown_with_options(&options);
3007 assert_eq!(md, expected, "page break {placeholder:?}");
3008 for size in 1..=doc.nodes.len() {
3011 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3012 .with_export_options(&options)
3013 .with_page_break_placeholder(placeholder.map(Into::into));
3014 let mut out = String::new();
3015 for chunk in doc.nodes.chunks(size) {
3016 out.push_str(&streamer.push(chunk, &[]));
3017 }
3018 out.push_str(&streamer.finish());
3019 assert_eq!(
3020 out, expected,
3021 "chunks of {size}, page break {placeholder:?}"
3022 );
3023 }
3024 }
3025 let mut doc = DoclingDocument::new("t");
3027 doc.push(para("a"));
3028 doc.push(picture(None));
3029 doc.push(para("b"));
3030 assert_eq!(doc.export_to_markdown(), "a\n\n<!-- image -->\n\nb\n");
3031 }
3032
3033 #[test]
3036 fn streamer_matches_buffered_export_with_options() {
3037 let doc = options_doc();
3038 let options = MarkdownExportOptions {
3039 layers: ContentLayers::ALL,
3040 traverse_pictures: true,
3041 escape_html: false,
3042 escape_underscores: false,
3043 image_placeholder: "(img)".into(),
3044 ..MarkdownExportOptions::default()
3045 };
3046 let (buffered, _) = doc.export_to_markdown_with_options(&options);
3047 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3048 .with_export_options(&options);
3049 let mut out = String::new();
3050 for chunk in doc.nodes.chunks(2) {
3051 out.push_str(&streamer.push(chunk, &[]));
3052 }
3053 out.push_str(&streamer.finish());
3054 assert_eq!(out, buffered);
3055 assert!(
3056 buffered.contains("(img)") && buffered.contains("R&D _ report"),
3057 "{buffered}"
3058 );
3059 }
3060
3061 #[test]
3064 fn plain_text_reads_escaped_stars() {
3065 assert_eq!(plain_inline("A\\*\\*\\* GmbH"), "A*** GmbH");
3066 assert_eq!(
3067 plain_inline("**FN \\*0\\*271\\*h** and *h\\*.at*"),
3068 "FN *0*271*h and h*.at"
3069 );
3070 }
3071
3072 #[test]
3075 fn plain_text_drops_the_markdown_decoration() {
3076 let item = |text: &str, ordered: bool, number: u64, level: u8| Node::ListItem {
3077 ordered,
3078 number,
3079 first_in_list: number == 1 && level == 0,
3080 text: text.into(),
3081 level,
3082 marker: None,
3083 location: None,
3084 dclx: None,
3085 href: None,
3086 layer: None,
3087 };
3088 let mut doc = DoclingDocument::new("plain");
3089 doc.add_heading(1, "Title with **bold**");
3090 doc.add_heading(2, "Section");
3091 doc.add_paragraph("Some **bold**, *italic*, ~~gone~~ and `code` with a [link](https://x.org).\nNext line.");
3092 doc.push(item("*one*", false, 1, 0));
3093 doc.push(item("[two](https://y.org)", false, 2, 0));
3094 doc.push(item("nested", true, 1, 1));
3095 doc.push(Node::CheckboxItem {
3096 checked: true,
3097 text: "done".into(),
3098 });
3099 doc.push(Node::Code {
3100 language: Some("rust".into()),
3101 text: "let x = **y**;".into(),
3102 orig: None,
3103 pretty: None,
3104 });
3105 doc.push(Node::Picture {
3106 caption: Some("Figure *1*".into()),
3107 caption_href: None,
3108 image: None,
3109 classification: None,
3110 description: None,
3111 caption_parent: Default::default(),
3112 caption_location: None,
3113 });
3114 doc.push(Node::Caption {
3115 text: "Linked caption".into(),
3116 href: Some("https://z.org".into()),
3117 });
3118 doc.push(Node::Table(Table {
3119 rows: vec![
3120 vec!["**n**".into(), "rich".into()],
3121 vec!["**2**".into(), "a \nb".into()],
3122 ],
3123 ..Default::default()
3124 }));
3125 doc.add_paragraph("R&D \\_ x");
3126 let text = doc.export_to_text();
3127 assert_eq!(
3128 text,
3129 "Title with bold\n\n\
3130 Section\n\n\
3131 Some bold, italic, gone and code with a link.\nNext line.\n\n\
3132 - one\n- two\n 1. nested\n\n\
3133 - [x] done\n\n\
3134 let x = **y**;\n\n\
3135 Figure 1\n\n\
3136 Linked caption\n\n\
3137 | n | rich |\n\
3138 |-----|--------|\n\
3139 | 2 | a b |\n\n\
3140 R&D _ x"
3141 );
3142
3143 let options = MarkdownExportOptions::plain_text();
3144 let (buffered, _) = doc.export_to_markdown_with_options(&options);
3145 assert_eq!(buffered, format!("{text}\n"));
3146 let mut streamer = MarkdownStreamer::new(false, ImageMode::Placeholder, false)
3147 .with_export_options(&options);
3148 let mut out = String::new();
3149 for chunk in doc.nodes.chunks(3) {
3150 out.push_str(&streamer.push(chunk, &[]));
3151 }
3152 out.push_str(&streamer.finish());
3153 assert_eq!(out, buffered);
3154 }
3155
3156 #[test]
3158 fn plain_inline_keeps_literal_markers() {
3159 for (input, want) in [
3160 ("**bold** and *it* and ***both***", "bold and it and both"),
3161 ("**[label](https://a.b/c_(d))** end", "label end"),
3162 (
3163 "grant *USE and *OBJMGT authority",
3164 "grant *USE and *OBJMGT authority",
3165 ),
3166 ("2 * 3 * 4", "2 * 3 * 4"),
3167 ("a ** b", "a ** b"),
3168 ("the `quoted’ and `x` term", "the `quoted’ and x term"),
3169 ("see [12] and [docs](u)", "see [12] and docs"),
3170 ("formula $[a](b)$ stays", "formula $[a](b)$ stays"),
3171 ("price $5 and **bold**", "price $5 and bold"),
3172 ("[outer [inner] text](u)", "outer [inner] text"),
3173 ("unclosed **bold", "unclosed **bold"),
3174 ] {
3175 assert_eq!(plain_inline(input), want, "{input}");
3176 }
3177 }
3178}