1use std::collections::{BTreeMap, HashMap, HashSet};
68
69use serde_json::Value;
70
71use crate::document::{ContentLayers, DoclingDocument};
72use crate::json::{TableData, TableView};
73use crate::markdown::ImageMode;
74use crate::TrackChanges;
75
76pub const PANDOC_API_VERSION: [u32; 4] = [1, 23, 1, 1];
78
79pub type PandocOutput = (String, Vec<(String, Vec<u8>)>);
82
83#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
85pub enum HeadingDepth {
86 #[default]
94 Docling,
95 Source,
104}
105
106impl HeadingDepth {
107 pub const NAMES: [&'static str; 2] = ["docling", "source"];
109
110 pub fn as_str(self) -> &'static str {
111 match self {
112 HeadingDepth::Docling => "docling",
113 HeadingDepth::Source => "source",
114 }
115 }
116}
117
118impl std::str::FromStr for HeadingDepth {
119 type Err = String;
120
121 fn from_str(s: &str) -> Result<Self, String> {
122 match s.trim().to_ascii_lowercase().as_str() {
123 "docling" | "" => Ok(HeadingDepth::Docling),
124 "source" => Ok(HeadingDepth::Source),
125 other => Err(format!(
126 "unknown heading depth {other:?} (expected docling or source)"
127 )),
128 }
129 }
130}
131
132#[derive(Debug, Clone)]
134pub struct PandocExportOptions {
135 pub image_mode: ImageMode,
138 pub artifacts_dir: String,
141 pub layers: ContentLayers,
143 pub api_version: Option<String>,
146 pub heading_depth: HeadingDepth,
149 pub track_changes: Option<TrackChanges>,
155}
156
157impl Default for PandocExportOptions {
158 fn default() -> Self {
159 Self {
160 image_mode: ImageMode::Embedded,
161 artifacts_dir: "artifacts".to_string(),
162 layers: ContentLayers::BODY,
163 api_version: None,
164 heading_depth: HeadingDepth::Docling,
165 track_changes: None,
166 }
167 }
168}
169
170#[derive(Debug, Clone, PartialEq, Eq)]
172pub enum PandocError {
173 UnsupportedApiVersion { requested: String },
175}
176
177impl std::fmt::Display for PandocError {
178 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
179 match self {
180 PandocError::UnsupportedApiVersion { requested } => write!(
181 f,
182 "unsupported Pandoc API version '{requested}': only {} (pandoc-types {}, Pandoc 3.x) is supported",
183 version_string(&PANDOC_API_VERSION[..2]),
184 version_string(&PANDOC_API_VERSION),
185 ),
186 }
187 }
188}
189
190impl std::error::Error for PandocError {}
191
192fn version_string(parts: &[u32]) -> String {
193 parts
194 .iter()
195 .map(u32::to_string)
196 .collect::<Vec<_>>()
197 .join(".")
198}
199
200pub fn check_api_version(requested: &str) -> Result<(), PandocError> {
204 let err = || PandocError::UnsupportedApiVersion {
205 requested: requested.to_string(),
206 };
207 let parts: Vec<u32> = requested
208 .trim()
209 .split('.')
210 .map(|p| p.parse::<u32>().map_err(|_| err()))
211 .collect::<Result<_, _>>()?;
212 if parts.len() < 2 || parts[..2] != PANDOC_API_VERSION[..2] {
213 return Err(err());
214 }
215 if parts.len() > PANDOC_API_VERSION.len()
216 || parts
217 .iter()
218 .zip(PANDOC_API_VERSION.iter())
219 .skip(2)
220 .any(|(a, b)| a > b)
221 {
222 return Err(err());
224 }
225 Ok(())
226}
227
228pub fn to_pandoc(
232 doc: &DoclingDocument,
233 options: &PandocExportOptions,
234) -> Result<PandocOutput, PandocError> {
235 let (json, tables) = crate::json::to_json_with_notes(doc);
238 render_json(&json, &tables, options)
239}
240
241pub fn from_docling_json(
245 json: &Value,
246 options: &PandocExportOptions,
247) -> Result<PandocOutput, PandocError> {
248 render_json(json, &[], options)
249}
250
251fn render_json(
254 json: &Value,
255 tables: &[TableData],
256 options: &PandocExportOptions,
257) -> Result<PandocOutput, PandocError> {
258 if let Some(v) = &options.api_version {
259 check_api_version(v)?;
260 }
261 let render = || {
264 let mut ser = Serializer::new(json, options);
265 ser.tables = tables;
266 let blocks = ser.body();
267 let doc = ast::document_with_meta(meta_of(json), blocks);
268 (
269 serde_json::to_string(&doc).expect("Pandoc JSON is always serializable"),
270 ser.artifacts,
271 )
272 };
273 #[cfg(not(target_arch = "wasm32"))]
274 {
275 Ok(std::thread::scope(|scope| {
276 std::thread::Builder::new()
277 .name("docling-pandoc".into())
278 .stack_size(256 << 20)
279 .spawn_scoped(scope, render)
280 .expect("spawn the pandoc serializer thread")
281 .join()
282 .expect("pandoc serializer thread panicked")
283 }))
284 }
285 #[cfg(target_arch = "wasm32")]
286 {
287 Ok(render())
288 }
289}
290
291pub fn auto_identifier(text: &str) -> String {
298 let kept: String = text
299 .chars()
300 .filter(|c| c.is_alphanumeric() || c.is_whitespace() || matches!(c, '_' | '-' | '.'))
301 .flat_map(char::to_lowercase)
302 .collect();
303 let joined = kept.split_whitespace().collect::<Vec<_>>().join("-");
304 let id = joined.trim_start_matches(|c: char| !c.is_alphabetic());
305 if id.is_empty() {
306 "section".to_string()
307 } else {
308 id.to_string()
309 }
310}
311
312pub mod ast {
322 use serde_json::{json, Value};
323
324 use super::PANDOC_API_VERSION;
325
326 fn arr<const N: usize>(items: [Value; N]) -> Value {
328 Value::Array(items.into())
329 }
330
331 pub fn document(blocks: Vec<Value>) -> Value {
332 document_with_meta(json!({}), blocks)
333 }
334 pub fn document_with_meta(meta: Value, blocks: Vec<Value>) -> Value {
336 let mut m = serde_json::Map::with_capacity(3);
337 m.insert("pandoc-api-version".into(), json!(PANDOC_API_VERSION));
338 m.insert("meta".into(), meta);
339 m.insert("blocks".into(), Value::Array(blocks));
340 Value::Object(m)
341 }
342 pub fn meta_inlines(inlines: Vec<Value>) -> Value {
343 node("MetaInlines", Value::Array(inlines))
344 }
345 pub fn meta_string(s: &str) -> Value {
346 node("MetaString", json!(s))
347 }
348 pub fn meta_list(items: Vec<Value>) -> Value {
349 node("MetaList", Value::Array(items))
350 }
351 pub fn span(classes: &[&str], attrs: &[(String, String)], inlines: Vec<Value>) -> Value {
353 let kv: Vec<Value> = attrs.iter().map(|(k, v)| json!([k, v])).collect();
354 node(
355 "Span",
356 arr([json!(["", classes, kv]), Value::Array(inlines)]),
357 )
358 }
359
360 pub fn attr(classes: &[&str]) -> Value {
362 json!(["", classes, []])
363 }
364
365 fn node(t: &str, c: Value) -> Value {
366 let mut m = serde_json::Map::with_capacity(2);
367 m.insert("t".into(), Value::String(t.into()));
368 m.insert("c".into(), c);
369 Value::Object(m)
370 }
371
372 pub fn para(inlines: Vec<Value>) -> Value {
375 node("Para", Value::Array(inlines))
376 }
377 pub fn plain(inlines: Vec<Value>) -> Value {
378 node("Plain", Value::Array(inlines))
379 }
380 pub fn header(level: usize, inlines: Vec<Value>) -> Value {
381 node(
382 "Header",
383 arr([level.into(), attr(&[]), Value::Array(inlines)]),
384 )
385 }
386 pub fn header_with_id(level: usize, id: &str, inlines: Vec<Value>) -> Value {
388 node(
389 "Header",
390 arr([level.into(), json!([id, [], []]), Value::Array(inlines)]),
391 )
392 }
393 pub fn anchor(id: &str) -> Value {
396 node("Span", json!([[id, ["anchor"], []], []]))
397 }
398 pub fn code_block(language: Option<&str>, text: &str) -> Value {
399 let classes: Vec<&str> = language.into_iter().collect();
400 node("CodeBlock", arr([attr(&classes), text.into()]))
401 }
402 pub fn bullet_list(items: Vec<Vec<Value>>) -> Value {
403 node(
404 "BulletList",
405 Value::Array(items.into_iter().map(Value::Array).collect()),
406 )
407 }
408 pub fn ordered_list(start: u64, style: &str, delim: &str, items: Vec<Vec<Value>>) -> Value {
412 node(
413 "OrderedList",
414 arr([
415 json!([start, { "t": style }, { "t": delim }]),
416 Value::Array(items.into_iter().map(Value::Array).collect()),
417 ]),
418 )
419 }
420 pub fn definition_list(entries: Vec<(Vec<Value>, Vec<Vec<Value>>)>) -> Value {
421 node(
422 "DefinitionList",
423 Value::Array(
424 entries
425 .into_iter()
426 .map(|(term, defs)| {
427 arr([
428 Value::Array(term),
429 Value::Array(defs.into_iter().map(Value::Array).collect()),
430 ])
431 })
432 .collect(),
433 ),
434 )
435 }
436 pub fn raw_block(format: &str, text: &str) -> Value {
439 node("RawBlock", json!([format, text]))
440 }
441 pub fn div(classes: &[&str], blocks: Vec<Value>) -> Value {
442 node("Div", arr([attr(classes), Value::Array(blocks)]))
443 }
444 pub fn caption(blocks: Vec<Value>) -> Value {
446 arr([Value::Null, Value::Array(blocks)])
447 }
448 pub fn figure(classes: &[&str], caption: Value, blocks: Vec<Value>) -> Value {
449 node(
450 "Figure",
451 arr([attr(classes), caption, Value::Array(blocks)]),
452 )
453 }
454 pub fn cell(row_span: usize, col_span: usize, blocks: Vec<Value>) -> Value {
456 arr([
457 attr(&[]),
458 json!({ "t": "AlignDefault" }),
459 row_span.into(),
460 col_span.into(),
461 Value::Array(blocks),
462 ])
463 }
464 pub fn row(cells: Vec<Value>) -> Value {
465 arr([attr(&[]), Value::Array(cells)])
466 }
467 pub fn table(caption: Value, num_cols: usize, head: Vec<Value>, body: Vec<Value>) -> Value {
469 let colspecs: Vec<Value> = (0..num_cols)
470 .map(|_| json!([{ "t": "AlignDefault" }, { "t": "ColWidthDefault" }]))
471 .collect();
472 node(
473 "Table",
474 arr([
475 attr(&[]),
476 caption,
477 Value::Array(colspecs),
478 arr([attr(&[]), Value::Array(head)]),
479 arr([arr([attr(&[]), 0.into(), json!([]), Value::Array(body)])]),
480 arr([attr(&[]), json!([])]),
481 ]),
482 )
483 }
484
485 pub fn str_(s: &str) -> Value {
488 node("Str", json!(s))
489 }
490 pub fn space() -> Value {
491 json!({ "t": "Space" })
492 }
493 pub fn line_break() -> Value {
494 json!({ "t": "LineBreak" })
495 }
496 pub fn wrap(t: &str, inlines: Vec<Value>) -> Value {
498 node(t, Value::Array(inlines))
499 }
500 pub fn code(text: &str) -> Value {
501 node("Code", arr([attr(&[]), text.into()]))
502 }
503 pub fn math(display: bool, tex: &str) -> Value {
504 let kind = if display { "DisplayMath" } else { "InlineMath" };
505 node("Math", json!([{ "t": kind }, tex]))
506 }
507 pub fn link(inlines: Vec<Value>, url: &str) -> Value {
508 node(
509 "Link",
510 arr([attr(&[]), Value::Array(inlines), json!([url, ""])]),
511 )
512 }
513 pub fn image(classes: &[&str], alt: Vec<Value>, src: &str) -> Value {
514 image_sized(classes, &[], alt, src)
515 }
516 pub fn image_sized(
518 classes: &[&str],
519 attrs: &[(String, String)],
520 alt: Vec<Value>,
521 src: &str,
522 ) -> Value {
523 let kv: Vec<Value> = attrs.iter().map(|(k, v)| json!([k, v])).collect();
524 node(
525 "Image",
526 arr([
527 json!(["", classes, kv]),
528 Value::Array(alt),
529 json!([src, ""]),
530 ]),
531 )
532 }
533 pub fn note(blocks: Vec<Value>) -> Value {
534 node("Note", Value::Array(blocks))
535 }
536}
537
538fn text_inlines(text: &str) -> Vec<Value> {
544 let mut out = Vec::new();
545 for (i, line) in text.trim().split('\n').enumerate() {
546 if i > 0 {
547 out.push(ast::line_break());
548 }
549 let is_ws = |c: char| c.is_whitespace() && c != '\u{a0}';
550 let line_start = out.len();
551 let mut pending_space = false;
554 let mut rest = line;
555 while !rest.is_empty() {
556 let (plain, link) = match markdown_link(rest) {
557 Some((before, label, target, after)) => {
558 rest = after;
559 (before, Some((label, target)))
560 }
561 None => (std::mem::take(&mut rest), None),
562 };
563 let words: Vec<&str> = plain.split(is_ws).filter(|w| !w.is_empty()).collect();
564 if words.is_empty() {
565 pending_space |= !plain.is_empty();
566 } else {
567 if out.len() > line_start && (pending_space || plain.starts_with(is_ws)) {
568 out.push(ast::space());
569 }
570 for (k, word) in words.iter().enumerate() {
571 if k > 0 {
572 out.push(ast::space());
573 }
574 out.push(ast::str_(word));
575 }
576 pending_space = plain.ends_with(is_ws);
577 }
578 if let Some((label, target)) = link {
579 if out.len() > line_start && pending_space {
580 out.push(ast::space());
581 }
582 let mut label_inlines = Vec::new();
583 for (k, word) in label.split_whitespace().enumerate() {
584 if k > 0 {
585 label_inlines.push(ast::space());
586 }
587 label_inlines.push(ast::str_(word));
588 }
589 if crate::hyperlink::is_safe_hyperlink(target) {
592 out.push(ast::link(label_inlines, target));
593 } else {
594 out.extend(label_inlines);
595 }
596 pending_space = false;
597 }
598 }
599 }
600 out
601}
602
603fn markdown_link(text: &str) -> Option<(&str, &str, &str, &str)> {
608 let mut from = 0;
609 while let Some(open) = text[from..].find('[') {
610 let open = from + open;
611 from = open + 1;
612 if text[..open].ends_with('!') {
613 continue;
614 }
615 let close = text[open + 1..].find(']')?;
616 let close = open + 1 + close;
617 let label = &text[open + 1..close];
618 if label.is_empty() || label.contains(['[', '\n']) || !text[close + 1..].starts_with('(') {
619 continue;
620 }
621 let Some(end) = text[close + 2..].find(')') else {
622 continue;
623 };
624 let end = close + 2 + end;
625 let target = &text[close + 2..end];
626 if target.is_empty() || target.contains(|c: char| c.is_whitespace() || c == '(') {
627 continue;
628 }
629 return Some((&text[..open], label, target, &text[end + 1..]));
630 }
631 None
632}
633
634fn text_with_notes(text: &str, notes: &[Value]) -> Vec<Value> {
639 let mut calls: Vec<(usize, &str)> = notes
640 .iter()
641 .filter_map(|n| Some((n.get(0)?.as_u64()? as usize, n.get(1)?.as_str()?)))
642 .collect();
643 calls.sort_by_key(|&(offset, _)| offset);
644 let chars: Vec<char> = text.chars().collect();
645 let mut out: Vec<Value> = Vec::new();
646 let push_segment = |out: &mut Vec<Value>, seg: &str| {
647 let inlines = text_inlines(seg);
648 if inlines.is_empty() {
649 if !out.is_empty() && seg.chars().any(char::is_whitespace) {
650 out.push(ast::space());
651 }
652 return;
653 }
654 if !out.is_empty() && seg.starts_with(char::is_whitespace) {
655 out.push(ast::space());
656 }
657 out.extend(inlines);
658 if seg.ends_with(char::is_whitespace) {
659 out.push(ast::space());
660 }
661 };
662 let mut prev = 0usize;
663 for (offset, note) in calls {
664 let at = offset.clamp(prev, chars.len());
665 let seg: String = chars[prev..at].iter().collect();
666 push_segment(&mut out, &seg);
667 out.push(ast::note(vec![ast::para(text_inlines(note))]));
668 prev = at;
669 }
670 let rest: String = chars[prev..].iter().collect();
671 push_segment(&mut out, &rest);
672 while out
673 .last()
674 .is_some_and(|v| v.get("t").and_then(Value::as_str) == Some("Space"))
675 {
676 out.pop();
677 }
678 out
679}
680
681fn concat_runs(runs: Vec<Value>) -> Vec<Value> {
685 let is_space = |v: &Value| v.get("t").and_then(Value::as_str) == Some("Space");
686 let mut out: Vec<Value> = Vec::with_capacity(runs.len());
687 for v in runs {
688 if is_space(&v) && out.last().is_none_or(is_space) {
689 continue;
690 }
691 out.push(v);
692 }
693 while out.last().is_some_and(is_space) {
694 out.pop();
695 }
696 out
697}
698
699fn join_inlines(parts: Vec<Vec<Value>>) -> Vec<Value> {
705 fn edge_char(inlines: &[Value], first: bool) -> Option<char> {
706 let node = if first {
707 inlines.first()
708 } else {
709 inlines.last()
710 }?;
711 match node.get("t").and_then(Value::as_str)? {
712 "Str" => {
713 let s = node.get("c")?.as_str()?;
714 if first {
715 s.chars().next()
716 } else {
717 s.chars().last()
718 }
719 }
720 "Strong" | "Emph" | "Underline" | "Strikeout" | "Subscript" | "Superscript" => {
722 edge_char(node.get("c")?.as_array()?, first)
723 }
724 "Link" => edge_char(node.get("c")?.get(1)?.as_array()?, first),
725 _ => None,
726 }
727 }
728 let mut out: Vec<Value> = Vec::new();
729 for part in parts.into_iter().filter(|p| !p.is_empty()) {
730 if !out.is_empty() {
731 let closes = edge_char(&part, true)
732 .is_some_and(|c| ".,;:!?)]}%\u{bb}\u{201d}\u{2019}".contains(c));
733 let opens = edge_char(&out, false).is_some_and(|c| "([{\u{ab}\u{201c}".contains(c));
734 if !closes && !opens {
735 out.push(ast::space());
736 }
737 }
738 out.extend(part);
739 }
740 out
741}
742
743fn meta_of(json: &Value) -> Value {
752 let mut meta = serde_json::Map::new();
753 let Some(m) = json.get("metadata").and_then(Value::as_object) else {
754 return Value::Object(meta);
755 };
756 let text = |key: &str| {
757 m.get(key)
758 .and_then(Value::as_str)
759 .map(text_inlines)
760 .filter(|t| !t.is_empty())
761 };
762 let list = |key: &str| -> Vec<Value> {
763 m.get(key)
764 .and_then(Value::as_array)
765 .into_iter()
766 .flatten()
767 .filter_map(Value::as_str)
768 .map(text_inlines)
769 .filter(|t| !t.is_empty())
770 .map(ast::meta_inlines)
771 .collect()
772 };
773 for (key, field) in [
774 ("title", "title"),
775 ("subject", "subject"),
776 ("description", "description"),
777 ("publisher", "publisher"),
778 ("rights", "rights"),
779 ("identifier", "identifier"),
780 ] {
781 if let Some(t) = text(field) {
782 meta.insert(key.into(), ast::meta_inlines(t));
783 }
784 }
785 let authors = list("authors");
786 if !authors.is_empty() {
787 meta.insert("author".into(), ast::meta_list(authors));
788 }
789 if let Some(date) = text("date").or_else(|| text("created")) {
790 meta.insert("date".into(), ast::meta_inlines(date));
791 }
792 let keywords = list("keywords");
793 if !keywords.is_empty() {
794 meta.insert("keywords".into(), ast::meta_list(keywords));
795 }
796 if let Some(lang) = m.get("language").and_then(Value::as_str).map(str::trim) {
797 if !lang.is_empty() {
798 meta.insert("lang".into(), ast::meta_string(lang));
799 }
800 }
801 Value::Object(meta)
802}
803
804fn split_comment(note: &str) -> (Option<String>, Option<String>, String) {
808 let Some(rest) = note.strip_prefix('[') else {
809 return (None, None, note.to_string());
810 };
811 let Some((head, body)) = rest.split_once(']') else {
812 return (None, None, note.to_string());
813 };
814 let body = body.strip_prefix(':').unwrap_or(body).trim().to_string();
815 let mut author = None;
816 let mut date = None;
817 for part in head.split(", ") {
818 if let Some(a) = part.strip_prefix("author: ") {
819 author = Some(a.trim().to_string());
820 } else if let Some(t) = part.strip_prefix("time: ") {
821 date = Some(t.trim().to_string());
822 }
823 }
824 (author, date, body)
825}
826
827fn inches(v: f64) -> String {
830 let s = format!("{v:.4}");
831 let s = s.trim_end_matches('0').trim_end_matches('.');
832 format!("{}in", if s.is_empty() { "0" } else { s })
833}
834
835fn list_attributes(markers: &[&str]) -> (u64, &'static str, &'static str) {
846 fn split(marker: &str) -> (&str, &'static str) {
847 let m = marker.trim();
848 if let Some(inner) = m.strip_prefix('(').and_then(|r| r.strip_suffix(')')) {
849 return (inner.trim(), "TwoParens");
850 }
851 if let Some(inner) = m.strip_suffix(')') {
852 return (inner.trim(), "OneParen");
853 }
854 if let Some(inner) = m.strip_suffix('.') {
855 return (inner.trim(), "Period");
856 }
857 (m, "DefaultDelim")
858 }
859 fn roman(token: &str) -> Option<u64> {
860 let value = |c: char| match c.to_ascii_lowercase() {
861 'i' => Some(1),
862 'v' => Some(5),
863 'x' => Some(10),
864 'l' => Some(50),
865 'c' => Some(100),
866 'd' => Some(500),
867 'm' => Some(1000),
868 _ => None,
869 };
870 let values: Vec<i64> = token.chars().map(value).collect::<Option<_>>()?;
871 if values.is_empty() {
872 return None;
873 }
874 let mut total = 0i64;
875 for (i, &v) in values.iter().enumerate() {
876 if values.get(i + 1).is_some_and(|&next| next > v) {
878 total -= v;
879 } else {
880 total += v;
881 }
882 }
883 (total > 0).then_some(total as u64)
884 }
885 fn alpha(token: &str) -> Option<u64> {
886 let first = token.chars().next()?;
887 if !first.is_ascii_alphabetic() || !token.chars().all(|c| c == first) {
888 return None;
889 }
890 let ordinal = (first.to_ascii_lowercase() as u64) - ('a' as u64) + 1;
891 Some((token.chars().count() as u64 - 1) * 26 + ordinal)
892 }
893 const DEFAULT: (u64, &str, &str) = (1, "Decimal", "Period");
894 let Some(first) = markers.first() else {
895 return DEFAULT;
896 };
897 let (token, delim) = split(first);
898 if token.is_empty() {
899 return DEFAULT;
900 }
901 if token.chars().all(|c| c.is_ascii_digit()) {
902 return token
903 .parse()
904 .map(|n| (n, "Decimal", delim))
905 .unwrap_or(DEFAULT);
906 }
907 if !token.chars().all(|c| c.is_ascii_alphabetic()) {
908 return DEFAULT;
909 }
910 let upper = token.chars().next().is_some_and(|c| c.is_ascii_uppercase());
911 let letters = |t: &str| t.chars().all(|c| c.is_ascii_alphabetic());
912 let reads_roman = roman(token).is_some()
913 && (token.len() > 1
914 || token.eq_ignore_ascii_case("i")
915 || markers[1..].iter().any(|m| {
916 let (t, _) = split(m);
917 t.len() > 1 && letters(t) && roman(t).is_some()
918 }));
919 if reads_roman {
920 let style = if upper { "UpperRoman" } else { "LowerRoman" };
921 return (roman(token).unwrap_or(1), style, delim);
922 }
923 match alpha(token) {
924 Some(n) => (n, if upper { "UpperAlpha" } else { "LowerAlpha" }, delim),
925 None => DEFAULT,
926 }
927}
928
929struct GridCell {
930 cell: usize,
932 start_row: usize,
933 start_col: usize,
934 row_span: usize,
935 col_span: usize,
936}
937
938struct Serializer<'a> {
939 json: &'a Value,
940 tables: &'a [TableData],
943 image_mode: ImageMode,
944 artifacts_dir: String,
945 layers: ContentLayers,
946 heading_depth: HeadingDepth,
947 track_changes: Option<TrackChanges>,
948 artifacts: Vec<(String, Vec<u8>)>,
949 pic_index: usize,
950 visited: HashSet<String>,
951 owned_captions: HashSet<String>,
954 owned_footnotes: HashSet<String>,
955}
956
957impl<'a> Serializer<'a> {
958 fn new(json: &'a Value, options: &PandocExportOptions) -> Self {
959 let mut owned_captions = HashSet::new();
960 let mut owned_footnotes = HashSet::new();
961 for bucket in [
962 "pictures",
963 "tables",
964 "key_value_items",
965 "form_items",
966 "texts",
967 ] {
968 for item in Self::array(json.get(bucket)) {
969 for (key, set) in [
970 ("captions", &mut owned_captions),
971 ("footnotes", &mut owned_footnotes),
972 ] {
973 for r in Self::array(item.get(key)) {
974 if let Some(cref) = r.get("$ref").and_then(Value::as_str) {
975 set.insert(cref.to_string());
976 }
977 }
978 }
979 }
980 }
981 Self {
982 json,
983 tables: &[],
984 image_mode: options.image_mode,
985 artifacts_dir: options.artifacts_dir.clone(),
986 layers: options.layers,
987 heading_depth: options.heading_depth,
988 track_changes: options.track_changes,
989 artifacts: Vec::new(),
990 pic_index: 0,
991 visited: HashSet::new(),
992 owned_captions,
993 owned_footnotes,
994 }
995 }
996
997 fn array(v: Option<&'a Value>) -> impl Iterator<Item = &'a Value> {
998 v.and_then(Value::as_array).into_iter().flatten()
999 }
1000
1001 fn resolve(&self, cref: &str) -> Option<&'a Value> {
1002 let rest = cref.strip_prefix("#/")?;
1003 if rest == "body" {
1004 return self.json.get("body");
1005 }
1006 let (bucket, idx) = rest.split_once('/')?;
1007 self.json.get(bucket)?.get(idx.parse::<usize>().ok()?)
1008 }
1009
1010 fn refs(v: Option<&'a Value>) -> Vec<&'a str> {
1011 Self::array(v)
1012 .filter_map(|r| r.get("$ref").and_then(Value::as_str))
1013 .collect()
1014 }
1015
1016 fn children(item: &'a Value) -> Vec<&'a str> {
1017 Self::refs(item.get("children"))
1018 }
1019
1020 fn self_ref(item: &Value) -> &str {
1021 item.get("self_ref").and_then(Value::as_str).unwrap_or("")
1022 }
1023
1024 fn label(item: &Value) -> &str {
1025 item.get("label").and_then(Value::as_str).unwrap_or("")
1026 }
1027
1028 fn text(item: &Value) -> &str {
1029 item.get("text").and_then(Value::as_str).unwrap_or("")
1030 }
1031
1032 fn anchors(item: &Value) -> Vec<String> {
1034 item.get("_anchors")
1035 .and_then(Value::as_array)
1036 .map(|a| {
1037 a.iter()
1038 .filter_map(Value::as_str)
1039 .map(str::to_string)
1040 .collect()
1041 })
1042 .unwrap_or_default()
1043 }
1044
1045 fn is_group(item: &Value) -> bool {
1046 Self::self_ref(item).starts_with("#/groups/")
1047 }
1048
1049 fn excluded(&self, item: &Value) -> bool {
1050 let layer = item
1051 .get("content_layer")
1052 .and_then(Value::as_str)
1053 .unwrap_or("body");
1054 !self.layers.contains_name(layer)
1055 }
1056
1057 fn descendants(&self, root: &'a Value, out: &mut Vec<&'a Value>) {
1062 let picture = Self::self_ref(root).starts_with("#/pictures/");
1063 let captions = if picture {
1064 Self::refs(root.get("captions"))
1065 } else {
1066 Vec::new()
1067 };
1068 for cref in Self::children(root) {
1069 if picture && !captions.contains(&cref) {
1070 continue;
1071 }
1072 if self.owned_captions.contains(cref) || self.owned_footnotes.contains(cref) {
1076 continue;
1077 }
1078 let Some(child) = self.resolve(cref) else {
1079 continue;
1080 };
1081 if !self.excluded(child) {
1082 out.push(child);
1083 }
1084 self.descendants(child, out);
1085 }
1086 }
1087
1088 fn blocks_of(&mut self, item: &'a Value) -> Vec<Value> {
1092 let mut nodes = Vec::new();
1093 self.descendants(item, &mut nodes);
1094 let mut out = Vec::new();
1095 for node in nodes {
1096 if self.visited.insert(Self::self_ref(node).to_string()) {
1097 out.extend(self.serialize(node));
1098 }
1099 }
1100 out
1101 }
1102
1103 fn body(&mut self) -> Vec<Value> {
1104 let Some(body) = self.json.get("body") else {
1105 return Vec::new();
1106 };
1107 self.visited.insert("#/body".to_string());
1108 let mut blocks = self.blocks_of(body);
1109 let notes = self.unplaced_notes();
1110 blocks.extend(notes);
1111 blocks
1112 }
1113
1114 fn unplaced_notes(&mut self) -> Vec<Value> {
1120 let notes: Vec<&'a Value> = Self::array(self.json.get("texts"))
1121 .filter(|t| Self::label(t) == "footnote" && !Self::is_note_body(t))
1122 .filter(|t| t.get("content_layer").and_then(Value::as_str) == Some("furniture"))
1123 .filter(|t| self.excluded(t) && !self.owned_footnotes.contains(Self::self_ref(t)))
1124 .collect();
1125 let mut out = Vec::new();
1126 for t in notes {
1127 let (text, _, _) = self.own_inlines(t);
1129 if !text.is_empty() {
1130 out.push(ast::para(vec![ast::note(vec![ast::para(text)])]));
1131 }
1132 }
1133 out
1134 }
1135
1136 fn serialize(&mut self, item: &'a Value) -> Vec<Value> {
1138 let sref = Self::self_ref(item);
1139 if Self::is_group(item) {
1140 return match Self::label(item) {
1141 "comment_section" => Vec::new(),
1145 "list" => self.list_group(item),
1146 "inline" => {
1147 let (inlines, mut rest) = self.inline_group(item);
1148 let mut out = Vec::new();
1149 if !inlines.is_empty() {
1150 out.push(ast::para(inlines));
1151 }
1152 out.append(&mut rest);
1153 out
1154 }
1155 _ => self.blocks_of(item),
1156 };
1157 }
1158 if sref.starts_with("#/texts/") {
1159 if self.owned_captions.contains(sref) || self.owned_footnotes.contains(sref) {
1160 return Vec::new();
1161 }
1162 if self.excluded(item) || Self::is_note_body(item) {
1164 return Vec::new();
1165 }
1166 return self.text_item(item);
1167 }
1168 if sref.starts_with("#/tables/") {
1169 return self.table(item);
1170 }
1171 if sref.starts_with("#/pictures/") {
1172 return self.picture(item);
1173 }
1174 if sref.starts_with("#/key_value_items/") {
1175 return self.graph_item(item, "key-value-region");
1176 }
1177 if sref.starts_with("#/form_items/") {
1178 return self.graph_item(item, "form-container");
1179 }
1180 if sref.starts_with("#/field_regions/") {
1181 return self.field_region(item);
1182 }
1183 if sref.starts_with("#/field_items/") {
1184 let entries = vec![self.field_item(item)];
1185 return Self::field_blocks(entries);
1186 }
1187 Vec::new()
1188 }
1189
1190 fn field_region(&mut self, region: &'a Value) -> Vec<Value> {
1194 let mut entries = Vec::new();
1195 for cref in Self::children(region) {
1196 let Some(child) = self.resolve(cref) else {
1197 continue;
1198 };
1199 if !self.visited.insert(cref.to_string()) {
1200 continue;
1201 }
1202 if cref.starts_with("#/field_items/") {
1203 entries.push(self.field_item(child));
1204 } else {
1205 let mut blocks = if self.excluded(child) {
1206 Vec::new()
1207 } else {
1208 self.serialize(child)
1209 };
1210 blocks.extend(self.blocks_of(child));
1211 entries.push((Vec::new(), blocks));
1212 }
1213 }
1214 let blocks = Self::field_blocks(entries);
1215 if blocks.is_empty() {
1216 Vec::new()
1217 } else {
1218 vec![ast::div(&["field-region"], blocks)]
1219 }
1220 }
1221
1222 fn field_item(&mut self, field: &'a Value) -> (Vec<Value>, Vec<Value>) {
1225 let mut nodes = Vec::new();
1226 self.descendants(field, &mut nodes);
1227 let (mut term, mut defs) = (Vec::new(), Vec::new());
1228 for node in nodes {
1229 if !self.visited.insert(Self::self_ref(node).to_string()) {
1230 continue;
1231 }
1232 match Self::label(node) {
1233 "marker" | "field_key" => {
1234 let inlines = self.formatted(node);
1235 if !inlines.is_empty() {
1236 term = join_inlines(vec![term, inlines]);
1237 }
1238 }
1239 "field_value" => {
1240 let inlines = self.formatted(node);
1241 if !inlines.is_empty() {
1242 defs.push(ast::plain(inlines));
1243 }
1244 }
1245 _ => defs.extend(self.serialize(node)),
1246 }
1247 }
1248 (term, defs)
1249 }
1250
1251 fn field_blocks(entries: Vec<(Vec<Value>, Vec<Value>)>) -> Vec<Value> {
1254 let mut out = Vec::new();
1255 let mut run: Vec<(Vec<Value>, Vec<Vec<Value>>)> = Vec::new();
1256 for (term, defs) in entries {
1257 if term.is_empty() {
1258 if !run.is_empty() {
1259 out.push(ast::definition_list(std::mem::take(&mut run)));
1260 }
1261 out.extend(defs);
1262 } else {
1263 let defs = defs.into_iter().map(|d| vec![d]).collect();
1264 run.push((term, defs));
1265 }
1266 }
1267 if !run.is_empty() {
1268 out.push(ast::definition_list(run));
1269 }
1270 out
1271 }
1272
1273 fn own_inlines(&mut self, item: &'a Value) -> (Vec<Value>, Vec<Value>, bool) {
1280 let children = Self::children(item);
1281 if Self::text(item).is_empty() {
1282 if let Some(group) = children
1283 .first()
1284 .and_then(|c| self.resolve(c))
1285 .filter(|c| Self::is_group(c) && Self::label(c) == "inline")
1286 {
1287 self.visited.insert(Self::self_ref(group).to_string());
1288 let (inlines, rest) = self.inline_group(group);
1289 return (inlines, rest, true);
1290 }
1291 }
1292 (self.formatted(item), Vec::new(), false)
1293 }
1294
1295 fn with_comments(&mut self, item: &'a Value, inlines: Vec<Value>) -> Vec<Value> {
1304 let refs = Self::refs(item.get("comments"));
1305 if refs.is_empty() {
1306 return inlines;
1307 }
1308 if matches!(
1311 self.track_changes,
1312 Some(TrackChanges::Accept | TrackChanges::Reject)
1313 ) {
1314 return inlines;
1315 }
1316 let mut starts = Vec::new();
1317 let mut ends = Vec::new();
1318 for cref in refs {
1319 let Some(group) = self.resolve(cref) else {
1320 continue;
1321 };
1322 if !Self::is_group(group) || Self::label(group) != "comment_section" {
1323 continue;
1324 }
1325 self.visited.insert(cref.to_string());
1326 let id = group
1327 .get("name")
1328 .and_then(Value::as_str)
1329 .and_then(|n| n.strip_prefix("comment-"))
1330 .unwrap_or("0")
1331 .to_string();
1332 let mut k = 0;
1335 for r in Self::children(group) {
1336 self.visited.insert(r.to_string());
1337 let Some(node) = self.resolve(r).filter(|_| r.starts_with("#/texts/")) else {
1338 continue;
1339 };
1340 let (author, date, text) = split_comment(Self::text(node));
1341 let comment_id = if k == 0 {
1342 id.clone()
1343 } else {
1344 format!("{id}-{k}")
1345 };
1346 k += 1;
1347 let mut attrs = vec![("comment-id".to_string(), comment_id.clone())];
1348 if let Some(a) = author {
1349 attrs.push(("author".to_string(), a));
1350 }
1351 if let Some(d) = date {
1352 attrs.push(("date".to_string(), d));
1353 }
1354 starts.push(ast::span(&["comment-start"], &attrs, text_inlines(&text)));
1355 ends.push(ast::span(
1356 &["comment-end"],
1357 &[("comment-id".to_string(), comment_id)],
1358 Vec::new(),
1359 ));
1360 }
1361 }
1362 if starts.is_empty() {
1363 return inlines;
1364 }
1365 let mut out = starts;
1366 out.extend(inlines);
1367 out.extend(ends);
1368 out
1369 }
1370
1371 fn formatted(&self, item: &Value) -> Vec<Value> {
1374 let base = match Self::label(item) {
1375 "code" => vec![ast::code(Self::text(item))],
1376 "formula" if !Self::text(item).is_empty() => {
1377 vec![ast::math(false, Self::text(item).trim())]
1378 }
1379 _ => {
1380 let (text, skipped) = Self::authored_text(item);
1381 match item.get("_notes").and_then(Value::as_array) {
1382 Some(notes) if !notes.is_empty() => {
1383 if skipped == 0 {
1384 text_with_notes(text, notes)
1385 } else {
1386 let shifted: Vec<Value> = notes
1388 .iter()
1389 .map(|n| {
1390 let off = n[0].as_u64().unwrap_or(0).saturating_sub(skipped);
1391 serde_json::json!([off, n[1]])
1392 })
1393 .collect();
1394 text_with_notes(text, &shifted)
1395 }
1396 }
1397 _ => text_inlines(text),
1398 }
1399 }
1400 };
1401 Self::decorate(base, item)
1402 }
1403
1404 fn authored_text(item: &Value) -> (&str, u64) {
1407 let text = Self::text(item);
1408 if let Some(number) = item.get("_number").and_then(Value::as_str) {
1409 if let Some(rest) = text
1410 .strip_prefix(number)
1411 .and_then(|rest| rest.strip_prefix(' '))
1412 {
1413 return (rest, (number.chars().count() + 1) as u64);
1414 }
1415 }
1416 (text, 0)
1417 }
1418
1419 fn is_note_body(item: &Value) -> bool {
1422 item.get("_note_body").and_then(Value::as_bool) == Some(true)
1423 }
1424
1425 fn decorate(mut inlines: Vec<Value>, item: &Value) -> Vec<Value> {
1426 if inlines.is_empty() {
1427 return inlines;
1428 }
1429 if let Some(f) = item.get("formatting") {
1430 let on = |k: &str| f.get(k).and_then(Value::as_bool).unwrap_or(false);
1431 if on("bold") {
1432 inlines = vec![ast::wrap("Strong", inlines)];
1433 }
1434 if on("italic") {
1435 inlines = vec![ast::wrap("Emph", inlines)];
1436 }
1437 if on("underline") {
1438 inlines = vec![ast::wrap("Underline", inlines)];
1439 }
1440 if on("strikethrough") {
1441 inlines = vec![ast::wrap("Strikeout", inlines)];
1442 }
1443 match f.get("script").and_then(Value::as_str) {
1444 Some("sub") => inlines = vec![ast::wrap("Subscript", inlines)],
1445 Some("super") => inlines = vec![ast::wrap("Superscript", inlines)],
1446 _ => {}
1447 }
1448 }
1449 if let Some(url) = item
1451 .get("hyperlink")
1452 .and_then(Value::as_str)
1453 .filter(|u| crate::hyperlink::is_safe_hyperlink(u))
1454 {
1455 inlines = vec![ast::link(inlines, url)];
1456 }
1457 if let Some(rev) = item.get("_revision") {
1460 if let Some(kind) = rev.get("kind").and_then(Value::as_str) {
1461 let mut attrs = Vec::new();
1462 for key in ["author", "date"] {
1463 if let Some(v) = rev.get(key).and_then(Value::as_str) {
1464 attrs.push((key.to_string(), v.to_string()));
1465 }
1466 }
1467 inlines = vec![ast::span(&[kind], &attrs, inlines)];
1468 }
1469 }
1470 inlines
1471 }
1472
1473 fn inline_group(&mut self, group: &'a Value) -> (Vec<Value>, Vec<Value>) {
1478 let mut nodes = Vec::new();
1479 self.descendants(group, &mut nodes);
1480 let mut runs: Vec<Value> = Vec::new();
1481 let mut rest: Vec<Value> = Vec::new();
1482 for node in nodes {
1483 let r = Self::self_ref(node).to_string();
1484 if self.visited.contains(&r) {
1485 continue;
1486 }
1487 self.visited.insert(r.clone());
1488 if r.starts_with("#/texts/") {
1489 if self.owned_captions.contains(&r)
1490 || self.owned_footnotes.contains(&r)
1491 || Self::is_note_body(node)
1492 {
1493 continue;
1494 }
1495 let mut run: Vec<Value> = Self::anchors(node)
1496 .iter()
1497 .map(|id| ast::anchor(id))
1498 .collect();
1499 let (lead, _, trail) = crate::inline::split_edges(Self::text(node));
1502 let edges = Self::label(node) != "code";
1503 if edges && !lead.is_empty() {
1504 runs.push(ast::space());
1505 }
1506 runs.append(&mut run);
1507 runs.append(&mut self.formatted(node));
1508 if edges && !trail.is_empty() {
1509 runs.push(ast::space());
1510 }
1511 continue;
1512 }
1513 if Self::is_group(node) && Self::label(node) == "inline" {
1514 let (mut inl, mut more) = self.inline_group(node);
1515 runs.append(&mut inl);
1516 rest.append(&mut more);
1517 continue;
1518 }
1519 rest.extend(self.serialize(node));
1520 }
1521 (concat_runs(runs), rest)
1522 }
1523
1524 fn text_item(&mut self, item: &'a Value) -> Vec<Value> {
1525 let label = Self::label(item);
1526 let mut out = Vec::new();
1527 match label {
1528 "list_item" => {
1529 let blocks = self.list_item_blocks(item);
1531 if !blocks.is_empty() {
1532 out.push(ast::bullet_list(vec![blocks]));
1533 }
1534 return out;
1535 }
1536 "code" => {
1537 let lang = item
1538 .get("code_language")
1539 .and_then(Value::as_str)
1540 .filter(|l| !l.is_empty() && *l != "unknown")
1541 .map(str::to_ascii_lowercase);
1542 out.push(ast::code_block(lang.as_deref(), Self::text(item)));
1543 }
1544 "formula" => {
1545 let tex = Self::text(item).trim();
1546 if !tex.is_empty() {
1547 out.push(ast::para(vec![ast::math(true, tex)]));
1548 }
1549 }
1550 _ => {
1551 let (inlines, rest, _) = self.own_inlines(item);
1552 let mut inlines = self.with_comments(item, inlines);
1553 let mut anchors = Self::anchors(item);
1557 let header_id =
1558 if matches!(label, "title" | "section_header") && !anchors.is_empty() {
1559 Some(anchors.remove(0))
1560 } else {
1561 None
1562 };
1563 if !anchors.is_empty() && (header_id.is_some() || !inlines.is_empty()) {
1564 let mut with_anchors: Vec<Value> =
1565 anchors.iter().map(|id| ast::anchor(id)).collect();
1566 with_anchors.append(&mut inlines);
1567 inlines = with_anchors;
1568 }
1569 match label {
1570 "title" | "section_header" => {
1571 let level = if label == "title" {
1572 1
1573 } else {
1574 let n = item.get("level").and_then(Value::as_u64).unwrap_or(1) as usize;
1575 match self.heading_depth {
1576 HeadingDepth::Docling => (n + 1).min(6),
1579 HeadingDepth::Source => n.clamp(1, 6),
1583 }
1584 };
1585 if !inlines.is_empty() {
1586 out.push(match &header_id {
1587 Some(id) => ast::header_with_id(level, id, inlines),
1588 None => ast::header(level, inlines),
1589 });
1590 }
1591 }
1592 "text" | "paragraph" => {
1593 if !inlines.is_empty() {
1594 out.push(ast::para(inlines));
1595 }
1596 }
1597 "checkbox_selected" | "checkbox_unselected" => {
1598 let mark = if label == "checkbox_selected" {
1599 "☒"
1600 } else {
1601 "☐"
1602 };
1603 let mut with_mark = vec![ast::str_(mark)];
1604 if !inlines.is_empty() {
1605 with_mark.push(ast::space());
1606 with_mark.append(&mut inlines);
1607 }
1608 out.push(ast::para(with_mark));
1609 }
1610 other => {
1611 if !inlines.is_empty() {
1612 let class = format!("docling-{other}");
1613 out.push(ast::div(&[&class], vec![ast::para(inlines)]));
1614 }
1615 }
1616 }
1617 out.extend(rest);
1618 }
1619 }
1620 out.extend(self.blocks_of(item));
1622 out
1623 }
1624
1625 fn list_item_blocks(&mut self, item: &'a Value) -> Vec<Value> {
1628 let (inlines, rest, _) = self.own_inlines(item);
1629 let inlines = self.with_comments(item, inlines);
1630 let mut blocks = Vec::new();
1631 if !inlines.is_empty() {
1632 blocks.push(ast::plain(inlines));
1633 }
1634 blocks.extend(rest);
1635 blocks.extend(self.blocks_of(item));
1636 blocks
1637 }
1638
1639 fn list_group(&mut self, group: &'a Value) -> Vec<Value> {
1640 let first = Self::children(group)
1641 .first()
1642 .and_then(|r| self.resolve(r))
1643 .filter(|c| Self::label(c) == "list_item");
1644 let enumerated = first.is_some_and(|c| {
1645 c.get("enumerated")
1646 .and_then(Value::as_bool)
1647 .unwrap_or(false)
1648 });
1649 let markers: Vec<&str> = Self::children(group)
1652 .iter()
1653 .filter_map(|r| self.resolve(r))
1654 .filter(|c| Self::label(c) == "list_item")
1655 .filter_map(|c| c.get("marker").and_then(Value::as_str))
1656 .collect();
1657 let (start, style, delim) = list_attributes(&markers);
1658
1659 let mut nodes = Vec::new();
1660 self.descendants(group, &mut nodes);
1661 let mut items: Vec<Vec<Value>> = Vec::new();
1662 for node in nodes {
1663 let r = Self::self_ref(node).to_string();
1664 if !self.visited.insert(r.clone()) {
1665 continue;
1666 }
1667 if Self::label(node) == "list_item"
1668 && r.starts_with("#/texts/")
1669 && !self.owned_captions.contains(&r)
1670 {
1671 let blocks = self.list_item_blocks(node);
1672 items.push(blocks);
1673 continue;
1674 }
1675 let blocks = self.serialize(node);
1678 if blocks.is_empty() {
1679 continue;
1680 }
1681 match items.last_mut() {
1682 Some(last) => last.extend(blocks),
1683 None => items.push(blocks),
1684 }
1685 }
1686 if items.is_empty() {
1687 return Vec::new();
1688 }
1689 if enumerated {
1690 vec![ast::ordered_list(start, style, delim, items)]
1691 } else {
1692 vec![ast::bullet_list(items)]
1693 }
1694 }
1695
1696 fn caption(&mut self, item: &'a Value) -> Value {
1702 let mut inlines = self.caption_text(item);
1703 for cref in Self::refs(item.get("footnotes")) {
1704 if let Some(note) = self.resolve(cref) {
1705 if cref.starts_with("#/texts/") && !self.excluded(note) {
1706 let (text, _, _) = self.own_inlines(note);
1707 if !text.is_empty() {
1708 inlines.push(ast::note(vec![ast::para(text)]));
1709 }
1710 }
1711 }
1712 }
1713 if inlines.is_empty() {
1714 ast::caption(Vec::new())
1715 } else {
1716 ast::caption(vec![ast::plain(inlines)])
1717 }
1718 }
1719
1720 fn caption_text(&mut self, item: &'a Value) -> Vec<Value> {
1723 let caps: Vec<&'a Value> = Self::refs(item.get("captions"))
1724 .into_iter()
1725 .filter(|cref| cref.starts_with("#/texts/"))
1726 .filter_map(|cref| self.resolve(cref))
1727 .filter(|cap| !self.excluded(cap))
1728 .collect();
1729 let mut parts = Vec::new();
1730 for cap in caps {
1731 let (inlines, _, _) = self.own_inlines(cap);
1732 parts.push(inlines);
1733 }
1734 join_inlines(parts)
1735 }
1736
1737 fn grid(table: &TableView<'_>) -> (usize, Vec<Vec<Option<GridCell>>>) {
1739 let (num_rows, num_cols) = (table.num_rows, table.num_cols);
1740 let mut grid: Vec<Vec<Option<GridCell>>> = (0..num_rows)
1741 .map(|_| (0..num_cols).map(|_| None).collect())
1742 .collect();
1743 for (i, cell) in table.cells.iter().enumerate() {
1744 let (r0, c0) = (cell.start_row, cell.start_col);
1745 let (r1, c1) = (cell.end_row, cell.end_col);
1746 if r0 >= num_rows || c0 >= num_cols {
1747 continue;
1748 }
1749 let (r1, c1) = (r1.clamp(r0 + 1, num_rows), c1.clamp(c0 + 1, num_cols));
1750 for row in grid.iter_mut().take(r1).skip(r0) {
1751 for slot in row.iter_mut().take(c1).skip(c0) {
1752 *slot = Some(GridCell {
1753 cell: i,
1754 start_row: r0,
1755 start_col: c0,
1756 row_span: r1 - r0,
1757 col_span: c1 - c0,
1758 });
1759 }
1760 }
1761 }
1762 (num_cols, grid)
1763 }
1764
1765 fn table_from_data(&mut self, data: &'a Value, caption: Value) -> Option<Value> {
1766 let table = crate::json::table_view(data, self.tables);
1767 let (num_cols, grid) = Self::grid(&table);
1768 if grid.is_empty() || num_cols == 0 {
1769 return None;
1770 }
1771 let mut rows: Vec<(bool, Value)> = Vec::new();
1772 for (i, row) in grid.iter().enumerate() {
1773 let mut cells = Vec::new();
1774 let mut all_header = true;
1775 let mut anchored = 0;
1776 for (j, slot) in row.iter().enumerate() {
1777 let Some(g) = slot else {
1778 cells.push(ast::cell(1, 1, Vec::new()));
1781 continue;
1782 };
1783 if g.start_row != i || g.start_col != j {
1784 continue;
1785 }
1786 anchored += 1;
1787 let cell = &table.cells[g.cell];
1788 all_header &= cell.column_header;
1789 let blocks = match cell.cell_ref {
1790 Some(cref) => match self.resolve(cref) {
1791 Some(target) => {
1792 self.visited.insert(cref.to_string());
1793 self.serialize(target)
1794 }
1795 None => Vec::new(),
1796 },
1797 None => {
1798 let inl = text_inlines(cell.text);
1799 if inl.is_empty() {
1800 Vec::new()
1801 } else {
1802 vec![ast::plain(inl)]
1803 }
1804 }
1805 };
1806 cells.push(ast::cell(g.row_span, g.col_span, blocks));
1807 }
1808 rows.push((all_header && anchored > 0, ast::row(cells)));
1809 }
1810 let head_len = rows.iter().take_while(|(h, _)| *h).count();
1812 let head_len = if head_len == rows.len() { 0 } else { head_len };
1813 let mut head = Vec::new();
1814 let mut body = Vec::new();
1815 for (i, (_, row)) in rows.into_iter().enumerate() {
1816 if i < head_len {
1817 head.push(row);
1818 } else {
1819 body.push(row);
1820 }
1821 }
1822 Some(ast::table(caption, num_cols, head, body))
1823 }
1824
1825 fn table(&mut self, item: &'a Value) -> Vec<Value> {
1826 let caption = self.caption(item);
1827 if self.excluded(item) {
1828 return Vec::new();
1829 }
1830 match item.get("data") {
1831 Some(data) => self.table_from_data(data, caption).into_iter().collect(),
1832 None => Vec::new(),
1833 }
1834 }
1835
1836 fn picture(&mut self, item: &'a Value) -> Vec<Value> {
1837 if self.excluded(item) {
1838 return Vec::new();
1839 }
1840 let caption = self.caption(item);
1841 let has_caption = caption
1842 .get(1)
1843 .and_then(Value::as_array)
1844 .is_some_and(|b| !b.is_empty());
1845 let uri = item
1846 .get("image")
1847 .and_then(|i| i.get("uri"))
1848 .and_then(Value::as_str);
1849 let src = match (self.image_mode, uri) {
1850 (ImageMode::Embedded, Some(uri)) if uri.starts_with("data:") => Some(uri.to_string()),
1851 (ImageMode::Referenced, Some(uri)) => self
1852 .reference_image(uri)
1853 .or_else(|| (!uri.starts_with("data:")).then(|| uri.to_string())),
1854 _ => None,
1855 };
1856 let description = item
1865 .get("meta")
1866 .and_then(|m| m.get("description"))
1867 .and_then(|d| d.get("text"))
1868 .and_then(Value::as_str)
1869 .map(text_inlines)
1870 .filter(|t| !t.is_empty());
1871 let alt = match description {
1872 Some(alt) => alt,
1873 None => self.caption_text(item),
1874 };
1875 let attrs: Vec<(String, String)> = item
1878 .get("image")
1879 .and_then(|i| i.get("_display"))
1880 .and_then(Value::as_array)
1881 .and_then(|d| Some((d.first()?.as_f64()?, d.get(1)?.as_f64()?)))
1882 .map(|(w, h)| {
1883 vec![
1884 ("width".to_string(), inches(w)),
1885 ("height".to_string(), inches(h)),
1886 ]
1887 })
1888 .unwrap_or_default();
1889 let image = match &src {
1890 Some(src) => ast::image_sized(&[], &attrs, alt, src),
1891 None => ast::image_sized(&["docling-placeholder"], &attrs, alt, ""),
1892 };
1893 let chart = item
1895 .get("meta")
1896 .and_then(|m| m.get("tabular_chart"))
1897 .and_then(|t| t.get("chart_data"))
1898 .and_then(|chart| self.table_from_data(chart, ast::caption(Vec::new())));
1899 if chart.is_none() && !has_caption {
1900 return vec![ast::para(vec![image])];
1902 }
1903 let mut blocks = vec![ast::plain(vec![image])];
1904 blocks.extend(chart);
1905 vec![ast::figure(&[], caption, blocks)]
1906 }
1907
1908 fn reference_image(&mut self, uri: &str) -> Option<String> {
1911 let rest = uri.strip_prefix("data:")?;
1912 let (mime, payload) = rest.split_once(";base64,")?;
1913 let bytes = crate::base64::decode(payload)?;
1914 let path = format!(
1915 "{}/image_{:06}.{}",
1916 self.artifacts_dir,
1917 self.pic_index,
1918 crate::markdown::ext_for(mime)
1919 );
1920 self.pic_index += 1;
1921 self.artifacts.push((path.clone(), bytes));
1922 Some(path)
1923 }
1924
1925 fn graph_item(&mut self, item: &'a Value, class: &str) -> Vec<Value> {
1930 let mut out = Vec::new();
1931 if !self.excluded(item) {
1932 if let Some(graph) = item.get("graph") {
1933 if let Some(block) = Self::graph(graph) {
1934 out.push(ast::div(&[class], vec![block]));
1935 }
1936 }
1937 }
1938 let caption = self.caption(item);
1940 for block in caption
1941 .get(1)
1942 .and_then(Value::as_array)
1943 .into_iter()
1944 .flatten()
1945 {
1946 if let Some(inlines) = block.get("c").and_then(Value::as_array) {
1947 out.push(ast::para(inlines.clone()));
1948 }
1949 }
1950 out
1951 }
1952
1953 fn graph(graph: &'a Value) -> Option<Value> {
1954 let cells: HashMap<u64, &Value> = Self::array(graph.get("cells"))
1955 .filter_map(|c| Some((c.get("cell_id")?.as_u64()?, c)))
1956 .collect();
1957 let order: Vec<u64> = Self::array(graph.get("cells"))
1958 .filter_map(|c| c.get("cell_id")?.as_u64())
1959 .collect();
1960 let mut child_links: HashMap<u64, Vec<u64>> = HashMap::new();
1961 let mut value_links: BTreeMap<u64, Vec<u64>> = BTreeMap::new();
1962 let mut value_order: Vec<u64> = Vec::new();
1963 let mut parents: HashSet<u64> = HashSet::new();
1964 for link in Self::array(graph.get("links")) {
1965 let (Some(s), Some(t)) = (
1966 link.get("source_cell_id").and_then(Value::as_u64),
1967 link.get("target_cell_id").and_then(Value::as_u64),
1968 ) else {
1969 continue;
1970 };
1971 if !cells.contains_key(&s) || !cells.contains_key(&t) {
1972 continue;
1973 }
1974 match link.get("label").and_then(Value::as_str) {
1975 Some("to_child") => {
1976 child_links.entry(s).or_default().push(t);
1977 parents.insert(t);
1978 }
1979 Some("to_value") => {
1980 if !value_links.contains_key(&s) {
1981 value_order.push(s);
1982 }
1983 value_links.entry(s).or_default().push(t);
1984 }
1985 _ => {}
1986 }
1987 }
1988 let cell_text =
1989 |id: u64| text_inlines(cells[&id].get("text").and_then(Value::as_str).unwrap_or(""));
1990 if child_links.is_empty() {
1991 if value_order.is_empty() {
1992 return None;
1993 }
1994 let entries = value_order
1995 .iter()
1996 .map(|&k| {
1997 let defs = value_links[&k]
1998 .iter()
1999 .map(|&v| vec![ast::plain(cell_text(v))])
2000 .collect();
2001 (cell_text(k), defs)
2002 })
2003 .collect();
2004 return Some(ast::definition_list(entries));
2005 }
2006 fn node(
2008 id: u64,
2009 cell_text: &dyn Fn(u64) -> Vec<Value>,
2010 child_links: &HashMap<u64, Vec<u64>>,
2011 value_links: &BTreeMap<u64, Vec<u64>>,
2012 path: &mut HashSet<u64>,
2013 ) -> Option<Vec<Value>> {
2014 if !path.insert(id) {
2015 return None;
2016 }
2017 let mut inlines = cell_text(id);
2018 if let Some(values) = value_links.get(&id) {
2019 let vals = join_inlines(values.iter().map(|&v| cell_text(v)).collect());
2020 inlines = vec![ast::wrap("Strong", inlines)];
2021 inlines.push(ast::str_(":"));
2022 if !vals.is_empty() {
2023 inlines.push(ast::space());
2024 inlines.extend(vals);
2025 }
2026 }
2027 let mut blocks = vec![ast::plain(inlines)];
2028 let kids: Vec<Vec<Value>> = child_links
2029 .get(&id)
2030 .into_iter()
2031 .flatten()
2032 .filter_map(|&c| node(c, cell_text, child_links, value_links, path))
2033 .collect();
2034 if !kids.is_empty() {
2035 blocks.push(ast::bullet_list(kids));
2036 }
2037 path.remove(&id);
2038 Some(blocks)
2039 }
2040 let mut path = HashSet::new();
2041 let items: Vec<Vec<Value>> = order
2042 .iter()
2043 .filter(|id| !parents.contains(id))
2044 .filter_map(|&id| node(id, &cell_text, &child_links, &value_links, &mut path))
2045 .collect();
2046 (!items.is_empty()).then(|| ast::bullet_list(items))
2047 }
2048}
2049
2050#[cfg(test)]
2051mod tests {
2052 use super::*;
2053 use serde_json::json;
2054
2055 fn doc(body: &[&str], texts: Value, groups: Value, tables: Value) -> Value {
2057 json!({
2058 "schema_name": "DoclingDocument",
2059 "name": "t",
2060 "body": {"self_ref": "#/body", "children": body.iter().map(|r| json!({"$ref": r})).collect::<Vec<_>>()},
2061 "texts": texts, "groups": groups, "tables": tables, "pictures": []
2062 })
2063 }
2064
2065 fn text(i: usize, label: &str, text: &str, extra: Value) -> Value {
2066 let mut v = json!({"self_ref": format!("#/texts/{i}"), "children": [], "label": label, "text": text, "content_layer": "body"});
2067 for (k, x) in extra.as_object().unwrap() {
2068 v[k] = x.clone();
2069 }
2070 v
2071 }
2072
2073 fn blocks(json: &Value) -> Vec<Value> {
2074 let (s, _) = from_docling_json(json, &PandocExportOptions::default()).unwrap();
2075 let v: Value = serde_json::from_str(&s).unwrap();
2076 assert_eq!(v["pandoc-api-version"], json!(PANDOC_API_VERSION));
2077 v["blocks"].as_array().unwrap().clone()
2078 }
2079
2080 #[test]
2084 fn note_calls_become_notes_at_their_call_sites() {
2085 let j = doc(
2086 &["#/texts/0", "#/texts/1"],
2087 json!([
2088 text(
2089 0,
2090 "text",
2091 "debut suite end",
2092 json!({"_notes": [[5, "first"], [15, "last"]]})
2093 ),
2094 text(
2095 1,
2096 "footnote",
2097 "first",
2098 json!({"content_layer": "furniture", "_note_body": true})
2099 ),
2100 ]),
2101 json!([]),
2102 json!([]),
2103 );
2104 let note = |t: &str| ast::note(vec![ast::para(vec![ast::str_(t)])]);
2105 assert_eq!(
2106 blocks(&j),
2107 vec![ast::para(vec![
2108 ast::str_("debut"),
2109 note("first"),
2110 ast::space(),
2111 ast::str_("suite"),
2112 ast::space(),
2113 ast::str_("end"),
2114 note("last"),
2115 ])]
2116 );
2117 let opts = PandocExportOptions {
2119 layers: ContentLayers::ALL,
2120 ..Default::default()
2121 };
2122 let (s, _) = from_docling_json(&j, &opts).unwrap();
2123 assert!(!s.contains("docling-footnote"), "{s}");
2124 }
2125
2126 #[test]
2129 fn uncalled_furniture_footnotes_trail_as_notes() {
2130 let j = doc(
2131 &["#/texts/0", "#/texts/1"],
2132 json!([
2133 text(0, "text", "body", json!({})),
2134 text(
2135 1,
2136 "footnote",
2137 "MARKFN",
2138 json!({"content_layer": "furniture"})
2139 ),
2140 ]),
2141 json!([]),
2142 json!([]),
2143 );
2144 assert_eq!(
2145 blocks(&j),
2146 vec![
2147 ast::para(vec![ast::str_("body")]),
2148 ast::para(vec![ast::note(vec![ast::para(vec![ast::str_("MARKFN")])])]),
2149 ]
2150 );
2151 }
2152
2153 #[test]
2158 fn pictures_are_always_images() {
2159 let mut j = doc(
2160 &["#/pictures/0", "#/pictures/1", "#/pictures/2"],
2161 json!([text(0, "caption", "A duck", json!({})),]),
2162 json!([]),
2163 json!([]),
2164 );
2165 j["pictures"] = json!([
2166 {"self_ref": "#/pictures/0", "children": [], "label": "picture", "content_layer": "body",
2167 "image": {"uri": "data:image/png;base64,AAAA"}, "captions": []},
2168 {"self_ref": "#/pictures/1", "children": [], "label": "picture", "content_layer": "body",
2169 "image": {"uri": "data:image/png;base64,BBBB"}, "captions": [{"$ref": "#/texts/0"}]},
2170 {"self_ref": "#/pictures/2", "children": [], "label": "picture", "content_layer": "body",
2171 "captions": []},
2172 ]);
2173 let b = blocks(&j);
2174 assert_eq!(
2175 b[0],
2176 ast::para(vec![ast::image(&[], vec![], "data:image/png;base64,AAAA")])
2177 );
2178 assert_eq!(
2179 b[1],
2180 ast::figure(
2181 &[],
2182 ast::caption(vec![ast::plain(vec![
2183 ast::str_("A"),
2184 ast::space(),
2185 ast::str_("duck")
2186 ])]),
2187 vec![ast::plain(vec![ast::image(
2188 &[],
2189 vec![ast::str_("A"), ast::space(), ast::str_("duck")],
2190 "data:image/png;base64,BBBB"
2191 )])]
2192 )
2193 );
2194 assert_eq!(
2195 b[2],
2196 ast::para(vec![ast::image(&["docling-placeholder"], vec![], "")])
2197 );
2198 assert_eq!(b.len(), 3);
2199 let opts = PandocExportOptions {
2201 image_mode: ImageMode::Placeholder,
2202 ..Default::default()
2203 };
2204 let (s, _) = from_docling_json(&j, &opts).unwrap();
2205 assert!(!s.contains("base64") && !s.contains("RawBlock"), "{s}");
2206 assert_eq!(s.matches("docling-placeholder").count(), 3, "{s}");
2207 }
2208
2209 #[test]
2216 fn captions_headings_and_items_keep_their_inline_groups() {
2217 let mut j = doc(
2218 &["#/texts/0", "#/tables/0", "#/texts/3", "#/groups/2"],
2219 json!([
2220 text(
2221 0,
2222 "caption",
2223 "",
2224 json!({"children": [{"$ref": "#/groups/0"}]})
2225 ),
2226 text(
2227 1,
2228 "text",
2229 "Table 1: ",
2230 json!({"formatting": {"bold": true}})
2231 ),
2232 text(
2233 2,
2234 "text",
2235 "Multi-run caption",
2236 json!({"hyperlink": "https://x.org/c"})
2237 ),
2238 text(
2239 3,
2240 "section_header",
2241 "",
2242 json!({"level": 1, "children": [{"$ref": "#/groups/1"}, {"$ref": "#/texts/6"}]})
2243 ),
2244 text(4, "text", "1.1 ", json!({"formatting": {"bold": true}})),
2245 text(5, "text", "Formatted header", json!({})),
2246 text(6, "text", "Body under header.", json!({})),
2247 text(
2248 7,
2249 "list_item",
2250 "",
2251 json!({"children": [{"$ref": "#/groups/3"}, {"$ref": "#/groups/4"}]})
2252 ),
2253 text(
2254 8,
2255 "text",
2256 "Bold run ",
2257 json!({"formatting": {"bold": true}})
2258 ),
2259 text(9, "text", "and tail", json!({})),
2260 text(10, "list_item", "Nested child", json!({})),
2261 text(
2262 11,
2263 "caption",
2264 "Linked bold",
2265 json!({"formatting": {"bold": true}, "hyperlink": "https://x.org/spec"})
2266 ),
2267 ]),
2268 json!([
2269 {"self_ref": "#/groups/0", "children": [{"$ref": "#/texts/1"}, {"$ref": "#/texts/2"}], "label": "inline", "name": "group"},
2270 {"self_ref": "#/groups/1", "children": [{"$ref": "#/texts/4"}, {"$ref": "#/texts/5"}], "label": "inline", "name": "group"},
2271 {"self_ref": "#/groups/2", "children": [{"$ref": "#/texts/7"}], "label": "list", "name": "list"},
2272 {"self_ref": "#/groups/3", "children": [{"$ref": "#/texts/8"}, {"$ref": "#/texts/9"}], "label": "inline", "name": "group"},
2273 {"self_ref": "#/groups/4", "children": [{"$ref": "#/texts/10"}], "label": "list", "name": "list"},
2274 ]),
2275 json!([{"self_ref": "#/tables/0", "children": [], "label": "table",
2276 "captions": [{"$ref": "#/texts/0"}, {"$ref": "#/texts/11"}],
2277 "data": {"num_rows": 1, "num_cols": 1, "table_cells": [
2278 {"text": "v", "row_span": 1, "col_span": 1, "start_row_offset_idx": 0, "end_row_offset_idx": 1,
2279 "start_col_offset_idx": 0, "end_col_offset_idx": 1, "column_header": false}]}}]),
2280 );
2281 j["texts"][0]["children"] = json!([{"$ref": "#/groups/0"}]);
2282 let b = blocks(&j);
2283 assert_eq!(b.len(), 4, "{b:?}");
2284 assert_eq!(b[0]["t"], "Table");
2285 let caption = &b[0]["c"][1][1][0];
2286 assert_eq!(caption["t"], "Plain");
2287 assert_eq!(
2288 caption["c"][0],
2289 ast::wrap(
2290 "Strong",
2291 vec![ast::str_("Table"), ast::space(), ast::str_("1:")]
2292 )
2293 );
2294 assert_eq!(caption["c"][2]["t"], "Link");
2295 let linked = caption["c"].as_array().unwrap().last().unwrap();
2296 assert_eq!(linked["t"], "Link");
2297 assert_eq!(linked["c"][1][0]["t"], "Strong");
2298 assert_eq!(b[1]["t"], "Header");
2299 assert_eq!(b[1]["c"][2][0], ast::wrap("Strong", vec![ast::str_("1.1")]));
2300 assert_eq!(
2301 b[2],
2302 ast::para(vec![
2303 ast::str_("Body"),
2304 ast::space(),
2305 ast::str_("under"),
2306 ast::space(),
2307 ast::str_("header.")
2308 ])
2309 );
2310 assert_eq!(b[3]["t"], "BulletList");
2311 let item = b[3]["c"][0].as_array().unwrap();
2312 assert_eq!(item.len(), 2);
2313 assert_eq!(item[0]["t"], "Plain");
2314 assert_eq!(item[0]["c"][0]["t"], "Strong");
2315 assert_eq!(item[1]["t"], "BulletList");
2316 }
2317
2318 #[test]
2322 fn pictures_carry_alt_text_and_display_size() {
2323 let mut j = doc(
2324 &["#/pictures/0", "#/pictures/1"],
2325 json!([text(0, "caption", "A duck", json!({})),]),
2326 json!([]),
2327 json!([]),
2328 );
2329 j["pictures"] = json!([
2330 {"self_ref": "#/pictures/0", "children": [], "label": "picture", "content_layer": "body",
2331 "meta": {"description": {"text": "Architecture diagram", "created_by": "alt_text"}},
2332 "image": {"uri": "data:image/png;base64,AAAA", "_display": [2.5, 1.25]}, "captions": [{"$ref": "#/texts/0"}]},
2333 {"self_ref": "#/pictures/1", "children": [], "label": "picture", "content_layer": "body",
2334 "image": {"uri": "data:image/png;base64,BBBB"}, "captions": [{"$ref": "#/texts/0"}]},
2335 ]);
2336 let b = blocks(&j);
2337 let image = &b[0]["c"][2][0]["c"][0];
2338 assert_eq!(image["t"], "Image");
2339 assert_eq!(
2340 image["c"][0],
2341 json!(["", [], [["width", "2.5in"], ["height", "1.25in"]]])
2342 );
2343 assert_eq!(
2344 image["c"][1],
2345 json!([
2346 ast::str_("Architecture"),
2347 ast::space(),
2348 ast::str_("diagram")
2349 ])
2350 );
2351 let image = &b[1]["c"][2][0]["c"][0];
2352 assert_eq!(image["c"][0], json!(["", [], []]));
2353 assert_eq!(
2354 image["c"][1],
2355 json!([ast::str_("A"), ast::space(), ast::str_("duck")])
2356 );
2357 assert_eq!(inches(2.5), "2.5in");
2358 assert_eq!(inches(1.0), "1in");
2359 assert_eq!(inches(0.33333), "0.3333in");
2360 }
2361
2362 #[test]
2364 fn list_attributes_follow_the_markers() {
2365 assert_eq!(
2366 list_attributes(&["a)", "b)"]),
2367 (1, "LowerAlpha", "OneParen")
2368 );
2369 assert_eq!(
2370 list_attributes(&["iii.", "iv."]),
2371 (3, "LowerRoman", "Period")
2372 );
2373 assert_eq!(
2374 list_attributes(&["(3)", "(4)"]),
2375 (3, "Decimal", "TwoParens")
2376 );
2377 assert_eq!(
2378 list_attributes(&["c)", "d)"]),
2379 (3, "LowerAlpha", "OneParen")
2380 );
2381 assert_eq!(list_attributes(&["i."]), (1, "LowerRoman", "Period"));
2382 assert_eq!(list_attributes(&["v.", "vi."]), (5, "LowerRoman", "Period"));
2383 assert_eq!(list_attributes(&["x.", "y."]), (24, "LowerAlpha", "Period"));
2384 assert_eq!(list_attributes(&["B."]), (2, "UpperAlpha", "Period"));
2385 assert_eq!(list_attributes(&["IV)"]), (4, "UpperRoman", "OneParen"));
2386 assert_eq!(list_attributes(&["aa)"]), (27, "LowerAlpha", "OneParen"));
2387 assert_eq!(list_attributes(&["12"]), (12, "Decimal", "DefaultDelim"));
2388 assert_eq!(list_attributes(&["3."]), (3, "Decimal", "Period"));
2389 assert_eq!(list_attributes(&["1.2."]), (1, "Decimal", "Period"));
2390 assert_eq!(list_attributes(&[]), (1, "Decimal", "Period"));
2391 let j = doc(
2392 &["#/groups/0"],
2393 json!([
2394 text(
2395 0,
2396 "list_item",
2397 "third",
2398 json!({"enumerated": true, "marker": "(3)"})
2399 ),
2400 text(
2401 1,
2402 "list_item",
2403 "fourth",
2404 json!({"enumerated": true, "marker": "(4)"})
2405 ),
2406 ]),
2407 json!([{"self_ref": "#/groups/0", "children": [{"$ref": "#/texts/0"}, {"$ref": "#/texts/1"}], "label": "list", "name": "list"}]),
2408 json!([]),
2409 );
2410 let b = blocks(&j);
2411 assert_eq!(
2412 b[0]["c"][0],
2413 json!([3, {"t": "Decimal"}, {"t": "TwoParens"}])
2414 );
2415 }
2416
2417 #[test]
2422 fn metadata_and_comments_reach_the_ast() {
2423 let mut j = doc(
2424 &["#/texts/0", "#/groups/0"],
2425 json!([
2426 text(
2427 0,
2428 "text",
2429 "The timeout is 30 s.",
2430 json!({"comments": [{"$ref": "#/groups/0"}]})
2431 ),
2432 text(
2433 1,
2434 "text",
2435 "[author: Reviewer (SR), time: 2026-01-15T12:00:00Z]: Verify it.",
2436 json!({"content_layer": "notes"})
2437 ),
2438 text(2, "text", "Agreed", json!({"content_layer": "notes"})),
2439 ]),
2440 json!([{"self_ref": "#/groups/0", "children": [{"$ref": "#/texts/1"}, {"$ref": "#/texts/2"}], "label": "comment_section", "name": "comment-3", "content_layer": "notes"}]),
2441 json!([]),
2442 );
2443 j["metadata"] = json!({
2444 "title": "Synthetic Spec", "authors": ["A. Author", "B. Author"], "keywords": ["architecture", "protocol"],
2445 "language": "en-US", "created": "2026-01-15T10:00:00Z", "subject": "Gateways"
2446 });
2447 let (s, _) = from_docling_json(&j, &PandocExportOptions::default()).unwrap();
2448 let v: Value = serde_json::from_str(&s).unwrap();
2449 assert_eq!(
2450 v["meta"]["title"],
2451 ast::meta_inlines(vec![
2452 ast::str_("Synthetic"),
2453 ast::space(),
2454 ast::str_("Spec")
2455 ])
2456 );
2457 assert_eq!(v["meta"]["author"]["t"], "MetaList");
2458 assert_eq!(v["meta"]["author"]["c"].as_array().unwrap().len(), 2);
2459 assert_eq!(
2460 v["meta"]["keywords"]["c"][1],
2461 ast::meta_inlines(vec![ast::str_("protocol")])
2462 );
2463 assert_eq!(v["meta"]["lang"], ast::meta_string("en-US"));
2464 assert_eq!(
2465 v["meta"]["date"],
2466 ast::meta_inlines(vec![ast::str_("2026-01-15T10:00:00Z")])
2467 );
2468 assert_eq!(v["meta"]["subject"]["t"], "MetaInlines");
2469 let b = v["blocks"].as_array().unwrap();
2470 assert_eq!(b.len(), 1, "{b:?}");
2471 let inl = b[0]["c"].as_array().unwrap();
2472 assert_eq!(inl[0]["t"], "Span");
2473 assert_eq!(
2474 inl[0]["c"][0],
2475 json!([
2476 "",
2477 ["comment-start"],
2478 [
2479 ["comment-id", "3"],
2480 ["author", "Reviewer (SR)"],
2481 ["date", "2026-01-15T12:00:00Z"]
2482 ]
2483 ])
2484 );
2485 assert_eq!(
2486 inl[0]["c"][1],
2487 json!([ast::str_("Verify"), ast::space(), ast::str_("it.")])
2488 );
2489 assert_eq!(
2490 inl[1]["c"][0],
2491 json!(["", ["comment-start"], [["comment-id", "3-1"]]])
2492 );
2493 assert_eq!(inl[2], ast::str_("The"));
2494 let last = inl.last().unwrap();
2495 assert_eq!(
2496 last["c"][0],
2497 json!(["", ["comment-end"], [["comment-id", "3-1"]]])
2498 );
2499 assert_eq!(
2500 inl[inl.len() - 2]["c"][0],
2501 json!(["", ["comment-end"], [["comment-id", "3"]]])
2502 );
2503 let opts = PandocExportOptions {
2505 layers: ContentLayers::ALL,
2506 ..Default::default()
2507 };
2508 let (s, _) = from_docling_json(&j, &opts).unwrap();
2509 assert_eq!(s.matches("Verify").count(), 1, "{s}");
2510 assert_eq!(
2511 split_comment("[time: t]: x"),
2512 (None, Some("t".into()), "x".into())
2513 );
2514 assert_eq!(split_comment("bare"), (None, None, "bare".into()));
2515 assert_eq!(meta_of(&json!({})), json!({}));
2516 }
2517
2518 #[test]
2519 fn text_splits_into_words_spaces_and_line_breaks() {
2520 assert_eq!(
2521 text_inlines(" two words\nnext\u{a0}line "),
2522 vec![
2523 ast::str_("two"),
2524 ast::space(),
2525 ast::str_("words"),
2526 ast::line_break(),
2527 ast::str_("next\u{a0}line"),
2528 ]
2529 );
2530 assert!(text_inlines(" ").is_empty());
2531 }
2532
2533 #[test]
2534 fn headings_paragraphs_and_lists() {
2535 let j = doc(
2536 &["#/texts/0", "#/texts/1", "#/texts/2", "#/groups/0"],
2537 json!([
2538 text(0, "title", "Ducks", json!({})),
2539 text(1, "section_header", "Diet", json!({"level": 1})),
2540 text(
2541 2,
2542 "text",
2543 "Ducks eat plants.",
2544 json!({"formatting": {"bold": true}, "hyperlink": "https://x.org"})
2545 ),
2546 text(
2547 3,
2548 "list_item",
2549 "seeds",
2550 json!({"enumerated": true, "marker": "3."})
2551 ),
2552 text(
2553 4,
2554 "list_item",
2555 "insects",
2556 json!({"enumerated": true, "marker": "4."})
2557 ),
2558 ]),
2559 json!([{"self_ref": "#/groups/0", "children": [{"$ref": "#/texts/3"}, {"$ref": "#/texts/4"}], "label": "list", "name": "list"}]),
2560 json!([]),
2561 );
2562 let b = blocks(&j);
2563 assert_eq!(b[0], ast::header(1, vec![ast::str_("Ducks")]));
2564 assert_eq!(b[1], ast::header(2, vec![ast::str_("Diet")]));
2565 assert_eq!(b[2]["t"], "Para");
2567 assert_eq!(b[2]["c"][0]["t"], "Link");
2568 assert_eq!(b[2]["c"][0]["c"][1][0]["t"], "Strong");
2569 assert_eq!(b[3]["t"], "OrderedList");
2570 assert_eq!(b[3]["c"][0][0], 3, "start from the first marker");
2571 assert_eq!(
2572 b[3]["c"][1],
2573 json!([
2574 [ast::plain(vec![ast::str_("seeds")])],
2575 [ast::plain(vec![ast::str_("insects")])]
2576 ])
2577 );
2578 }
2579
2580 #[test]
2585 fn heading_depths_under_both_projections() {
2586 let j = doc(
2587 &[
2588 "#/texts/0",
2589 "#/texts/1",
2590 "#/texts/2",
2591 "#/texts/3",
2592 "#/texts/4",
2593 "#/texts/5",
2594 "#/texts/6",
2595 "#/texts/7",
2596 ],
2597 json!([
2598 text(0, "title", "T", json!({})),
2599 text(1, "section_header", "d1", json!({"level": 1})),
2600 text(2, "section_header", "d2", json!({"level": 2})),
2601 text(3, "section_header", "d5", json!({"level": 5})),
2602 text(4, "section_header", "d6", json!({"level": 6})),
2603 text(5, "section_header", "none", json!({})),
2604 text(6, "section_header", "zero", json!({"level": 0})),
2605 text(7, "section_header", "seven", json!({"level": 7})),
2606 ]),
2607 json!([]),
2608 json!([]),
2609 );
2610 let depths = |heading_depth: HeadingDepth| -> Vec<u64> {
2611 let options = PandocExportOptions {
2612 heading_depth,
2613 ..Default::default()
2614 };
2615 let (s, _) = from_docling_json(&j, &options).unwrap();
2616 let v: Value = serde_json::from_str(&s).unwrap();
2617 v["blocks"]
2618 .as_array()
2619 .unwrap()
2620 .iter()
2621 .map(|b| b["c"][0].as_u64().unwrap())
2622 .collect()
2623 };
2624 assert_eq!(depths(HeadingDepth::Docling), [1, 2, 3, 6, 6, 2, 1, 6]);
2625 assert_eq!(depths(HeadingDepth::Source), [1, 1, 2, 5, 6, 1, 1, 6]);
2626 assert_eq!("Source".parse::<HeadingDepth>(), Ok(HeadingDepth::Source));
2627 assert!("deep".parse::<HeadingDepth>().is_err());
2628 }
2629
2630 #[test]
2633 fn markdown_links_in_text_become_links() {
2634 assert_eq!(
2635 text_inlines("Jump to [target](#target-section) now [x] (y)"),
2636 vec![
2637 ast::str_("Jump"),
2638 ast::space(),
2639 ast::str_("to"),
2640 ast::space(),
2641 ast::link(vec![ast::str_("target")], "#target-section"),
2642 ast::space(),
2643 ast::str_("now"),
2644 ast::space(),
2645 ast::str_("[x]"),
2646 ast::space(),
2647 ast::str_("(y)"),
2648 ]
2649 );
2650 assert_eq!(
2651 text_inlines("[a](https://x.org/a)[b](https://x.org/b)"),
2652 vec![
2653 ast::link(vec![ast::str_("a")], "https://x.org/a"),
2654 ast::link(vec![ast::str_("b")], "https://x.org/b"),
2655 ]
2656 );
2657 assert_eq!(
2658 text_inlines("see  here"),
2659 vec"),
2663 ast::space(),
2664 ast::str_("here"),
2665 ]
2666 );
2667 }
2668
2669 #[test]
2672 fn revisions_become_spans_and_views_drop_comments() {
2673 let j = doc(
2674 &["#/texts/0", "#/texts/1"],
2675 json!([
2676 text(
2677 0,
2678 "text",
2679 "old",
2680 json!({"_revision": {"kind": "deletion", "author": "Fixture", "date": "2025-01-01T00:00:00Z"}})
2681 ),
2682 text(
2683 1,
2684 "text",
2685 "Commented",
2686 json!({"comments": [{"$ref": "#/groups/0"}]})
2687 ),
2688 text(
2689 2,
2690 "text",
2691 "[author: Fixture, time: 2025-01-01T00:00:00Z]: Review note",
2692 json!({"content_layer": "notes"})
2693 ),
2694 ]),
2695 json!([{"self_ref": "#/groups/0", "children": [{"$ref": "#/texts/2"}], "label": "comment_section", "name": "comment-3", "content_layer": "notes"}]),
2696 json!([]),
2697 );
2698 let b = blocks(&j);
2699 assert_eq!(
2700 b[0],
2701 ast::para(vec![ast::span(
2702 &["deletion"],
2703 &[
2704 ("author".into(), "Fixture".into()),
2705 ("date".into(), "2025-01-01T00:00:00Z".into())
2706 ],
2707 vec![ast::str_("old")]
2708 )])
2709 );
2710 assert_eq!(
2711 b[1]["c"][0]["t"], "Span",
2712 "comments written without a policy"
2713 );
2714 let accept = PandocExportOptions {
2715 track_changes: Some(TrackChanges::Accept),
2716 ..Default::default()
2717 };
2718 let (s, _) = from_docling_json(&j, &accept).unwrap();
2719 let v: Value = serde_json::from_str(&s).unwrap();
2720 assert_eq!(v["blocks"][1], ast::para(vec![ast::str_("Commented")]));
2721 }
2722
2723 #[test]
2726 fn numbered_headings_write_the_authored_text() {
2727 let j = doc(
2728 &["#/texts/0", "#/texts/1", "#/texts/2"],
2729 json!([
2730 text(
2731 0,
2732 "section_header",
2733 "1.1 2026 Results",
2734 json!({"level": 2, "_number": "1.1"})
2735 ),
2736 text(1, "section_header", "2026 Results", json!({"level": 1})),
2737 text(
2738 2,
2739 "section_header",
2740 "3 Odd",
2741 json!({"level": 1, "_number": "4"})
2742 ),
2743 ]),
2744 json!([]),
2745 json!([]),
2746 );
2747 let b = blocks(&j);
2748 assert_eq!(
2749 b[0],
2750 ast::header(
2751 3,
2752 vec![ast::str_("2026"), ast::space(), ast::str_("Results")]
2753 )
2754 );
2755 assert_eq!(
2756 b[1],
2757 ast::header(
2758 2,
2759 vec![ast::str_("2026"), ast::space(), ast::str_("Results")]
2760 )
2761 );
2762 assert_eq!(
2764 b[2],
2765 ast::header(2, vec![ast::str_("3"), ast::space(), ast::str_("Odd")])
2766 );
2767 }
2768
2769 #[test]
2773 fn anchors_become_header_ids_and_anchor_spans() {
2774 let j = doc(
2775 &["#/texts/0", "#/texts/1", "#/texts/2"],
2776 json!([
2777 text(
2778 0,
2779 "section_header",
2780 "Target section",
2781 json!({"level": 1, "_anchors": ["target-section", "Alias"]})
2782 ),
2783 text(1, "text", "Jump", json!({"hyperlink": "#target-section"})),
2784 text(2, "text", "Body mark", json!({"_anchors": ["BodyMark"]})),
2785 ]),
2786 json!([]),
2787 json!([]),
2788 );
2789 let b = blocks(&j);
2790 assert_eq!(
2791 b[0],
2792 ast::header_with_id(
2793 2,
2794 "target-section",
2795 vec![
2796 ast::anchor("Alias"),
2797 ast::str_("Target"),
2798 ast::space(),
2799 ast::str_("section")
2800 ]
2801 )
2802 );
2803 assert_eq!(
2804 b[1],
2805 ast::para(vec![ast::link(vec![ast::str_("Jump")], "#target-section")])
2806 );
2807 assert_eq!(
2808 b[2],
2809 ast::para(vec![
2810 ast::anchor("BodyMark"),
2811 ast::str_("Body"),
2812 ast::space(),
2813 ast::str_("mark")
2814 ])
2815 );
2816 assert_eq!(auto_identifier("Target section"), "target-section");
2817 assert_eq!(auto_identifier("2026 Results"), "results");
2818 assert_eq!(auto_identifier("1.1 Été & hiver!"), "été-hiver");
2819 assert_eq!(auto_identifier("***"), "section");
2820 }
2821
2822 #[test]
2823 fn code_formulas_checkboxes_and_unmapped_labels() {
2824 let j = doc(
2825 &["#/texts/0", "#/texts/1", "#/texts/2", "#/texts/3"],
2826 json!([
2827 text(0, "code", "print(1)", json!({"code_language": "Python"})),
2828 text(1, "formula", "E = mc^2", json!({})),
2829 text(2, "checkbox_selected", "done", json!({})),
2830 text(3, "reference", "[1] Duck, D. (2020).", json!({})),
2831 ]),
2832 json!([]),
2833 json!([]),
2834 );
2835 let b = blocks(&j);
2836 assert_eq!(b[0], ast::code_block(Some("python"), "print(1)"));
2837 assert_eq!(b[1], ast::para(vec![ast::math(true, "E = mc^2")]));
2838 assert_eq!(b[2]["c"][0], ast::str_("☒"));
2839 assert_eq!(b[3]["t"], "Div");
2840 assert_eq!(b[3]["c"][0][1], json!(["docling-reference"]));
2841 }
2842
2843 #[test]
2844 fn tables_keep_spans_header_rows_and_captions() {
2845 let cell = |r0: usize, c0: usize, rs: usize, cs: usize, t: &str, header: bool| {
2846 json!({"text": t, "row_span": rs, "col_span": cs,
2847 "start_row_offset_idx": r0, "end_row_offset_idx": r0 + rs,
2848 "start_col_offset_idx": c0, "end_col_offset_idx": c0 + cs,
2849 "column_header": header})
2850 };
2851 let j = doc(
2852 &["#/tables/0"],
2853 json!([text(0, "caption", "Table 1: Ducks", json!({}))]),
2854 json!([]),
2855 json!([{"self_ref": "#/tables/0", "children": [{"$ref": "#/texts/0"}], "label": "table",
2856 "captions": [{"$ref": "#/texts/0"}],
2857 "data": {"num_rows": 2, "num_cols": 2, "table_cells": [
2858 cell(0, 0, 1, 2, "Species", true),
2859 cell(1, 0, 1, 1, "Mallard", false),
2860 cell(1, 1, 1, 1, "Anas", false)]}}]),
2861 );
2862 let b = blocks(&j);
2863 assert_eq!(b.len(), 1, "the caption is rendered by its table only");
2864 let t = &b[0];
2865 assert_eq!(t["t"], "Table");
2866 assert_eq!(
2867 t["c"][1][1][0],
2868 ast::plain(vec![
2869 ast::str_("Table"),
2870 ast::space(),
2871 ast::str_("1:"),
2872 ast::space(),
2873 ast::str_("Ducks")
2874 ])
2875 );
2876 assert_eq!(t["c"][2].as_array().unwrap().len(), 2);
2877 let head = t["c"][3][1].as_array().unwrap();
2878 assert_eq!(head.len(), 1);
2879 assert_eq!(head[0][1][0][3], 2, "colspan");
2880 assert_eq!(t["c"][4][0][3].as_array().unwrap().len(), 1, "one body row");
2881 }
2882
2883 #[test]
2884 fn api_version_is_checked() {
2885 for ok in ["1.23", "1.23.1", "1.23.1.1"] {
2886 assert!(check_api_version(ok).is_ok(), "{ok}");
2887 }
2888 for bad in ["1.22", "2.0", "1", "1.23.2", "1.23.1.2", "x.y", ""] {
2889 let e = check_api_version(bad).unwrap_err();
2890 assert!(e.to_string().contains("only 1.23"), "{bad}: {e}");
2891 }
2892 let err = to_pandoc(
2893 &DoclingDocument::new("t"),
2894 &PandocExportOptions {
2895 api_version: Some("1.22".into()),
2896 ..Default::default()
2897 },
2898 )
2899 .unwrap_err();
2900 assert_eq!(
2901 err,
2902 PandocError::UnsupportedApiVersion {
2903 requested: "1.22".into()
2904 }
2905 );
2906 }
2907
2908 #[test]
2909 #[cfg(feature = "preserve-order")] fn empty_document_is_a_valid_pandoc_document() {
2911 let (s, artifacts) =
2912 to_pandoc(&DoclingDocument::new("e"), &PandocExportOptions::default()).unwrap();
2913 assert!(artifacts.is_empty());
2914 assert_eq!(
2915 s,
2916 r#"{"pandoc-api-version":[1,23,1,1],"meta":{},"blocks":[]}"#
2917 );
2918 assert_eq!(s, DoclingDocument::new("e").export_to_pandoc_json());
2919 }
2920}