use std::path::{Path, PathBuf};
use roxmltree::{Document, Node as XmlNode, ParsingOptions};
use crate::backend::images::ImagePolicy;
use crate::backend::markdown::escape_text;
use crate::backend::DeclarativeBackend;
use crate::error::ConversionError;
use crate::source::SourceDocument;
use docling_core::tree::{Formatting, ItemTree, ListMeta, TreeKind};
use docling_core::{
inline_paragraph_node, DoclingDocument, InlineRun, ListItemDclx, Node, PictureImage, Script,
Table, TableStructure,
};
#[derive(Default)]
pub struct JatsBackend {
pub images: ImagePolicy,
}
const SKIP_TEXT: &[&str] = &["term", "disp-formula", "inline-formula"];
#[derive(Clone, Copy, Default, PartialEq, Eq)]
struct Fmt {
bold: bool,
italic: bool,
underline: bool,
strike: bool,
script: Script,
}
impl Fmt {
fn with_tag(self, tag: &str) -> Fmt {
match tag {
"bold" => Fmt { bold: true, ..self },
"italic" => Fmt {
italic: true,
..self
},
"underline" => Fmt {
underline: true,
..self
},
"strike" => Fmt {
strike: true,
..self
},
"sub" => Fmt {
script: Script::Sub,
..self
},
"sup" => Fmt {
script: Script::Super,
..self
},
_ => self,
}
}
fn to_inline_run(self, text: &str) -> InlineRun {
InlineRun {
text: text.to_string(),
bold: self.bold,
italic: self.italic,
underline: self.underline,
strike: self.strike,
script: self.script,
code: false,
formula: false,
}
}
}
struct Seg {
formula: bool,
text: String,
fmt: Fmt,
hyperlink: Option<String>,
}
const FLUSH_TAGS: &[&str] = &["ack", "sec", "list", "boxed-text", "disp-formula", "fig"];
const DEFAULT_HEADER_ACKNOWLEDGMENTS: &str = "Acknowledgments";
const DEFAULT_HEADER_FOOTNOTES: &str = "Footnotes";
const DEFAULT_HEADER_REFERENCES: &str = "References";
const DEFAULT_TEXT_ETAL: &str = "et al.";
const ORDERED_LIST_TYPES: &[&str] = &[
"order",
"alpha-lower",
"alpha-upper",
"roman-lower",
"roman-upper",
];
struct Out<'a> {
doc: &'a mut DoclingDocument,
tree: ItemTree,
}
impl Out<'_> {
#[allow(clippy::too_many_arguments)]
fn text(
&mut self,
parent: Option<usize>,
label: &str,
text: &str,
formatting: Option<Formatting>,
hyperlink: Option<String>,
level: Option<u8>,
list: Option<ListMeta>,
) -> usize {
self.tree.add(
parent,
None,
TreeKind::Text {
label: label.into(),
text: text.into(),
orig: None,
formatting,
hyperlink,
level,
list,
},
)
}
fn heading(&mut self, parent: Option<usize>, text: &str, level: i32) -> usize {
self.text(
parent,
"section_header",
text,
None,
None,
Some(level.clamp(1, 100) as u8),
None,
)
}
fn group(&mut self, parent: Option<usize>, label: &str, name: &str) -> usize {
self.tree.add(
parent,
None,
TreeKind::Group {
label: label.into(),
name: name.into(),
},
)
}
fn list_item(&mut self, parent: Option<usize>, text: &str) -> usize {
self.text(
parent,
"list_item",
text,
None,
None,
None,
Some(ListMeta::default()),
)
}
fn list_item_meta(&mut self, parent: Option<usize>, text: &str, meta: ListMeta) -> usize {
self.text(parent, "list_item", text, None, None, None, Some(meta))
}
fn is_list(&self, parent: Option<usize>) -> bool {
parent.is_some_and(|p| {
matches!(&self.tree.items[p].kind, TreeKind::Group { label, .. } if label == "list")
})
}
}
fn tree_formatting(fmt: Fmt) -> Option<Formatting> {
(fmt != Fmt::default()).then_some(Formatting {
bold: fmt.bold,
italic: fmt.italic,
underline: fmt.underline,
strikethrough: fmt.strike,
script: fmt.script,
})
}
impl DeclarativeBackend for JatsBackend {
fn convert(&self, source: &SourceDocument) -> Result<DoclingDocument, ConversionError> {
let xml = source.text()?;
super::xml_depth::check(&xml, "jats")?;
let opts = ParsingOptions {
allow_dtd: true,
..Default::default()
};
let dom = Document::parse_with_options(&xml, opts)
.map_err(|e| ConversionError::with_source("jats", e))?;
let mut doc = DoclingDocument::new(&source.name);
let mut out = Out {
doc: &mut doc,
tree: ItemTree::default(),
};
let title = parse_title(&dom);
if let Some(title) = &title {
out.doc.push(Node::Heading {
level: 1,
text: escape_text(title),
});
}
let root = out.text(
None,
"title",
title.as_deref().unwrap_or(""),
None,
None,
None,
None,
);
let root = Some(root);
let (authors, affiliations) = parse_authors(&dom);
if !authors.is_empty() {
let joined = authors.join(", ");
out.doc.push(Node::Paragraph {
text: escape_text(&joined),
});
out.text(root, "paragraph", &joined, None, None, None, None);
}
if !affiliations.is_empty() {
let joined = affiliations.join("; ");
out.doc.push(Node::Paragraph {
text: escape_text(&joined),
});
out.text(root, "paragraph", &joined, None, None, None, None);
}
let fig_base = if self.images.sources.local() {
source.base_dir()
} else {
None
};
for abs in parse_abstracts(&dom) {
if abs.paragraphs.is_empty() && abs.sections.is_empty() {
continue;
}
let label = if abs.label.is_empty() {
"Abstract"
} else {
&abs.label
};
out.doc.push(Node::Heading {
level: 2,
text: escape_text(label),
});
let abs_heading = Some(out.heading(root, label, 1));
let mut no_sections = 0;
for p in &abs.paragraphs {
walk_linear(
*p,
abs_heading,
Fmt::default(),
None,
&mut no_sections,
fig_base,
&mut out,
);
}
for (title, paragraphs) in &abs.sections {
let section_parent = if title.is_empty() {
abs_heading
} else {
out.doc.push(Node::Heading {
level: 3,
text: escape_text(title),
});
Some(out.heading(abs_heading, title, 2))
};
for p in paragraphs {
walk_linear(
*p,
section_parent,
Fmt::default(),
None,
&mut no_sections,
fig_base,
&mut out,
);
}
}
}
let mut hlevel: i32 = 0;
for tag in ["body", "back"] {
if let Some(node) = dom
.descendants()
.find(|n| n.has_tag_name(tag) && !in_sub_article(*n))
{
walk_linear(
node,
root,
Fmt::default(),
None,
&mut hlevel,
fig_base,
&mut out,
);
}
}
let subs: Vec<XmlNode> = dom
.descendants()
.filter(|n| is_sub_article(*n) && !in_sub_article(*n))
.collect();
for sub in subs {
add_sub_article(sub, root, &mut hlevel, fig_base, &mut out);
}
let tree = out.tree;
doc.tree = Some(tree);
Ok(doc)
}
}
fn is_sub_article(n: XmlNode) -> bool {
n.has_tag_name("sub-article") || n.has_tag_name("response")
}
fn in_sub_article(n: XmlNode) -> bool {
n.ancestors().skip(1).any(is_sub_article)
}
fn add_sub_article(
node: XmlNode,
parent: Option<usize>,
hlevel: &mut i32,
fig_base: Option<&std::path::Path>,
out: &mut Out,
) {
let title = node
.children()
.filter(|c| c.has_tag_name("front-stub") || c.has_tag_name("front"))
.flat_map(|front| {
let meta = if front.has_tag_name("front") {
front.children().find(|c| c.has_tag_name("article-meta"))
} else {
Some(front)
};
meta.into_iter()
.flat_map(|m| m.children().filter(|c| c.has_tag_name("title-group")))
.flat_map(|g| g.children().filter(|c| c.has_tag_name("article-title")))
})
.next();
let text = title.map(|t| normalize(&get_text(t))).unwrap_or_default();
let mut new_parent = parent;
if !text.is_empty() {
*hlevel += 1;
out.doc.push(Node::Heading {
level: fw_level(*hlevel),
text: escape_text(&text),
});
new_parent = Some(out.heading(parent, &text, *hlevel));
}
for child in node
.children()
.filter(|c| c.has_tag_name("body") || c.has_tag_name("back"))
{
walk_linear(
child,
new_parent,
Fmt::default(),
None,
hlevel,
fig_base,
out,
);
}
for child in node.children().filter(|c| is_sub_article(*c)) {
add_sub_article(child, new_parent, hlevel, fig_base, out);
}
if !text.is_empty() {
*hlevel -= 1;
}
}
fn raw_text(node: XmlNode, out: &mut String) {
if let Some(t) = node.text() {
out.push_str(&t.replace('\n', " "));
}
for child in node.children() {
if child.is_element() {
if !SKIP_TEXT.contains(&child.tag_name().name()) {
raw_text(child, out);
}
if let Some(tail) = child.tail() {
out.push_str(&tail.replace('\n', " "));
}
} else if child.is_text() {
}
}
}
pub(crate) fn convert_generic(source: &SourceDocument) -> Result<DoclingDocument, ConversionError> {
let xml = source.text()?;
super::xml_depth::check(&xml, "xml")?;
let opts = ParsingOptions {
allow_dtd: true,
..Default::default()
};
let dom = Document::parse_with_options(&xml, opts)
.map_err(|e| ConversionError::with_source("xml", e))?;
let mut doc = DoclingDocument::new(&source.name);
walk_generic(dom.root_element(), &mut doc);
Ok(doc)
}
fn walk_generic(node: XmlNode, doc: &mut DoclingDocument) {
for child in node.children().filter(XmlNode::is_element) {
let tag = child.tag_name().name();
if tag == "table-wrap" {
let mut out = Out {
doc,
tree: ItemTree::default(),
};
add_table(&mut out, None, child);
continue;
}
if tag == "table" {
if let Some(t) = parse_jats_table(child) {
doc.push(Node::Table(t));
}
continue;
}
let has_direct_text = child
.children()
.any(|c| c.is_text() && !c.text().unwrap_or("").trim().is_empty());
let has_element_children = child.children().any(|c| c.is_element());
if !has_element_children || has_direct_text {
let t = normalize(&sanitize_generic(&generic_text(child)));
if !t.trim().is_empty() {
doc.push(Node::Paragraph {
text: escape_text(&t),
});
}
} else {
walk_generic(child, doc);
}
}
}
fn sanitize_generic(s: &str) -> String {
s.chars()
.map(|c| match c {
'\u{2014}' | '\u{2013}' => '-',
'\u{2019}' | '\u{2018}' => '\'',
'\u{201C}' | '\u{201D}' => '"',
c => c,
})
.collect()
}
fn generic_text(node: XmlNode) -> String {
let mut s = String::new();
for child in node.children() {
if child.is_text() {
s.push_str(child.text().unwrap_or(""));
} else if child.is_element() {
s.push(' ');
s.push_str(&generic_text(child));
s.push(' ');
}
}
s
}
fn node_text(node: XmlNode) -> String {
let mut s = String::new();
raw_text(node, &mut s);
normalize(&s)
}
fn normalize(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut pending_space = false;
for c in s.trim().chars() {
if c.is_ascii_whitespace() {
pending_space = true;
} else {
if pending_space && !out.is_empty() {
out.push(' ');
}
pending_space = false;
out.push(c);
}
}
out
}
fn parse_title(dom: &Document) -> Option<String> {
const METAS: [&str; 4] = [
"article-meta",
"collection-meta",
"book-meta",
"book-part-meta",
];
const NAMES: [&str; 4] = ["article-title", "subtitle", "title", "label"];
let titles: Vec<String> = dom
.descendants()
.filter(|n| {
n.has_tag_name("title-group")
&& n.parent()
.is_some_and(|p| METAS.contains(&p.tag_name().name()))
})
.map(|group| {
group
.children()
.filter(|c| c.is_element() && NAMES.contains(&c.tag_name().name()))
.map(|c| direct_text(c).replace('\n', " ").trim().to_string())
.collect::<Vec<_>>()
.join(" ")
.trim()
.to_string()
})
.collect();
let text = titles.join(" - ");
(!text.is_empty()).then_some(text)
}
fn direct_text<'a>(node: XmlNode<'a, 'a>) -> &'a str {
node.first_child()
.filter(XmlNode::is_text)
.and_then(|c| c.text())
.unwrap_or("")
}
fn parse_authors(dom: &Document) -> (Vec<String>, Vec<String>) {
let Some(meta) = dom.descendants().find(|n| n.has_tag_name("article-meta")) else {
return (Vec::new(), Vec::new());
};
let mut aff_by_id = std::collections::HashMap::new();
for aff in meta.descendants().filter(|n| n.has_tag_name("aff")) {
let Some(id) = aff.attribute("id") else {
continue;
};
let mut name = aff
.descendants()
.filter(|n| n.is_text())
.filter_map(|n| n.text())
.map(|t| t.trim())
.filter(|t| !t.is_empty())
.collect::<Vec<_>>()
.join(", ")
.replace('\n', " ");
if let Some(label) = aff
.children()
.find(|c| c.has_tag_name("label"))
.and_then(|l| l.text())
{
name = name
.strip_prefix(&format!("{label}, "))
.unwrap_or(&name)
.to_string();
}
aff_by_id.insert(id.to_string(), name);
}
let mut authors = Vec::new();
let mut affiliations = Vec::new();
for contrib in meta
.descendants()
.filter(|n| n.has_tag_name("contrib") && n.attribute("contrib-type") == Some("author"))
{
let name = contrib_name(contrib);
if name.is_empty() {
continue;
}
authors.push(name);
for xref in contrib
.children()
.filter(|c| c.has_tag_name("xref") && c.attribute("ref-type") == Some("aff"))
{
if let Some(aff) = xref.attribute("rid").and_then(|id| aff_by_id.get(id)) {
if !affiliations.contains(aff) {
affiliations.push(aff.clone());
}
}
}
}
(authors, affiliations)
}
fn contrib_name(contrib: XmlNode) -> String {
let name = contrib.children().find(|c| c.has_tag_name("name"));
let Some(name) = name else {
return String::new();
};
["prefix", "given-names", "surname", "suffix"]
.iter()
.filter_map(|tag| {
name.children()
.find(|c| c.has_tag_name(*tag))
.and_then(|c| c.text())
.map(str::trim)
.filter(|s| !s.is_empty())
})
.collect::<Vec<_>>()
.join(" ")
}
struct Abstract<'a, 'i> {
label: String,
paragraphs: Vec<XmlNode<'a, 'i>>,
sections: Vec<(String, Vec<XmlNode<'a, 'i>>)>,
}
fn has_text(p: XmlNode) -> bool {
p.descendants()
.filter(|n| n.is_text())
.any(|n| n.text().is_some_and(|t| !t.trim().is_empty()))
}
fn parse_abstracts<'a, 'i>(dom: &'a Document<'i>) -> Vec<Abstract<'a, 'i>> {
let mut out = Vec::new();
for abs in dom.descendants().filter(|n| n.has_tag_name("abstract")) {
let mut paragraphs = Vec::new();
let mut sections = Vec::new();
for child in abs.children().filter(XmlNode::is_element) {
match child.tag_name().name() {
"p" if has_text(child) => paragraphs.push(child),
"sec" => {
let section = abstract_section(child);
if !section.1.is_empty() {
sections.push(section);
}
}
_ => {}
}
}
out.push(Abstract {
label: title_or_label(abs),
paragraphs,
sections,
});
}
out
}
fn abstract_section<'a, 'i>(section: XmlNode<'a, 'i>) -> (String, Vec<XmlNode<'a, 'i>>) {
let paragraphs = section
.children()
.filter(|c| c.has_tag_name("p") && has_text(*c))
.collect();
(title_or_label(section), paragraphs)
}
fn title_or_label(node: XmlNode) -> String {
node.children()
.find(|c| c.has_tag_name("title") || c.has_tag_name("label"))
.map(node_text)
.unwrap_or_default()
}
fn get_text(node: XmlNode) -> String {
let mut s = String::new();
raw_text(node, &mut s);
s
}
fn norm_text(node: XmlNode) -> String {
normalize(&get_text(node))
}
fn fw_level(dl: i32) -> u8 {
(dl + 1).clamp(1, 6) as u8
}
fn header_text(child: XmlNode) -> Option<String> {
child
.children()
.find(|c| c.has_tag_name("title") || c.has_tag_name("label"))
.map(get_text)
.map(|s| normalize(&s))
.filter(|s| !s.is_empty())
.or_else(|| {
child
.has_tag_name("ack")
.then(|| DEFAULT_HEADER_ACKNOWLEDGMENTS.to_string())
})
}
fn add_citation(out: &mut Out, parent: Option<usize>, text: &str) {
if text.is_empty() {
return;
}
if out.is_list(parent) {
out.doc.push(Node::ListItem {
ordered: false,
number: 0,
first_in_list: false,
text: escape_text(text),
level: 0,
marker: None,
location: None,
dclx: None,
href: None,
layer: None,
});
out.list_item(parent, text);
} else {
out.doc.push(Node::Paragraph {
text: escape_text(text),
});
out.text(parent, "text", text, None, None, None, None);
}
}
fn walk_linear(
node: XmlNode,
parent: Option<usize>,
fmt: Fmt,
hyperlink: Option<&str>,
hlevel: &mut i32,
fig_base: Option<&Path>,
out: &mut Out,
) -> Vec<Seg> {
let node_tag = node.tag_name().name();
let current = fmt.with_tag(node_tag);
let own_link = (node_tag == "ext-link")
.then(|| ext_link_href(node))
.flatten();
let current_link: Option<&str> = own_link.as_deref().or(hyperlink);
let mut segments: Vec<Seg> = Vec::new();
if node_tag != "term" {
if let Some(t) = node.text() {
append_run(&mut segments, t, current, current_link);
}
}
let mut new_parent = parent;
for child in node.children().filter(XmlNode::is_element) {
let mut stop_walk = false;
let ctag = child.tag_name().name();
if node_tag == "p" && FLUSH_TAGS.contains(&ctag) {
emit_inline(out, parent, std::mem::take(&mut segments));
}
let mut opened_section = false;
match ctag {
"sec" | "ack" => {
if let Some(text) = header_text(child) {
*hlevel += 1;
out.doc.push(Node::Heading {
level: fw_level(*hlevel),
text: escape_text(&text),
});
new_parent = Some(out.heading(parent, &text, *hlevel));
opened_section = true;
}
}
"list" => {
new_parent = Some(out.group(parent, "list", "list"));
}
"list-item" => {
add_list_item(out, parent, child, 0);
stop_walk = true;
}
"fig" => {
add_figure(out, parent, child, fig_base);
stop_walk = true;
}
"table-wrap" => {
add_table(out, parent, child);
stop_walk = true;
}
"supplementary-material" => {
stop_walk = true;
}
"fn-group" => {
add_footnote_group(out, parent, child, *hlevel);
stop_walk = true;
}
"ref-list" if node_tag != "ref-list" => {
let text = child
.children()
.find(|c| c.has_tag_name("title") || c.has_tag_name("label"))
.map(|h| normalize(&get_text(h)))
.filter(|s| !s.is_empty())
.unwrap_or_else(|| DEFAULT_HEADER_REFERENCES.to_string());
out.doc.push(Node::Heading {
level: fw_level(1),
text: escape_text(&text),
});
let heading = out.heading(parent, &text, 1);
new_parent = Some(out.group(Some(heading), "list", "list"));
}
"element-citation" => {
let text = parse_element_citation(child);
add_citation(out, parent, &text);
stop_walk = true;
}
"mixed-citation" => {
let text = norm_text(child);
add_citation(out, parent, &text);
stop_walk = true;
}
"tex-math" => {
add_equation(out, parent, child);
stop_walk = true;
}
"inline-formula" => {
extend_segments(
&mut segments,
walk_inline_formula(child, current, current_link),
);
stop_walk = true;
}
_ => {}
}
if !stop_walk {
let child_segments = walk_linear(
child,
new_parent,
current,
current_link,
hlevel,
fig_base,
out,
);
let parent_is_p = node.parent().map(|p| p.has_tag_name("p")).unwrap_or(false);
if !(parent_is_p && FLUSH_TAGS.contains(&node_tag)) {
extend_segments(&mut segments, child_segments);
}
if opened_section {
*hlevel -= 1;
}
}
if let Some(tail) = child.tail() {
append_run(&mut segments, tail, current, current_link);
}
}
if node_tag == "p" {
emit_inline(out, parent, segments);
Vec::new()
} else {
segments
}
}
fn walk_inline_formula(node: XmlNode, fmt: Fmt, hyperlink: Option<&str>) -> Vec<Seg> {
let current = fmt.with_tag(node.tag_name().name());
let mut segments = Vec::new();
if let Some(t) = node.text() {
append_run(&mut segments, t, current, hyperlink);
}
for child in node.children().filter(XmlNode::is_element) {
if child.tag_name().name() == "tex-math" {
if let Some(formula) = extract_tex_math(child) {
segments.push(Seg {
formula: true,
text: formula,
fmt: Fmt::default(),
hyperlink: hyperlink.map(str::to_string),
});
}
} else {
extend_segments(
&mut segments,
walk_inline_formula(child, current, hyperlink),
);
}
if let Some(tail) = child.tail() {
append_run(&mut segments, tail, current, hyperlink);
}
}
segments
}
fn ext_link_href(node: XmlNode) -> Option<String> {
let href = node
.attributes()
.find(|a| a.name() == "href")
.map(|a| a.value().trim())
.filter(|v| !v.is_empty())?;
Some(crate::backend::html::normalize_url(href))
}
fn extract_tex_math(node: XmlNode) -> Option<String> {
let text = node.text()?.trim().to_string();
for delim in ["$$", "$"] {
if text.len() > 2 * delim.len() && text.starts_with(delim) && text.ends_with(delim) {
let inner = text[delim.len()..text.len() - delim.len()]
.trim()
.to_string();
return (!inner.is_empty()).then_some(inner);
}
}
(!text.is_empty()).then_some(text)
}
fn xml_line_breaks(text: &str) -> std::borrow::Cow<'_, str> {
if !text.contains('\n') {
return std::borrow::Cow::Borrowed(text);
}
let chunks: Vec<&str> = text.split('\n').collect();
let last = chunks.len() - 1;
let mut parts: Vec<&str> = vec![chunks[0].trim_end_matches([' ', '\t'])];
parts.extend(
chunks[1..last]
.iter()
.map(|c| c.trim_matches([' ', '\t']))
.filter(|c| !c.is_empty()),
);
parts.push(chunks[last].trim_start_matches([' ', '\t']));
std::borrow::Cow::Owned(parts.join(" "))
}
fn append_run(segments: &mut Vec<Seg>, text: &str, fmt: Fmt, hyperlink: Option<&str>) {
let text = xml_line_breaks(text);
let text = text.as_ref();
if text.is_empty() {
return;
}
if let Some(last) = segments.last_mut() {
if !last.formula && last.fmt == fmt && last.hyperlink.as_deref() == hyperlink {
last.text.push_str(text);
return;
}
}
segments.push(Seg {
formula: false,
text: text.to_string(),
fmt,
hyperlink: hyperlink.map(str::to_string),
});
}
fn extend_segments(segments: &mut Vec<Seg>, more: Vec<Seg>) {
for seg in more {
if seg.formula {
segments.push(seg);
} else {
append_run(segments, &seg.text, seg.fmt, seg.hyperlink.as_deref());
}
}
}
fn strip_segments(segments: Vec<Seg>) -> Vec<Seg> {
segments
.into_iter()
.filter_map(|s| {
let text = s.text.trim().to_string();
(!text.is_empty()).then_some(Seg { text, ..s })
})
.collect()
}
fn emit_inline(out: &mut Out, parent: Option<usize>, segments: Vec<Seg>) {
let stripped = strip_segments(segments);
if stripped.is_empty() {
return;
}
let md_text = stripped
.iter()
.map(seg_markdown)
.collect::<Vec<_>>()
.join(" ");
let runs = stripped
.iter()
.map(|s| {
if s.formula {
InlineRun {
text: s.text.clone(),
formula: true,
..InlineRun::default()
}
} else {
s.fmt.to_inline_run(&s.text)
}
})
.collect();
out.doc.push(inline_paragraph_node(md_text, runs, true));
let container = if stripped.len() > 1 {
Some(out.group(parent, "inline", "group"))
} else {
parent
};
for seg in &stripped {
let label = if seg.formula { "formula" } else { "text" };
out.text(
container,
label,
&seg.text,
tree_formatting(seg.fmt),
seg.hyperlink.clone(),
None,
None,
);
}
}
fn seg_markdown(s: &Seg) -> String {
if s.formula {
return format!("${}$", s.text);
}
let mut out = escape_text(&s.text);
if s.fmt.bold {
out = format!("**{out}**");
}
if s.fmt.italic {
out = format!("*{out}*");
}
if s.fmt.strike {
out = format!("~~{out}~~");
}
if let Some(url) = &s.hyperlink {
out = docling_core::hyperlink::markdown_link(&out, url);
}
out
}
fn add_list_item(out: &mut Out, parent: Option<usize>, item: XmlNode, level: u8) {
let list = item.parent_element();
let enumerated = list.is_some_and(is_enumerated_list);
let number = list.map_or(1, |l| {
l.children()
.filter(|c| c.has_tag_name("list-item"))
.position(|c| c == item)
.unwrap_or(0) as u64
+ 1
});
let label = item_label(item);
let mut text = String::new();
for p in item.children().filter(|c| c.has_tag_name("p")) {
let t = norm_text(p);
if t.is_empty() {
continue;
}
if !text.is_empty() {
text.push(' ');
}
text.push_str(&t);
}
if !text.is_empty() || !label.is_empty() {
let shape = markdown_shape(enumerated, &label, &text);
out.doc.push(Node::ListItem {
ordered: shape.ordered,
number,
first_in_list: number == 1,
text: escape_text(&shape.text),
level,
marker: shape.marker,
location: None,
dclx: shape.dclx.then(|| ListItemDclx {
ordered: enumerated,
marker: (!label.is_empty()).then(|| label.clone()),
text: escape_text(&text),
runs: Vec::new(),
}),
href: None,
layer: None,
});
}
let item_id = Some(out.list_item_meta(
parent,
&text,
ListMeta {
enumerated,
marker: label,
},
));
for nested in item.children().filter(|c| c.has_tag_name("list")) {
let group = Some(out.group(item_id, "list", "list"));
for sub in nested.children().filter(|c| c.has_tag_name("list-item")) {
add_list_item(out, group, sub, level.saturating_add(1));
}
}
}
fn is_enumerated_list(list: XmlNode) -> bool {
list.attribute("list-type")
.is_some_and(|t| ORDERED_LIST_TYPES.contains(&t.trim().to_lowercase().as_str()))
}
fn item_label(item: XmlNode) -> String {
item.children()
.find(|c| c.has_tag_name("label"))
.map(|l| get_text(l).trim().to_string())
.unwrap_or_default()
}
struct MarkdownShape {
ordered: bool,
marker: Option<String>,
text: String,
dclx: bool,
}
fn markdown_shape(enumerated: bool, label: &str, text: &str) -> MarkdownShape {
let digits_dot = label
.strip_suffix('.')
.is_some_and(|d| !d.is_empty() && d.chars().all(char::is_numeric));
if label.is_empty() {
MarkdownShape {
ordered: enumerated,
marker: None,
text: text.to_string(),
dclx: false,
}
} else if digits_dot {
MarkdownShape {
ordered: true,
marker: Some(label.to_string()),
text: text.to_string(),
dclx: !enumerated,
}
} else if label.chars().any(char::is_alphanumeric) {
MarkdownShape {
ordered: false,
marker: None,
text: format!("{label} {text}"),
dclx: true,
}
} else {
MarkdownShape {
ordered: false,
marker: None,
text: text.to_string(),
dclx: true,
}
}
}
fn add_equation(out: &mut Out, parent: Option<usize>, node: XmlNode) {
if let Some(formula) = extract_tex_math(node) {
out.doc.push(Node::Formula {
orig: formula.clone(),
latex: formula.clone(),
location: None,
});
out.text(parent, "formula", &formula, None, None, None, None);
}
}
fn add_figure(out: &mut Out, parent: Option<usize>, node: XmlNode, fig_base: Option<&Path>) {
let label = node
.children()
.find(|c| c.has_tag_name("label"))
.map(|l| get_text(l).trim().to_string())
.unwrap_or_default();
let caption = node
.children()
.find(|c| c.has_tag_name("caption"))
.map(caption_text)
.unwrap_or_default();
let sep = if !label.is_empty() && !caption.is_empty() {
" "
} else {
""
};
let fig_text = format!("{label}{sep}{caption}");
let image = fig_base.and_then(|base| load_figure_image(node, base));
out.doc.push(Node::Picture {
caption: (!fig_text.is_empty()).then(|| escape_text(&fig_text)),
caption_href: None,
image: image.clone(),
classification: None,
description: None,
caption_parent: Default::default(),
caption_location: None,
});
let captions = if fig_text.is_empty() {
Vec::new()
} else {
vec![out.text(None, "caption", &fig_text, None, None, None, None)]
};
out.tree.add(
parent,
None,
TreeKind::Picture {
captions,
image,
extra_classes: Vec::new(),
classification: None,
description: None,
confidence: None,
chart: None,
dpi: None,
},
);
}
const RASTER_IMAGE_SUFFIXES: [&str; 6] = [".jpg", ".jpeg", ".png", ".tif", ".tiff", ".gif"];
fn load_figure_image(fig: XmlNode, base: &Path) -> Option<PictureImage> {
let graphics = fig.children().filter(XmlNode::is_element).flat_map(|c| {
let own = std::iter::once(c).filter(|c| c.has_tag_name("graphic"));
let alternatives = c
.children()
.filter(move |g| c.has_tag_name("alternatives") && g.has_tag_name("graphic"));
own.chain(alternatives)
});
let mut missing: Vec<String> = Vec::new();
for graphic in graphics {
let Some(href) = graphic
.attributes()
.find(|a| a.name() == "href")
.map(|a| a.value().trim())
.filter(|v| !v.is_empty())
else {
continue;
};
if !is_local_path(href) {
continue;
}
if is_absolute_path(href) {
eprintln!(
"docling: warning: Could not process an image from {href}: \
Absolute paths are not allowed with local base_path."
);
continue;
}
let suffix = Path::new(href)
.extension()
.and_then(|e| e.to_str())
.map(str::to_ascii_lowercase);
if suffix.as_deref() == Some("svg") {
continue;
}
let mut candidates = vec![href.to_string()];
if suffix.is_none() {
candidates.extend(RASTER_IMAGE_SUFFIXES.iter().map(|s| format!("{href}{s}")));
}
let mut found = false;
for candidate in &candidates {
let Some(resolved) = confined_path(base, candidate) else {
eprintln!(
"docling: warning: Could not process an image from {href}: \
Path traversal blocked: '{candidate}' resolves outside base directory"
);
return None;
};
if !resolved.is_file() {
continue;
}
found = true;
let decoded = std::fs::read(&resolved)
.ok()
.and_then(|data| crate::backend::ooxml::picture_image(candidate, data));
match decoded {
Some(image) => return Some(image),
None => eprintln!(
"docling: warning: Could not process an image from {}: \
cannot identify image file",
resolved.display()
),
}
}
if !found {
missing.push(href.to_string());
}
}
if !missing.is_empty() {
eprintln!(
"docling: warning: Could not process JATS figure image(s) {}: \
no matching local file exists.",
missing.join(", ")
);
}
None
}
fn is_local_path(value: &str) -> bool {
let Some(colon) = value.find(':') else {
return !value.starts_with("//");
};
let scheme = &value[..colon];
let is_scheme = scheme
.chars()
.next()
.is_some_and(|c| c.is_ascii_alphabetic())
&& scheme
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'));
if !is_scheme {
return !value.starts_with("//");
}
let rest = &value[colon + 1..];
!rest.starts_with("//") && scheme.len() == 1
}
fn is_absolute_path(value: &str) -> bool {
Path::new(value).has_root()
|| (value.len() >= 2
&& value.as_bytes()[1] == b':'
&& value.as_bytes()[0].is_ascii_alphabetic()
&& !value[2..].starts_with("//"))
}
fn confined_path(base: &Path, rel: &str) -> Option<PathBuf> {
use std::path::Component;
let mut out = base.to_path_buf();
let mut depth = 0usize;
for comp in Path::new(rel).components() {
match comp {
Component::CurDir => {}
Component::ParentDir => {
if depth == 0 {
return None;
}
out.pop();
depth -= 1;
}
Component::Normal(seg) => {
out.push(seg);
depth += 1;
}
Component::RootDir | Component::Prefix(_) => return None,
}
}
Some(out)
}
fn caption_text(caption: XmlNode) -> String {
let mut out = String::new();
for par in caption.children().filter(XmlNode::is_element) {
if par
.descendants()
.any(|d| d.has_tag_name("supplementary-material"))
{
continue;
}
out.push_str(get_text(par).trim());
out.push(' ');
}
out.trim().to_string()
}
fn add_table(out: &mut Out, parent: Option<usize>, node: XmlNode) {
let content = node
.children()
.find(|c| c.has_tag_name("table"))
.or_else(|| {
node.children()
.find(|c| c.has_tag_name("alternatives"))
.and_then(|a| a.children().find(|c| c.has_tag_name("table")))
});
let label = node
.children()
.find(|c| c.has_tag_name("label"))
.and_then(|l| l.text())
.map(|t| t.trim().to_string())
.unwrap_or_default();
let caption = node
.children()
.find(|c| c.has_tag_name("caption"))
.map(caption_text)
.unwrap_or_default();
let sep = if !label.is_empty() && !caption.is_empty() {
" "
} else {
""
};
let cap_text = format!("{label}{sep}{caption}");
let Some(table_node) = content else { return };
let captions = if cap_text.is_empty() {
Vec::new()
} else {
vec![out.text(None, "caption", &cap_text, None, None, None, None)]
};
let Some(mut table) = parse_jats_table(table_node) else {
return;
};
out.tree.add(
parent,
None,
TreeKind::Table {
table: table.clone(),
rich_cells: Vec::new(),
captions,
},
);
table.caption = (!cap_text.is_empty()).then(|| escape_text(&cap_text));
out.doc.push(Node::Table(table));
}
fn parse_jats_table(table: XmlNode) -> Option<Table> {
let rows_nodes: Vec<XmlNode> = table
.descendants()
.filter(|n| n.has_tag_name("tr"))
.collect();
if rows_nodes.iter().any(|r| {
r.descendants()
.any(|d| d.has_tag_name("table") && d != table)
}) {
return None;
}
let num_cols = rows_nodes
.iter()
.map(|r| {
r.children()
.filter(|c| c.has_tag_name("td") || c.has_tag_name("th"))
.map(col_span)
.sum::<usize>()
})
.max()
.unwrap_or(0);
if rows_nodes.is_empty() || num_cols == 0 {
return None;
}
let nrows = rows_nodes.len();
let mut grid: Vec<Vec<String>> = vec![vec![String::new(); num_cols]; nrows];
let mut filled: Vec<Vec<bool>> = vec![vec![false; num_cols]; nrows];
let mut col_header: Vec<Vec<bool>> = vec![vec![false; num_cols]; nrows];
let mut col_continuation: Vec<Vec<bool>> = vec![vec![false; num_cols]; nrows];
let mut row_continuation: Vec<Vec<bool>> = vec![vec![false; num_cols]; nrows];
for (ri, row) in rows_nodes.iter().enumerate() {
let mut ci = 0usize;
for cell in row
.children()
.filter(|c| c.has_tag_name("td") || c.has_tag_name("th"))
{
while ci < num_cols && filled[ri][ci] {
ci += 1;
}
if ci >= num_cols {
break;
}
let cs = col_span(cell);
let rs = row_span(cell);
let text = normalize(&get_text(cell));
let header =
cell.has_tag_name("th") || cell.ancestors().any(|a| a.has_tag_name("thead"));
for r in ri..(ri + rs).min(nrows) {
for c in ci..(ci + cs).min(num_cols) {
grid[r][c] = text.clone();
filled[r][c] = true;
if c > ci {
col_continuation[r][c] = true;
}
if r > ri {
row_continuation[r][c] = true;
}
}
}
col_header[ri][ci] = header;
ci += cs;
}
}
let has_header = col_header.iter().flatten().any(|&h| h);
let has_span = col_continuation
.iter()
.chain(&row_continuation)
.flatten()
.any(|&s| s);
let structure = (has_header || has_span).then(|| TableStructure {
header_row: Vec::new(),
col_continuation,
row_continuation,
row_header: Vec::new(),
col_header,
});
Some(Table {
rows: grid,
location: None,
structure,
cell_blocks: None,
cells: None,
caption: None,
caption_parent: Default::default(),
caption_location: None,
})
}
fn col_span(cell: XmlNode) -> usize {
cell.attribute("colspan")
.and_then(|v| v.parse().ok())
.filter(|&n: &usize| n >= 1)
.unwrap_or(1)
}
fn row_span(cell: XmlNode) -> usize {
cell.attribute("rowspan")
.and_then(|v| v.parse().ok())
.filter(|&n: &usize| n >= 1)
.unwrap_or(1)
}
fn add_footnote_group(out: &mut Out, parent: Option<usize>, node: XmlNode, hlevel: i32) {
let footnotes: Vec<Vec<Seg>> = node
.children()
.filter(|c| c.has_tag_name("fn"))
.map(|f| strip_segments(walk_inline_formula(f, Fmt::default(), None)))
.filter(|segs| !segs.is_empty())
.collect();
if footnotes.is_empty() {
return;
}
let title = node
.children()
.find(|c| c.has_tag_name("title"))
.map(|t| normalize(&get_text(t)))
.filter(|s| !s.is_empty())
.unwrap_or_else(|| DEFAULT_HEADER_FOOTNOTES.to_string());
out.doc.push(Node::Heading {
level: fw_level(hlevel + 1),
text: escape_text(&title),
});
let heading = Some(out.heading(parent, &title, hlevel + 1));
let group = Some(out.group(heading, "list", "footnotes"));
for segs in footnotes {
let li = Some(out.list_item(group, ""));
let container = if segs.len() > 1 {
Some(out.group(li, "inline", "group"))
} else {
li
};
for seg in &segs {
let label = if seg.formula { "formula" } else { "footnote" };
out.text(
container,
label,
&seg.text,
tree_formatting(seg.fmt),
seg.hyperlink.clone(),
None,
None,
);
}
let text = segs.iter().map(seg_markdown).collect::<Vec<_>>().join(" ");
out.doc.push(Node::ListItem {
ordered: false,
number: 0,
first_in_list: false,
text,
level: 0,
marker: None,
location: None,
dclx: None,
href: None,
layer: None,
});
}
}
fn parse_element_citation(node: XmlNode) -> String {
let mut names: Vec<String> = Vec::new();
for name in node.descendants().filter(|n| n.has_tag_name("name")) {
let surname = name
.children()
.find(|c| c.has_tag_name("surname"))
.and_then(|c| c.text())
.map(|t| t.replace('\n', " "))
.map(|t| t.trim().to_string());
let given = name
.children()
.find(|c| c.has_tag_name("given-names"))
.and_then(|c| c.text())
.map(|t| t.replace('\n', " "))
.map(|t| t.trim().to_string());
let name_str = [surname, given]
.into_iter()
.flatten()
.filter(|s| !s.is_empty())
.collect::<Vec<_>>()
.join(" ");
if !name_str.is_empty() {
names.push(name_str);
}
}
if let Some(etal) = node.descendants().find(|n| n.has_tag_name("etal")) {
let etal_text = etal
.text()
.map(str::trim)
.filter(|s| !s.is_empty())
.unwrap_or(DEFAULT_TEXT_ETAL);
names.push(etal_text.to_string());
}
let author_names = names.join(", ");
let title = [
"article-title",
"chapter-title",
"data-title",
"issue-title",
"part-title",
"trans-title",
]
.iter()
.find_map(|t| node.children().find(|c| c.has_tag_name(*t)))
.map(get_text)
.unwrap_or_else(|| {
node.text()
.map(|t| t.replace('\n', " ").trim().to_string())
.unwrap_or_default()
});
let field = |name: &str| -> String {
node.children()
.find(|c| c.has_tag_name(name))
.and_then(|c| c.text())
.map(|t| t.replace('\n', " ").trim().to_string())
.unwrap_or_default()
};
let source = field("source");
let year = field("year");
let publisher_name = field("publisher-name");
let publisher_loc = field("publisher-loc");
let volume = field("volume");
let mut pub_ids: Vec<String> = Vec::new();
for id in node.children().filter(|c| c.has_tag_name("pub-id")) {
let id_type = id
.attribute("assigning-authority")
.or_else(|| id.attribute("pub-id-type"));
if let (Some(t), Some(text)) = (id_type, id.text()) {
pub_ids.push(format!(
"{}: {}",
t.replace('\n', " ").trim().to_uppercase(),
text.replace('\n', " ").trim()
));
}
}
let pub_id = pub_ids.join(", ");
let page = if let Some(e) = node.children().find(|c| c.has_tag_name("elocation-id")) {
e.text()
.map(|t| t.replace('\n', " ").trim().to_string())
.unwrap_or_default()
} else if let Some(f) = node.children().find(|c| c.has_tag_name("fpage")) {
let mut p = f
.text()
.map(|t| t.replace('\n', " ").trim().to_string())
.unwrap_or_default();
if let Some(l) = node
.children()
.find(|c| c.has_tag_name("lpage"))
.and_then(|l| l.text())
.map(|t| t.replace('\n', " ").trim().to_string())
.filter(|l| !l.is_empty())
{
p.push('\u{2013}');
p.push_str(&l);
}
p
} else {
String::new()
};
let mut text = String::new();
if !author_names.is_empty() {
text.push_str(author_names.trim_end_matches('.'));
text.push_str(". ");
}
if !title.is_empty() {
text.push_str(title.trim());
text.push_str(". ");
}
if !source.is_empty() {
text.push_str(&source);
text.push_str(". ");
}
if !publisher_name.is_empty() {
if !publisher_loc.is_empty() {
text.push_str(&format!("{publisher_loc}: "));
}
text.push_str(&publisher_name);
text.push_str(". ");
}
if !volume.is_empty() {
rstrip_dot_space(&mut text);
text.push_str(&format!(" {volume}. "));
}
if !page.is_empty() {
rstrip_dot_space(&mut text);
if !volume.is_empty() {
text.push(':');
}
text.push_str(&page);
text.push_str(". ");
}
if !year.is_empty() {
rstrip_dot_space(&mut text);
text.push_str(&format!(" ({year})."));
}
if !pub_id.is_empty() {
while text.ends_with('.') {
text.pop();
}
text.push_str(". ");
text.push_str(&pub_id);
}
text
}
fn rstrip_dot_space(s: &mut String) {
while matches!(s.chars().last(), Some('.') | Some(' ')) {
s.pop();
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backend::images::ImageSources;
use crate::format::InputFormat;
fn body_doc(body: &str) -> DoclingDocument {
let xml = format!(
"<article><front><article-meta><title-group><article-title>Body Test\
</article-title></title-group></article-meta></front><body>{body}</body></article>"
);
let src = SourceDocument::from_bytes("b", InputFormat::XmlJats, xml.into_bytes());
JatsBackend::default().convert(&src).unwrap()
}
fn json_list_items(doc: &DoclingDocument) -> Vec<(String, bool, String)> {
let v: serde_json::Value = serde_json::from_str(&doc.export_to_json()).unwrap();
v["texts"]
.as_array()
.unwrap()
.iter()
.filter(|t| t["label"] == "list_item")
.map(|t| {
(
t["text"].as_str().unwrap().to_string(),
t["enumerated"].as_bool().unwrap(),
t["marker"].as_str().unwrap().to_string(),
)
})
.collect()
}
fn two_items(list_attr: &str) -> String {
format!(
"<sec><title>Steps</title><list{list_attr}>\
<list-item><p>first</p></list-item><list-item><p>second</p></list-item>\
</list></sec>"
)
}
#[test]
fn list_type_sets_enumerated() {
let cases = [
(Some("order"), true),
(Some("alpha-lower"), true),
(Some("alpha-upper"), true),
(Some("roman-lower"), true),
(Some("roman-upper"), true),
(Some(" Order "), true),
(Some("ROMAN-UPPER"), true),
(Some("bullet"), false),
(Some("simple"), false),
(Some("none"), false),
(Some("bogus"), false),
(None, false),
];
for (list_type, enumerated) in cases {
let attr = list_type.map_or(String::new(), |t| format!(" list-type=\"{t}\""));
let doc = body_doc(&two_items(&attr));
assert_eq!(
json_list_items(&doc),
vec![
("first".to_string(), enumerated, String::new()),
("second".to_string(), enumerated, String::new()),
],
"{list_type:?}"
);
let want = if enumerated {
"1. first\n2. second"
} else {
"- first\n- second"
};
let md = doc.export_to_markdown();
assert!(md.contains(want), "{list_type:?}: {md}");
let dclx = doc.export_to_doclang();
assert_eq!(
dclx.contains("<list class=\"ordered\">"),
enumerated,
"{list_type:?}: {dclx}"
);
}
}
#[test]
fn nested_list_takes_its_own_list_type() {
let doc = body_doc(
"<sec><title>Steps</title><list list-type=\"bullet\"><list-item><p>outer</p>\
<list list-type=\"order\"><list-item><p>inner one</p></list-item>\
<list-item><p>inner two</p></list-item></list></list-item></list></sec>",
);
let enumerated: Vec<(String, bool)> = json_list_items(&doc)
.into_iter()
.map(|(t, e, _)| (t, e))
.collect();
assert_eq!(
enumerated,
vec![
("outer".into(), false),
("inner one".into(), true),
("inner two".into(), true),
]
);
assert!(
doc.export_to_markdown()
.contains("- outer\n 1. inner one\n 2. inner two"),
"{}",
doc.export_to_markdown()
);
let doc = body_doc(
"<sec><title>Steps</title><list list-type=\"order\"><list-item><p>outer</p>\
<list><list-item><p>inner one</p></list-item><list-item><p>inner two</p></list-item>\
</list></list-item><list-item><p>outer two</p></list-item></list></sec>",
);
assert!(
doc.export_to_markdown()
.contains("1. outer\n - inner one\n - inner two\n2. outer two"),
"{}",
doc.export_to_markdown()
);
}
#[test]
fn list_item_label_becomes_marker() {
let doc = body_doc(
"<sec><title>Conditions</title><list list-type=\"simple\">\
<list-item><label>(a)</label><p>first</p></list-item>\
<list-item><label>(b)</label><p>second</p></list-item></list></sec>",
);
assert_eq!(
json_list_items(&doc),
vec![
("first".to_string(), false, "(a)".to_string()),
("second".to_string(), false, "(b)".to_string()),
]
);
assert!(
doc.export_to_markdown()
.contains("- (a) first\n- (b) second"),
"{}",
doc.export_to_markdown()
);
let dclx = doc.export_to_doclang();
assert!(
dclx.contains(
"<list>\n <ldiv>\n <marker>(a)</marker>\n </ldiv>\n first\n"
),
"{dclx}"
);
}
#[test]
fn labels_in_a_numbered_list() {
let doc = body_doc(
"<sec><title>C</title><list list-type=\"order\">\
<list-item><label>1.</label><p>first</p></list-item>\
<list-item><label>(ii)</label><p>second</p></list-item>\
<list-item><p>third</p></list-item></list></sec>",
);
assert_eq!(
json_list_items(&doc),
vec![
("first".to_string(), true, "1.".to_string()),
("second".to_string(), true, "(ii)".to_string()),
("third".to_string(), true, String::new()),
]
);
let md = doc.export_to_markdown();
assert!(md.contains("1. first\n- (ii) second\n3. third"), "{md}");
let dclx = doc.export_to_doclang();
assert!(
dclx.contains(
"<list class=\"ordered\">\n <ldiv>\n <marker>1.</marker>\n </ldiv>\n first\n <ldiv>\n <marker>(ii)</marker>\n </ldiv>\n second\n <ldiv/>\n third\n </list>"
),
"{dclx}"
);
}
#[test]
fn list_item_text_is_its_paragraphs() {
let doc = body_doc(
"<sec><title>C</title><list list-type=\"bullet\">\
<list-item>loose text<p>para one</p><p>para two</p></list-item>\
<list-item><label>x</label></list-item></list></sec>",
);
assert_eq!(
json_list_items(&doc),
vec![
("para one para two".to_string(), false, String::new()),
(String::new(), false, "x".to_string()),
]
);
let md = doc.export_to_markdown();
assert!(md.contains("- para one para two\n- x \n"), "{md:?}");
}
#[test]
fn ext_links_nbsp_and_display_formulas() {
let xml = r#"<article xmlns:xlink="http://www.w3.org/1999/xlink"><front><article-meta>
<title-group><article-title>T</article-title></title-group>
</article-meta></front>
<body><sec><title>S</title>
<p>See RRID: <ext-link ext-link-type="uri" xlink:href="https://scicrunch.org/resolver/AB_1">AB_1</ext-link> here.</p>
<p>Plain <ext-link xlink:href=" ">blank</ext-link> link.</p>
<disp-formula><tex-math><![CDATA[$$\begin{eqnarray}
a=b
\end{eqnarray}$$]]></tex-math></disp-formula>
</sec></body>
<back><ref-list><title>References</title>
<ref><mixed-citation>A. S. de Castro, Phys. Lett. A. 346 (2005).</mixed-citation></ref>
</ref-list></back></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let doc = JatsBackend::default().convert(&src).unwrap();
let md = doc.export_to_markdown();
assert!(
md.contains("See RRID: [AB\\_1](https://scicrunch.org/resolver/AB_1) here."),
"{md}"
);
assert!(
md.contains("Plain blank link."),
"blank href → no link: {md}"
);
assert!(
md.contains("$$\\begin{eqnarray}\na=b\n\\end{eqnarray}$$"),
"{md}"
);
assert!(md.contains("A. S. de\u{a0}Castro"), "{md}");
assert!(doc
.nodes
.iter()
.any(|n| matches!(n, Node::Formula { latex, .. } if latex.starts_with("\\begin"))));
}
#[test]
fn metadata_and_sections() {
let xml = r#"<article><front><article-meta>
<title-group><article-title>My Paper</article-title></title-group>
<contrib-group>
<contrib contrib-type="author"><name><surname>Doe</surname><given-names>Jane</given-names></name>
<xref ref-type="aff" rid="a1"/></contrib>
</contrib-group>
<aff id="a1"><label>1</label>Acme & Co</aff>
<abstract><p>Short summary.</p></abstract>
</article-meta></front>
<body><sec><title>Intro</title><p>Body text.</p></sec></body></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let md = JatsBackend::default()
.convert(&src)
.unwrap()
.export_to_markdown();
assert!(md.starts_with("# My Paper\n\nJane Doe\n\nAcme & Co\n\n## Abstract\n\nShort summary.\n\n## Intro\n\nBody text."), "got:\n{md}");
}
fn json_of(xml: &str) -> serde_json::Value {
let src = SourceDocument::from_bytes("t", InputFormat::XmlJats, xml.as_bytes().to_vec());
serde_json::from_str(
&JatsBackend::default()
.convert(&src)
.unwrap()
.export_to_json(),
)
.unwrap()
}
#[test]
fn tree_mirrors_docling_parent_threading() {
let xml = r#"<article><front><article-meta><title-group><article-title>T</article-title></title-group>
<contrib-group><contrib contrib-type="author"><name><surname>Doe</surname><given-names>J</given-names></name></contrib></contrib-group>
<abstract><p>Abs.</p></abstract></article-meta></front>
<body><sec><title>Intro</title><p>Plain <italic>it</italic> end.</p>
<list><list-item><p>one</p></list-item><list-item><p>two</p></list-item></list>
<fig><label>Fig 1</label><caption><p>Cap.</p></caption><graphic xlink:href="a.png" xmlns:xlink="http://www.w3.org/1999/xlink"/></fig></sec></body>
<back><fn-group><fn><p>Note.</p></fn></fn-group>
<ref-list><title>References</title><ref><mixed-citation>Ref one.</mixed-citation></ref></ref-list></back></article>"#;
let v = json_of(xml);
let texts = v["texts"].as_array().unwrap();
let by = |i: usize| {
(
texts[i]["label"].as_str().unwrap(),
texts[i]["parent"]["$ref"].as_str().unwrap(),
)
};
assert_eq!(by(0), ("title", "#/body"));
assert_eq!(by(1), ("paragraph", "#/texts/0")); assert_eq!(by(2), ("section_header", "#/texts/0")); assert_eq!(texts[2]["level"], 1);
assert_eq!(by(3), ("text", "#/texts/2"));
assert_eq!(by(4), ("section_header", "#/texts/0")); assert_eq!(v["groups"][0]["label"], "inline");
assert_eq!(v["groups"][0]["parent"]["$ref"], "#/texts/4");
assert_eq!(by(5), ("text", "#/groups/0"));
assert_eq!(texts[6]["formatting"]["italic"], true);
assert_eq!(v["groups"][1]["label"], "list");
assert_eq!(v["groups"][1]["parent"]["$ref"], "#/texts/4");
assert_eq!(by(8), ("list_item", "#/groups/1"));
assert_eq!(by(10), ("caption", "#/body"));
assert_eq!(texts[10]["text"], "Fig 1 Cap.");
assert_eq!(v["pictures"][0]["parent"]["$ref"], "#/texts/4");
assert_eq!(v["pictures"][0]["captions"][0]["$ref"], "#/texts/10");
assert_eq!(by(11), ("section_header", "#/texts/0"));
assert_eq!(texts[11]["text"], "Footnotes");
assert_eq!(v["groups"][2]["name"], "footnotes");
assert_eq!(by(12), ("list_item", "#/groups/2"));
assert_eq!(texts[12]["text"], "");
assert_eq!(by(13), ("footnote", "#/texts/12"));
assert_eq!(by(14), ("section_header", "#/texts/0"));
assert_eq!(v["groups"][3]["label"], "list");
assert_eq!(v["groups"][3]["parent"]["$ref"], "#/texts/14");
assert_eq!(by(15), ("list_item", "#/groups/3"));
}
#[test]
fn siblings_after_a_section_inherit_its_heading() {
let xml = r#"<article><body><sec><title>Funding</title><p>a</p></sec><p>after</p></body></article>"#;
let v = json_of(xml);
let texts = v["texts"].as_array().unwrap();
assert_eq!(texts[1]["text"], "Funding");
assert_eq!(texts[3]["text"], "after");
assert_eq!(texts[3]["parent"]["$ref"], "#/texts/1");
}
fn md_of(xml: &str) -> String {
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
JatsBackend::default()
.convert(&src)
.unwrap()
.export_to_markdown()
}
#[test]
fn structured_abstract_keeps_its_sections() {
let md = md_of(
r#"<article><front><article-meta>
<title-group><article-title>T</article-title></title-group>
<abstract>
<sec><title>Background</title><p>B one.</p><p>B two.</p></sec>
<sec><p>No title here.</p></sec>
<sec><title>Empty</title></sec>
<sec><title>Outer</title><p>O.</p><sec><title>Inner</title><p>I.</p></sec></sec>
</abstract>
</article-meta></front><body/></article>"#,
);
assert_eq!(
md,
"# T
## Abstract
### Background
B one.
B two.
No title here.
### Outer
O.
"
);
}
#[test]
fn sub_articles_are_converted() {
let md = md_of(
r#"<article><front><article-meta>
<title-group><article-title>T</article-title></title-group>
</article-meta></front><body><p>Main.</p></body>
<sub-article><front-stub><title-group><article-title>Decision letter</article-title></title-group></front-stub>
<body><p>Reviewed.</p></body>
<sub-article><front-stub><title-group><article-title>Reply</article-title></title-group></front-stub>
<body><p>Thanks.</p></body></sub-article>
</sub-article></article>"#,
);
assert_eq!(
md,
"# T\n\nMain.\n\n## Decision letter\n\nReviewed.\n\n### Reply\n\nThanks.\n"
);
}
#[test]
fn abstract_label_and_skip_rules_follow_docling() {
let md = md_of(
r#"<article><front><article-meta>
<title-group><article-title>T</article-title></title-group>
<abstract><label>Summary</label><p>S.</p></abstract>
<abstract abstract-type="graphical"><title>Graphical</title><sec><title>X</title></sec></abstract>
<abstract><p>Plain.</p><sec><title>Also</title><p>Sectioned.</p></sec></abstract>
</article-meta></front><body/></article>"#,
);
assert_eq!(
md,
"# T
## Summary
S.
## Abstract
Plain.
### Also
Sectioned.
"
);
}
#[test]
fn title_is_the_direct_text_of_the_title_group_children() {
let md = md_of(
r#"<article><front><article-meta>
<title-group><article-title>Response of <italic>Y. pestis</italic> to stress</article-title>
<subtitle>A sub</subtitle></title-group>
</article-meta></front><body/></article>"#,
);
assert!(
md.starts_with(
"# Response of A sub
"
),
"{md}"
);
}
#[test]
fn body_tables_figures_and_references() {
let xml = r#"<article><front><article-meta>
<title-group><article-title>T</article-title></title-group>
</article-meta></front>
<body><sec><title>S</title>
<fig><label>Fig 1</label><caption><p>A caption.</p></caption><graphic/></fig>
<table-wrap><label>Table 1</label><caption><p>Table cap.</p></caption>
<table><thead><tr><th>Name</th><th>N</th></tr></thead>
<tbody><tr><td>a</td><td>1</td></tr></tbody></table></table-wrap>
</sec></body>
<back><ref-list><title>References</title>
<ref><mixed-citation>Doe J. A title. 2020.</mixed-citation></ref>
</ref-list></back></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let md = JatsBackend::default()
.convert(&src)
.unwrap()
.export_to_markdown();
assert!(
md.contains("Fig 1 A caption.\n\n<!-- image -->"),
"figure:\n{md}"
);
assert!(md.contains("Table 1 Table cap."), "table caption:\n{md}");
assert!(md.contains("| Name"), "table grid:\n{md}");
assert!(md.contains("## References"), "refs heading:\n{md}");
assert!(md.contains("- Doe J. A title. 2020."), "citation:\n{md}");
}
#[test]
fn table_caption_and_span_structure() {
let xml = r#"<article><body><sec><title>S</title>
<table-wrap><label>Table 1</label><caption><p>Cap.</p></caption>
<table>
<thead><tr><th colspan="2">Group</th><th>N</th></tr></thead>
<tbody><tr><td>a</td><td>b</td><td>1</td></tr></tbody>
</table></table-wrap>
</sec></body></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let doc = JatsBackend::default().convert(&src).unwrap();
let dclx = doc.export_to_doclang();
assert!(
dclx.contains("<table>\n <caption>Table 1 Cap.</caption>"),
"caption inside table:\n{dclx}"
);
assert!(
dclx.contains("<ched/>\n Group\n <lcel/>\n <ched/>\n N"),
"colspan header → ched + lcel:\n{dclx}"
);
let md = doc.export_to_markdown();
assert!(md.contains("Table 1 Cap."), "md caption:\n{md}");
}
#[test]
fn emphasis_and_inline_formula() {
let xml = r#"<article><body><sec><title>S</title>
<p>We combined <italic>B</italic>. <italic>malayi</italic> with a
<bold>strong</bold> effect and mass <inline-formula><tex-math>$m c^2$</tex-math></inline-formula> energy.</p>
</sec></body></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let md = JatsBackend::default()
.convert(&src)
.unwrap()
.export_to_markdown();
assert!(
md.contains(
"We combined *B* . *malayi* with a **strong** effect and mass $m c^2$ energy."
),
"emphasis + inline formula:\n{md}"
);
}
#[test]
fn nested_lists_keep_structure() {
let xml = r#"<article><body><sec><title>S</title>
<list>
<list-item><p>Item 1</p>
<list>
<list-item><p>Subitem A</p></list-item>
<list-item><p>Subitem B</p></list-item>
</list>
</list-item>
<list-item><p>Item 2</p></list-item>
</list></sec></body></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let doc = JatsBackend::default().convert(&src).unwrap();
let items: Vec<(String, u8)> = doc
.nodes
.iter()
.filter_map(|n| match n {
Node::ListItem { text, level, .. } => Some((text.clone(), *level)),
_ => None,
})
.collect();
assert_eq!(
items,
vec![
("Item 1".to_string(), 0),
("Subitem A".to_string(), 1),
("Subitem B".to_string(), 1),
("Item 2".to_string(), 0),
],
"nested structure: {items:?}"
);
let md = doc.export_to_markdown();
assert!(
md.contains("- Item 1\n - Subitem A"),
"markdown nesting:\n{md}"
);
}
#[test]
fn empty_display_formula_does_not_truncate() {
let xml = r#"<article><body><sec><title>S</title>
<p>Before.</p>
<disp-formula><tex-math/></disp-formula>
<p>After.</p></sec></body></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let md = JatsBackend::default()
.convert(&src)
.unwrap()
.export_to_markdown();
assert!(md.contains("Before."), "got:\n{md}");
assert!(
md.contains("After."),
"content after the empty formula lost:\n{md}"
);
assert!(!md.contains("$$"), "no phantom formula:\n{md}");
}
struct TempDir(PathBuf);
impl TempDir {
fn new(tag: &str) -> Self {
let dir = std::env::temp_dir().join(format!(
"docling-jats-{tag}-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
TempDir(dir)
}
fn path(&self) -> &Path {
&self.0
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn write_png(path: &Path, w: u32, h: u32, rgb: [u8; 3]) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel(w, h, image::Rgb(rgb)))
.save_with_format(path, image::ImageFormat::Png)
.unwrap();
}
fn write_jpg(path: &Path, w: u32, h: u32, rgb: [u8; 3]) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel(w, h, image::Rgb(rgb)))
.save_with_format(path, image::ImageFormat::Jpeg)
.unwrap();
}
fn jats_body(body: &str) -> String {
format!(
r#"<article xmlns:xlink="http://www.w3.org/1999/xlink"><front><article-meta>
<title-group><article-title>T</article-title></title-group>
</article-meta></front><body><sec><title>S</title>{body}</sec></body></article>"#
)
}
fn convert_file(dir: &Path, body: &str, fetch_images: bool) -> DoclingDocument {
let path = dir.join("article.nxml");
std::fs::write(&path, jats_body(body)).unwrap();
let src = SourceDocument::from_file(&path).unwrap();
JatsBackend {
images: local_images(fetch_images),
}
.convert(&src)
.unwrap()
}
fn local_images(on: bool) -> ImagePolicy {
ImagePolicy::new(if on {
ImageSources::Local
} else {
ImageSources::None
})
}
fn picture_images(doc: &DoclingDocument) -> Vec<Option<&docling_core::PictureImage>> {
doc.nodes
.iter()
.filter_map(|n| match n {
Node::Picture { image, .. } => Some(image.as_ref()),
_ => None,
})
.collect()
}
fn size_and_pixel(img: &docling_core::PictureImage) -> ((u32, u32), [u8; 3]) {
let rgb = image::load_from_memory(&img.data).unwrap().to_rgb8();
((img.width, img.height), rgb.get_pixel(0, 0).0)
}
#[test]
fn figure_image_is_embedded_only_when_fetching() {
let dir = TempDir::new("embed");
write_png(&dir.path().join("images/figure.png"), 7, 5, [255, 0, 0]);
let body = r#"<fig><label>Figure 1</label><caption><p>A red rectangle.</p></caption>
<graphic xlink:href="images/figure.png"/></fig>"#;
let doc = convert_file(dir.path(), body, false);
assert_eq!(picture_images(&doc), [None]);
let doc = convert_file(dir.path(), body, true);
let pics = picture_images(&doc);
assert_eq!(pics.len(), 1);
let img = pics[0].expect("embedded");
assert_eq!(size_and_pixel(img), ((7, 5), [255, 0, 0]));
assert_eq!(img.mimetype, "image/png");
let caption = doc.nodes.iter().find_map(|n| match n {
Node::Picture { caption, .. } => caption.clone(),
_ => None,
});
assert_eq!(caption.as_deref(), Some("Figure 1 A red rectangle."));
assert!(doc.export_to_markdown().contains("<!-- image -->"));
let json: serde_json::Value = serde_json::from_str(&doc.export_to_json()).unwrap();
assert!(json["pictures"][0]["image"]["uri"]
.as_str()
.is_some_and(|u| u.starts_with("data:image/png;base64,")));
}
#[test]
fn figure_image_needs_a_local_source_file() {
let dir = TempDir::new("stream");
write_png(&dir.path().join("figure.png"), 7, 5, [255, 0, 0]);
let body = r#"<fig><graphic xlink:href="figure.png"/></fig>"#;
let xml = jats_body(body).into_bytes();
let stream = SourceDocument::from_bytes("article.nxml", InputFormat::XmlJats, xml.clone());
let doc = JatsBackend {
images: local_images(true),
}
.convert(&stream)
.unwrap();
assert_eq!(picture_images(&doc), [None]);
let remote = SourceDocument::from_bytes("article.nxml", InputFormat::XmlJats, xml.clone())
.with_base_url("https://example.com/article.nxml");
let doc = JatsBackend {
images: local_images(true),
}
.convert(&remote)
.unwrap();
assert_eq!(picture_images(&doc), [None]);
let mut with_path = SourceDocument::from_bytes("article.nxml", InputFormat::XmlJats, xml);
with_path.path = Some(dir.path().join("source.nxml"));
let doc = JatsBackend {
images: local_images(true),
}
.convert(&with_path)
.unwrap();
assert!(picture_images(&doc)[0].is_some());
}
#[test]
fn figure_image_resolves_an_extensionless_href() {
let dir = TempDir::new("extless");
write_jpg(&dir.path().join("images/figure.jpg"), 9, 6, [0, 0, 255]);
for graphic in [
r#"<graphic xlink:href="images/figure"/>"#,
r#"<alternatives><graphic xlink:href="images/unsupported.svg"/><graphic xlink:href="images/figure"/></alternatives>"#,
] {
let doc = convert_file(dir.path(), &format!("<fig>{graphic}</fig>"), true);
let pics = picture_images(&doc);
assert_eq!(pics.len(), 1, "{graphic}");
let img = pics[0].expect("probed .jpg");
assert_eq!((img.width, img.height), (9, 6));
assert_eq!(img.mimetype, "image/jpeg");
}
}
#[test]
fn figure_image_falls_back_past_undecodable_and_absolute_renditions() {
let dir = TempDir::new("fallback");
std::fs::create_dir_all(dir.path().join("images")).unwrap();
std::fs::write(dir.path().join("images/broken.png"), b"not an image").unwrap();
write_png(&dir.path().join("images/figure.png"), 9, 6, [0, 0, 255]);
write_png(&dir.path().join("absolute.png"), 7, 5, [255, 0, 0]);
let absolute = dir.path().join("absolute.png");
for body in [
r#"<fig><alternatives><graphic xlink:href="images/broken.png"/><graphic xlink:href="images/figure.png"/></alternatives></fig>"#.to_string(),
format!(
r#"<fig><alternatives><graphic xlink:href="{}"/><graphic xlink:href="images/figure.png"/></alternatives></fig>"#,
absolute.display()
),
] {
let doc = convert_file(dir.path(), &body, true);
let img = picture_images(&doc)[0].expect("fell back");
assert_eq!(size_and_pixel(img), ((9, 6), [0, 0, 255]), "{body}");
}
}
#[test]
fn figure_image_skips_unavailable_renditions() {
let dir = TempDir::new("skip");
for graphic in [
"",
"<graphic/>",
r#"<graphic xlink:href=" "/>"#,
r#"<graphic xlink:href="https://example.com/figure.png"/>"#,
r#"<graphic xlink:href="figure.svg"/>"#,
r#"<graphic xlink:href="missing.png"/>"#,
] {
let doc = convert_file(
dir.path(),
&format!("<fig>{graphic}</fig><p>Content after the unavailable figure.</p>"),
true,
);
assert_eq!(picture_images(&doc), [None], "{graphic}");
assert!(doc
.export_to_markdown()
.contains("Content after the unavailable figure."));
}
}
#[test]
fn figure_image_blocks_path_traversal() {
let dir = TempDir::new("traversal");
write_png(&dir.path().join("outside.png"), 7, 5, [255, 0, 0]);
let article_dir = dir.path().join("article");
write_png(&article_dir.join("fallback.png"), 9, 6, [0, 0, 255]);
let doc = convert_file(
&article_dir,
r#"<fig><alternatives><graphic xlink:href="../outside.png"/><graphic xlink:href="fallback.png"/></alternatives></fig>
<p>Content after the blocked figure.</p>"#,
true,
);
assert_eq!(picture_images(&doc), [None]);
assert!(doc
.export_to_markdown()
.contains("Content after the blocked figure."));
}
#[test]
fn figure_path_helpers_follow_docling() {
assert!(is_local_path("images/a.png"));
assert!(is_local_path("/abs/a.png"));
assert!(is_local_path(r"C:\a.png"));
assert!(!is_local_path("https://x/a.png"));
assert!(!is_local_path("//cdn/a.png"));
assert!(!is_local_path("data:image/png;base64,AA=="));
assert!(is_absolute_path("/abs/a.png"));
assert!(is_absolute_path("C:/a.png"));
assert!(!is_absolute_path("images/a.png"));
let base = Path::new("/base");
assert_eq!(
confined_path(base, "a/../b.png"),
Some(PathBuf::from("/base/b.png"))
);
assert_eq!(confined_path(base, "../b.png"), None);
assert_eq!(confined_path(base, "/etc/passwd"), None);
}
}