use std::path::{Path, PathBuf};
use roxmltree::{Document, Node as XmlNode, ParsingOptions};
use crate::backend::markdown::escape_text;
use crate::backend::DeclarativeBackend;
use crate::error::ConversionError;
use crate::source::SourceDocument;
use docling_core::{
inline_paragraph_node, DoclingDocument, InlineRun, Node, PictureImage, Script, Table,
TableStructure,
};
#[derive(Default)]
pub struct JatsBackend {
pub fetch_images: bool,
}
const SKIP_TEXT: &[&str] = &["term", "disp-formula", "inline-formula"];
#[derive(Clone, Copy, Default, PartialEq, Eq)]
struct Fmt {
bold: bool,
italic: bool,
underline: bool,
strike: bool,
script: Script,
}
impl Fmt {
fn with_tag(self, tag: &str) -> Fmt {
match tag {
"bold" => Fmt { bold: true, ..self },
"italic" => Fmt {
italic: true,
..self
},
"underline" => Fmt {
underline: true,
..self
},
"strike" => Fmt {
strike: true,
..self
},
"sub" => Fmt {
script: Script::Sub,
..self
},
"sup" => Fmt {
script: Script::Super,
..self
},
_ => self,
}
}
fn to_inline_run(self, text: &str) -> InlineRun {
InlineRun {
text: text.to_string(),
bold: self.bold,
italic: self.italic,
underline: self.underline,
strike: self.strike,
script: self.script,
code: false,
formula: false,
}
}
}
struct Seg {
formula: bool,
text: String,
fmt: Fmt,
hyperlink: Option<String>,
}
const FLUSH_TAGS: &[&str] = &["ack", "sec", "list", "boxed-text", "disp-formula", "fig"];
const DEFAULT_HEADER_ACKNOWLEDGMENTS: &str = "Acknowledgments";
const DEFAULT_HEADER_FOOTNOTES: &str = "Footnotes";
const DEFAULT_HEADER_REFERENCES: &str = "References";
const DEFAULT_TEXT_ETAL: &str = "et al.";
impl DeclarativeBackend for JatsBackend {
fn convert(&self, source: &SourceDocument) -> Result<DoclingDocument, ConversionError> {
let xml = source.text()?;
let opts = ParsingOptions {
allow_dtd: true,
..Default::default()
};
let dom = Document::parse_with_options(xml, opts)
.map_err(|e| ConversionError::with_source("jats", e))?;
let mut doc = DoclingDocument::new(&source.name);
if let Some(title) = parse_title(&dom) {
doc.push(Node::Heading {
level: 1,
text: escape_text(&title),
});
}
let (authors, affiliations) = parse_authors(&dom);
if !authors.is_empty() {
doc.push(Node::Paragraph {
text: escape_text(&authors.join(", ")),
});
}
if !affiliations.is_empty() {
doc.push(Node::Paragraph {
text: escape_text(&affiliations.join("; ")),
});
}
for abs in parse_abstracts(&dom) {
if abs.plain.is_empty() && abs.sections.is_empty() {
continue;
}
let label = if abs.label.is_empty() {
"Abstract"
} else {
&abs.label
};
doc.push(Node::Heading {
level: 2,
text: escape_text(label),
});
if abs.sections.is_empty() {
doc.push(Node::Paragraph {
text: escape_text(&abs.plain),
});
} else {
for (title, paragraphs) in &abs.sections {
if !title.is_empty() {
doc.push(Node::Heading {
level: 3,
text: escape_text(title),
});
}
for p in paragraphs {
doc.push(Node::Paragraph {
text: escape_text(p),
});
}
}
}
}
let fig_base = if self.fetch_images {
source.base_dir()
} else {
None
};
let mut hlevel: i32 = 0;
for tag in ["body", "back"] {
if let Some(node) = dom.descendants().find(|n| n.has_tag_name(tag)) {
walk_linear(
node,
false,
Fmt::default(),
None,
&mut hlevel,
fig_base,
&mut doc,
);
}
}
Ok(doc)
}
}
fn raw_text(node: XmlNode, out: &mut String) {
if let Some(t) = node.text() {
out.push_str(&t.replace('\n', " "));
}
for child in node.children() {
if child.is_element() {
if !SKIP_TEXT.contains(&child.tag_name().name()) {
raw_text(child, out);
}
if let Some(tail) = child.tail() {
out.push_str(&tail.replace('\n', " "));
}
} else if child.is_text() {
}
}
}
pub(crate) fn convert_generic(source: &SourceDocument) -> Result<DoclingDocument, ConversionError> {
let xml = source.text()?;
let opts = ParsingOptions {
allow_dtd: true,
..Default::default()
};
let dom = Document::parse_with_options(xml, opts)
.map_err(|e| ConversionError::with_source("xml", e))?;
let mut doc = DoclingDocument::new(&source.name);
walk_generic(dom.root_element(), &mut doc);
Ok(doc)
}
fn walk_generic(node: XmlNode, doc: &mut DoclingDocument) {
for child in node.children().filter(XmlNode::is_element) {
let tag = child.tag_name().name();
if tag == "table-wrap" {
add_table(doc, child);
continue;
}
if tag == "table" {
if let Some(t) = parse_jats_table(child) {
doc.push(Node::Table(t));
}
continue;
}
let has_direct_text = child
.children()
.any(|c| c.is_text() && !c.text().unwrap_or("").trim().is_empty());
let has_element_children = child.children().any(|c| c.is_element());
if !has_element_children || has_direct_text {
let t = normalize(&sanitize_generic(&generic_text(child)));
if !t.trim().is_empty() {
doc.push(Node::Paragraph {
text: escape_text(&t),
});
}
} else {
walk_generic(child, doc);
}
}
}
fn sanitize_generic(s: &str) -> String {
s.chars()
.map(|c| match c {
'\u{2014}' | '\u{2013}' => '-',
'\u{2019}' | '\u{2018}' => '\'',
'\u{201C}' | '\u{201D}' => '"',
c => c,
})
.collect()
}
fn generic_text(node: XmlNode) -> String {
let mut s = String::new();
for child in node.children() {
if child.is_text() {
s.push_str(child.text().unwrap_or(""));
} else if child.is_element() {
s.push(' ');
s.push_str(&generic_text(child));
s.push(' ');
}
}
s
}
fn node_text(node: XmlNode) -> String {
let mut s = String::new();
raw_text(node, &mut s);
normalize(&s)
}
fn normalize(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut pending_space = false;
for c in s.trim().chars() {
if c.is_ascii_whitespace() {
pending_space = true;
} else {
if pending_space && !out.is_empty() {
out.push(' ');
}
pending_space = false;
out.push(c);
}
}
out
}
fn parse_title(dom: &Document) -> Option<String> {
const METAS: [&str; 4] = [
"article-meta",
"collection-meta",
"book-meta",
"book-part-meta",
];
const NAMES: [&str; 4] = ["article-title", "subtitle", "title", "label"];
let titles: Vec<String> = dom
.descendants()
.filter(|n| {
n.has_tag_name("title-group")
&& n.parent()
.is_some_and(|p| METAS.contains(&p.tag_name().name()))
})
.map(|group| {
group
.children()
.filter(|c| c.is_element() && NAMES.contains(&c.tag_name().name()))
.map(|c| direct_text(c).replace('\n', " ").trim().to_string())
.collect::<Vec<_>>()
.join(" ")
.trim()
.to_string()
})
.collect();
let text = titles.join(" - ");
(!text.is_empty()).then_some(text)
}
fn direct_text<'a>(node: XmlNode<'a, 'a>) -> &'a str {
node.first_child()
.filter(XmlNode::is_text)
.and_then(|c| c.text())
.unwrap_or("")
}
fn parse_authors(dom: &Document) -> (Vec<String>, Vec<String>) {
let Some(meta) = dom.descendants().find(|n| n.has_tag_name("article-meta")) else {
return (Vec::new(), Vec::new());
};
let mut aff_by_id = std::collections::HashMap::new();
for aff in meta.descendants().filter(|n| n.has_tag_name("aff")) {
let Some(id) = aff.attribute("id") else {
continue;
};
let mut name = aff
.descendants()
.filter(|n| n.is_text())
.filter_map(|n| n.text())
.map(|t| t.trim())
.filter(|t| !t.is_empty())
.collect::<Vec<_>>()
.join(", ")
.replace('\n', " ");
if let Some(label) = aff
.children()
.find(|c| c.has_tag_name("label"))
.and_then(|l| l.text())
{
name = name
.strip_prefix(&format!("{label}, "))
.unwrap_or(&name)
.to_string();
}
aff_by_id.insert(id.to_string(), name);
}
let mut authors = Vec::new();
let mut affiliations = Vec::new();
for contrib in meta
.descendants()
.filter(|n| n.has_tag_name("contrib") && n.attribute("contrib-type") == Some("author"))
{
let name = contrib_name(contrib);
if name.is_empty() {
continue;
}
authors.push(name);
for xref in contrib
.children()
.filter(|c| c.has_tag_name("xref") && c.attribute("ref-type") == Some("aff"))
{
if let Some(aff) = xref.attribute("rid").and_then(|id| aff_by_id.get(id)) {
if !affiliations.contains(aff) {
affiliations.push(aff.clone());
}
}
}
}
(authors, affiliations)
}
fn contrib_name(contrib: XmlNode) -> String {
let name = contrib.children().find(|c| c.has_tag_name("name"));
let Some(name) = name else {
return String::new();
};
["prefix", "given-names", "surname", "suffix"]
.iter()
.filter_map(|tag| {
name.children()
.find(|c| c.has_tag_name(*tag))
.and_then(|c| c.text())
.map(str::trim)
.filter(|s| !s.is_empty())
})
.collect::<Vec<_>>()
.join(" ")
}
struct Abstract {
label: String,
plain: String,
sections: Vec<(String, Vec<String>)>,
}
fn parse_abstracts(dom: &Document) -> Vec<Abstract> {
let mut out = Vec::new();
for abs in dom.descendants().filter(|n| n.has_tag_name("abstract")) {
let mut plain = Vec::new();
let mut sections = Vec::new();
for child in abs.children().filter(XmlNode::is_element) {
match child.tag_name().name() {
"p" => {
let t = node_text(child);
if !t.is_empty() {
plain.push(t);
}
}
"sec" => {
let section = abstract_section(child);
if !section.1.is_empty() {
sections.push(section);
}
}
_ => {}
}
}
out.push(Abstract {
label: title_or_label(abs),
plain: normalize(&plain.join(" ")),
sections,
});
}
out
}
fn abstract_section(section: XmlNode) -> (String, Vec<String>) {
let paragraphs = section
.children()
.filter(|c| c.has_tag_name("p"))
.map(node_text)
.filter(|t| !t.is_empty())
.collect();
(title_or_label(section), paragraphs)
}
fn title_or_label(node: XmlNode) -> String {
node.children()
.find(|c| c.has_tag_name("title") || c.has_tag_name("label"))
.map(node_text)
.unwrap_or_default()
}
fn get_text(node: XmlNode) -> String {
let mut s = String::new();
raw_text(node, &mut s);
s
}
fn norm_text(node: XmlNode) -> String {
normalize(&get_text(node))
}
fn fw_level(dl: i32) -> u8 {
(dl + 1).clamp(1, 6) as u8
}
fn header_text(child: XmlNode) -> Option<String> {
child
.children()
.find(|c| c.has_tag_name("title") || c.has_tag_name("label"))
.map(get_text)
.map(|s| normalize(&s))
.filter(|s| !s.is_empty())
.or_else(|| {
child
.has_tag_name("ack")
.then(|| DEFAULT_HEADER_ACKNOWLEDGMENTS.to_string())
})
}
fn add_citation(doc: &mut DoclingDocument, parent_is_list: bool, text: &str) {
if text.is_empty() {
return;
}
if parent_is_list {
doc.push(Node::ListItem {
ordered: false,
number: 0,
first_in_list: false,
text: escape_text(text),
level: 0,
marker: None,
location: None,
dclx: None,
href: None,
layer: None,
});
} else {
doc.push(Node::Paragraph {
text: escape_text(text),
});
}
}
fn walk_linear(
node: XmlNode,
parent_is_list: bool,
fmt: Fmt,
hyperlink: Option<&str>,
hlevel: &mut i32,
fig_base: Option<&Path>,
doc: &mut DoclingDocument,
) -> Vec<Seg> {
let node_tag = node.tag_name().name();
let current = fmt.with_tag(node_tag);
let own_link = (node_tag == "ext-link")
.then(|| ext_link_href(node))
.flatten();
let current_link: Option<&str> = own_link.as_deref().or(hyperlink);
let mut segments: Vec<Seg> = Vec::new();
if node_tag != "term" {
if let Some(t) = node.text() {
append_run(&mut segments, &t.replace('\n', " "), current, current_link);
}
}
for child in node.children().filter(XmlNode::is_element) {
let mut stop_walk = false;
let ctag = child.tag_name().name();
if node_tag == "p" && FLUSH_TAGS.contains(&ctag) {
emit_inline(doc, std::mem::take(&mut segments));
}
let mut child_in_list = parent_is_list;
let mut opened_section = false;
match ctag {
"sec" | "ack" => {
if let Some(text) = header_text(child) {
*hlevel += 1;
doc.push(Node::Heading {
level: fw_level(*hlevel),
text: escape_text(&text),
});
opened_section = true;
}
}
"list" => {
child_in_list = true;
}
"list-item" => {
add_list_item(doc, child, 0);
stop_walk = true;
}
"fig" => {
add_figure(doc, child, fig_base);
stop_walk = true;
}
"table-wrap" => {
add_table(doc, child);
stop_walk = true;
}
"supplementary-material" => {
stop_walk = true;
}
"fn-group" => {
add_footnote_group(doc, child, *hlevel);
stop_walk = true;
}
"ref-list" if node_tag != "ref-list" => {
let text = child
.children()
.find(|c| c.has_tag_name("title") || c.has_tag_name("label"))
.map(|h| normalize(&get_text(h)))
.filter(|s| !s.is_empty())
.unwrap_or_else(|| DEFAULT_HEADER_REFERENCES.to_string());
doc.push(Node::Heading {
level: fw_level(1),
text: escape_text(&text),
});
child_in_list = true;
}
"element-citation" => {
let text = parse_element_citation(child);
add_citation(doc, parent_is_list, &text);
stop_walk = true;
}
"mixed-citation" => {
let text = norm_text(child);
add_citation(doc, parent_is_list, &text);
stop_walk = true;
}
"tex-math" => {
add_equation(doc, child);
stop_walk = true;
}
"inline-formula" => {
extend_segments(
&mut segments,
walk_inline_formula(child, current, current_link),
);
stop_walk = true;
}
_ => {}
}
if !stop_walk {
let child_segments = walk_linear(
child,
child_in_list,
current,
current_link,
hlevel,
fig_base,
doc,
);
let parent_is_p = node.parent().map(|p| p.has_tag_name("p")).unwrap_or(false);
if !(parent_is_p && FLUSH_TAGS.contains(&node_tag)) {
extend_segments(&mut segments, child_segments);
}
if opened_section {
*hlevel -= 1;
}
}
if let Some(tail) = child.tail() {
append_run(
&mut segments,
&tail.replace('\n', " "),
current,
current_link,
);
}
}
if node_tag == "p" {
emit_inline(doc, segments);
Vec::new()
} else {
segments
}
}
fn walk_inline_formula(node: XmlNode, fmt: Fmt, hyperlink: Option<&str>) -> Vec<Seg> {
let current = fmt.with_tag(node.tag_name().name());
let mut segments = Vec::new();
if let Some(t) = node.text() {
append_run(&mut segments, &t.replace('\n', " "), current, hyperlink);
}
for child in node.children().filter(XmlNode::is_element) {
if child.tag_name().name() == "tex-math" {
if let Some(formula) = extract_tex_math(child) {
segments.push(Seg {
formula: true,
text: formula,
fmt: Fmt::default(),
hyperlink: hyperlink.map(str::to_string),
});
}
} else {
extend_segments(
&mut segments,
walk_inline_formula(child, current, hyperlink),
);
}
if let Some(tail) = child.tail() {
append_run(&mut segments, &tail.replace('\n', " "), current, hyperlink);
}
}
segments
}
fn ext_link_href(node: XmlNode) -> Option<String> {
let href = node
.attributes()
.find(|a| a.name() == "href")
.map(|a| a.value().trim())
.filter(|v| !v.is_empty())?;
Some(crate::backend::html::normalize_url(href))
}
fn extract_tex_math(node: XmlNode) -> Option<String> {
let text = node.text()?.trim().to_string();
for delim in ["$$", "$"] {
if text.len() > 2 * delim.len() && text.starts_with(delim) && text.ends_with(delim) {
let inner = text[delim.len()..text.len() - delim.len()]
.trim()
.to_string();
return (!inner.is_empty()).then_some(inner);
}
}
(!text.is_empty()).then_some(text)
}
fn append_run(segments: &mut Vec<Seg>, text: &str, fmt: Fmt, hyperlink: Option<&str>) {
if text.is_empty() {
return;
}
if let Some(last) = segments.last_mut() {
if !last.formula && last.fmt == fmt && last.hyperlink.as_deref() == hyperlink {
last.text.push_str(text);
return;
}
}
segments.push(Seg {
formula: false,
text: text.to_string(),
fmt,
hyperlink: hyperlink.map(str::to_string),
});
}
fn extend_segments(segments: &mut Vec<Seg>, more: Vec<Seg>) {
for seg in more {
if seg.formula {
segments.push(seg);
} else {
append_run(segments, &seg.text, seg.fmt, seg.hyperlink.as_deref());
}
}
}
fn emit_inline(doc: &mut DoclingDocument, segments: Vec<Seg>) {
let stripped: Vec<Seg> = segments
.into_iter()
.filter_map(|s| {
let text = s.text.trim().to_string();
(!text.is_empty()).then_some(Seg { text, ..s })
})
.collect();
if stripped.is_empty() {
return;
}
let md_text = stripped
.iter()
.map(seg_markdown)
.collect::<Vec<_>>()
.join(" ");
let runs = stripped
.iter()
.map(|s| {
if s.formula {
InlineRun {
text: s.text.clone(),
formula: true,
..InlineRun::default()
}
} else {
s.fmt.to_inline_run(&s.text)
}
})
.collect();
doc.push(inline_paragraph_node(md_text, runs, true));
}
fn seg_markdown(s: &Seg) -> String {
if s.formula {
return format!("${}$", s.text);
}
let mut out = escape_text(&s.text);
if s.fmt.bold {
out = format!("**{out}**");
}
if s.fmt.italic {
out = format!("*{out}*");
}
if s.fmt.strike {
out = format!("~~{out}~~");
}
if let Some(url) = &s.hyperlink {
out = format!("[{out}]({url})");
}
out
}
fn add_list_item(doc: &mut DoclingDocument, item: XmlNode, level: u8) {
let mut text = String::new();
for part in item.children() {
if part.has_tag_name("list") {
continue;
}
let t = if part.is_text() {
normalize(part.text().unwrap_or(""))
} else {
norm_text(part)
};
if t.trim().is_empty() {
continue;
}
if !text.is_empty() {
text.push(' ');
}
text.push_str(t.trim());
}
if !text.is_empty() {
doc.push(Node::ListItem {
ordered: false,
number: 0,
first_in_list: false,
text: escape_text(&text),
level,
marker: None,
location: None,
dclx: None,
href: None,
layer: None,
});
}
for nested in item.children().filter(|c| c.has_tag_name("list")) {
for sub in nested.children().filter(|c| c.has_tag_name("list-item")) {
add_list_item(doc, sub, level.saturating_add(1));
}
}
}
fn add_equation(doc: &mut DoclingDocument, node: XmlNode) {
if let Some(formula) = extract_tex_math(node) {
doc.push(Node::Formula {
orig: formula.clone(),
latex: formula,
location: None,
});
}
}
fn add_figure(doc: &mut DoclingDocument, node: XmlNode, fig_base: Option<&Path>) {
let label = node
.children()
.find(|c| c.has_tag_name("label"))
.map(|l| get_text(l).trim().to_string())
.unwrap_or_default();
let caption = node
.children()
.find(|c| c.has_tag_name("caption"))
.map(caption_text)
.unwrap_or_default();
let sep = if !label.is_empty() && !caption.is_empty() {
" "
} else {
""
};
let fig_text = format!("{label}{sep}{caption}");
doc.push(Node::Picture {
caption: (!fig_text.is_empty()).then(|| escape_text(&fig_text)),
caption_href: None,
image: fig_base.and_then(|base| load_figure_image(node, base)),
classification: None,
caption_parent: Default::default(),
});
}
const RASTER_IMAGE_SUFFIXES: [&str; 6] = [".jpg", ".jpeg", ".png", ".tif", ".tiff", ".gif"];
fn load_figure_image(fig: XmlNode, base: &Path) -> Option<PictureImage> {
let graphics = fig.children().filter(XmlNode::is_element).flat_map(|c| {
let own = std::iter::once(c).filter(|c| c.has_tag_name("graphic"));
let alternatives = c
.children()
.filter(move |g| c.has_tag_name("alternatives") && g.has_tag_name("graphic"));
own.chain(alternatives)
});
let mut missing: Vec<String> = Vec::new();
for graphic in graphics {
let Some(href) = graphic
.attributes()
.find(|a| a.name() == "href")
.map(|a| a.value().trim())
.filter(|v| !v.is_empty())
else {
continue;
};
if !is_local_path(href) {
continue;
}
if is_absolute_path(href) {
eprintln!(
"docling: warning: Could not process an image from {href}: \
Absolute paths are not allowed with local base_path."
);
continue;
}
let suffix = Path::new(href)
.extension()
.and_then(|e| e.to_str())
.map(str::to_ascii_lowercase);
if suffix.as_deref() == Some("svg") {
continue;
}
let mut candidates = vec![href.to_string()];
if suffix.is_none() {
candidates.extend(RASTER_IMAGE_SUFFIXES.iter().map(|s| format!("{href}{s}")));
}
let mut found = false;
for candidate in &candidates {
let Some(resolved) = confined_path(base, candidate) else {
eprintln!(
"docling: warning: Could not process an image from {href}: \
Path traversal blocked: '{candidate}' resolves outside base directory"
);
return None;
};
if !resolved.is_file() {
continue;
}
found = true;
let decoded = std::fs::read(&resolved)
.ok()
.and_then(|data| crate::backend::ooxml::picture_image(candidate, data));
match decoded {
Some(image) => return Some(image),
None => eprintln!(
"docling: warning: Could not process an image from {}: \
cannot identify image file",
resolved.display()
),
}
}
if !found {
missing.push(href.to_string());
}
}
if !missing.is_empty() {
eprintln!(
"docling: warning: Could not process JATS figure image(s) {}: \
no matching local file exists.",
missing.join(", ")
);
}
None
}
fn is_local_path(value: &str) -> bool {
let Some(colon) = value.find(':') else {
return !value.starts_with("//");
};
let scheme = &value[..colon];
let is_scheme = scheme
.chars()
.next()
.is_some_and(|c| c.is_ascii_alphabetic())
&& scheme
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'));
if !is_scheme {
return !value.starts_with("//");
}
let rest = &value[colon + 1..];
!rest.starts_with("//") && scheme.len() == 1
}
fn is_absolute_path(value: &str) -> bool {
Path::new(value).is_absolute()
|| (value.len() >= 2
&& value.as_bytes()[1] == b':'
&& value.as_bytes()[0].is_ascii_alphabetic()
&& !value[2..].starts_with("//"))
}
fn confined_path(base: &Path, rel: &str) -> Option<PathBuf> {
use std::path::Component;
let mut out = base.to_path_buf();
let mut depth = 0usize;
for comp in Path::new(rel).components() {
match comp {
Component::CurDir => {}
Component::ParentDir => {
if depth == 0 {
return None;
}
out.pop();
depth -= 1;
}
Component::Normal(seg) => {
out.push(seg);
depth += 1;
}
Component::RootDir | Component::Prefix(_) => return None,
}
}
Some(out)
}
fn caption_text(caption: XmlNode) -> String {
let mut out = String::new();
for par in caption.children().filter(XmlNode::is_element) {
if par
.descendants()
.any(|d| d.has_tag_name("supplementary-material"))
{
continue;
}
out.push_str(get_text(par).trim());
out.push(' ');
}
out.trim().to_string()
}
fn add_table(doc: &mut DoclingDocument, node: XmlNode) {
let content = node
.children()
.find(|c| c.has_tag_name("table"))
.or_else(|| {
node.children()
.find(|c| c.has_tag_name("alternatives"))
.and_then(|a| a.children().find(|c| c.has_tag_name("table")))
});
let Some(table_node) = content else { return };
let Some(mut table) = parse_jats_table(table_node) else {
return;
};
let label = node
.children()
.find(|c| c.has_tag_name("label"))
.and_then(|l| l.text())
.map(|t| t.trim().to_string())
.unwrap_or_default();
let caption = node
.children()
.find(|c| c.has_tag_name("caption"))
.map(caption_text)
.unwrap_or_default();
let sep = if !label.is_empty() && !caption.is_empty() {
" "
} else {
""
};
let cap_text = format!("{label}{sep}{caption}");
table.caption = (!cap_text.is_empty()).then(|| escape_text(&cap_text));
doc.push(Node::Table(table));
}
fn parse_jats_table(table: XmlNode) -> Option<Table> {
let rows_nodes: Vec<XmlNode> = table
.descendants()
.filter(|n| n.has_tag_name("tr"))
.collect();
if rows_nodes.iter().any(|r| {
r.descendants()
.any(|d| d.has_tag_name("table") && d != table)
}) {
return None;
}
let num_cols = rows_nodes
.iter()
.map(|r| {
r.children()
.filter(|c| c.has_tag_name("td") || c.has_tag_name("th"))
.map(col_span)
.sum::<usize>()
})
.max()
.unwrap_or(0);
if rows_nodes.is_empty() || num_cols == 0 {
return None;
}
let nrows = rows_nodes.len();
let mut grid: Vec<Vec<String>> = vec![vec![String::new(); num_cols]; nrows];
let mut filled: Vec<Vec<bool>> = vec![vec![false; num_cols]; nrows];
let mut col_header: Vec<Vec<bool>> = vec![vec![false; num_cols]; nrows];
let mut col_continuation: Vec<Vec<bool>> = vec![vec![false; num_cols]; nrows];
let mut row_continuation: Vec<Vec<bool>> = vec![vec![false; num_cols]; nrows];
for (ri, row) in rows_nodes.iter().enumerate() {
let mut ci = 0usize;
for cell in row
.children()
.filter(|c| c.has_tag_name("td") || c.has_tag_name("th"))
{
while ci < num_cols && filled[ri][ci] {
ci += 1;
}
if ci >= num_cols {
break;
}
let cs = col_span(cell);
let rs = row_span(cell);
let text = normalize(&get_text(cell));
let header =
cell.has_tag_name("th") || cell.ancestors().any(|a| a.has_tag_name("thead"));
for r in ri..(ri + rs).min(nrows) {
for c in ci..(ci + cs).min(num_cols) {
grid[r][c] = text.clone();
filled[r][c] = true;
if c > ci {
col_continuation[r][c] = true;
}
if r > ri {
row_continuation[r][c] = true;
}
}
}
col_header[ri][ci] = header;
ci += cs;
}
}
let has_header = col_header.iter().flatten().any(|&h| h);
let has_span = col_continuation
.iter()
.chain(&row_continuation)
.flatten()
.any(|&s| s);
let structure = (has_header || has_span).then(|| TableStructure {
header_row: Vec::new(),
col_continuation,
row_continuation,
row_header: Vec::new(),
col_header,
});
Some(Table {
rows: grid,
location: None,
structure,
cell_blocks: None,
cells: None,
caption: None,
caption_parent: Default::default(),
})
}
fn col_span(cell: XmlNode) -> usize {
cell.attribute("colspan")
.and_then(|v| v.parse().ok())
.filter(|&n: &usize| n >= 1)
.unwrap_or(1)
}
fn row_span(cell: XmlNode) -> usize {
cell.attribute("rowspan")
.and_then(|v| v.parse().ok())
.filter(|&n: &usize| n >= 1)
.unwrap_or(1)
}
fn add_footnote_group(doc: &mut DoclingDocument, node: XmlNode, hlevel: i32) {
let footnotes: Vec<String> = node
.children()
.filter(|c| c.has_tag_name("fn"))
.map(norm_text)
.filter(|s| !s.is_empty())
.collect();
if footnotes.is_empty() {
return;
}
let title = node
.children()
.find(|c| c.has_tag_name("title"))
.map(|t| normalize(&get_text(t)))
.filter(|s| !s.is_empty())
.unwrap_or_else(|| DEFAULT_HEADER_FOOTNOTES.to_string());
doc.push(Node::Heading {
level: fw_level(hlevel + 1),
text: escape_text(&title),
});
for item in footnotes {
doc.push(Node::ListItem {
ordered: false,
number: 0,
first_in_list: false,
text: escape_text(&item),
level: 0,
marker: None,
location: None,
dclx: None,
href: None,
layer: None,
});
}
}
fn parse_element_citation(node: XmlNode) -> String {
let mut names: Vec<String> = Vec::new();
for name in node.descendants().filter(|n| n.has_tag_name("name")) {
let surname = name
.children()
.find(|c| c.has_tag_name("surname"))
.and_then(|c| c.text())
.map(|t| t.replace('\n', " "))
.map(|t| t.trim().to_string());
let given = name
.children()
.find(|c| c.has_tag_name("given-names"))
.and_then(|c| c.text())
.map(|t| t.replace('\n', " "))
.map(|t| t.trim().to_string());
if let (Some(s), Some(g)) = (surname, given) {
names.push(format!("{s} {g}"));
}
}
if let Some(etal) = node.descendants().find(|n| n.has_tag_name("etal")) {
let etal_text = etal
.text()
.map(str::trim)
.filter(|s| !s.is_empty())
.unwrap_or(DEFAULT_TEXT_ETAL);
names.push(etal_text.to_string());
}
let author_names = names.join(", ");
let title = [
"article-title",
"chapter-title",
"data-title",
"issue-title",
"part-title",
"trans-title",
]
.iter()
.find_map(|t| node.children().find(|c| c.has_tag_name(*t)))
.map(get_text)
.unwrap_or_else(|| {
node.text()
.map(|t| t.replace('\n', " ").trim().to_string())
.unwrap_or_default()
});
let field = |name: &str| -> String {
node.children()
.find(|c| c.has_tag_name(name))
.and_then(|c| c.text())
.map(|t| t.replace('\n', " ").trim().to_string())
.unwrap_or_default()
};
let source = field("source");
let year = field("year");
let publisher_name = field("publisher-name");
let publisher_loc = field("publisher-loc");
let volume = field("volume");
let mut pub_ids: Vec<String> = Vec::new();
for id in node.children().filter(|c| c.has_tag_name("pub-id")) {
let id_type = id
.attribute("assigning-authority")
.or_else(|| id.attribute("pub-id-type"));
if let (Some(t), Some(text)) = (id_type, id.text()) {
pub_ids.push(format!(
"{}: {}",
t.replace('\n', " ").trim().to_uppercase(),
text.replace('\n', " ").trim()
));
}
}
let pub_id = pub_ids.join(", ");
let page = if let Some(e) = node.children().find(|c| c.has_tag_name("elocation-id")) {
e.text()
.map(|t| t.replace('\n', " ").trim().to_string())
.unwrap_or_default()
} else if let Some(f) = node.children().find(|c| c.has_tag_name("fpage")) {
let mut p = f
.text()
.map(|t| t.replace('\n', " ").trim().to_string())
.unwrap_or_default();
if let Some(l) = node.children().find(|c| c.has_tag_name("lpage")) {
p.push('\u{2013}');
p.push_str(
l.text()
.map(|t| t.replace('\n', " "))
.unwrap_or_default()
.trim(),
);
}
p
} else {
String::new()
};
let mut text = String::new();
if !author_names.is_empty() {
text.push_str(author_names.trim_end_matches('.'));
text.push_str(". ");
}
if !title.is_empty() {
text.push_str(title.trim());
text.push_str(". ");
}
if !source.is_empty() {
text.push_str(&source);
text.push_str(". ");
}
if !publisher_name.is_empty() {
if !publisher_loc.is_empty() {
text.push_str(&format!("{publisher_loc}: "));
}
text.push_str(&publisher_name);
text.push_str(". ");
}
if !volume.is_empty() {
rstrip_dot_space(&mut text);
text.push_str(&format!(" {volume}. "));
}
if !page.is_empty() {
rstrip_dot_space(&mut text);
if !volume.is_empty() {
text.push(':');
}
text.push_str(&page);
text.push_str(". ");
}
if !year.is_empty() {
rstrip_dot_space(&mut text);
text.push_str(&format!(" ({year})."));
}
if !pub_id.is_empty() {
while text.ends_with('.') {
text.pop();
}
text.push_str(". ");
text.push_str(&pub_id);
}
text
}
fn rstrip_dot_space(s: &mut String) {
while matches!(s.chars().last(), Some('.') | Some(' ')) {
s.pop();
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::InputFormat;
#[test]
fn ext_links_nbsp_and_display_formulas() {
let xml = r#"<article xmlns:xlink="http://www.w3.org/1999/xlink"><front><article-meta>
<title-group><article-title>T</article-title></title-group>
</article-meta></front>
<body><sec><title>S</title>
<p>See RRID: <ext-link ext-link-type="uri" xlink:href="https://scicrunch.org/resolver/AB_1">AB_1</ext-link> here.</p>
<p>Plain <ext-link xlink:href=" ">blank</ext-link> link.</p>
<disp-formula><tex-math><![CDATA[$$\begin{eqnarray}
a=b
\end{eqnarray}$$]]></tex-math></disp-formula>
</sec></body>
<back><ref-list><title>References</title>
<ref><mixed-citation>A. S. de Castro, Phys. Lett. A. 346 (2005).</mixed-citation></ref>
</ref-list></back></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let doc = JatsBackend::default().convert(&src).unwrap();
let md = doc.export_to_markdown();
assert!(
md.contains("See RRID: [AB\\_1](https://scicrunch.org/resolver/AB_1) here."),
"{md}"
);
assert!(
md.contains("Plain blank link."),
"blank href → no link: {md}"
);
assert!(
md.contains("$$\\begin{eqnarray}\na=b\n\\end{eqnarray}$$"),
"{md}"
);
assert!(md.contains("A. S. de\u{a0}Castro"), "{md}");
assert!(doc
.nodes
.iter()
.any(|n| matches!(n, Node::Formula { latex, .. } if latex.starts_with("\\begin"))));
}
#[test]
fn metadata_and_sections() {
let xml = r#"<article><front><article-meta>
<title-group><article-title>My Paper</article-title></title-group>
<contrib-group>
<contrib contrib-type="author"><name><surname>Doe</surname><given-names>Jane</given-names></name>
<xref ref-type="aff" rid="a1"/></contrib>
</contrib-group>
<aff id="a1"><label>1</label>Acme & Co</aff>
<abstract><p>Short summary.</p></abstract>
</article-meta></front>
<body><sec><title>Intro</title><p>Body text.</p></sec></body></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let md = JatsBackend::default()
.convert(&src)
.unwrap()
.export_to_markdown();
assert!(md.starts_with("# My Paper\n\nJane Doe\n\nAcme & Co\n\n## Abstract\n\nShort summary.\n\n## Intro\n\nBody text."), "got:\n{md}");
}
fn md_of(xml: &str) -> String {
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
JatsBackend::default()
.convert(&src)
.unwrap()
.export_to_markdown()
}
#[test]
fn structured_abstract_keeps_its_sections() {
let md = md_of(
r#"<article><front><article-meta>
<title-group><article-title>T</article-title></title-group>
<abstract>
<sec><title>Background</title><p>B one.</p><p>B two.</p></sec>
<sec><p>No title here.</p></sec>
<sec><title>Empty</title></sec>
<sec><title>Outer</title><p>O.</p><sec><title>Inner</title><p>I.</p></sec></sec>
</abstract>
</article-meta></front><body/></article>"#,
);
assert_eq!(
md,
"# T
## Abstract
### Background
B one.
B two.
No title here.
### Outer
O.
"
);
}
#[test]
fn abstract_label_and_skip_rules_follow_docling() {
let md = md_of(
r#"<article><front><article-meta>
<title-group><article-title>T</article-title></title-group>
<abstract><label>Summary</label><p>S.</p></abstract>
<abstract abstract-type="graphical"><title>Graphical</title><sec><title>X</title></sec></abstract>
<abstract><p>Plain.</p><sec><title>Also</title><p>Sectioned.</p></sec></abstract>
</article-meta></front><body/></article>"#,
);
assert_eq!(
md,
"# T
## Summary
S.
## Abstract
### Also
Sectioned.
"
);
}
#[test]
fn title_is_the_direct_text_of_the_title_group_children() {
let md = md_of(
r#"<article><front><article-meta>
<title-group><article-title>Response of <italic>Y. pestis</italic> to stress</article-title>
<subtitle>A sub</subtitle></title-group>
</article-meta></front><body/></article>"#,
);
assert!(
md.starts_with(
"# Response of A sub
"
),
"{md}"
);
}
#[test]
fn body_tables_figures_and_references() {
let xml = r#"<article><front><article-meta>
<title-group><article-title>T</article-title></title-group>
</article-meta></front>
<body><sec><title>S</title>
<fig><label>Fig 1</label><caption><p>A caption.</p></caption><graphic/></fig>
<table-wrap><label>Table 1</label><caption><p>Table cap.</p></caption>
<table><thead><tr><th>Name</th><th>N</th></tr></thead>
<tbody><tr><td>a</td><td>1</td></tr></tbody></table></table-wrap>
</sec></body>
<back><ref-list><title>References</title>
<ref><mixed-citation>Doe J. A title. 2020.</mixed-citation></ref>
</ref-list></back></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let md = JatsBackend::default()
.convert(&src)
.unwrap()
.export_to_markdown();
assert!(
md.contains("Fig 1 A caption.\n\n<!-- image -->"),
"figure:\n{md}"
);
assert!(md.contains("Table 1 Table cap."), "table caption:\n{md}");
assert!(md.contains("| Name"), "table grid:\n{md}");
assert!(md.contains("## References"), "refs heading:\n{md}");
assert!(md.contains("- Doe J. A title. 2020."), "citation:\n{md}");
}
#[test]
fn table_caption_and_span_structure() {
let xml = r#"<article><body><sec><title>S</title>
<table-wrap><label>Table 1</label><caption><p>Cap.</p></caption>
<table>
<thead><tr><th colspan="2">Group</th><th>N</th></tr></thead>
<tbody><tr><td>a</td><td>b</td><td>1</td></tr></tbody>
</table></table-wrap>
</sec></body></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let doc = JatsBackend::default().convert(&src).unwrap();
let dclx = doc.export_to_doclang();
assert!(
dclx.contains("<table>\n <caption>Table 1 Cap.</caption>"),
"caption inside table:\n{dclx}"
);
assert!(
dclx.contains("<ched/>\n Group\n <lcel/>\n <ched/>\n N"),
"colspan header → ched + lcel:\n{dclx}"
);
let md = doc.export_to_markdown();
assert!(md.contains("Table 1 Cap."), "md caption:\n{md}");
}
#[test]
fn emphasis_and_inline_formula() {
let xml = r#"<article><body><sec><title>S</title>
<p>We combined <italic>B</italic>. <italic>malayi</italic> with a
<bold>strong</bold> effect and mass <inline-formula><tex-math>$m c^2$</tex-math></inline-formula> energy.</p>
</sec></body></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let md = JatsBackend::default()
.convert(&src)
.unwrap()
.export_to_markdown();
assert!(
md.contains(
"We combined *B* . *malayi* with a **strong** effect and mass $m c^2$ energy."
),
"emphasis + inline formula:\n{md}"
);
}
#[test]
fn nested_lists_keep_structure() {
let xml = r#"<article><body><sec><title>S</title>
<list>
<list-item><p>Item 1</p>
<list>
<list-item><p>Subitem A</p></list-item>
<list-item><p>Subitem B</p></list-item>
</list>
</list-item>
<list-item><p>Item 2</p></list-item>
</list></sec></body></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let doc = JatsBackend::default().convert(&src).unwrap();
let items: Vec<(String, u8)> = doc
.nodes
.iter()
.filter_map(|n| match n {
Node::ListItem { text, level, .. } => Some((text.clone(), *level)),
_ => None,
})
.collect();
assert_eq!(
items,
vec![
("Item 1".to_string(), 0),
("Subitem A".to_string(), 1),
("Subitem B".to_string(), 1),
("Item 2".to_string(), 0),
],
"nested structure: {items:?}"
);
let md = doc.export_to_markdown();
assert!(
md.contains("- Item 1\n - Subitem A"),
"markdown nesting:\n{md}"
);
}
#[test]
fn empty_display_formula_does_not_truncate() {
let xml = r#"<article><body><sec><title>S</title>
<p>Before.</p>
<disp-formula><tex-math/></disp-formula>
<p>After.</p></sec></body></article>"#;
let src = SourceDocument::from_bytes("p", InputFormat::XmlJats, xml.as_bytes().to_vec());
let md = JatsBackend::default()
.convert(&src)
.unwrap()
.export_to_markdown();
assert!(md.contains("Before."), "got:\n{md}");
assert!(
md.contains("After."),
"content after the empty formula lost:\n{md}"
);
assert!(!md.contains("$$"), "no phantom formula:\n{md}");
}
struct TempDir(PathBuf);
impl TempDir {
fn new(tag: &str) -> Self {
let dir = std::env::temp_dir().join(format!(
"docling-jats-{tag}-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
TempDir(dir)
}
fn path(&self) -> &Path {
&self.0
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn write_png(path: &Path, w: u32, h: u32, rgb: [u8; 3]) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel(w, h, image::Rgb(rgb)))
.save_with_format(path, image::ImageFormat::Png)
.unwrap();
}
fn write_jpg(path: &Path, w: u32, h: u32, rgb: [u8; 3]) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
image::DynamicImage::ImageRgb8(image::RgbImage::from_pixel(w, h, image::Rgb(rgb)))
.save_with_format(path, image::ImageFormat::Jpeg)
.unwrap();
}
fn jats_body(body: &str) -> String {
format!(
r#"<article xmlns:xlink="http://www.w3.org/1999/xlink"><front><article-meta>
<title-group><article-title>T</article-title></title-group>
</article-meta></front><body><sec><title>S</title>{body}</sec></body></article>"#
)
}
fn convert_file(dir: &Path, body: &str, fetch_images: bool) -> DoclingDocument {
let path = dir.join("article.nxml");
std::fs::write(&path, jats_body(body)).unwrap();
let src = SourceDocument::from_file(&path).unwrap();
JatsBackend { fetch_images }.convert(&src).unwrap()
}
fn picture_images(doc: &DoclingDocument) -> Vec<Option<&docling_core::PictureImage>> {
doc.nodes
.iter()
.filter_map(|n| match n {
Node::Picture { image, .. } => Some(image.as_ref()),
_ => None,
})
.collect()
}
fn size_and_pixel(img: &docling_core::PictureImage) -> ((u32, u32), [u8; 3]) {
let rgb = image::load_from_memory(&img.data).unwrap().to_rgb8();
((img.width, img.height), rgb.get_pixel(0, 0).0)
}
#[test]
fn figure_image_is_embedded_only_when_fetching() {
let dir = TempDir::new("embed");
write_png(&dir.path().join("images/figure.png"), 7, 5, [255, 0, 0]);
let body = r#"<fig><label>Figure 1</label><caption><p>A red rectangle.</p></caption>
<graphic xlink:href="images/figure.png"/></fig>"#;
let doc = convert_file(dir.path(), body, false);
assert_eq!(picture_images(&doc), [None]);
let doc = convert_file(dir.path(), body, true);
let pics = picture_images(&doc);
assert_eq!(pics.len(), 1);
let img = pics[0].expect("embedded");
assert_eq!(size_and_pixel(img), ((7, 5), [255, 0, 0]));
assert_eq!(img.mimetype, "image/png");
let caption = doc.nodes.iter().find_map(|n| match n {
Node::Picture { caption, .. } => caption.clone(),
_ => None,
});
assert_eq!(caption.as_deref(), Some("Figure 1 A red rectangle."));
assert!(doc.export_to_markdown().contains("<!-- image -->"));
let json: serde_json::Value = serde_json::from_str(&doc.export_to_json()).unwrap();
assert!(json["pictures"][0]["image"]["uri"]
.as_str()
.is_some_and(|u| u.starts_with("data:image/png;base64,")));
}
#[test]
fn figure_image_needs_a_local_source_file() {
let dir = TempDir::new("stream");
write_png(&dir.path().join("figure.png"), 7, 5, [255, 0, 0]);
let body = r#"<fig><graphic xlink:href="figure.png"/></fig>"#;
let xml = jats_body(body).into_bytes();
let stream = SourceDocument::from_bytes("article.nxml", InputFormat::XmlJats, xml.clone());
let doc = JatsBackend { fetch_images: true }.convert(&stream).unwrap();
assert_eq!(picture_images(&doc), [None]);
let remote = SourceDocument::from_bytes("article.nxml", InputFormat::XmlJats, xml.clone())
.with_base_url("https://example.com/article.nxml");
let doc = JatsBackend { fetch_images: true }.convert(&remote).unwrap();
assert_eq!(picture_images(&doc), [None]);
let mut with_path = SourceDocument::from_bytes("article.nxml", InputFormat::XmlJats, xml);
with_path.path = Some(dir.path().join("source.nxml"));
let doc = JatsBackend { fetch_images: true }
.convert(&with_path)
.unwrap();
assert!(picture_images(&doc)[0].is_some());
}
#[test]
fn figure_image_resolves_an_extensionless_href() {
let dir = TempDir::new("extless");
write_jpg(&dir.path().join("images/figure.jpg"), 9, 6, [0, 0, 255]);
for graphic in [
r#"<graphic xlink:href="images/figure"/>"#,
r#"<alternatives><graphic xlink:href="images/unsupported.svg"/><graphic xlink:href="images/figure"/></alternatives>"#,
] {
let doc = convert_file(dir.path(), &format!("<fig>{graphic}</fig>"), true);
let pics = picture_images(&doc);
assert_eq!(pics.len(), 1, "{graphic}");
let img = pics[0].expect("probed .jpg");
assert_eq!((img.width, img.height), (9, 6));
assert_eq!(img.mimetype, "image/jpeg");
}
}
#[test]
fn figure_image_falls_back_past_undecodable_and_absolute_renditions() {
let dir = TempDir::new("fallback");
std::fs::create_dir_all(dir.path().join("images")).unwrap();
std::fs::write(dir.path().join("images/broken.png"), b"not an image").unwrap();
write_png(&dir.path().join("images/figure.png"), 9, 6, [0, 0, 255]);
write_png(&dir.path().join("absolute.png"), 7, 5, [255, 0, 0]);
let absolute = dir.path().join("absolute.png");
for body in [
r#"<fig><alternatives><graphic xlink:href="images/broken.png"/><graphic xlink:href="images/figure.png"/></alternatives></fig>"#.to_string(),
format!(
r#"<fig><alternatives><graphic xlink:href="{}"/><graphic xlink:href="images/figure.png"/></alternatives></fig>"#,
absolute.display()
),
] {
let doc = convert_file(dir.path(), &body, true);
let img = picture_images(&doc)[0].expect("fell back");
assert_eq!(size_and_pixel(img), ((9, 6), [0, 0, 255]), "{body}");
}
}
#[test]
fn figure_image_skips_unavailable_renditions() {
let dir = TempDir::new("skip");
for graphic in [
"",
"<graphic/>",
r#"<graphic xlink:href=" "/>"#,
r#"<graphic xlink:href="https://example.com/figure.png"/>"#,
r#"<graphic xlink:href="figure.svg"/>"#,
r#"<graphic xlink:href="missing.png"/>"#,
] {
let doc = convert_file(
dir.path(),
&format!("<fig>{graphic}</fig><p>Content after the unavailable figure.</p>"),
true,
);
assert_eq!(picture_images(&doc), [None], "{graphic}");
assert!(doc
.export_to_markdown()
.contains("Content after the unavailable figure."));
}
}
#[test]
fn figure_image_blocks_path_traversal() {
let dir = TempDir::new("traversal");
write_png(&dir.path().join("outside.png"), 7, 5, [255, 0, 0]);
let article_dir = dir.path().join("article");
write_png(&article_dir.join("fallback.png"), 9, 6, [0, 0, 255]);
let doc = convert_file(
&article_dir,
r#"<fig><alternatives><graphic xlink:href="../outside.png"/><graphic xlink:href="fallback.png"/></alternatives></fig>
<p>Content after the blocked figure.</p>"#,
true,
);
assert_eq!(picture_images(&doc), [None]);
assert!(doc
.export_to_markdown()
.contains("Content after the blocked figure."));
}
#[test]
fn figure_path_helpers_follow_docling() {
assert!(is_local_path("images/a.png"));
assert!(is_local_path("/abs/a.png"));
assert!(is_local_path(r"C:\a.png"));
assert!(!is_local_path("https://x/a.png"));
assert!(!is_local_path("//cdn/a.png"));
assert!(!is_local_path("data:image/png;base64,AA=="));
assert!(is_absolute_path("/abs/a.png"));
assert!(is_absolute_path("C:/a.png"));
assert!(!is_absolute_path("images/a.png"));
let base = Path::new("/base");
assert_eq!(
confined_path(base, "a/../b.png"),
Some(PathBuf::from("/base/b.png"))
);
assert_eq!(confined_path(base, "../b.png"), None);
assert_eq!(confined_path(base, "/etc/passwd"), None);
}
}