use docling_core::tree::{Formatting, ItemTree, ListMeta, TreeKind};
use docling_core::{DoclingDocument, InlineRun, Node, Table, TableCell};
use crate::backend::images::{FsImageResolver, ImagePolicy, ImageResolver, ImageSources, NoFetch};
use crate::backend::markdown::escape_text;
use crate::backend::DeclarativeBackend;
use crate::error::ConversionError;
use crate::source::SourceDocument;
const CELL_SPEC: &str = r"(?:(?:\d+(?:\.\d+)?|\.\d+)[*+])*[<^>]?(?:\.[<^>])?[adehlms]?";
const LIST_ITEM: &str = r"^(\s*)(\*+|-|\.+|\d+\.|[a-zA-Z]\.)\s+(.*)";
const LITERAL_FENCE: &str = "....";
const LISTING_FENCE: &str = "----";
const COMMENT_BLOCK: &str = "////";
const CONTENT_BLOCK_DELIMITERS: &[&str] = &["====", "****", "____", "--", "++++"];
const SOURCE_ATTR: &str = r"^\[source(?:,\s*([\w+#.-]+))?[^\]]*\]$";
#[derive(Default)]
pub struct AsciiDocBackend {
pub images: ImagePolicy,
}
impl DeclarativeBackend for AsciiDocBackend {
fn convert(&self, source: &SourceDocument) -> Result<DoclingDocument, ConversionError> {
let text = source.text()?;
if self.images.sources != ImageSources::None {
let resolver = FsImageResolver::new(
source.base_dir().map(|p| p.to_path_buf()),
source.base_url.clone(),
self.images.clone(),
);
Ok(parse(&text, &source.name, &resolver))
} else {
Ok(parse(&text, &source.name, &NoFetch))
}
}
}
fn parse(text: &str, name: &str, images: &dyn ImageResolver) -> DoclingDocument {
let mut doc = DoclingDocument::new(name);
let mut p = Parser {
images: Some(images),
parents: vec![None; LEVELS],
indents: vec![None; LEVELS],
list_kinds: vec![None; LEVELS],
..Parser::default()
};
for block in blocks(text) {
match block {
Block::Line(line) => p.feed(line, &mut doc),
Block::Literal { code, language } => p.feed_literal(code, language, &mut doc),
}
}
p.finish(&mut doc);
doc.tree = Some(std::mem::take(&mut p.tree));
doc
}
const LEVELS: usize = 10;
enum Block<'a> {
Line(&'a str),
Literal {
code: String,
language: Option<String>,
},
}
fn blocks(text: &str) -> Vec<Block<'_>> {
let lines: Vec<&str> = text
.split('\n')
.map(|l| l.strip_suffix('\r').unwrap_or(l))
.collect();
let source_attr = cached_regex!(SOURCE_ATTR);
let mut out = Vec::new();
let mut in_comment_block = false;
let mut i = 0;
while i < lines.len() {
let stripped = lines[i].trim();
if stripped == LITERAL_FENCE || stripped == LISTING_FENCE {
let (body, next) = delimited_body(&lines, i + 1, stripped);
out.push(Block::Literal {
code: body,
language: None,
});
i = next;
continue;
}
if in_comment_block {
in_comment_block = stripped != COMMENT_BLOCK;
i += 1;
continue;
}
if stripped == COMMENT_BLOCK {
in_comment_block = true;
i += 1;
continue;
}
if lines[i].starts_with("//") {
i += 1;
continue;
}
if CONTENT_BLOCK_DELIMITERS.contains(&stripped) {
if lines[i + 1..].iter().any(|l| l.trim() == stripped) {
i += 1;
while i < lines.len() && lines[i].trim() != stripped {
out.push(Block::Line(lines[i]));
i += 1;
}
}
out.push(Block::Line(""));
i += 1;
continue;
}
if let Some(caps) = source_attr.captures(stripped) {
if lines.get(i + 1).is_some_and(|l| l.trim() == LISTING_FENCE) {
let language = caps.get(1).map(|m| m.as_str().to_string());
let (body, next) = delimited_body(&lines, i + 2, LISTING_FENCE);
out.push(Block::Literal {
code: body,
language,
});
i = next;
continue;
}
}
out.push(Block::Line(lines[i]));
i += 1;
}
out
}
fn delimited_body(lines: &[&str], start: usize, delimiter: &str) -> (String, usize) {
let mut i = start;
let mut body: Vec<&str> = Vec::new();
while i < lines.len() && lines[i].trim() != delimiter {
body.push(lines[i].trim_end_matches(['\r', '\n']));
i += 1;
}
(body.join("\n"), (i + 1).min(lines.len()))
}
#[derive(Default)]
struct Parser<'r> {
text_data: Vec<String>,
caption_data: Vec<String>,
table_data: Vec<Vec<String>>,
in_list: bool,
in_table: bool,
in_delimited_table: bool,
fresh_list: bool,
last_item: Option<usize>,
list_continuation: bool,
images: Option<&'r dyn ImageResolver>,
tree: ItemTree,
parents: Vec<Option<usize>>,
indents: Vec<Option<usize>>,
list_kinds: Vec<Option<char>>,
last_item_tree: Option<usize>,
raw_caption: String,
}
enum Trigger<'a> {
Line(&'a str),
Literal,
}
impl Parser<'_> {
fn current_level(&self) -> usize {
(1..LEVELS)
.find(|&k| self.parents[k].is_none())
.map_or(0, |k| k - 1)
}
fn current_parent(&self) -> Option<usize> {
(1..LEVELS)
.find(|&k| self.parents[k].is_none())
.and_then(|k| self.parents[k - 1])
}
fn is_list_group(&self, id: usize) -> bool {
matches!(&self.tree.items[id].kind, TreeKind::Group { label, .. } if label == "list")
}
fn add_tree_text(
&mut self,
parent: Option<usize>,
label: &str,
text: &str,
formatting: Option<Formatting>,
level: Option<u8>,
list: Option<ListMeta>,
) -> usize {
self.tree.add(
parent,
None,
TreeKind::Text {
label: label.into(),
text: text.into(),
orig: None,
formatting,
hyperlink: None,
level,
list,
},
)
}
fn open_list_group(&mut self, level: usize, parent: Option<usize>, indent: usize, kind: char) {
if level + 1 >= LEVELS {
return;
}
let g = self.tree.add(
parent,
None,
TreeKind::Group {
label: "list".into(),
name: "list".into(),
},
);
self.parents[level + 1] = Some(g);
self.indents[level + 1] = Some(indent);
self.list_kinds[level + 1] = Some(kind);
}
fn add_tree_caption(&mut self, text: &str) -> usize {
self.add_tree_text(None, "caption", text, None, None, None)
}
fn close_list_if_needed(&mut self, trigger: Trigger) {
if !self.in_list {
return;
}
let keep = match trigger {
Trigger::Literal => self.list_continuation,
Trigger::Line(line) => {
let stripped = line.trim();
list_item(line).is_some()
|| stripped.is_empty()
|| stripped == "+"
|| (self.list_continuation && is_picture(line))
}
};
if !keep {
let mut level = self.current_level();
while level > 0 && self.parents[level].is_some_and(|g| self.is_list_group(g)) {
self.parents[level] = None;
level -= 1;
}
self.end_list();
}
}
fn feed_literal(&mut self, code: String, language: Option<String>, doc: &mut DoclingDocument) {
let language = language
.map(|hint| docling_core::code_language_label(&hint))
.filter(|l| *l != "unknown")
.map(str::to_string);
self.close_list_if_needed(Trigger::Literal);
self.flush_text(doc);
let caption = self.take_caption();
let tree_caption = if caption.is_some() {
let raw = self.raw_caption.clone();
Some(self.add_tree_caption(&raw))
} else {
None
};
let parent = if self.in_list {
self.last_item_tree
} else {
self.current_parent()
};
self.tree.add(
parent,
None,
TreeKind::Code {
text: code.clone(),
orig: None,
language: language.clone(),
formatting: None,
hyperlink: None,
captions: tree_caption.into_iter().collect(),
},
);
if !(self.in_list && self.fold_child(doc, &format!("```\n{code}\n```"))) {
doc.push(Node::Code {
language,
text: code,
orig: None,
pretty: None,
});
}
if let Some(text) = caption {
doc.push(Node::Caption { text, href: None });
}
self.list_continuation = false;
}
fn feed(&mut self, line: &str, doc: &mut DoclingDocument) {
let stripped = line.trim();
if self.in_delimited_table && self.continue_table(line) {
return;
}
self.close_list_if_needed(Trigger::Line(line));
if cached_regex!(r"^\[\[.+\]\]$").is_match(stripped)
|| cached_regex!(r"^\[[^\[\]]+\]$").is_match(stripped)
|| cached_regex!(r"^:[\w.-]+:(\s.*)?$").is_match(stripped)
{
return;
}
if let Some(rest) = is_title(line) {
doc.push(Node::Heading {
level: 1,
text: escape_text(rest.trim()),
});
let id = self.add_tree_text(None, "title", rest.trim(), None, None, None);
self.parents[0] = Some(id);
return;
}
if let Some((n, text)) = section_header(line) {
self.flush_text(doc);
doc.push(Node::Heading {
level: n.min(6),
text: escape_text(text.trim()),
});
let level = (usize::from(n) - 1).min(LEVELS - 1);
let ancestor = (0..level).rev().find_map(|k| self.parents[k]);
let id = self.add_tree_text(
ancestor,
"section_header",
text.trim(),
None,
Some(level as u8),
None,
);
self.parents[level] = Some(id);
for k in level + 1..LEVELS {
self.parents[k] = None;
}
return;
}
if let Some(item) = list_item(line) {
self.push_list_item(item, doc);
return;
}
if self.in_list && stripped == "+" {
self.list_continuation = true;
return;
}
if stripped == "|===" {
if self.in_table {
self.flush_table(doc);
} else {
self.in_table = true;
self.in_delimited_table = true;
}
return;
}
if is_table_line(line) {
self.in_table = true;
self.table_data.push(parse_table_line(line));
return;
}
if self.in_table {
self.flush_table(doc);
return;
}
if let Some(uri) = picture_uri(line) {
self.push_picture(&uri, doc);
return;
}
if let Some(rest) = is_caption(line) {
if self.caption_data.is_empty() {
self.caption_data.push(rest.to_string());
return;
}
}
if !stripped.is_empty() && !self.caption_data.is_empty() {
self.caption_data.push(stripped.to_string());
return;
}
if stripped.is_empty() {
self.flush_text(doc);
self.flush_caption_as_paragraph(doc);
} else {
self.text_data.push(stripped.to_string());
}
}
fn continue_table(&mut self, line: &str) -> bool {
let stripped = line.trim();
if stripped == "|===" || is_table_line(line) {
return false;
}
if stripped.is_empty() {
return true;
}
let Some(cell) = self.table_data.last_mut().and_then(|row| row.last_mut()) else {
return false;
};
*cell = format!("{cell} {stripped}").trim_start().to_string();
true
}
fn flush_caption_as_paragraph_under(
&mut self,
parent: Option<usize>,
doc: &mut DoclingDocument,
) {
if let Some(text) = self.take_caption() {
doc.push(Node::Paragraph {
text: format!("**{text}**"),
});
let raw = self.raw_caption.clone();
self.add_tree_text(
parent,
"paragraph",
&raw,
Some(Formatting {
bold: true,
..Formatting::default()
}),
None,
None,
);
}
}
fn flush_caption_as_paragraph(&mut self, doc: &mut DoclingDocument) {
let parent = self.current_parent();
self.flush_caption_as_paragraph_under(parent, doc);
}
fn push_list_item(&mut self, item: ListItem<'_>, doc: &mut DoclingDocument) {
let mut level = self.current_level();
let kind = item.kind();
if !self.in_list {
self.in_list = true;
let parent = self.parents[level];
self.flush_caption_as_paragraph_under(parent, doc);
self.list_kinds = vec![None; LEVELS];
self.open_list_group(level, parent, item.indent, kind);
self.fresh_list = true;
} else if item.indent > self.indents[level].unwrap_or(0) {
let parent = self.last_item_tree;
self.open_list_group(level, parent, item.indent, kind);
} else if item.indent < self.indents[level].unwrap_or(0) {
while level > 0 && item.indent < self.indents[level].unwrap_or(0) {
if self.indents[level - 1].is_none() {
break;
}
self.parents[level] = None;
self.indents[level] = None;
level -= 1;
}
}
self.match_list_kind(&item, kind);
let tree_parent = self.current_parent();
let explicit = numeric_marker(item.marker);
let tree_marker = explicit.map_or(String::new(), |_| item.marker.to_string());
let id = self.add_tree_text(
tree_parent,
"list_item",
item.text.trim(),
None,
None,
Some(ListMeta {
enumerated: item.numbered,
marker: tree_marker,
}),
);
self.last_item_tree = Some(id);
let (depth, position, group_enumerated) = self.list_position(id);
let (ordered, number) = match explicit {
Some(n) => (true, n),
None => (group_enumerated, position),
};
doc.push(Node::ListItem {
ordered,
number,
first_in_list: self.fresh_list,
text: escape_text(item.text.trim()),
level: depth,
marker: explicit.map(|_| item.marker.to_string()),
location: None,
dclx: None,
href: None,
layer: None,
});
self.last_item = Some(doc.nodes.len() - 1);
self.fresh_list = false;
self.list_continuation = false;
}
fn match_list_kind(&mut self, item: &ListItem<'_>, kind: char) {
let level = self.current_level();
if self.list_kinds[level] == Some(kind) {
return;
}
let mut outer = level;
while outer > 0 {
outer -= 1;
let Some(outer_kind) = self.list_kinds[outer] else {
break;
};
if self.indents[outer] != Some(item.indent) {
break;
}
if outer_kind == kind {
for closed in outer + 1..=level {
self.parents[closed] = None;
self.indents[closed] = None;
}
return;
}
}
let parent = self.last_item_tree;
self.open_list_group(level, parent, item.indent, kind);
}
fn group_depth(&self, group: usize) -> u8 {
let mut depth = 0u8;
let mut at = self.tree.items[group].parent;
while let Some(a) = at {
if self.is_list_group(a) {
depth += 1;
}
at = self.tree.items[a].parent;
}
depth
}
fn list_position(&self, id: usize) -> (u8, u64, bool) {
let group = self.tree.items[id].parent;
let mut depth = 0u8;
let mut at = group.and_then(|g| self.tree.items[g].parent);
while let Some(a) = at {
if self.is_list_group(a) {
depth += 1;
}
at = self.tree.items[a].parent;
}
let Some(group) = group else {
return (depth, 1, false);
};
let children = &self.tree.items[group].children;
let position = children.iter().position(|&c| c == id).map_or(1, |p| p + 1) as u64;
let first_enumerated = children.first().is_some_and(|&c| {
matches!(
&self.tree.items[c].kind,
TreeKind::Text { list: Some(l), .. } if l.enumerated
)
});
(depth, position, first_enumerated)
}
fn push_picture(&mut self, uri: &str, doc: &mut DoclingDocument) {
let cap = self.take_caption();
let image = self.images.and_then(|r| r.resolve(uri));
let captions: Vec<usize> = if cap.is_some() {
let raw = self.raw_caption.clone();
vec![self.add_tree_caption(&raw)]
} else {
Vec::new()
};
let parent = if self.in_list {
self.last_item_tree
} else {
self.current_parent()
};
self.tree.add(
parent,
None,
TreeKind::Picture {
captions,
image: image.clone(),
extra_classes: Vec::new(),
classification: None,
description: None,
confidence: None,
chart: None,
dpi: None,
},
);
if self.in_list {
let mut block = String::new();
if let Some(cap) = &cap {
block.push_str(cap);
block.push('\n');
}
block.push_str("<!-- image -->");
if self.fold_child(doc, &block) {
self.list_continuation = false;
return;
}
}
doc.push(Node::Picture {
caption: cap,
caption_href: None,
image,
classification: None,
description: None,
caption_parent: Default::default(),
caption_location: None,
});
self.list_continuation = false;
}
fn fold_child(&mut self, doc: &mut DoclingDocument, block: &str) -> bool {
let Some(idx) = self.last_item else {
return false;
};
let Some(Node::ListItem { text, level, .. }) = doc.nodes.get_mut(idx) else {
return false;
};
text.push('\n');
text.push_str(&" ".repeat(*level as usize));
text.push_str(block);
true
}
fn end_list(&mut self) {
self.in_list = false;
self.last_item = None;
self.last_item_tree = None;
self.list_continuation = false;
}
fn take_caption(&mut self) -> Option<String> {
if self.caption_data.is_empty() {
return None;
}
let cap = self.caption_data.join(" ");
self.caption_data.clear();
self.raw_caption = cap.clone();
Some(escape_text(&cap))
}
fn flush_text(&mut self, doc: &mut DoclingDocument) {
if self.text_data.is_empty() {
return;
}
let text = self.text_data.join(" ");
self.text_data.clear();
let parent = self.current_parent();
let list_level = parent
.filter(|&p| self.is_list_group(p))
.map(|g| self.group_depth(g));
let runs = parse_inline(&text, false, false);
if !runs.iter().any(|r| r.bold || r.italic || r.code) {
let escaped = escape_text(&text);
doc.push(match list_level {
Some(level) => Node::ListText {
level,
text: escaped,
glue: false,
},
None => Node::Paragraph { text: escaped },
});
self.add_tree_text(parent, "paragraph", &text, None, None, None);
return;
}
let group = self.tree.add(
parent,
None,
TreeKind::Group {
label: "inline".into(),
name: "group".into(),
},
);
let mut md = Vec::new();
let mut node_runs = Vec::new();
for run in runs {
let t = run.text.trim();
if t.is_empty() {
continue;
}
if run.code {
self.tree.add(
Some(group),
None,
TreeKind::Code {
text: t.to_string(),
orig: None,
language: None,
formatting: None,
hyperlink: None,
captions: Vec::new(),
},
);
md.push(format!("`{t}`"));
} else {
let formatting = (run.bold || run.italic).then(|| Formatting {
bold: run.bold,
italic: run.italic,
..Formatting::default()
});
self.add_tree_text(Some(group), "text", t, formatting, None, None);
let escaped = escape_text(t);
md.push(match (run.bold, run.italic) {
(true, true) => format!("***{escaped}***"),
(true, false) => format!("**{escaped}**"),
(false, true) => format!("*{escaped}*"),
(false, false) => escaped,
});
}
node_runs.push(InlineRun {
text: t.to_string(),
bold: run.bold,
italic: run.italic,
code: run.code,
..InlineRun::default()
});
}
let md_text = md.join(" ");
if let Some(level) = list_level {
doc.push(Node::ListText {
level,
text: md_text,
glue: true,
});
return;
}
let unwrapped =
parent.is_some_and(|p| matches!(self.tree.items[p].kind, TreeKind::Text { .. }));
doc.push(Node::InlineGroup {
unwrapped,
runs: node_runs,
md_text,
});
}
fn flush_table(&mut self, doc: &mut DoclingDocument) {
if !self.table_data.is_empty() {
let mut captions = Vec::new();
if let Some(cap) = self.take_caption() {
doc.push(Node::Paragraph { text: cap });
let raw = self.raw_caption.clone();
captions.push(self.add_tree_caption(&raw));
}
let num_cols = self.table_data.iter().map(Vec::len).max().unwrap_or(0);
let raw_rows: Vec<Vec<String>> = self.table_data.clone();
let rows: Vec<Vec<String>> = self
.table_data
.drain(..)
.map(|mut r| {
r.resize(num_cols, String::new());
r
})
.collect();
let table = Table {
rows,
location: None,
structure: None,
cell_blocks: None,
cells: None,
caption: None,
caption_parent: Default::default(),
caption_location: None,
};
let cells: Vec<TableCell> = raw_rows
.iter()
.enumerate()
.flat_map(|(r, row)| {
row.iter().enumerate().map(move |(c, text)| TableCell {
text: text.clone(),
bbox: None,
start_row: r,
start_col: c,
row_span: 1,
col_span: 1,
column_header: r == 0,
row_header: false,
row_section: false,
})
})
.collect();
let parent = self.current_parent();
self.tree.add(
parent,
None,
TreeKind::Table {
table: Table {
cells: Some(cells),
..table.clone()
},
rich_cells: Vec::new(),
captions,
},
);
doc.push(Node::Table(table));
}
self.in_table = false;
self.in_delimited_table = false;
self.table_data.clear();
}
fn finish(&mut self, doc: &mut DoclingDocument) {
self.flush_text(doc);
if self.in_table {
self.flush_table(doc);
} else {
self.flush_caption_as_paragraph(doc);
}
}
}
struct Run {
text: String,
bold: bool,
italic: bool,
code: bool,
}
fn is_word(c: char) -> bool {
c.is_alphanumeric() || c == '_'
}
fn match_mark(
chars: &[char],
at: usize,
mark: char,
doubled: bool,
) -> Option<(usize, std::ops::Range<usize>)> {
if doubled {
if chars.get(at) != Some(&mark) || chars.get(at + 1) != Some(&mark) {
return None;
}
let start = at + 2;
(start + 1..chars.len().saturating_sub(1))
.find(|&e| chars[e] == mark && chars[e + 1] == mark)
.map(|e| (e + 2, start..e))
} else {
if chars.get(at) != Some(&mark) || (at > 0 && is_word(chars[at - 1])) {
return None;
}
let start = at + 1;
if chars.get(start).is_none_or(|c| c.is_whitespace()) {
return None;
}
(start + 1..chars.len())
.find(|&e| {
chars[e] == mark
&& !chars[e - 1].is_whitespace()
&& chars.get(e + 1).is_none_or(|&c| !is_word(c))
})
.map(|e| (e + 1, start..e))
}
}
fn parse_inline(text: &str, bold: bool, italic: bool) -> Vec<Run> {
let chars: Vec<char> = text.chars().collect();
let mut runs = Vec::new();
let plain = |from: usize, to: usize| Run {
text: chars[from..to].iter().collect(),
bold,
italic,
code: false,
};
let mut position = 0;
let mut at = 0;
while at < chars.len() {
let found = [
('`', true, 0),
('*', true, 1),
('_', true, 2),
('`', false, 0),
('*', false, 1),
('_', false, 2),
]
.into_iter()
.find_map(|(mark, doubled, kind)| {
match_mark(&chars, at, mark, doubled).map(|(end, inner)| (end, inner, kind))
});
let Some((end, inner, kind)) = found else {
at += 1;
continue;
};
if at > position {
runs.push(plain(position, at));
}
let inner_text: String = chars[inner].iter().collect();
match kind {
0 => runs.push(Run {
text: inner_text,
bold: false,
italic: false,
code: true,
}),
1 => runs.extend(parse_inline(&inner_text, true, italic)),
_ => runs.extend(parse_inline(&inner_text, bold, true)),
}
position = end;
at = end;
}
if position < chars.len() {
runs.push(plain(position, chars.len()));
}
runs
}
fn is_title(line: &str) -> Option<&str> {
line.strip_prefix("= ")
}
fn section_header(line: &str) -> Option<(u8, String)> {
if !line.starts_with("==") {
return None;
}
let caps = cached_regex!(r"^(=+)\s+(.*)").captures(line)?;
let level = caps.get(1)?.as_str().len() as u8;
Some((level, caps.get(2)?.as_str().to_string()))
}
struct ListItem<'a> {
indent: usize,
marker: &'a str,
numbered: bool,
text: &'a str,
}
impl ListItem<'_> {
fn kind(&self) -> char {
if self.numbered {
'.'
} else {
self.marker.chars().next().unwrap_or('-')
}
}
}
fn list_item(line: &str) -> Option<ListItem<'_>> {
let caps = cached_regex!(LIST_ITEM).captures(line)?;
let marker = caps.get(2)?.as_str();
let mut indent = caps.get(1)?.as_str().len();
if marker.starts_with(['.', '*']) {
indent += marker.len() - 1;
}
Some(ListItem {
indent,
marker,
numbered: !(marker.starts_with('*') || marker == "-"),
text: caps.get(3)?.as_str(),
})
}
fn numeric_marker(marker: &str) -> Option<u64> {
let digits = marker.strip_suffix('.')?;
if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
digits.parse().ok()
}
fn is_table_line(line: &str) -> bool {
cached_regex!(&format!(r"^{CELL_SPEC}\|")).is_match(line)
}
fn parse_table_line(line: &str) -> Vec<String> {
let cleaned = cached_regex!(&format!(r"(^|\s){CELL_SPEC}(\|)")).replace_all(line, "$1$2");
let mut cells = Vec::new();
let mut cell = String::new();
let mut chars = cleaned.chars().peekable();
while let Some(c) = chars.next() {
match c {
'\\' if chars.peek() == Some(&'|') => {
cell.push('|');
chars.next();
}
'|' => cells.push(std::mem::take(&mut cell)),
_ => cell.push(c),
}
}
cells.push(cell);
cells
.into_iter()
.skip(1)
.map(|c| c.trim().to_string())
.collect()
}
fn is_picture(line: &str) -> bool {
line.starts_with("image::")
}
fn picture_uri(line: &str) -> Option<String> {
if !is_picture(line) {
return None;
}
Some(
cached_regex!(r"^image::(.+)\[(.*)\]$")
.captures(line)
.and_then(|c| c.get(1).map(|m| m.as_str().trim().to_string()))
.unwrap_or_else(|| line.to_string()),
)
}
fn is_caption(line: &str) -> Option<&str> {
let rest = line.strip_prefix('.')?;
(!rest.starts_with(char::is_whitespace) && !rest.is_empty()).then_some(rest)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backend::images::NoFetch;
fn md(src: &str) -> String {
parse(src, "t", &NoFetch).export_to_markdown()
}
fn json(src: &str) -> serde_json::Value {
serde_json::from_str(&parse(src, "t", &NoFetch).export_to_json()).unwrap()
}
#[test]
fn tree_mirrors_docling_parents_and_indents() {
let v = json(
"= Title\n\nAbstract.\n\n== Section 1\n\n* a\n * b\n* c\n\n\
=== Deep\n\n.Steps\n. one\n. two\n\n.Table title\n|===\n|H|I|\n|1|2|\n|===\n",
);
let texts = v["texts"].as_array().unwrap();
let by = |i: usize| {
(
texts[i]["label"].as_str().unwrap(),
texts[i]["parent"]["$ref"].as_str().unwrap(),
)
};
assert_eq!(by(0), ("title", "#/body"));
assert_eq!(by(1), ("paragraph", "#/texts/0"));
assert_eq!(by(2), ("section_header", "#/texts/0"));
assert_eq!(texts[2]["level"], 1);
assert_eq!(by(3), ("list_item", "#/groups/0"));
assert_eq!(by(4), ("list_item", "#/groups/1"));
assert_eq!(by(5), ("list_item", "#/groups/0"));
assert_eq!(v["groups"][0]["parent"]["$ref"], "#/texts/2");
assert_eq!(v["groups"][1]["parent"]["$ref"], "#/texts/3");
assert_eq!(texts[3]["marker"], "");
assert_eq!(texts[3]["enumerated"], false);
assert_eq!(by(6), ("section_header", "#/texts/2"));
assert_eq!(texts[6]["level"], 2);
assert_eq!(by(7), ("paragraph", "#/texts/6"));
assert_eq!(texts[7]["formatting"]["bold"], true);
assert_eq!(v["groups"][2]["parent"]["$ref"], "#/texts/6");
assert_eq!(by(8), ("list_item", "#/groups/2"));
assert_eq!(texts[8]["enumerated"], true);
assert_eq!(texts[8]["marker"], "");
assert_eq!(by(10), ("caption", "#/body"));
assert_eq!(v["tables"][0]["parent"]["$ref"], "#/texts/6");
assert_eq!(v["tables"][0]["captions"][0]["$ref"], "#/texts/10");
assert_eq!(
v["tables"][0]["data"]["table_cells"][0]["column_header"],
true
);
}
#[test]
fn tree_nests_continued_pictures_and_keeps_ragged_rows() {
let v = json("= T\n\n. step\n+\nimage::a.png[]\n\n|A|B|C|\n|1|\n");
assert_eq!(v["pictures"][0]["parent"]["$ref"], "#/texts/1");
assert_eq!(v["texts"][1]["children"][0]["$ref"], "#/pictures/0");
let cells = v["tables"][0]["data"]["table_cells"].as_array().unwrap();
assert_eq!(cells.len(), 6, "4 header cells + 2, no padding");
assert_eq!(v["tables"][0]["data"]["num_cols"], 4);
assert_eq!(
cells
.iter()
.filter(|c| c["start_row_offset_idx"] == 1)
.count(),
2
);
}
#[test]
fn literal_block_becomes_a_code_item() {
assert_eq!(
md("= T\n\npara\n\n....\nraw literal\n indented\n....\n\nafter\n"),
"# T\n\npara\n\n```\nraw literal\n indented\n```\n\nafter\n"
);
assert_eq!(md("....\nno close\n"), "```\nno close\n\n```\n");
assert_eq!(md("....\nno close"), "```\nno close\n```\n");
assert_eq!(
md(".Literal example\n....\nraw\n....\n"),
"```\nraw\n```\n\nLiteral example\n"
);
assert_eq!(
md("= T\n\npara\n\n.Cap here\n....\nraw\n....\n\nafter\n"),
"# T\n\npara\n\n```\nraw\n```\n\nCap here\n\nafter\n"
);
}
#[test]
fn plus_keeps_a_literal_block_and_an_image_inside_the_item() {
assert_eq!(
md(". one\n+\n....\ncode\n....\n. two\n+\nimage::a.png[]\n. three\n"),
"1. one\n```\ncode\n```\n2. two\n<!-- image -->\n3. three\n"
);
assert_eq!(
md("* one\n\n....\ncode\n....\n"),
"- one\n\n```\ncode\n```\n"
);
}
#[test]
fn a_child_block_is_indented_to_its_item_depth() {
assert_eq!(
md(". outer\n.. inner\n+\n....\ndeep\n....\n.. inner two\n+\nimage::a.png[]\n"),
"1. outer\n 1. inner\n ```\ndeep\n```\n 2. inner two\n <!-- image -->\n"
);
}
#[test]
fn dotted_and_lettered_markers_open_ordered_items() {
assert_eq!(
md(". one\n. two\n.. nested\n. three\na. lettered\n"),
"1. one\n2. two\n 1. nested\n3. three\n4. lettered\n"
);
}
#[test]
fn an_explicit_numeric_marker_is_printed_verbatim() {
assert_eq!(
md("1. one\n2. two\n.. sub\n"),
"1. one\n2. two\n 1. sub\n"
);
assert_eq!(md("* bullet\n. dotted\n"), "- bullet\n 1. dotted\n");
}
#[test]
fn a_broken_run_of_explicit_markers_stays_one_list() {
assert_eq!(md("1. one\n5. five\n"), "1. one\n5. five\n");
assert_eq!(
md("* bullet one\n1. explicit one\n* bullet two\n"),
"- bullet one\n 1. explicit one\n- bullet two\n"
);
}
#[test]
fn a_block_title_in_front_of_a_list_is_a_bold_paragraph() {
assert_eq!(
md(".Procedure\n\n. one\n\n.Verification\n\n* check\n"),
"**Procedure**\n\n1. one\n\n**Verification**\n\n- check\n"
);
}
#[test]
fn skipped_levels_and_pictures_keep_reading_order() {
assert_eq!(
md("= T\n\n== A\n\n==== Deep\n\ntext\n\n== B\n\n.Cap\nimage::x.png[]\n\nafter\n"),
"# T\n\n## A\n\n#### Deep\n\ntext\n\n## B\n\nCap\n\n<!-- image -->\n\nafter\n"
);
}
#[test]
fn rowspan_only_cell_specifiers_are_stripped() {
assert_eq!(parse_table_line(".2+|A |B"), vec!["A", "B"]);
assert_eq!(parse_table_line("2.3+|D ^.^h|E"), vec!["D", "E"]);
assert!(is_table_line(".2+|A |B"));
assert_eq!(
md("|===\n.2+|A |B\n|===\n"),
"| A | B |\n|-----|-----|\n"
);
}
#[test]
fn a_non_list_line_ends_the_list_without_being_swallowed() {
assert_eq!(
md("= T\n\n* one\n* two\n\n== Head\n\npara\n"),
"# T\n\n- one\n- two\n\n## Head\n\npara\n"
);
}
#[test]
fn a_heading_flushes_the_pending_paragraph_into_its_own_section() {
let doc = parse(
"= T\n\n== S1\n\n=== S1.1\n\nbody\n== S2\n\nbody2\n",
"t",
&NoFetch,
);
assert_eq!(
doc.export_to_markdown().trim(),
"# T\n\n## S1\n\n### S1.1\n\nbody\n\n## S2\n\nbody2"
);
let json: serde_json::Value = serde_json::from_str(&doc.export_to_json()).unwrap();
let texts = json["texts"].as_array().unwrap();
let body = texts.iter().find(|t| t["text"] == "body").unwrap();
let s11 = texts.iter().find(|t| t["text"] == "S1.1").unwrap();
assert_eq!(body["parent"]["$ref"], s11["self_ref"]);
let body2 = texts.iter().find(|t| t["text"] == "body2").unwrap();
let s2 = texts.iter().find(|t| t["text"] == "S2").unwrap();
assert_eq!(body2["parent"]["$ref"], s2["self_ref"]);
}
#[test]
fn a_lone_word_and_period_stays_a_paragraph() {
assert_eq!(md("= T\n\nIntro.\n"), "# T\n\nIntro.\n");
assert_eq!(md("Fig. 1 shows the result\n"), "Fig. 1 shows the result\n");
assert_eq!(md("b. option\n"), "1. option\n");
}
#[test]
fn inline_marks_split_runs() {
let runs = |t: &str| -> Vec<(String, u8)> {
parse_inline(t, false, false)
.into_iter()
.map(|r| {
let k = u8::from(r.bold) | u8::from(r.italic) << 1 | u8::from(r.code) << 2;
(r.text, k)
})
.collect()
};
assert_eq!(
runs("a *b* _c_ `d` **e**f"),
[
("a ".into(), 0),
("b".into(), 1),
(" ".into(), 0),
("c".into(), 2),
(" ".into(), 0),
("d".into(), 4),
(" ".into(), 0),
("e".into(), 1),
("f".into(), 0),
]
);
assert_eq!(runs("*_both_*"), [("both".into(), 3)]);
assert_eq!(
runs("snake_case_name 2*3*4 a* b*"),
[("snake_case_name 2*3*4 a* b*".into(), 0)]
);
assert_eq!(
md("= T\n\nSome *bold* and `x`.\n"),
"# T\n\nSome **bold** and `x` .\n"
);
}
#[test]
fn tables_and_stray_delimiters_follow_docling() {
assert_eq!(
md("|===\n|A |B\n\n|a\\|b |c\nwrapped\n|===\n"),
"| A | B |\n|----------|-----------|\n| a|b | c wrapped |\n"
);
assert_eq!(md("before\n====\nafter\n"), "before\n\nafter\n");
assert_eq!(
md("// note\n[NOTE]\n////\nhidden\n////\n:attr: x\ntext\n"),
"text\n"
);
}
#[test]
fn table_ends_and_pending_paragraphs_follow_docling() {
assert_eq!(
md("|a|b\n|c|d\ngone\nkept\n"),
"| a | b |\n|-----|-----|\n| c | d |\n\nkept\n"
);
assert_eq!(md("para\n* a\n\n* b\n"), "- a\npara\n- b\n");
assert_eq!(md("some *x*\n. one\n"), "1. onesome **x**\n");
let v = json("para\n* a\n");
assert_eq!(v["texts"][1]["label"], "paragraph");
assert_eq!(v["texts"][1]["parent"]["$ref"], "#/groups/0");
}
#[test]
fn an_image_carries_no_imageref_until_images_are_fetched() {
let png = {
let img = image::RgbImage::from_pixel(2, 3, image::Rgb([1, 2, 3]));
let mut buf = std::io::Cursor::new(Vec::new());
image::DynamicImage::ImageRgb8(img)
.write_to(&mut buf, image::ImageFormat::Png)
.unwrap();
buf.into_inner()
};
let dir = std::env::temp_dir().join(format!("docling-adoc-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(dir.join("a.png"), &png).unwrap();
let src = dir.join("doc.asciidoc");
std::fs::write(&src, "= T\n\nimage::a.png[Alt]\n").unwrap();
let picture = |doc: DoclingDocument| {
doc.nodes
.into_iter()
.find_map(|n| match n {
Node::Picture { image, .. } => Some(image),
_ => None,
})
.expect("a picture")
};
let source = SourceDocument::from_file(&src).unwrap();
assert!(picture(AsciiDocBackend::default().convert(&source).unwrap()).is_none());
let fetched = picture(
AsciiDocBackend {
images: ImagePolicy::new(ImageSources::Local),
}
.convert(&source)
.unwrap(),
)
.expect("the local image is read");
assert_eq!((fetched.width, fetched.height), (2, 3));
let _ = std::fs::remove_dir_all(&dir);
}
}