use crate::backend::images::{FsImageResolver, ImagePolicy, ImageResolver, ImageSources, NoFetch};
use docling_core::{DoclingDocument, Node, Table};
use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, Options, Parser, Tag, TagEnd};
use crate::backend::DeclarativeBackend;
use crate::error::ConversionError;
use crate::source::SourceDocument;
#[derive(Default)]
pub struct MarkdownBackend {
pub strict: bool,
pub images: ImagePolicy,
}
pub(super) fn marko_strikethrough<'a>(
raw: Vec<(Event<'a>, std::ops::Range<usize>)>,
text: &'a str,
) -> Vec<(Event<'a>, std::ops::Range<usize>)> {
let mut kept: Vec<bool> = Vec::new();
let tildes = |r: std::ops::Range<usize>| (Event::Text(text[r.clone()].into()), r);
let raw: Vec<_> = raw
.into_iter()
.map(|(event, range)| match event {
Event::Start(Tag::Strikethrough) => {
let src = &text[range.clone()];
let keep = src.len() > 4
&& src.starts_with("~~")
&& src.ends_with("~~")
&& !src[2..src.len() - 2].contains('~')
&& !text[..range.start].ends_with('~')
&& !text[range.end..].starts_with('~');
kept.push(keep);
if keep {
(event, range)
} else {
let n = src.len() - src.trim_start_matches('~').len();
tildes(range.start..range.start + n)
}
}
Event::End(TagEnd::Strikethrough) => {
if kept.pop().unwrap_or(true) {
(event, range)
} else {
let src = &text[range.clone()];
let n = src.len() - src.trim_end_matches('~').len();
tildes(range.end - n..range.end)
}
}
_ => (event, range),
})
.collect();
let bytes = text.as_bytes();
let mut out = Vec::with_capacity(raw.len());
let mut opaque = 0usize;
let mut k = 0;
while k < raw.len() {
match &raw[k].0 {
Event::Start(Tag::CodeBlock(_) | Tag::HtmlBlock | Tag::MetadataBlock(_)) => opaque += 1,
Event::End(TagEnd::CodeBlock | TagEnd::HtmlBlock | TagEnd::MetadataBlock(_)) => {
opaque = opaque.saturating_sub(1)
}
_ => {}
}
if opaque > 0 || !matches!(raw[k].0, Event::Text(_)) {
out.push(raw[k].clone());
k += 1;
continue;
}
let (start, mut end, first) = (raw[k].1.start, raw[k].1.start, k);
while let Some((Event::Text(t), r)) = raw.get(k) {
if r.start != end || text.get(r.clone()) != Some(&**t) {
break;
}
end = r.end;
k += 1;
}
if k == first {
out.push(raw[k].clone());
k += 1;
continue;
}
let mut spans = Vec::new();
let mut i = start;
while i + 1 < end {
if bytes[i] == b'~' && bytes[i + 1] == b'~' && (i == 0 || bytes[i - 1] != b'~') {
let mut j = i + 2;
while j < end && bytes[j] != b'~' {
j += 1;
}
if j > i + 2
&& j + 1 < end
&& bytes[j + 1] == b'~'
&& bytes.get(j + 2) != Some(&b'~')
{
spans.push((i, j + 2));
i = j + 2;
continue;
}
}
i += 1;
}
if spans.is_empty() {
out.extend(raw[first..k].iter().cloned());
continue;
}
let piece = |r: std::ops::Range<usize>| (Event::Text(text[r.clone()].into()), r);
let mut at = start;
for (s, e) in spans {
if s > at {
out.push(piece(at..s));
}
out.push((Event::Start(Tag::Strikethrough), s..e));
out.push(piece(s + 2..e - 2));
out.push((Event::End(TagEnd::Strikethrough), s..e));
at = e;
}
if end > at {
out.push(piece(at..end));
}
}
out
}
#[derive(Clone, Copy, PartialEq)]
enum BareUrl {
Url,
Www,
Email,
}
fn find_bare_urls(
text: &str,
run: std::ops::Range<usize>,
in_table: bool,
) -> Vec<(usize, usize, BareUrl)> {
let url = cached_regex!(r"(?:https?|ftp)://([\w.\-]*?\.[\w.\-]+)[^<\s]*");
let www = cached_regex!(r"www\.([\w.\-]*?\.[\w.\-]+)[^<\s]*");
let email = cached_regex!(r"[\w.\-+]+@[\w.\-]*?\.[\w.\-]*[a-zA-Z0-9]");
let email_whole = cached_regex!(r"^[\w.\-+]+@[\w.\-]*?\.[\w.\-]*[a-zA-Z0-9]$");
let entity_tail = cached_regex!(r"&[a-zA-Z]+;$");
let hay_end = text[run.end..]
.find(|c: char| c.is_whitespace() || c == '<')
.map_or(text.len(), |k| run.end + k);
let starts_ok = |at: usize| match text[..at].chars().next_back() {
Some('(') if text[..at - 1].ends_with(']') => false,
None => true,
Some(c) => {
c.is_whitespace()
|| matches!(c, '*' | '_' | '~' | '(')
|| ('\u{ff00}'..='\u{ffef}').contains(&c)
|| (in_table && c == '|')
}
};
let email_end = |start: usize, end: usize| -> Option<usize> {
(start + 1..=end).rev().find(|&e| {
text.is_char_boundary(e)
&& email_whole.is_match(&text[start..e])
&& text[e..]
.chars()
.next()
.is_none_or(|c| c.is_whitespace() || c == '.' || c == '<')
})
};
let domain_ok = |domain: &str| {
let parts: Vec<&str> = domain.split('.').collect();
parts.len() >= 2 && !parts[parts.len() - 2..].iter().any(|p| p.contains('_'))
};
let mut found = Vec::new();
let mut pos = run.start;
while pos < run.end {
let hay = &text[pos..hay_end];
let candidates = [
url.captures(hay).map(|c| {
(
c.get(0).unwrap(),
c.get(1).map(|d| d.as_str()),
BareUrl::Url,
)
}),
www.captures(hay).map(|c| {
(
c.get(0).unwrap(),
c.get(1).map(|d| d.as_str()),
BareUrl::Www,
)
}),
email.find(hay).map(|m| (m, None, BareUrl::Email)),
];
let Some((m, domain, kind)) = candidates
.into_iter()
.flatten()
.min_by_key(|(m, _, kind)| (m.start(), *kind != BareUrl::Url))
else {
break;
};
let start = pos + m.start();
if start >= run.end {
break;
}
let next = |at: usize| at + text[at..].chars().next().map_or(1, char::len_utf8);
if !starts_ok(start) {
pos = next(start);
continue;
}
let mut full_end = pos + m.end();
if kind == BareUrl::Email {
match email_end(start, full_end) {
Some(e) => full_end = e,
None => {
pos = next(start);
continue;
}
}
}
pos = full_end;
if domain.is_some_and(|d| !domain_ok(d)) {
continue;
}
let link = &text[start..full_end];
let mut end = full_end;
if link.ends_with(['?', '!', '.', ',', ':', '*', '_', '~']) {
end -= 1;
} else if link.ends_with(')') && link.matches(')').count() > link.matches('(').count() {
end -= link.matches(')').count() - link.matches('(').count();
} else if let Some(m) = entity_tail.find(link) {
end -= m.len();
}
if end <= run.end && end > start {
found.push((start, end, kind));
}
}
found
}
pub(super) fn link_bare_urls<'a>(
raw: Vec<(Event<'a>, std::ops::Range<usize>)>,
text: &'a str,
) -> Vec<(Event<'a>, std::ops::Range<usize>)> {
let mut out = Vec::with_capacity(raw.len());
let mut opaque = 0usize;
let mut in_table = false;
let mut k = 0;
while k < raw.len() {
match &raw[k].0 {
Event::Start(
Tag::Link { .. }
| Tag::Image { .. }
| Tag::CodeBlock(_)
| Tag::HtmlBlock
| Tag::MetadataBlock(_),
) => opaque += 1,
Event::End(
TagEnd::Link
| TagEnd::Image
| TagEnd::CodeBlock
| TagEnd::HtmlBlock
| TagEnd::MetadataBlock(_),
) => opaque = opaque.saturating_sub(1),
Event::Start(Tag::Table(_)) => in_table = true,
Event::End(TagEnd::Table) => in_table = false,
_ => {}
}
if opaque > 0 || !matches!(raw[k].0, Event::Text(_)) {
out.push(raw[k].clone());
k += 1;
continue;
}
let first = k;
let mut run_end = raw[k].1.start;
while let Some((Event::Text(_), r)) = raw.get(k) {
if r.start != run_end {
break;
}
run_end = r.end;
k += 1;
}
let run = &raw[first..k];
let urls = find_bare_urls(text, run[0].1.start..run_end, in_table);
let plain = |t: &str, r: &std::ops::Range<usize>| text.get(r.clone()) == Some(t);
let urls: Vec<_> = urls
.into_iter()
.filter(|&(s, e, _)| {
run.iter().all(|(ev, r)| match ev {
Event::Text(t) if !plain(t, r) => {
let inside = |c: usize| r.start < c && c < r.end;
!inside(s) && !inside(e)
}
_ => true,
})
})
.collect();
if urls.is_empty() {
out.extend(run.iter().cloned());
continue;
}
let mut pieces: Vec<(String, std::ops::Range<usize>)> = Vec::new();
for (ev, r) in run {
let Event::Text(t) = ev else { continue };
if !plain(t, r) {
pieces.push((t.to_string(), r.clone()));
continue;
}
let mut cuts: Vec<usize> = urls
.iter()
.flat_map(|&(s, e, _)| [s, e])
.filter(|&c| r.start < c && c < r.end)
.collect();
cuts.sort_unstable();
let mut at = r.start;
for c in cuts.into_iter().chain([r.end]) {
if c > at {
pieces.push((text[at..c].to_string(), at..c));
}
at = c;
}
}
let mut p = 0;
for &(s, e, kind) in &urls {
while p < pieces.len() && pieces[p].1.end <= s {
out.push((Event::Text(pieces[p].0.clone().into()), pieces[p].1.clone()));
p += 1;
}
let inner: Vec<_> = pieces[p..]
.iter()
.take_while(|(_, r)| r.end <= e)
.cloned()
.collect();
p += inner.len();
let label: String = inner.iter().map(|(t, _)| t.as_str()).collect();
let (link_type, dest) = match kind {
BareUrl::Url => (pulldown_cmark::LinkType::Autolink, label.clone()),
BareUrl::Www => (
pulldown_cmark::LinkType::Autolink,
format!("http://{label}"),
),
BareUrl::Email => (pulldown_cmark::LinkType::Email, label.clone()),
};
out.push((
Event::Start(Tag::Link {
link_type,
dest_url: dest.into(),
title: "".into(),
id: "".into(),
}),
s..e,
));
for (t, r) in inner {
out.push((Event::Text(t.into()), r));
}
out.push((Event::End(TagEnd::Link), s..e));
}
for (t, r) in &pieces[p..] {
out.push((Event::Text(t.clone().into()), r.clone()));
}
}
out
}
pub(super) fn fold_math_into_text<'a>(
raw: Vec<(Event<'a>, std::ops::Range<usize>)>,
text: &'a str,
) -> Vec<(Event<'a>, std::ops::Range<usize>)> {
let standalone = |k: usize| {
matches!(
raw.get(k.wrapping_sub(1)),
Some((Event::Start(Tag::Paragraph), _))
) && matches!(raw.get(k + 1), Some((Event::End(TagEnd::Paragraph), _)))
};
raw.iter()
.enumerate()
.map(|(k, (event, range))| match event {
Event::DisplayMath(_) if standalone(k) => (event.clone(), range.clone()),
Event::InlineMath(_) | Event::DisplayMath(_) => (
Event::Text(text.get(range.clone()).unwrap_or("").into()),
range.clone(),
),
_ => (event.clone(), range.clone()),
})
.collect()
}
fn is_delimiter_row(line: &str) -> bool {
let line = line.trim();
if !line.contains('|') {
return false;
}
line.trim_start_matches('|')
.trim_end_matches('|')
.split('|')
.all(|cell| {
let dashes = cell.trim().trim_start_matches(':').trim_end_matches(':');
!dashes.is_empty() && dashes.bytes().all(|b| b == b'-')
})
}
fn normalize_table_edge_pipes(text: &str) -> std::borrow::Cow<'_, str> {
let cell_count = |line: &str| {
line.trim()
.trim_start_matches('|')
.trim_end_matches('|')
.split('|')
.count()
};
let is_fence = |line: &str| {
let t = line.trim_start();
t.starts_with("```") || t.starts_with("~~~")
};
let lines: Vec<&str> = text.split('\n').collect();
let mut needs_fix = false;
{
let mut in_fence = false;
for w in lines.windows(2) {
let head = w[0].trim_end_matches('\r');
if is_fence(head) {
in_fence = !in_fence;
continue;
}
let delim = w[1].trim_end_matches('\r');
if !in_fence
&& head.contains('|')
&& !head.trim_start().starts_with('|')
&& is_delimiter_row(delim)
&& cell_count(head) == cell_count(delim)
{
needs_fix = true;
break;
}
}
}
if !needs_fix {
return std::borrow::Cow::Borrowed(text);
}
let mut out: Vec<String> = Vec::with_capacity(lines.len());
let mut in_fence = false;
let mut i = 0;
while i < lines.len() {
let head = lines[i].trim_end_matches('\r');
if is_fence(head) {
in_fence = !in_fence;
out.push(lines[i].to_string());
i += 1;
continue;
}
let is_table_start = !in_fence
&& head.contains('|')
&& !head.trim_start().starts_with('|')
&& lines
.get(i + 1)
.is_some_and(|d| is_delimiter_row(d.trim_end_matches('\r')))
&& cell_count(head) == cell_count(lines[i + 1].trim_end_matches('\r'));
if !is_table_start {
out.push(lines[i].to_string());
i += 1;
continue;
}
while i < lines.len() {
let (body, cr) = match lines[i].strip_suffix('\r') {
Some(b) => (b, "\r"),
None => (lines[i], ""),
};
if !body.contains('|') || body.trim().is_empty() {
break;
}
let t = body.trim();
let mut fixed = String::with_capacity(t.len() + 4);
if !t.starts_with('|') {
fixed.push_str("| ");
}
fixed.push_str(t);
if !t.ends_with('|') {
fixed.push_str(" |");
}
fixed.push_str(cr);
out.push(fixed);
i += 1;
}
}
std::borrow::Cow::Owned(out.join("\n"))
}
impl DeclarativeBackend for MarkdownBackend {
fn convert(&self, source: &SourceDocument) -> Result<DoclingDocument, ConversionError> {
let text = source.text()?;
let normalized = normalize_table_edge_pipes(&text);
let text = normalized.as_ref();
let mut opts = Options::empty();
opts.insert(Options::ENABLE_TABLES);
opts.insert(Options::ENABLE_STRIKETHROUGH);
opts.insert(Options::ENABLE_MATH);
let raw: Vec<(Event, std::ops::Range<usize>)> =
Parser::new_ext(text, opts).into_offset_iter().collect();
let raw = marko_strikethrough(raw, text);
let raw = link_bare_urls(fold_math_into_text(raw, text), text);
let mut events: Vec<Event> = Vec::with_capacity(raw.len());
let mut k = 0;
let mut in_table = false;
while k < raw.len() {
match &raw[k].0 {
Event::Start(Tag::Table(_)) => in_table = true,
Event::End(TagEnd::Table) => in_table = false,
_ => {}
}
if matches!(raw[k].0, Event::Text(_)) {
let mut merged = String::new();
let mut end = raw[k].1.start;
while let Some((Event::Text(t), range)) = raw.get(k) {
let escape_gap = in_table
&& range.start == end + 1
&& text.as_bytes().get(end) == Some(&b'\\');
if range.start != end && !escape_gap {
break;
}
merged.push_str(t);
end = range.end;
k += 1;
}
events.push(Event::Text(merged.into()));
} else {
events.push(raw[k].0.clone());
k += 1;
}
}
let fs_resolver;
let images: &dyn ImageResolver = if self.images.sources == ImageSources::None {
&NoFetch
} else {
fs_resolver = FsImageResolver::new(
source.base_dir().map(|p| p.to_path_buf()),
source.base_url.clone(),
self.images.clone(),
);
&fs_resolver
};
let mut doc = DoclingDocument::new(&source.name);
let mut i = 0;
self.parse_blocks(&events, &mut i, &mut doc.nodes, 0, Stop::Eof, 0, images);
doc.tree = super::md_tree::build_tree(text, images);
Ok(doc)
}
}
#[derive(Clone, Copy, PartialEq)]
enum Stop {
Eof,
BlockQuote,
}
const MAX_NESTING: u16 = 100;
pub(super) fn skip_subtree(events: &[Event], i: &mut usize) {
let mut depth = 0usize;
while *i < events.len() {
match &events[*i] {
Event::Start(_) => depth += 1,
Event::End(_) => {
depth = depth.saturating_sub(1);
if depth == 0 {
*i += 1;
return;
}
}
_ => {}
}
*i += 1;
}
}
impl MarkdownBackend {
#[allow(clippy::too_many_arguments)]
fn parse_blocks(
&self,
events: &[Event],
i: &mut usize,
out: &mut Vec<Node>,
list_level: u8,
stop: Stop,
depth: u16,
images: &dyn ImageResolver,
) {
while *i < events.len() {
match &events[*i] {
Event::Start(Tag::BlockQuote(_) | Tag::List(_)) if depth >= MAX_NESTING => {
skip_subtree(events, i);
}
Event::End(TagEnd::BlockQuote(_)) if stop == Stop::BlockQuote => {
*i += 1;
return;
}
Event::End(_) => {
*i += 1;
}
Event::Start(Tag::HtmlBlock) => {
*i += 1;
let mut html = String::new();
while *i < events.len() && !matches!(events[*i], Event::End(TagEnd::HtmlBlock))
{
if let Event::Html(t) = &events[*i] {
html.push_str(t);
}
*i += 1;
}
consume_end(events, i);
super::html::append_fragment(&html, out, images);
}
Event::Start(Tag::BlockQuote(_)) => {
*i += 1;
self.parse_blocks(
events,
i,
out,
list_level,
Stop::BlockQuote,
depth + 1,
images,
);
}
Event::Start(Tag::Paragraph) => {
if let Some(Event::DisplayMath(latex)) = events.get(*i + 1) {
if matches!(events.get(*i + 2), Some(Event::End(TagEnd::Paragraph))) {
let latex = latex.trim().to_string();
if !latex.is_empty() {
out.push(Node::Formula {
orig: latex.clone(),
latex,
location: None,
});
}
*i += 3;
continue;
}
}
if !self.strict
&& matches!(events.get(*i + 1), Some(Event::Code(_)))
&& matches!(events.get(*i + 2), Some(Event::End(TagEnd::Paragraph)))
{
if let Some(Event::Code(c)) = events.get(*i + 1) {
let text = unescape_entities(c.trim());
if !text.is_empty() {
out.push(Node::Code {
language: None,
text,
orig: None,
pretty: None,
});
}
}
*i += 3;
continue;
}
*i += 1;
if let Some(nodes) = self.lone_image_picture(events, *i, images) {
skip_subtree(events, i); consume_end(events, i); out.extend(nodes);
continue;
}
let runs = self.collect_inline(events, i, false);
consume_end(events, i);
let text = self.join_runs(&runs, false);
if !text.is_empty() {
out.push(Node::Paragraph { text });
}
}
Event::Start(Tag::Heading { level, .. }) => {
let lvl = heading_level(*level);
*i += 1;
let runs = self.collect_inline(events, i, false);
consume_end(events, i);
let text = self.join_runs(&runs, false);
if !text.is_empty() {
out.push(Node::Heading { level: lvl, text });
}
}
Event::Start(Tag::List(start)) => {
let ordered = start.is_some();
let start_num = start.unwrap_or(1);
*i += 1;
self.parse_list(events, i, out, list_level, ordered, start_num, depth + 1);
}
Event::Start(Tag::CodeBlock(kind)) => {
let language = match kind {
CodeBlockKind::Fenced(info) => {
let lang = info.split_whitespace().next().unwrap_or("");
(!lang.is_empty()).then(|| lang.to_string())
}
CodeBlockKind::Indented => None,
};
*i += 1;
let mut code = String::new();
while *i < events.len() && !matches!(events[*i], Event::End(TagEnd::CodeBlock))
{
if let Event::Text(t) = &events[*i] {
code.push_str(t);
}
*i += 1;
}
consume_end(events, i);
let trimmed = code.trim_end();
let text = if self.strict {
trimmed.to_string()
} else {
unescape_entities(trimmed)
};
if !text.is_empty() {
out.push(Node::Code {
language,
text,
orig: None,
pretty: None,
});
}
}
Event::Start(Tag::Table(_)) => {
*i += 1;
self.parse_table(events, i, out);
}
_ => {
*i += 1;
}
}
}
}
#[allow(clippy::too_many_arguments)]
fn parse_list(
&self,
events: &[Event],
i: &mut usize,
out: &mut Vec<Node>,
level: u8,
ordered: bool,
start: u64,
depth: u16,
) {
let mut number = start;
let mut first = true;
while *i < events.len() {
match &events[*i] {
Event::Start(Tag::Item) => {
*i += 1;
self.parse_item(events, i, out, level, ordered, number, first, depth);
number += 1;
first = false;
}
Event::End(TagEnd::List(_)) => {
*i += 1;
return;
}
_ => {
*i += 1;
}
}
}
}
fn lone_image_picture(
&self,
events: &[Event],
start: usize,
images: &dyn ImageResolver,
) -> Option<Vec<Node>> {
let Some(Event::Start(Tag::Image {
dest_url, title, ..
})) = events.get(start)
else {
return None;
};
let mut end = start;
skip_subtree(events, &mut end);
if !matches!(events.get(end), Some(Event::End(TagEnd::Paragraph))) {
return None;
}
let mut j = start + 1;
let alt = self.join_runs(&self.collect_inline(events, &mut j, false), false);
let mut nodes = vec![Node::Picture {
caption: Some(escape_text(title)).filter(|t| !t.is_empty()),
caption_href: None,
image: images.resolve(dest_url),
classification: None,
description: None,
caption_parent: docling_core::CaptionParent::Container,
caption_location: None,
}];
if !alt.is_empty() {
nodes.push(Node::Paragraph { text: alt });
}
Some(nodes)
}
#[allow(clippy::too_many_arguments)]
fn parse_item(
&self,
events: &[Event],
i: &mut usize,
out: &mut Vec<Node>,
level: u8,
ordered: bool,
number: u64,
first_in_list: bool,
depth: u16,
) {
let mut emitted = false;
while *i < events.len() {
match &events[*i] {
Event::End(TagEnd::Item) => {
*i += 1;
break;
}
Event::Start(Tag::List(_) | Tag::BlockQuote(_)) if depth >= MAX_NESTING => {
skip_subtree(events, i);
}
Event::Start(Tag::List(start)) => {
let nested_ordered = start.is_some();
let nested_start = start.unwrap_or(1);
*i += 1;
self.parse_list(
events,
i,
out,
level + 1,
nested_ordered,
nested_start,
depth + 1,
);
}
Event::Start(Tag::Paragraph) => {
*i += 1;
let runs = self.collect_inline(events, i, false);
consume_end(events, i);
emit_item(
out,
&mut emitted,
ordered,
number,
first_in_list,
self.join_runs(&runs, false),
level,
);
}
ev if is_inline_start(ev) => {
let runs = self.collect_inline(events, i, false);
emit_item(
out,
&mut emitted,
ordered,
number,
first_in_list,
self.join_runs(&runs, false),
level,
);
}
_ => {
*i += 1;
}
}
}
}
fn parse_table(&self, events: &[Event], i: &mut usize, out: &mut Vec<Node>) {
let mut rows: Vec<Vec<String>> = Vec::new();
while *i < events.len() {
match &events[*i] {
Event::Start(Tag::TableHead) | Event::Start(Tag::TableRow) => {
*i += 1;
rows.push(self.parse_table_row(events, i));
}
Event::End(TagEnd::Table) => {
*i += 1;
break;
}
_ => {
*i += 1;
}
}
}
if !rows.is_empty() {
out.push(Node::Table(Table {
rows,
location: None,
structure: None,
cell_blocks: None,
cells: None,
caption: None,
caption_parent: Default::default(),
caption_location: None,
}));
}
}
fn parse_table_row(&self, events: &[Event], i: &mut usize) -> Vec<String> {
let mut cells = Vec::new();
while *i < events.len() {
match &events[*i] {
Event::Start(Tag::TableCell) => {
*i += 1;
let runs = self.collect_inline(events, i, true);
consume_end(events, i);
cells.push(self.join_runs(&runs, true));
}
Event::End(TagEnd::TableHead) | Event::End(TagEnd::TableRow) => {
*i += 1;
break;
}
_ => {
*i += 1;
}
}
}
cells
}
fn collect_inline(&self, events: &[Event], i: &mut usize, table: bool) -> Vec<String> {
let mut runs: Vec<String> = Vec::new();
let mut pending: Option<Break> = None;
while *i < events.len() {
match &events[*i] {
Event::Text(t) => {
let before = runs.len();
self.push_text(t, table, &mut runs);
if runs.len() > before {
self.apply_break(&mut runs, pending.take(), table);
}
*i += 1;
}
Event::SoftBreak | Event::HardBreak => {
let hard = matches!(events[*i], Event::HardBreak);
if table {
} else if self.strict {
runs.push(if hard { "\n" } else { " " }.to_string());
} else {
pending = Some(if hard { Break::Hard } else { Break::Soft });
}
*i += 1;
}
Event::InlineHtml(_) | Event::Html(_) => {
*i += 1;
}
Event::Code(t) => {
let content = if self.strict && !table {
t.to_string()
} else {
unescape_entities(t)
};
runs.push(if table {
content
} else {
docling_core::code_span(&content)
});
*i += 1;
}
Event::Start(Tag::Emphasis) => {
runs.push(self.wrap_inline(events, i, table, "*"));
self.apply_break(&mut runs, pending.take(), table);
}
Event::Start(Tag::Strong) => {
runs.push(self.wrap_inline(events, i, table, "**"));
self.apply_break(&mut runs, pending.take(), table);
}
Event::Start(Tag::Strikethrough) => {
runs.push(self.wrap_inline(events, i, table, "~~"));
self.apply_break(&mut runs, pending.take(), table);
}
Event::Start(Tag::Link {
link_type,
dest_url,
..
}) => {
let url = link_dest(*link_type, dest_url);
*i += 1;
let inner = self.join_runs(&self.collect_inline(events, i, table), table);
consume_end(events, i);
runs.push(docling_core::hyperlink::markdown_link(&inner, &url));
}
Event::Start(Tag::Image {
dest_url, title, ..
}) => {
let url = dest_url.to_string();
*i += 1;
let alt = self.join_runs(&self.collect_inline(events, i, table), table);
consume_end(events, i);
runs.push(if table {
if title.is_empty() {
"<!-- image -->".to_string()
} else {
format!("{title} <!-- image -->")
}
} else {
format!("")
});
}
_ => break,
}
}
runs
}
fn apply_break(&self, runs: &mut Vec<String>, pending: Option<Break>, table: bool) {
let Some(brk) = pending else {
return;
};
if table || self.strict || runs.len() < 2 {
return;
}
let new = runs.pop().expect("two runs");
let prev = runs.pop().expect("two runs");
let sep = match brk {
Break::Hard => "\n",
Break::Soft => " ",
};
let marker_of = |r: &str| -> Option<&'static str> {
["**", "~~", "*"]
.into_iter()
.find(|m| r.len() > 2 * m.len() && r.starts_with(m) && r.ends_with(m))
};
let plain = |r: &str| !r.starts_with(['*', '~', '`', '[', '!']);
match (marker_of(&prev), marker_of(&new)) {
(None, None) if plain(&prev) && plain(&new) => {
runs.push(format!("{prev}{sep}{new}"));
}
(Some(m), Some(n)) if m == n => {
let inner_prev = &prev[m.len()..prev.len() - m.len()];
let inner_new = &new[m.len()..new.len() - m.len()];
runs.push(format!("{m}{inner_prev}{sep}{inner_new}{m}"));
}
_ => {
runs.push(prev);
runs.push(match brk {
Break::Hard => format!("\n{new}"),
Break::Soft => new,
});
}
}
}
fn wrap_inline(&self, events: &[Event], i: &mut usize, table: bool, marker: &str) -> String {
*i += 1;
let inner = self.join_runs(&self.collect_inline(events, i, table), table);
consume_end(events, i);
if table {
inner
} else {
format!("{marker}{inner}{marker}")
}
}
fn push_text(&self, text: &str, table: bool, runs: &mut Vec<String>) {
if table {
let trimmed = text.trim();
if !trimmed.is_empty() {
runs.push(trimmed.to_string());
}
} else if self.strict {
if !text.is_empty() {
runs.push(text.to_string());
}
} else {
let trimmed = text.trim();
if !trimmed.is_empty() {
runs.push(escape_html(&escape_underscores(trimmed)));
}
}
}
fn join_runs(&self, runs: &[String], table: bool) -> String {
if self.strict && !table {
runs.concat()
} else {
runs.iter()
.filter(|r| !r.is_empty())
.cloned()
.collect::<Vec<_>>()
.join(" ")
}
}
}
#[allow(clippy::too_many_arguments)]
fn emit_item(
out: &mut Vec<Node>,
emitted: &mut bool,
ordered: bool,
number: u64,
first_in_list: bool,
text: String,
level: u8,
) {
if !*emitted && !text.is_empty() {
out.push(Node::ListItem {
ordered,
number,
first_in_list,
text,
level,
marker: None,
location: None,
dclx: None,
href: None,
layer: None,
});
*emitted = true;
}
}
fn consume_end(events: &[Event], i: &mut usize) {
if matches!(events.get(*i), Some(Event::End(_))) {
*i += 1;
}
}
#[derive(Clone, Copy)]
enum Break {
Soft,
Hard,
}
pub(super) fn link_dest(link_type: pulldown_cmark::LinkType, dest: &str) -> String {
match link_type {
pulldown_cmark::LinkType::Email => format!("mailto:{dest}"),
_ => dest.to_string(),
}
}
fn is_inline_start(event: &Event) -> bool {
matches!(
event,
Event::Text(_)
| Event::Code(_)
| Event::SoftBreak
| Event::HardBreak
| Event::InlineHtml(_)
| Event::Start(Tag::Emphasis | Tag::Strong | Tag::Strikethrough)
| Event::Start(Tag::Link { .. } | Tag::Image { .. })
)
}
fn heading_level(level: HeadingLevel) -> u8 {
match level {
HeadingLevel::H1 => 1,
HeadingLevel::H2 => 2,
HeadingLevel::H3 => 3,
HeadingLevel::H4 => 4,
HeadingLevel::H5 => 5,
HeadingLevel::H6 => 6,
}
}
pub(crate) fn escape_html(text: &str) -> String {
text.replace('&', "&")
.replace('<', "<")
.replace('>', ">")
}
pub(crate) fn escape_text(text: &str) -> String {
escape_html(&escape_underscores(text))
}
pub(crate) fn escape_underscores(text: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut prev = '\0';
for ch in text.chars() {
if ch == '_' && prev != '\\' {
out.push('\\');
}
out.push(ch);
prev = ch;
}
out
}
pub(super) fn unescape_entities(text: &str) -> String {
let mut out = String::with_capacity(text.len());
let bytes = text.as_bytes();
let mut idx = 0;
while idx < bytes.len() {
if bytes[idx] == b'&' {
if let Some(semi) = text[idx..].find(';') {
let entity = &text[idx + 1..idx + semi];
if let Some(ch) = decode_entity(entity) {
out.push(ch);
idx += semi + 1;
continue;
}
}
}
let ch = text[idx..].chars().next().unwrap();
out.push(ch);
idx += ch.len_utf8();
}
out
}
fn decode_entity(entity: &str) -> Option<char> {
match entity {
"amp" => Some('&'),
"lt" => Some('<'),
"gt" => Some('>'),
"quot" => Some('"'),
"apos" | "#39" => Some('\''),
"vert" => Some('|'),
_ => {
let code = if let Some(hex) = entity.strip_prefix("#x").or(entity.strip_prefix("#X")) {
u32::from_str_radix(hex, 16).ok()?
} else if let Some(dec) = entity.strip_prefix('#') {
dec.parse().ok()?
} else {
return None;
};
char::from_u32(code)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::InputFormat;
#[test]
fn table_cells_keep_the_space_around_inline_emphasis() {
let doc = convert(
"| H | I |\n|---|---|\n| **C** Cadre | x |\n| foo **bar** | y |\n| **A** **B** | z |\n| *italic* and **bold** | w |\n",
);
let json: serde_json::Value = serde_json::from_str(&doc.export_to_json()).unwrap();
let cells: Vec<String> = json["tables"][0]["data"]["table_cells"]
.as_array()
.unwrap()
.iter()
.map(|c| c["text"].as_str().unwrap().to_string())
.collect();
assert_eq!(
cells,
[
"H",
"I",
"C Cadre",
"x",
"foo bar",
"y",
"A B",
"z",
"italic and bold",
"w"
]
);
}
fn convert(md: &str) -> DoclingDocument {
let src = SourceDocument::from_bytes("t", InputFormat::Md, md.as_bytes().to_vec());
MarkdownBackend {
strict: false,
images: Default::default(),
}
.convert(&src)
.unwrap()
}
fn convert_strict(md: &str) -> DoclingDocument {
let src = SourceDocument::from_bytes("t", InputFormat::Md, md.as_bytes().to_vec());
let mut doc = MarkdownBackend {
strict: true,
images: Default::default(),
}
.convert(&src)
.unwrap();
doc.strict_markdown = true;
doc
}
#[test]
fn pathological_nesting_converts_instead_of_overflowing() {
let quotes = ">".repeat(50_000) + " deep\n\nafter\n";
let md = convert("es).export_to_markdown();
assert!(md.contains("after"), "{md:?}");
let list: String = (0..5_000)
.map(|i| format!("{}- item {i}\n", " ".repeat(i)))
.collect();
let md = convert(&list).export_to_markdown();
assert!(md.contains("item 0") && md.contains("item 99"), "{md:?}");
let shallow: String = (0..20)
.map(|i| format!("{}- item {i}\n", " ".repeat(i)))
.collect();
let md = convert(&shallow).export_to_markdown();
assert!(md.contains("item 19"), "{md:?}");
}
#[test]
fn inline_runs_get_spaced_in_legacy() {
let doc = convert("Foo *emphasis* **strong** ***both***.\n");
assert_eq!(
doc.export_to_markdown(),
"Foo *emphasis* **strong** ***both*** .\n"
);
}
#[test]
fn strict_keeps_inline_clean() {
let doc = convert_strict("Foo *emphasis* **strong** ***both***.\n");
assert_eq!(
doc.export_to_markdown(),
"Foo *emphasis* **strong** ***both***.\n"
);
}
#[test]
fn display_math_is_a_formula_and_inline_math_stays_text() {
let doc = convert(
"Costs $5 and $6, with $x^2$ inline and [$$E=mc^2$$](https://e.org) linked.\n\n\
$$\n H(y) = -\\sum_i y_i \\log y_i \n$$\n\n\
$$ a + b $$\n\n- item with $$c$$ inside\n",
);
let nodes: Vec<String> = doc
.nodes
.iter()
.map(|n| match n {
Node::Paragraph { text } => format!("P:{text}"),
Node::Formula { latex, orig, .. } => format!("F:{latex}|{orig}"),
Node::ListItem { text, .. } => format!("L:{text}"),
other => format!("{other:?}"),
})
.collect();
assert_eq!(
nodes,
[
"P:Costs $5 and $6, with $x^2$ inline and [$$E=mc^2$$](https://e.org) linked.",
"F:H(y) = -\\sum_i y_i \\log y_i|H(y) = -\\sum_i y_i \\log y_i",
"F:a + b|a + b",
"L:item with $$c$$ inside",
]
);
let md = doc.export_to_markdown();
assert!(md.contains("$$H(y) = -\\sum_i y_i \\log y_i$$"), "{md}");
}
#[test]
fn strict_keeps_code_language_and_inline_code() {
let doc = convert_strict("```rust\nlet x = 1;\n```\n");
assert_eq!(doc.export_to_markdown(), "```rust\nlet x = 1;\n```\n");
}
#[test]
fn ordered_list_honors_start() {
let doc = convert("3. third\n4. fourth\n");
assert_eq!(doc.export_to_markdown(), "3. third\n4. fourth\n");
}
#[test]
fn parses_github_table_plain_cells() {
let doc = convert("| **A** | B |\n|---|---|\n| x | y |\n");
assert_eq!(
doc.export_to_markdown(),
"| A | B |\n|-----|-----|\n| x | y |\n"
);
}
#[test]
fn lone_code_span_becomes_code_block_in_legacy() {
let doc = convert("`& <`\n");
assert_eq!(
doc.nodes,
vec![Node::Code {
language: None,
text: "& <".into(),
orig: None,
pretty: None,
}]
);
}
}