use crate::blocks::{Block, BlockKind, ExtractOptions, PageSeparators, TableRenderMode};
#[cfg(test)]
use crate::table_trace::TableOrigin;
use crate::text::{HorizontalRule, TextSpan};
struct CellAt<'a> {
row: usize,
column: usize,
row_span: usize,
column_span: usize,
header: bool,
text: &'a str,
}
struct OwnedCell {
row: usize,
column: usize,
row_span: usize,
column_span: usize,
header: bool,
text: String,
}
fn cell_of(block: &Block) -> Option<CellAt<'_>> {
match block.kind {
BlockKind::TableCell {
row,
column,
row_span,
column_span,
header,
} => Some(CellAt {
row: row as usize,
column: column as usize,
row_span: (row_span as usize).max(1),
column_span: (column_span as usize).max(1),
header,
text: block.text.trim(),
}),
_ => None,
}
}
pub(crate) fn table_run(blocks: &[Block], from: usize) -> Option<usize> {
let first = cell_of(&blocks[from])?;
let mut last = (first.row, first.column);
let mut end = from + 1;
while end < blocks.len() && blocks[end].page == blocks[from].page {
match cell_of(&blocks[end]) {
Some(cell) if (cell.row, cell.column) > last => {
last = (cell.row, cell.column);
end += 1;
}
_ => break,
}
}
Some(end)
}
fn table_run_across(blocks: &[Block], from: usize) -> Option<usize> {
let mut end = table_run(blocks, from)?;
while end < blocks.len()
&& blocks[end].continues_previous
&& blocks[end].page != blocks[end - 1].page
&& cell_of(&blocks[end]).is_some()
{
end = table_run(blocks, end)?;
}
Some(end)
}
fn cells_across(blocks: &[Block]) -> Vec<CellAt<'_>> {
let mut out: Vec<CellAt> = Vec::new();
let mut offset = 0usize;
let mut segment_head: Option<usize> = None; let head: Vec<&str> = blocks
.iter()
.filter_map(cell_of)
.take_while(|c| c.row == 0)
.map(|c| c.text)
.collect();
for (k, block) in blocks.iter().enumerate() {
let Some(mut cell) = cell_of(block) else {
continue;
};
if k > 0 && block.continues_previous && block.page != blocks[k - 1].page {
let rows_so_far = out.iter().map(|c| c.row + c.row_span).max().unwrap_or(0);
let first_row = cell.row;
let this_head: Vec<&str> = blocks[k..]
.iter()
.take_while(|b| b.page == block.page)
.filter_map(cell_of)
.take_while(|c| c.row == first_row)
.map(|c| c.text)
.collect();
segment_head = (!head.is_empty() && this_head == head).then_some(first_row);
offset = rows_so_far - first_row.min(rows_so_far);
if segment_head.is_some() {
offset = rows_so_far.saturating_sub(first_row + 1);
}
}
if segment_head == Some(cell.row) {
continue;
}
cell.row += offset;
out.push(cell);
}
out
}
fn table(blocks: &[Block], switches: crate::learned::Switches, mode: TableRenderMode) -> String {
let mut owned: Vec<OwnedCell> = cells_across(blocks)
.iter()
.map(|cell| OwnedCell {
row: cell.row,
column: cell.column,
row_span: cell.row_span,
column_span: cell.column_span,
header: cell.header,
text: cell.text.to_string(),
})
.collect();
join_currency_columns(&mut owned);
let mut cells: Vec<CellAt> = owned
.iter()
.map(|cell| CellAt {
row: cell.row,
column: cell.column,
row_span: cell.row_span,
column_span: cell.column_span,
header: cell.header,
text: &cell.text,
})
.collect();
let head = header_block(&cells, switches);
for cell in cells.iter_mut().filter(|cell| cell.row < head) {
cell.header = true;
}
let columns = cells
.iter()
.map(|cell| cell.column + cell.column_span)
.max()
.unwrap_or(0);
let rows = cells
.iter()
.map(|cell| cell.row + cell.row_span)
.max()
.unwrap_or(0);
if columns == 0 || rows == 0 {
return String::new();
}
let rectangular = cells
.iter()
.all(|cell| cell.row_span == 1 && cell.column_span == 1)
&& (0..rows).all(|r| {
let mut seen: Vec<usize> = cells
.iter()
.filter(|cell| cell.row == r)
.map(|cell| cell.column)
.collect();
seen.sort_unstable();
seen == (0..columns).collect::<Vec<usize>>()
});
let pipe = match mode {
TableRenderMode::Markdown => true,
TableRenderMode::Html => false,
TableRenderMode::Auto => {
rectangular && head <= 1 && (head == 1 || !switches.markdown_headerless_html)
}
};
if pipe {
markdown_table(&cells, rows, columns)
} else {
html_table(&cells, rows, columns)
}
}
fn header_block(cells: &[CellAt], switches: crate::learned::Switches) -> usize {
let rows = cells.iter().map(|cell| cell.row + 1).max().unwrap_or(0);
if rows == 0 {
return 0;
}
let figures = |r: usize| {
if is_year_row(cells, r, switches.markdown_header_block_stub_years) {
return false;
}
let texts: Vec<&str> = cells
.iter()
.filter(|cell| cell.row == r)
.map(|cell| cell.text.trim())
.filter(|text| !text.is_empty())
.collect();
if texts.is_empty() {
return false;
}
let counted = texts
.iter()
.filter(|t| {
is_figure(t)
|| (switches.markdown_header_block_figures && is_stacked_figure_cell(t))
})
.count();
let valued = texts
.iter()
.filter(|t| {
(is_figure(t)
|| (switches.markdown_header_block_figures && is_stacked_figure_cell(t)))
&& !is_year_cell(t)
})
.count();
let mixed = valued > 0 && valued < texts.len();
2 * counted >= texts.len() || (switches.markdown_header_block_mixed_body && mixed)
};
let mut head = (0..rows).find(|&r| figures(r)).unwrap_or(rows);
if head == rows && switches.markdown_header_block_word_table && rows > 1 {
let first_has_text = cells
.iter()
.filter(|cell| cell.row == 0)
.any(|cell| !cell.text.trim().is_empty());
if first_has_text {
head = 1;
}
}
if head == rows || head > 4 {
return 0;
}
if cells.iter().any(|cell| cell.row_span > 1) {
if !switches.markdown_header_block_rowspan_crossing {
return 0;
}
let crosses = cells
.iter()
.any(|cell| cell.row < head && cell.row + cell.row_span > head);
if crosses {
return 0;
}
}
head
}
fn is_year_row(cells: &[CellAt], row: usize, stub_years: bool) -> bool {
let mut all_years = true;
let mut years = 0usize;
for cell in cells.iter().filter(|cell| cell.row == row) {
let text = cell.text.trim();
if text.is_empty() {
continue;
}
if !is_year_cell(text) {
all_years = false;
break;
}
years += 1;
}
if all_years && years >= 2 {
return true;
}
if !stub_years {
return false;
}
let mut valued = 0usize;
for cell in cells
.iter()
.filter(|cell| cell.row == row && cell.column != 0)
{
let text = cell.text.trim();
if text.is_empty() {
continue;
}
if !is_year_cell(text) {
return false;
}
valued += 1;
}
valued >= 2
}
fn is_year_cell(text: &str) -> bool {
let mut rest = text.trim();
if rest.is_empty() {
return false;
}
if let Some(without_star) = rest.strip_suffix('*') {
rest = without_star.trim_end();
if rest.is_empty() {
return false;
}
}
if let Some(open) = rest.find('(') {
let (year, note) = rest.split_at(open);
if !is_year(year.trim_end()) {
return false;
}
let note = note.trim();
if !(note.starts_with('(') && note.ends_with(')')) {
return false;
}
let inner = note[1..note.len() - 1].trim();
return !inner.is_empty() && inner.bytes().all(|b| b.is_ascii_digit());
}
if rest.contains(['(', ')']) {
return false;
}
is_year(rest)
}
fn is_year(text: &str) -> bool {
text.len() == 4
&& (text.starts_with("19") || text.starts_with("20"))
&& text.bytes().all(|b| b.is_ascii_digit())
}
fn is_currency_symbol(text: &str) -> bool {
matches!(text.trim(), "$" | "€" | "£" | "¥")
}
fn contains_the_join(cell: &OwnedCell, join: usize) -> bool {
cell.column_span > 1 && cell.column <= join && cell.column + cell.column_span >= join + 2
}
fn find_currency_column(cells: &[OwnedCell]) -> Option<usize> {
let columns = cells
.iter()
.map(|cell| cell.column + cell.column_span)
.max()
.unwrap_or(0);
if columns < 2 {
return None;
}
for join in 0..columns - 1 {
if cells.iter().any(|cell| {
cell.column_span > 1
&& cell.column < join + 2
&& cell.column + cell.column_span > join
&& !contains_the_join(cell, join)
}) {
continue;
}
let mut symbol = false;
let mut clean = true;
for cell in cells
.iter()
.filter(|cell| cell.column == join && !contains_the_join(cell, join))
{
let text = cell.text.trim();
if text.is_empty() {
continue;
}
if is_currency_symbol(text) {
symbol = true;
} else {
clean = false;
break;
}
}
if clean && symbol {
return Some(join);
}
}
None
}
fn join_currency_columns(cells: &mut Vec<OwnedCell>) {
while let Some(join) = find_currency_column(cells) {
let symbols: Vec<(usize, String, bool)> = cells
.iter()
.filter(|cell| cell.column == join)
.filter_map(|cell| {
let text = cell.text.trim();
if is_currency_symbol(text) {
Some((cell.row, text.to_string(), cell.header))
} else {
None
}
})
.collect();
for (row, symbol, header) in symbols {
match cells
.iter_mut()
.find(|cell| cell.row == row && cell.column == join + 1)
{
Some(right) => {
let text = right.text.trim();
right.text = if text.is_empty() {
symbol
} else {
format!("{symbol} {text}")
};
}
None => cells.push(OwnedCell {
row,
column: join + 1,
row_span: 1,
column_span: 1,
header,
text: symbol,
}),
}
}
cells.retain(|cell| cell.column != join || contains_the_join(cell, join));
for cell in cells
.iter_mut()
.filter(|cell| contains_the_join(cell, join))
{
cell.column_span -= 1;
}
for cell in cells.iter_mut().filter(|cell| cell.column > join) {
cell.column -= 1;
}
}
}
pub(crate) fn is_figure(text: &str) -> bool {
let t = text.trim();
if t.eq_ignore_ascii_case("n/a") || t.eq_ignore_ascii_case("na") {
return true;
}
if !t.is_empty() && t.chars().all(|c| matches!(c, '-' | '–' | '—')) {
return true;
}
let core = t
.trim_start_matches(|c: char| c.is_whitespace() || "$€£(-–—+".contains(c))
.trim_end_matches(|c: char| c.is_whitespace() || "%*)".contains(c));
!core.is_empty()
&& core.chars().any(|c| c.is_ascii_digit())
&& core
.chars()
.all(|c| c.is_ascii_digit() || c == '.' || c == ',')
}
fn is_stacked_figure_cell(text: &str) -> bool {
let mut substance = false;
for token in text.split_whitespace() {
if is_figure(token) {
substance = true;
continue;
}
let rest = token.trim_start_matches(['$', '€', '£', '¥', '¢', '₹']);
if rest.len() < token.len()
&& !rest.is_empty()
&& rest.chars().all(|c| matches!(c, '-' | '–' | '—'))
{
substance = true;
continue;
}
if !token.is_empty() && token.chars().all(|c| "$€£¥¢₹".contains(c)) {
continue;
}
return false;
}
substance
}
fn markdown_table(cells: &[CellAt], rows: usize, columns: usize) -> String {
let mut out = String::new();
for r in 0..rows {
out.push('|');
for c in 0..columns {
let text = cells
.iter()
.find(|cell| cell.row == r && cell.column == c)
.map(|cell| cell.text)
.unwrap_or_default();
out.push(' ');
out.push_str(&escape_cell(&cell_text(text, columns)));
out.push_str(" |");
}
out.push('\n');
if r == 0 {
out.push('|');
for _ in 0..columns {
out.push_str(" --- |");
}
out.push('\n');
}
}
out
}
fn html_table(cells: &[CellAt], rows: usize, columns: usize) -> String {
let mut taken = vec![vec![false; columns]; rows];
let mut out = String::from("<table>\n");
for r in 0..rows {
out.push_str("<tr>");
let mut c = 0usize;
while c < columns {
if taken[r][c] {
c += 1;
continue;
}
let Some(cell) = cells.iter().find(|cell| cell.row == r && cell.column == c) else {
out.push_str("<td></td>");
c += 1;
continue;
};
for row in taken.iter_mut().take((r + cell.row_span).min(rows)).skip(r) {
for held in row
.iter_mut()
.take((c + cell.column_span).min(columns))
.skip(c)
{
*held = true;
}
}
let tag = if cell.header { "th" } else { "td" };
out.push('<');
out.push_str(tag);
if cell.column_span > 1 {
out.push_str(&format!(" colspan=\"{}\"", cell.column_span));
}
if cell.row_span > 1 {
out.push_str(&format!(" rowspan=\"{}\"", cell.row_span));
}
out.push('>');
out.push_str(&escape_html(&cell_text(cell.text, columns)));
out.push_str(&format!("</{tag}>"));
c += cell.column_span.max(1);
}
out.push_str("</tr>\n");
}
out.push_str("</table>");
out
}
fn cell_text(text: &str, columns: usize) -> String {
if columns >= 3 {
without_leaders(text)
} else {
text.to_string()
}
}
fn without_leaders(text: &str) -> String {
let chars: Vec<char> = text.chars().collect();
let mut out = String::with_capacity(text.len());
let mut i = 0usize;
while i < chars.len() {
if chars[i] == '.' {
let mut j = i;
let mut dots = 0usize;
while j < chars.len() && chars[j] == '.' {
dots += 1;
j += 1;
if j < chars.len() && chars[j].is_whitespace() && chars.get(j + 1) == Some(&'.') {
j += 1;
}
}
if dots >= 4 {
out.push(' ');
i = j;
continue;
}
}
out.push(chars[i]);
i += 1;
}
out.split_whitespace().collect::<Vec<&str>>().join(" ")
}
fn escape_cell(text: &str) -> String {
let mut out = String::with_capacity(text.len());
for c in text.chars() {
match c {
'|' => out.push_str("\\|"),
'\n' | '\r' => out.push(' '),
_ => out.push(c),
}
}
out
}
fn escape_html(text: &str) -> String {
let mut out = String::with_capacity(text.len());
for c in text.chars() {
match c {
'&' => out.push_str("&"),
'<' => out.push_str("<"),
'>' => out.push_str(">"),
_ => out.push(c),
}
}
out
}
const ORDINAL_MAX_DIGITS: usize = 9;
fn opens_a_marker(line: &str, marker: char) -> bool {
let run = if marker == '#' {
line.chars().take_while(|c| *c == '#').count().min(6)
} else {
marker.len_utf8()
};
let rest = &line[run..];
if matches!(rest.chars().next(), None | Some(' ') | Some('\t')) {
return true;
}
matches!(marker, '-' | '*')
&& line.chars().filter(|c| !c.is_whitespace()).count() >= 3
&& line.chars().all(|c| c == marker || c.is_whitespace())
}
fn escape_leading_marker(text: &str) -> String {
let trimmed = text.trim_start();
let indent = &text[..text.len() - trimmed.len()];
for fence in ['`', '~'] {
if trimmed.starts_with([fence, fence, fence].iter().collect::<String>().as_str()) {
return format!("{indent}\\{trimmed}");
}
}
let mut chars = trimmed.chars();
let escaped = match chars.next() {
Some(c @ ('>' | '|' | '`' | '~' | '<')) => format!("\\{c}{}", chars.as_str()),
Some(c @ ('#' | '-' | '*' | '+')) if opens_a_marker(trimmed, c) => {
format!("\\{c}{}", chars.as_str())
}
Some(c) if c.is_ascii_digit() => {
let rest = chars.as_str();
let digits = trimmed.chars().take_while(|c| c.is_ascii_digit()).count();
let after = trimmed[digits..].chars().next();
let delimited = matches!(after, Some('.') | Some(')'))
&& opens_a_marker(&trimmed[digits..], after.unwrap_or('.'));
if digits <= ORDINAL_MAX_DIGITS && delimited {
format!("{}\\{}", &trimmed[..digits], &trimmed[digits..])
} else {
format!("{c}{rest}")
}
}
Some(c) => format!("{c}{}", chars.as_str()),
None => String::new(),
};
format!("{indent}{escaped}")
}
fn page_marker(page: usize) -> String {
format!("<!-- page {page} -->")
}
fn note_page(
out: &mut String,
open: &mut Vec<(usize, usize)>,
last: &mut Option<usize>,
page: usize,
separators: PageSeparators,
) {
if *last == Some(page) {
return;
}
if last.is_some() && !out.is_empty() && separators == PageSeparators::Comment {
out.push_str("\n\n");
out.push_str(&page_marker(page));
open.clear();
}
*last = Some(page);
}
fn is_right_to_left(c: char) -> bool {
matches!(c as u32, 0x0590..=0x08FF | 0xFB1D..=0xFDFF | 0xFE70..=0xFEFF)
}
fn whole_at(text: &str, run: &str, from: usize) -> Option<usize> {
let mut at = from;
while let Some(found) = text[at..].find(run) {
let start = at + found;
let end = start + run.len();
let before = text[..start].chars().next_back();
let after = text[end..].chars().next();
if !before.is_some_and(char::is_alphanumeric) && !after.is_some_and(char::is_alphanumeric) {
return Some(start);
}
at = start + run.chars().next().map_or(1, char::len_utf8);
}
None
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Set {
Bold,
Italic,
BoldItalic,
Sup,
Sub,
}
impl Set {
fn marks(self) -> (&'static str, &'static str) {
match self {
Set::Bold => ("**", "**"),
Set::Italic => ("*", "*"),
Set::BoldItalic => ("***", "***"),
Set::Sup => ("<sup>", "</sup>"),
Set::Sub => ("<sub>", "</sub>"),
}
}
}
type CodeSpan = (usize, usize);
type StyleMark = (usize, usize, Option<Set>, bool);
fn body_of(block: &Block, text: &str, rules: &[HorizontalRule], code: bool) -> String {
let (spans, styles) = rich_marks(block, text, rules, code);
let rich = emit_rich(text, &styles, &spans);
if spans.first().is_some_and(|(start, _)| *start == 0) {
return rich;
}
let escaped = escape_leading_marker(text);
if escaped != text {
escaped
} else {
escape_leading_marker(&rich)
}
}
fn rich_text(block: &Block, text: &str, rules: &[HorizontalRule], code: bool) -> String {
let (spans, styles) = rich_marks(block, text, rules, code);
emit_rich(text, &styles, &spans)
}
fn rich_marks(
block: &Block,
text: &str,
rules: &[HorizontalRule],
code: bool,
) -> (Vec<CodeSpan>, Vec<StyleMark>) {
let spans = if code && code_here(block, text) {
code_ranges(block, text)
} else {
Vec::new()
};
let mut styles = styled_ranges(block, text, rules);
styles.retain(|(start, end, _, _)| !spans.iter().any(|(cs, ce)| *start < *ce && *cs < *end));
(spans, styles)
}
fn code_here(block: &Block, text: &str) -> bool {
matches!(
block.kind,
BlockKind::Paragraph | BlockKind::ListItem { .. }
) && !mostly_digits(text)
&& !text.chars().any(is_right_to_left)
}
fn mostly_digits(text: &str) -> bool {
let mut total = 0usize;
let mut figures = 0usize;
for c in text.chars() {
if c.is_whitespace() {
continue;
}
total += 1;
if c.is_ascii_digit() || c.is_ascii_punctuation() {
figures += 1;
}
}
total > 0 && 2 * figures > total
}
fn code_ranges(block: &Block, text: &str) -> Vec<CodeSpan> {
let mut ranges: Vec<CodeSpan> = Vec::new();
let mut cursor = 0usize;
for span in &block.spans {
let run = span.text.trim();
if run.is_empty() {
continue;
}
let Some(found) = text[cursor..].find(run) else {
continue;
};
let start = cursor + found;
let end = start + run.len();
cursor = end;
if span.is_monospace != Some(true) {
continue;
}
match ranges.last_mut() {
Some(last) if text[last.1..start].trim().is_empty() => last.1 = end,
_ => ranges.push((start, end)),
}
}
if let Some(last) = ranges.last() {
if last.1 == text.len() && text.ends_with('-') {
ranges.pop();
}
}
ranges
}
fn longest_backtick_run(text: &str) -> usize {
let mut best = 0usize;
let mut run = 0usize;
for c in text.chars() {
if c == '`' {
run += 1;
best = best.max(run);
} else {
run = 0;
}
}
best
}
fn inline_code(content: &str) -> String {
let fence = "`".repeat(longest_backtick_run(content) + 1);
if content.starts_with('`') || content.ends_with('`') {
format!("{fence} {content} {fence}")
} else {
format!("{fence}{content}{fence}")
}
}
fn fenced_lines(block: &Block) -> Option<Vec<String>> {
if !matches!(block.kind, BlockKind::Paragraph) {
return None;
}
if block.text.chars().any(is_right_to_left) || mostly_digits(&block.text) {
return None;
}
if !block.spans.iter().any(|s| !s.text.trim().is_empty())
|| block
.spans
.iter()
.any(|s| !s.text.trim().is_empty() && s.is_monospace != Some(true))
{
return None;
}
let lines: Vec<String> = crate::text::lines_of_block(block)
.iter()
.map(|line| crate::text::line_text(block, line))
.map(|line| line.trim_end().to_string())
.filter(|line| !line.trim().is_empty())
.collect();
(lines.len() >= 2).then_some(lines)
}
fn fenced_block(lines: &[String]) -> String {
let fence = "`".repeat(
lines
.iter()
.map(|line| longest_backtick_run(line))
.max()
.unwrap_or(0)
.max(2)
+ 1,
);
format!("{fence}\n{}\n{fence}", lines.join("\n"))
}
fn underlined(span: &TextSpan, line: (f64, f64), rules: &[HorizontalRule]) -> bool {
let size = span.font_size;
if size <= 0.0 || span.width <= 0.0 {
return false;
}
let (left, right) = (span.x, span.x + span.width);
rules.iter().any(|r| {
r.y <= span.y + 0.05 * size
&& r.y >= span.y - 0.35 * size
&& r.left <= left + 0.1 * span.width
&& r.right >= right - 0.1 * span.width
&& r.left >= line.0 - size
&& r.right <= line.1 + size
})
}
fn line_of(block: &Block, span: &TextSpan) -> (f64, f64) {
block
.spans
.iter()
.filter(|s| (s.y - span.y).abs() <= 0.2 * span.font_size.max(1.0))
.fold((span.x, span.x + span.width), |(l, r), s| {
(l.min(s.x), r.max(s.x + s.width))
})
}
fn page_rules<'a>(rules: &'a [Vec<HorizontalRule>], block: &Block) -> &'a [HorizontalRule] {
rules.get(block.page).map_or(&[], Vec::as_slice)
}
#[cfg(test)]
fn styled(block: &Block, text: &str, rules: &[HorizontalRule]) -> String {
emit_rich(text, &styled_ranges(block, text, rules), &[])
}
fn styled_ranges(block: &Block, text: &str, rules: &[HorizontalRule]) -> Vec<StyleMark> {
if text.chars().any(is_right_to_left) {
return Vec::new();
}
let mut ranges: Vec<StyleMark> = Vec::new();
let mut cursor = 0usize;
let mut previous: Option<&crate::text::TextSpan> = None;
for (index, span) in block.spans.iter().enumerate() {
let run = span.text.trim();
let before = previous;
previous = Some(span);
if run.is_empty() || run.contains(['*', '_', '<', '>', '`', '[', ']', '\\', '~']) {
continue;
}
let raised = before.and_then(|prev| {
let small = span.font_size > 0.0
&& prev.font_size > 0.0
&& span.font_size <= 0.85 * prev.font_size
&& run.chars().count() <= 4;
if !small {
return None;
}
let shift = span.y - prev.y;
if shift.abs() >= 0.8 * prev.font_size || span.x < prev.x {
return None;
}
if shift >= 0.2 * prev.font_size {
Some(Set::Sup)
} else if shift <= -0.1 * prev.font_size {
Some(Set::Sub)
} else {
None
}
});
let set = raised.or(match (span.is_bold, span.is_italic) {
(true, true) => Some(Set::BoldItalic),
(true, false) => Some(Set::Bold),
(false, true) => Some(Set::Italic),
(false, false) => None,
});
let place = if matches!(set, Some(Set::Sup | Set::Sub)) {
text[cursor..].find(run).map(|found| cursor + found)
} else {
let rest = &text[cursor..];
let skipped = rest
.find(|c: char| !c.is_whitespace())
.unwrap_or(rest.len());
let at = cursor + skipped;
let joined = text[at..].starts_with(run)
&& (before.is_some_and(|prev| crate::words_join::spans_join(prev, span))
|| block
.spans
.get(index + 1)
.is_some_and(|next| crate::words_join::spans_join(span, next)));
if joined {
Some(at)
} else {
whole_at(text, run, cursor)
}
};
let Some(start) = place else {
continue;
};
let end = start + run.len();
cursor = end;
let under = !rules.is_empty() && underlined(span, line_of(block, span), rules);
if set.is_some() || under {
match ranges.last_mut() {
Some(last)
if last.2 == set
&& last.3 == under
&& text[last.1..start].trim().is_empty() =>
{
last.1 = end
}
_ => ranges.push((start, end, set, under)),
}
}
}
if let Some(last) = ranges.last() {
if last.1 == text.len() && text.ends_with('-') {
ranges.pop();
}
}
ranges
}
fn emit_rich(text: &str, styles: &[StyleMark], code: &[CodeSpan]) -> String {
if styles.is_empty() && code.is_empty() {
return text.to_string();
}
let mut out = String::with_capacity(text.len() + styles.len() * 6 + code.len() * 2);
let mut at = 0usize;
let mut styles = styles;
let mut code = code;
loop {
let take_style = match (styles.first(), code.first()) {
(Some((s, _, _, _)), Some((c, _))) => *s < *c,
(Some(_), None) => true,
(None, Some(_)) => false,
(None, None) => break,
};
if take_style {
let (start, end, set, under) = styles[0];
styles = &styles[1..];
let (open, close) = set.map_or(("", ""), Set::marks);
out.push_str(&text[at..start]);
out.push_str(open);
if under {
out.push_str("<u>");
}
out.push_str(&text[start..end]);
if under {
out.push_str("</u>");
}
out.push_str(close);
at = end;
} else {
let (start, end) = code[0];
code = &code[1..];
out.push_str(&text[at..start]);
out.push_str(&inline_code(&text[start..end]));
at = end;
}
}
out.push_str(&text[at..]);
out
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct MarkdownOptions {
pub include_headers: bool,
pub include_footers: bool,
pub page_separators: PageSeparators,
pub code: bool,
pub include_figures: bool,
pub table_render_mode: TableRenderMode,
}
impl Default for MarkdownOptions {
fn default() -> Self {
Self {
include_headers: false,
include_footers: false,
page_separators: PageSeparators::Comment,
code: default_code(),
include_figures: true,
table_render_mode: TableRenderMode::Auto,
}
}
}
impl From<&ExtractOptions> for MarkdownOptions {
fn from(options: &ExtractOptions) -> Self {
Self {
include_headers: options.include_headers,
include_footers: options.include_footers,
page_separators: options.page_separators,
code: options.code,
include_figures: options.include_figures,
table_render_mode: options.table_render_mode,
}
}
}
pub fn to_markdown(blocks: &[Block]) -> String {
to_markdown_with_options(blocks, &MarkdownOptions::default())
}
pub fn to_markdown_with_code(blocks: &[Block], code: bool) -> String {
let options = MarkdownOptions {
code,
..MarkdownOptions::default()
};
to_markdown_with_options(blocks, &options)
}
fn default_code() -> bool {
crate::blocks::ExtractOptions::default().code
}
#[doc(hidden)]
pub fn to_markdown_with_rules(blocks: &[Block], rules: &[Vec<HorizontalRule>]) -> String {
render_markdown(
blocks,
rules,
crate::learned::SWITCHES,
&MarkdownOptions::default(),
)
}
#[doc(hidden)]
pub fn to_markdown_with_switches(blocks: &[Block], switches: crate::learned::Switches) -> String {
render_markdown(blocks, &[], switches, &MarkdownOptions::default())
}
pub fn to_markdown_with_options(blocks: &[Block], options: &MarkdownOptions) -> String {
render_markdown(blocks, &[], crate::learned::SWITCHES, options)
}
fn document_is_set_in_monospace(blocks: &[Block]) -> bool {
blocks
.iter()
.flat_map(|block| block.spans.iter())
.all(|span| span.is_monospace == Some(true))
}
fn render_markdown(
blocks: &[Block],
rules: &[Vec<HorizontalRule>],
switches: crate::learned::Switches,
options: &MarkdownOptions,
) -> String {
let code = options.code;
let fence_code = code && !document_is_set_in_monospace(blocks);
let mut out = String::new();
let mut open: Vec<(usize, usize)> = Vec::new();
let mut after_paragraph = false;
let mut last_page: Option<usize> = None;
let mut figure_ordinals: std::collections::BTreeMap<usize, usize> =
std::collections::BTreeMap::new();
let mut at = 0usize;
while at < blocks.len() {
if let Some(end) = table_run_across(blocks, at) {
let page = blocks[at].page;
let written = table(&blocks[at..end], switches, options.table_render_mode);
at = end;
if written.is_empty() {
continue;
}
note_page(
&mut out,
&mut open,
&mut last_page,
page,
options.page_separators,
);
open.clear();
after_paragraph = false;
if !out.is_empty() {
out.push_str("\n\n");
}
out.push_str(written.trim_end_matches('\n'));
continue;
}
let block = &blocks[at];
at += 1;
let wanted = match block.kind {
BlockKind::Header => options.include_headers,
BlockKind::Footer => options.include_footers,
BlockKind::Figure => options.include_figures,
_ => true,
};
if !wanted {
continue;
}
if matches!(block.kind, BlockKind::Figure) {
let ordinal = figure_ordinals.get(&block.page).copied().unwrap_or(0);
let target = block.text.trim();
let marker;
let target = if target.is_empty() {
marker = crate::markdown_images::figure_name(block.page, ordinal);
marker.as_str()
} else {
target
};
figure_ordinals.insert(block.page, ordinal + 1);
note_page(
&mut out,
&mut open,
&mut last_page,
block.page,
options.page_separators,
);
if !out.is_empty() {
out.push_str("\n\n");
}
after_paragraph = false;
open.clear();
out.push_str("![](");
out.push_str(target);
out.push(')');
continue;
}
let is_furniture = matches!(block.kind, BlockKind::Header | BlockKind::Footer);
let text = block.text.trim();
if text.is_empty() {
continue;
}
let fence = if fence_code {
fenced_lines(block)
} else {
None
};
if block.continues_previous
&& after_paragraph
&& fence.is_none()
&& !out.is_empty()
&& !is_furniture
{
if crate::text::is_hyphen_wrap_candidate(&out, text) {
out.pop();
} else {
out.push(' ');
}
out.push_str(&rich_text(block, text, page_rules(rules, block), code));
continue;
}
note_page(
&mut out,
&mut open,
&mut last_page,
block.page,
options.page_separators,
);
if !out.is_empty() {
let tight = !open.is_empty() && matches!(block.kind, BlockKind::ListItem { .. });
out.push_str(if tight { "\n" } else { "\n\n" });
}
after_paragraph = matches!(block.kind, BlockKind::Paragraph);
match block.kind {
BlockKind::Heading { level } => {
open.clear();
let level = level.clamp(1, 6) as usize;
out.push_str(&"#".repeat(level));
out.push(' ');
out.push_str(text);
}
BlockKind::ListItem { level, ordinal } => {
let depth = (level as usize).min(open.len());
open.truncate(depth);
let indent = open.last().map_or(0, |(column, marker)| column + marker);
let marker = match ordinal {
Some(number) => format!("{number}. "),
None => "- ".to_string(),
};
out.push_str(&" ".repeat(indent));
out.push_str(&marker);
open.push((indent, marker.chars().count()));
out.push_str(&body_of(block, text, page_rules(rules, block), code));
}
BlockKind::Caption => {
open.clear();
out.push('*');
out.push_str(body_of(block, text, page_rules(rules, block), code).trim());
out.push('*');
}
_ => {
open.clear();
if let Some(lines) = &fence {
after_paragraph = false;
out.push_str(&fenced_block(lines));
} else {
out.push_str(&body_of(block, text, page_rules(rules, block), code));
}
}
}
}
if !out.is_empty() {
out.push('\n');
}
out
}
#[cfg(test)]
mod tests {
#[test]
fn a_cell_loses_its_dot_leaders_but_keeps_an_ellipsis_and_decimals() {
use super::without_leaders;
assert_eq!(
without_leaders("United States . . . . ........ $11,772"),
"United States $11,772"
);
assert_eq!(
without_leaders("2005 Plan ......................"),
"2005 Plan"
);
assert_eq!(without_leaders("Net sales"), "Net sales");
assert_eq!(
without_leaders("to be determined..."),
"to be determined..."
);
assert_eq!(without_leaders("1.25 . 3.5"), "1.25 . 3.5");
assert_eq!(without_leaders("....."), "");
assert_eq!(
super::cell_text("Dell . . . . . 16", 2),
"Dell . . . . . 16"
);
assert_eq!(super::cell_text("Revenues ...... 954", 3), "Revenues 954");
}
fn cell_at(row: usize, column: usize, text: &str) -> super::CellAt<'_> {
super::CellAt {
row,
column,
row_span: 1,
column_span: 1,
header: false,
text,
}
}
fn off() -> crate::learned::Switches {
crate::learned::Switches {
markdown_header_block_figures: false,
markdown_header_block_rowspan_crossing: false,
markdown_header_block_word_table: false,
markdown_header_block_mixed_body: false,
markdown_headerless_html: false,
markdown_header_block_stub_years: false,
..crate::learned::SWITCHES
}
}
#[test]
fn a_two_line_header_is_marked_and_a_table_of_words_is_not() {
use super::header_block;
let two = [
cell_at(0, 0, ""),
cell_at(0, 1, "Issue"),
cell_at(1, 0, "Class"),
cell_at(1, 1, "Age"),
cell_at(2, 0, "MN"),
cell_at(2, 1, "35"),
cell_at(3, 0, "MN"),
cell_at(3, 1, "$1,500.00"),
];
assert_eq!(header_block(&two, off()), 2);
let words = [
cell_at(0, 0, "Name"),
cell_at(0, 1, "Role"),
cell_at(1, 0, "Ann"),
cell_at(1, 1, "Chair"),
];
assert_eq!(header_block(&words, off()), 0);
let figures_first = [
cell_at(0, 0, "Total"),
cell_at(0, 1, "12"),
cell_at(1, 0, "Net"),
cell_at(1, 1, "(3)"),
];
assert_eq!(header_block(&figures_first, off()), 0);
assert!(super::is_figure("(1,234.5)") && super::is_figure("12%") && super::is_figure("—"));
assert!(!super::is_figure("Fair Value") && !super::is_figure("December 31,"));
}
#[test]
fn stacked_figure_cells_count_as_figures_under_the_switch() {
for text in [
"86,090 (4,305)",
"12,000 (1,000)",
"$—",
"$ —",
"$ 990",
"€ -",
"— —",
"(4,305)",
] {
assert!(
super::is_stacked_figure_cell(text),
"{text:?} reads as figures"
);
}
for text in [
"$",
"",
"December 31,",
"Fair Value",
"Contributions Distributions",
"86,090 and more",
] {
assert!(
!super::is_stacked_figure_cell(text),
"{text:?} reads as text"
);
}
}
#[test]
fn fused_figure_rows_end_the_header_block_under_the_switch() {
use super::header_block;
let cells = [
cell_at(0, 0, ""),
cell_at(0, 1, "Shares"),
cell_at(0, 2, "Amount"),
cell_at(1, 0, "Balance at December 31, 2001"),
cell_at(1, 1, "— —"),
cell_at(1, 2, "— —"),
cell_at(2, 0, "Contributions Distributions"),
cell_at(2, 1, "86,090 (4,305)"),
cell_at(2, 2, "12,000 (1,000)"),
cell_at(3, 0, "Net loss"),
cell_at(3, 1, "(5,000)"),
cell_at(3, 2, "(6,000)"),
];
assert_eq!(header_block(&cells, off()), 3);
assert_eq!(
header_block(
&cells,
crate::learned::Switches {
markdown_header_block_figures: true,
..off()
}
),
1
);
}
#[test]
fn a_row_of_years_does_not_end_the_header() {
use super::header_block;
fn spanned(row: usize, column: usize, column_span: usize, text: &str) -> super::CellAt<'_> {
super::CellAt {
row,
column,
row_span: 1,
column_span,
header: false,
text,
}
}
let years = [
cell_at(0, 0, ""),
spanned(0, 1, 3, "Year Ended December 31,"),
cell_at(1, 0, ""),
cell_at(1, 1, "2012"),
cell_at(1, 2, "2011"),
cell_at(1, 3, "2010"),
cell_at(2, 0, "Revenue"),
cell_at(2, 1, "1"),
cell_at(2, 2, "2"),
cell_at(2, 3, "3"),
];
assert_eq!(header_block(&years, off()), 2);
let footnote = [
cell_at(0, 0, ""),
spanned(0, 1, 2, "Year Ended December 31,"),
cell_at(1, 0, ""),
cell_at(1, 1, "2012 (1)"),
cell_at(1, 2, "2011"),
cell_at(2, 0, "Revenue"),
cell_at(2, 1, "1"),
cell_at(2, 2, "2"),
];
assert_eq!(header_block(&footnote, off()), 2);
let not_all_years = [
cell_at(0, 0, "Item"),
cell_at(0, 1, "Value"),
cell_at(0, 2, "Share"),
cell_at(1, 0, "Total"),
cell_at(1, 1, "2012"),
cell_at(1, 2, "35.0%"),
cell_at(2, 0, "Net"),
cell_at(2, 1, "3"),
cell_at(2, 2, "4%"),
];
assert_eq!(header_block(¬_all_years, off()), 1);
}
#[test]
fn a_rowspan_clear_of_the_boundary_keeps_the_header_under_the_switch() {
use super::header_block;
fn tall(row: usize, column: usize, row_span: usize, text: &str) -> super::CellAt<'_> {
super::CellAt {
row,
column,
row_span,
column_span: 1,
header: false,
text,
}
}
let on = || crate::learned::Switches {
markdown_header_block_rowspan_crossing: true,
..off()
};
let grouped = [
cell_at(0, 0, "Region"),
cell_at(0, 1, "Sales"),
cell_at(1, 0, "North"),
cell_at(1, 1, "Q1"),
cell_at(2, 0, "East"),
cell_at(2, 1, "12"),
tall(3, 0, 2, "West"),
cell_at(3, 1, "9"),
cell_at(4, 1, "7"),
];
assert_eq!(header_block(&grouped, off()), 0);
assert_eq!(header_block(&grouped, on()), 2);
let stub = [
tall(0, 0, 2, "Test"),
cell_at(0, 1, "Low"),
cell_at(1, 1, "High"),
cell_at(2, 0, "A"),
cell_at(2, 1, "3"),
];
assert_eq!(header_block(&stub, off()), 0);
assert_eq!(header_block(&stub, on()), 2);
let straddling = [
cell_at(0, 0, "Region"),
cell_at(0, 1, "Sales"),
tall(1, 0, 2, "North"),
cell_at(1, 1, "Q1"),
cell_at(2, 1, "12"),
];
assert_eq!(header_block(&straddling, off()), 0);
assert_eq!(header_block(&straddling, on()), 0);
}
#[test]
fn a_word_table_heads_its_first_row_under_the_switch() {
use super::header_block;
let on = || crate::learned::Switches {
markdown_header_block_word_table: true,
..off()
};
let words = [
cell_at(0, 0, ""),
cell_at(0, 1, "Enablers"),
cell_at(0, 2, "Barriers"),
cell_at(1, 0, "Hiring"),
cell_at(1, 1, "Referrals"),
cell_at(1, 2, "Costs"),
cell_at(2, 0, "Keeping"),
cell_at(2, 1, "Pay"),
cell_at(2, 2, "Hours"),
];
assert_eq!(header_block(&words, off()), 0);
assert_eq!(header_block(&words, on()), 1);
let single = [cell_at(0, 0, "Name"), cell_at(0, 1, "Role")];
assert_eq!(header_block(&single, on()), 0);
let blank = [
cell_at(0, 0, ""),
cell_at(0, 1, " "),
cell_at(1, 0, "Ann"),
cell_at(1, 1, "Chair"),
];
assert_eq!(header_block(&blank, on()), 0);
}
#[test]
fn a_mixed_row_ends_the_header_under_the_switch() {
use super::header_block;
let on = || crate::learned::Switches {
markdown_header_block_mixed_body: true,
..off()
};
let over = [
cell_at(0, 0, "Region"),
cell_at(0, 1, "North"),
cell_at(0, 2, "South"),
cell_at(0, 3, "Share"),
cell_at(1, 0, "Age (years)"),
cell_at(1, 1, "North"),
cell_at(1, 2, "South"),
cell_at(1, 3, "0.73"),
cell_at(2, 0, "Female"),
cell_at(2, 1, "26"),
cell_at(2, 2, "20"),
cell_at(2, 3, "0.22"),
];
assert_eq!(header_block(&over, off()), 2);
assert_eq!(header_block(&over, on()), 1);
let headerless = [
cell_at(0, 0, "Genome size"),
cell_at(0, 1, "North"),
cell_at(0, 2, "South"),
cell_at(0, 3, "14.6"),
cell_at(1, 0, "Assembled"),
cell_at(1, 1, "13"),
cell_at(1, 2, "12"),
cell_at(1, 3, "11"),
];
assert_eq!(header_block(&headerless, off()), 1);
assert_eq!(header_block(&headerless, on()), 0);
let stubbed_years = [
cell_at(0, 0, "Payments due by period"),
cell_at(0, 1, ""),
cell_at(0, 2, ""),
cell_at(1, 0, "Contractual Obligation"),
cell_at(1, 1, "2008"),
cell_at(1, 2, "2009 and 2010"),
cell_at(2, 0, "Long-term debt"),
cell_at(2, 1, "$ 629"),
cell_at(2, 2, "$ 2,334"),
];
assert_eq!(header_block(&stubbed_years, off()), 2);
assert_eq!(header_block(&stubbed_years, on()), 2);
}
#[test]
fn a_stubbed_year_row_heads_under_the_switch() {
use super::header_block;
let on = || crate::learned::Switches {
markdown_header_block_stub_years: true,
..off()
};
let stubbed = [
cell_at(0, 0, "($ in millions)"),
cell_at(0, 1, "2010"),
cell_at(0, 2, "2009"),
cell_at(1, 0, "Sales"),
cell_at(1, 1, "$ 914"),
cell_at(1, 2, "$ 850"),
];
assert_eq!(header_block(&stubbed, off()), 0);
assert_eq!(header_block(&stubbed, on()), 1);
let single = [
cell_at(0, 0, "As of"),
cell_at(0, 1, "December"),
cell_at(0, 2, "2010"),
cell_at(1, 0, "Cash"),
cell_at(1, 1, "1"),
cell_at(1, 2, "2"),
];
assert_eq!(header_block(&single, on()), 1);
let plain = [
cell_at(0, 0, "2010"),
cell_at(0, 1, "2009"),
cell_at(1, 0, "Sales"),
cell_at(1, 1, "914"),
];
assert_eq!(header_block(&plain, off()), 1);
assert_eq!(header_block(&plain, on()), 1);
}
use super::*;
fn block(kind: BlockKind, text: &str) -> Block {
Block {
kind,
page: 0,
bbox: [0.0; 4],
text: text.to_string(),
spans: Vec::new(),
index: 0,
continues_previous: false,
confidence: None,
grid: None,
table_origin: TableOrigin::Other,
trace_learned: None,
}
}
#[test]
fn a_caption_is_an_italic_paragraph_of_its_own() {
let md = to_markdown(&[
block(BlockKind::Paragraph, "The plant runs day and night."),
block(BlockKind::Caption, "Figure 1. The plant at night."),
]);
assert_eq!(
md,
"The plant runs day and night.\n\n*Figure 1. The plant at night.*\n"
);
}
fn run(text: &str, y: f64, size: f64, bold: bool, italic: bool) -> crate::text::TextSpan {
crate::text::TextSpan {
text: text.to_string(),
y,
font_size: size,
is_bold: bold,
is_italic: italic,
..Default::default()
}
}
fn set_block(text: &str, spans: Vec<crate::text::TextSpan>) -> Block {
Block {
spans,
..block(BlockKind::Paragraph, text)
}
}
fn placed(text: &str, x: f64, width: f64, bold: bool) -> crate::text::TextSpan {
crate::text::TextSpan {
x,
width,
..run(text, 700.0, 10.0, bold, false)
}
}
fn rule(left: f64, right: f64, y: f64) -> HorizontalRule {
HorizontalRule { left, right, y }
}
#[test]
fn a_rule_under_a_run_underlines_it_and_a_border_does_not() {
let b = set_block(
"See the terms of service before signing.",
vec![
placed("See the", 100.0, 35.0, false),
placed("terms of service", 140.0, 80.0, true),
placed("before signing.", 225.0, 75.0, false),
],
);
assert_eq!(
styled(&b, &b.text, &[rule(140.0, 220.0, 698.5)]),
"See the **<u>terms of service</u>** before signing."
);
assert_eq!(
styled(&b, &b.text, &[rule(40.0, 560.0, 698.5)]),
"See the **terms of service** before signing."
);
assert_eq!(
styled(&b, &b.text, &[rule(140.0, 220.0, 688.0)]),
"See the **terms of service** before signing."
);
assert_eq!(
styled(&b, &b.text, &[rule(225.0, 300.0, 699.0)]),
"See the **terms of service** <u>before signing.</u>"
);
}
#[test]
fn bold_and_italic_runs_are_marked_from_the_spans() {
let b = set_block(
"Pay the net amount within thirty days of the invoice.",
vec![
run("Pay the", 700.0, 10.0, false, false),
run("net amount", 700.0, 10.0, true, false),
run("within", 700.0, 10.0, false, false),
run("thirty days", 700.0, 10.0, false, true),
run("of the invoice.", 700.0, 10.0, false, false),
],
);
assert_eq!(
to_markdown(&[b]),
"Pay the **net amount** within *thirty days* of the invoice.\n"
);
}
#[test]
fn sub_word_spans_the_rebuild_joined_are_marked_whole() {
let b = set_block(
"2025",
vec![
placed("2", 100.0, 3.0, true),
placed("0", 103.0, 3.0, true),
placed("2", 106.0, 3.0, true),
placed("5", 109.0, 3.0, true),
],
);
assert_eq!(styled(&b, &b.text, &[]), "**2025**");
}
#[test]
fn a_hyphen_off_its_line_is_still_half_a_word() {
let b = set_block(
"hyphen-ated",
vec![
placed("hyphen-", 100.0, 35.0, true),
crate::text::TextSpan {
x: 100.0,
width: 20.0,
..run("ated", 688.0, 10.0, false, false)
},
],
);
assert_eq!(styled(&b, &b.text, &[]), "hyphen-ated");
}
#[test]
fn neighbouring_runs_of_one_setting_are_one_mark() {
let b = set_block(
"Two bold words here.",
vec![
run("Two", 700.0, 10.0, true, false),
run("bold", 700.0, 10.0, true, false),
run("words here.", 700.0, 10.0, false, false),
],
);
assert_eq!(to_markdown(&[b]), "**Two bold** words here.\n");
}
#[test]
fn raised_and_lowered_runs_are_sup_and_sub() {
let b = set_block(
"Water is H2O and the area is 4 m2 here.",
vec![
run("Water is H", 700.0, 10.0, false, false),
run("2", 697.0, 7.0, false, false),
run("O and the area is 4 m", 700.0, 10.0, false, false),
run("2", 703.5, 7.0, false, false),
run("here.", 700.0, 10.0, false, false),
],
);
assert_eq!(
to_markdown(&[b]),
"Water is H<sub>2</sub>O and the area is 4 m<sup>2</sup> here.\n"
);
}
#[test]
fn a_line_that_opens_like_a_list_item_keeps_its_escape_and_no_style() {
let b = set_block(
"3. The parties agree.",
vec![
run("3.", 700.0, 10.0, true, false),
run("The parties agree.", 700.0, 10.0, false, false),
],
);
assert_eq!(to_markdown(&[b]), "3\\. The parties agree.\n");
}
#[test]
fn a_small_run_opening_the_next_line_is_not_a_subscript() {
let b = set_block(
"the end of a line and 12 more words",
vec![
run("the end of a line and", 700.0, 10.0, false, false),
run("12", 688.0, 8.0, false, false),
run("more words", 688.0, 10.0, false, false),
],
);
assert_eq!(to_markdown(&[b]), "the end of a line and 12 more words\n");
}
#[test]
fn a_bold_run_ending_in_a_page_break_hyphen_still_joins() {
let mut first = set_block(
"the word is hyphen-",
vec![
run("the word is", 700.0, 10.0, false, false),
run("hyphen-", 700.0, 10.0, true, false),
],
);
first.page = 0;
let mut second = block(BlockKind::Paragraph, "ated across the page.");
second.page = 1;
second.continues_previous = true;
assert_eq!(
to_markdown(&[first, second]),
"the word is hyphenated across the page.\n"
);
}
#[test]
fn right_to_left_text_is_left_unmarked() {
let b = set_block(
"שלום עולם",
vec![
run("עולם", 700.0, 10.0, true, false),
run("שלום", 700.0, 10.0, false, false),
],
);
assert_eq!(to_markdown(&[b]), "שלום עולם\n");
}
#[test]
fn a_run_that_holds_markup_or_cannot_be_found_is_left_alone() {
let b = set_block(
"Use a*b and the hyphen-ated word.",
vec![
run("Use a*b", 700.0, 10.0, true, false),
run("hyphen-", 700.0, 10.0, true, false),
],
);
assert_eq!(to_markdown(&[b]), "Use a*b and the hyphen-ated word.\n");
}
fn mono(text: &str, x: f64, y: f64) -> crate::text::TextSpan {
crate::text::TextSpan {
text: text.to_string(),
x,
y,
width: text.len().max(1) as f64 * 6.0,
font_size: 10.0,
is_monospace: Some(true),
..Default::default()
}
}
fn plain(text: &str, x: f64, y: f64) -> crate::text::TextSpan {
crate::text::TextSpan {
text: text.to_string(),
x,
y,
width: text.len().max(1) as f64 * 6.0,
font_size: 10.0,
..Default::default()
}
}
#[test]
fn a_monospace_word_in_a_sentence_is_inline_code() {
let b = set_block(
"Use foo here.",
vec![
plain("Use ", 100.0, 700.0),
mono("foo", 130.0, 700.0),
plain(" here.", 150.0, 700.0),
],
);
assert_eq!(to_markdown(&[b]), "Use `foo` here.\n");
}
#[test]
fn neighbouring_monospace_runs_are_one_span() {
let b = set_block(
"Run foo bar now.",
vec![
plain("Run ", 100.0, 700.0),
mono("foo", 130.0, 700.0),
mono("bar", 155.0, 700.0),
plain(" now.", 180.0, 700.0),
],
);
assert_eq!(to_markdown(&[b]), "Run `foo bar` now.\n");
}
#[test]
fn a_backtick_in_code_lengthens_the_fence() {
let b = set_block(
"Use a`b here.",
vec![
plain("Use ", 100.0, 700.0),
mono("a`b", 130.0, 700.0),
plain(" here.", 155.0, 700.0),
],
);
assert_eq!(to_markdown(&[b]), "Use ``a`b`` here.\n");
}
#[test]
fn code_wins_over_bold_on_the_same_run() {
let mut code = mono("foo", 130.0, 700.0);
code.is_bold = true;
let b = set_block(
"Use foo here.",
vec![
plain("Use ", 100.0, 700.0),
code,
plain(" here.", 150.0, 700.0),
],
);
assert_eq!(to_markdown(&[b]), "Use `foo` here.\n");
}
#[test]
fn code_at_the_opening_needs_no_escape() {
let b = set_block("# foo", vec![mono("# foo", 100.0, 700.0)]);
assert_eq!(to_markdown(&[b]), "`# foo`\n");
}
#[test]
fn a_list_item_reads_its_monospace_inline() {
let item = block(
BlockKind::ListItem {
level: 0,
ordinal: None,
},
"Run foo now.",
);
let item = Block {
spans: vec![
plain("Run ", 100.0, 700.0),
mono("foo", 130.0, 700.0),
plain(" now.", 150.0, 700.0),
],
..item
};
assert_eq!(to_markdown(&[item]), "- Run `foo` now.\n");
}
#[test]
fn an_all_monospace_paragraph_over_two_lines_reads_inline() {
let b = set_block(
"let x = 1; let y = 2;",
vec![
mono("let x = 1;", 100.0, 700.0),
mono("let y = 2;", 100.0, 688.0),
],
);
assert_eq!(to_markdown(&[b]), "`let x = 1; let y = 2;`\n");
}
#[test]
fn a_fence_inside_an_unfenced_block_stays_inline() {
let b = set_block(
"``` code",
vec![mono("```", 100.0, 700.0), mono("code", 100.0, 688.0)],
);
assert_eq!(
to_markdown(std::slice::from_ref(&b)),
"```` ``` code ````\n"
);
let prose = set_block("words", vec![plain("words", 100.0, 712.0)]);
assert_eq!(to_markdown(&[prose, b]), "words\n\n````\n```\ncode\n````\n");
}
#[test]
fn a_single_monospace_line_stays_inline() {
let b = set_block("print(x)", vec![mono("print(x)", 100.0, 700.0)]);
assert_eq!(to_markdown(&[b]), "`print(x)`\n");
}
#[test]
fn a_paragraph_only_partly_monospace_stays_inline() {
let b = set_block(
"code_x words here",
vec![
mono("code_x", 100.0, 700.0),
plain("words here", 100.0, 688.0),
],
);
assert_eq!(to_markdown(&[b]), "`code_x` words here\n");
}
#[test]
fn monospace_figures_are_not_code() {
let b = set_block(
"1,234 5,678 9,012",
vec![mono("1,234 5,678 9,012", 100.0, 700.0)],
);
assert_eq!(to_markdown(&[b]), "1,234 5,678 9,012\n");
let b = set_block(
"1,234 5,678",
vec![mono("1,234", 100.0, 700.0), mono("5,678", 100.0, 688.0)],
);
assert_eq!(to_markdown(&[b]), "1,234 5,678\n");
}
#[test]
fn tables_headings_and_captions_never_carry_code() {
let mut code = cell(1, 0, false, "a1");
code.spans = vec![mono("a1", 100.0, 700.0)];
let table = vec![
cell(0, 0, true, "Code"),
cell(0, 1, true, "Means"),
code,
cell(1, 1, false, "either"),
];
assert_eq!(
to_markdown(&table),
"| Code | Means |\n| --- | --- |\n| a1 | either |\n"
);
let heading = block(BlockKind::Heading { level: 1 }, "Setup foo.");
let heading = Block {
spans: vec![
plain("Setup ", 100.0, 700.0),
mono("foo", 150.0, 700.0),
plain(".", 170.0, 700.0),
],
..heading
};
assert_eq!(to_markdown(&[heading]), "# Setup foo.\n");
let caption = block(BlockKind::Caption, "Figure 1 shows foo.");
let caption = Block {
spans: vec![
plain("Figure 1 shows ", 100.0, 700.0),
mono("foo", 200.0, 700.0),
plain(".", 220.0, 700.0),
],
..caption
};
assert_eq!(to_markdown(&[caption]), "*Figure 1 shows foo.*\n");
}
#[test]
fn code_false_writes_todays_bytes() {
let b = set_block(
"Use foo here.",
vec![
plain("Use ", 100.0, 700.0),
mono("foo", 130.0, 700.0),
plain(" here.", 150.0, 700.0),
],
);
assert_eq!(
to_markdown_with_code(std::slice::from_ref(&b), false),
"Use foo here.\n"
);
let fenced = set_block(
"let x = 1; let y = 2;",
vec![
mono("let x = 1;", 100.0, 700.0),
mono("let y = 2;", 100.0, 688.0),
],
);
assert_eq!(
to_markdown_with_code(&[fenced], false),
"let x = 1; let y = 2;\n"
);
assert_eq!(to_markdown(&[b]), "Use `foo` here.\n");
}
#[test]
fn a_code_span_never_closes_after_a_page_break_hyphen() {
let mut first = set_block(
"the setter is config-",
vec![
plain("the setter is ", 100.0, 700.0),
mono("config-", 190.0, 700.0),
],
);
first.page = 0;
let mut second = block(BlockKind::Paragraph, "file on the next page.");
second.page = 1;
second.continues_previous = true;
assert_eq!(
to_markdown(&[first, second]),
"the setter is configfile on the next page.\n"
);
}
#[test]
fn heading_levels_become_one_to_three_hashes() {
let blocks = vec![
block(BlockKind::Heading { level: 1 }, "Scope"),
block(BlockKind::Paragraph, "This document describes."),
block(BlockKind::Heading { level: 2 }, "Terms"),
block(BlockKind::Heading { level: 3 }, "Party"),
];
assert_eq!(
to_markdown(&blocks),
"# Scope\n\nThis document describes.\n\n## Terms\n\n### Party\n"
);
}
#[test]
fn a_paragraph_that_opens_like_a_marker_is_escaped() {
let blocks = vec![
block(BlockKind::Paragraph, "# not a heading"),
block(BlockKind::Paragraph, "- not an item"),
block(BlockKind::Paragraph, "1. not an item"),
block(BlockKind::Paragraph, "1984 was a year"),
];
assert_eq!(
to_markdown(&blocks),
"\\# not a heading\n\n\\- not an item\n\n1\\. not an item\n\n1984 was a year\n"
);
}
#[test]
fn a_figure_at_the_head_of_a_line_is_not_escaped() {
let blocks = vec![
block(BlockKind::Paragraph, "-3,503 6,200"),
block(BlockKind::Paragraph, "0.1 0.01 0.001"),
block(BlockKind::Paragraph, "#4 of the series"),
block(BlockKind::Paragraph, "+3HnN is the rate"),
block(BlockKind::Paragraph, "2.94 0.98 0.88"),
block(BlockKind::Paragraph, "####### not a heading"),
];
assert_eq!(
to_markdown(&blocks),
"-3,503 6,200\n\n0.1 0.01 0.001\n\n#4 of the series\n\n\
+3HnN is the rate\n\n2.94 0.98 0.88\n\n####### not a heading\n"
);
}
#[test]
fn a_heading_that_opens_with_a_run_of_hashes_is_still_escaped() {
let blocks = vec![
block(BlockKind::Paragraph, "## Terms"),
block(BlockKind::Paragraph, "###### Sixth"),
block(BlockKind::Paragraph, "###"),
];
assert_eq!(
to_markdown(&blocks),
"\\## Terms\n\n\\###### Sixth\n\n\\###\n"
);
}
#[test]
fn a_rule_of_three_is_escaped_whatever_follows_it() {
let blocks = vec![
block(BlockKind::Paragraph, "---"),
block(BlockKind::Paragraph, "***"),
block(BlockKind::Paragraph, "-- an em rule"),
];
assert_eq!(to_markdown(&blocks), "\\---\n\n\\***\n\n-- an em rule\n");
}
#[test]
fn a_paragraph_that_opens_a_fence_cannot_swallow_the_document() {
let blocks = vec![
block(BlockKind::Paragraph, "```rust"),
block(BlockKind::Heading { level: 1 }, "Scope"),
block(BlockKind::Paragraph, "~~~ a tilde fence"),
block(BlockKind::Paragraph, "`inline code` is not a fence"),
];
let markdown = to_markdown(&blocks);
assert!(
markdown.starts_with("\\```rust"),
"the fence opens a block nothing closes: {markdown}"
);
assert!(
markdown.contains("\n# Scope\n"),
"the heading after the fence was swallowed: {markdown}"
);
assert!(markdown.contains("\\~~~ a tilde fence"), "{markdown}");
assert!(markdown.contains("\\`inline code`"), "{markdown}");
}
fn cell(row: u16, column: u16, header: bool, text: &str) -> Block {
block(
BlockKind::TableCell {
row,
column,
row_span: 1,
column_span: 1,
header,
},
text,
)
}
#[test]
fn a_headerless_rectangle_is_html_under_the_switch() {
let blocks = vec![
cell(0, 0, false, "Genome size"),
cell(0, 1, false, "14.6"),
cell(1, 0, false, "Assembled"),
cell(1, 1, false, "13"),
];
assert_eq!(
super::to_markdown_with_switches(&blocks, off()),
"| Genome size | 14.6 |\n| --- | --- |\n| Assembled | 13 |\n"
);
let on = crate::learned::Switches {
markdown_headerless_html: true,
..off()
};
let html = super::to_markdown_with_switches(&blocks, on);
assert!(
html.starts_with("<table>"),
"a headerless table is HTML, not pipes: {html:?}"
);
assert!(
!html.contains("<th"),
"no cell is promoted to a header: {html:?}"
);
}
#[test]
fn a_pipe_in_a_cell_does_not_open_a_column() {
let blocks = vec![
cell(0, 0, true, "Code"),
cell(0, 1, true, "Means"),
cell(1, 0, false, "a|b"),
cell(1, 1, false, "either"),
];
assert_eq!(
to_markdown(&blocks),
"| Code | Means |\n| --- | --- |\n| a\\|b | either |\n"
);
}
#[test]
fn a_cell_that_holds_markup_is_escaped_in_the_html_form() {
let mut blocks = vec![
block(
BlockKind::TableCell {
row: 0,
column: 0,
row_span: 1,
column_span: 2,
header: true,
},
"a < b & c",
),
cell(1, 0, false, "left"),
cell(1, 1, false, "right"),
];
blocks.push(cell(2, 0, false, "under"));
blocks.push(cell(2, 1, false, "it"));
assert_eq!(
to_markdown(&blocks),
"<table>\n\
<tr><th colspan=\"2\">a < b & c</th></tr>\n\
<tr><td>left</td><td>right</td></tr>\n\
<tr><td>under</td><td>it</td></tr>\n\
</table>\n"
);
}
#[test]
fn a_currency_column_joins_the_column_to_its_right() {
let joined = vec![
cell(0, 0, false, "Service cost"),
cell(0, 1, false, "$"),
cell(0, 2, false, "115"),
cell(1, 0, false, "Interest"),
cell(1, 1, false, ""),
cell(1, 2, false, "82"),
];
let markdown = to_markdown(&joined);
assert!(markdown.contains("$ 115"), "{markdown}");
assert!(markdown.starts_with("<table>"), "{markdown}");
assert!(!markdown.contains("<th"), "{markdown}");
assert!(!markdown.contains("<td>$</td>"), "{markdown}");
let not_currency = vec![
cell(0, 0, false, "A"),
cell(0, 1, false, "$"),
cell(0, 2, false, "1"),
cell(1, 0, false, "B"),
cell(1, 1, false, "12"),
cell(1, 2, false, "2"),
];
let markdown = to_markdown(¬_currency);
assert!(markdown.contains("<td>$</td>"), "{markdown}");
assert!(!markdown.contains("$ 1"), "{markdown}");
let spanning = vec![
cell(0, 0, false, "Service"),
cell(0, 1, false, "$"),
block(
BlockKind::TableCell {
row: 0,
column: 2,
row_span: 1,
column_span: 2,
header: false,
},
"115",
),
cell(1, 0, false, "Interest"),
cell(1, 1, false, ""),
cell(1, 2, false, "82"),
cell(1, 3, false, "83"),
];
let markdown = to_markdown(&spanning);
assert!(!markdown.contains("$ 115"), "{markdown}");
}
#[test]
fn a_header_span_over_the_join_shrinks_with_it_and_a_partial_span_blocks() {
fn spanned(row: u16, column: u16, column_span: u16, text: &str) -> Block {
block(
BlockKind::TableCell {
row,
column,
row_span: 1,
column_span,
header: false,
},
text,
)
}
let under_span = vec![
cell(0, 0, false, "Item"),
spanned(0, 1, 6, "For the years ended December 31"),
cell(1, 0, false, "Net sales"),
cell(1, 1, false, "$"),
cell(1, 2, false, "114,750"),
cell(1, 3, false, "$"),
cell(1, 4, false, "101,029"),
cell(1, 5, false, "$"),
cell(1, 6, false, "90,547"),
cell(2, 0, false, "Cost of sales"),
cell(2, 1, false, ""),
cell(2, 2, false, "86,133"),
cell(2, 3, false, ""),
cell(2, 4, false, "74,920"),
cell(2, 5, false, ""),
cell(2, 6, false, "66,610"),
];
let markdown = to_markdown(&under_span);
assert!(
markdown.contains("<th colspan=\"3\">For the years ended December 31</th>"),
"the span shrinks with the three columns it covers: {markdown}"
);
assert!(markdown.contains("$ 114,750"), "{markdown}");
assert!(
!markdown.contains(">$<"),
"no bare currency column survives: {markdown}"
);
let partial = vec![
spanned(0, 0, 2, "Sales"),
cell(0, 2, false, "Amount"),
cell(1, 0, false, "Net"),
cell(1, 1, false, "$"),
cell(1, 2, false, "1"),
];
let markdown = to_markdown(&partial);
assert!(
!markdown.contains("$ 1"),
"a partial span still blocks the join: {markdown}"
);
assert!(
markdown.contains(">$<"),
"the blocked currency column survives: {markdown}"
);
}
#[test]
fn a_second_table_starts_where_the_grid_starts_again() {
let blocks = vec![
cell(0, 0, false, "one"),
cell(0, 1, false, "two"),
cell(1, 0, false, "three"),
cell(1, 1, false, "four"),
cell(0, 0, false, "five"),
cell(0, 1, false, "six"),
cell(1, 0, false, "seven"),
cell(1, 1, false, "eight"),
];
let markdown = to_markdown(&blocks);
assert_eq!(markdown.matches("| --- | --- |").count(), 2, "{markdown}");
assert!(markdown.contains("| four |\n\n| five |"), "{markdown}");
}
#[test]
fn a_second_table_that_opens_with_a_hole_is_still_a_second_table() {
let blocks = vec![
cell(0, 0, false, "one"),
cell(0, 1, false, "two"),
cell(1, 0, false, "three"),
cell(1, 1, false, "four"),
cell(0, 1, false, "six"),
cell(1, 0, false, "seven"),
cell(1, 1, false, "eight"),
];
let markdown = to_markdown(&blocks);
assert_eq!(
markdown.matches("<table>").count()
+ markdown
.lines()
.filter(|line| line.starts_with("| ---"))
.count(),
2,
"{markdown}"
);
for word in ["one", "four", "six", "eight"] {
assert!(markdown.contains(word), "{word} was dropped: {markdown}");
}
}
fn on_page(kind: BlockKind, text: &str, page: usize) -> Block {
Block {
page,
..block(kind, text)
}
}
#[test]
fn a_paragraph_that_quotes_the_page_marker_is_not_one() {
let blocks = vec![
block(BlockKind::Paragraph, "<!-- page 7 -->"),
block(BlockKind::Paragraph, "<table> opens a block too"),
];
let markdown = to_markdown(&blocks);
assert_eq!(
markdown,
"\\<!-- page 7 -->\n\n\\<table> opens a block too\n"
);
assert!(
!markdown.lines().any(|line| line == "<!-- page 7 -->"),
"document text came out as a marker: {markdown}"
);
}
#[test]
fn a_page_change_is_marked_and_the_first_page_is_not() {
let blocks = vec![
on_page(BlockKind::Paragraph, "The first page ends here.", 0),
on_page(BlockKind::Paragraph, "The second opens here.", 1),
];
assert_eq!(
to_markdown(&blocks),
"The first page ends here.\n\n<!-- page 1 -->\n\nThe second opens here.\n"
);
}
#[test]
fn the_number_in_the_marker_is_the_block_s_own_page() {
let blocks = vec![
on_page(BlockKind::Paragraph, "Opening.", 4),
on_page(BlockKind::Paragraph, "Next.", 7),
];
assert!(to_markdown(&blocks).contains("<!-- page 7 -->"));
}
#[test]
fn a_paragraph_the_break_cut_in_two_is_not_cut_again_by_the_marker() {
let mut middle = on_page(BlockKind::Paragraph, "to its end.", 1);
middle.continues_previous = true;
let blocks = vec![
on_page(BlockKind::Paragraph, "The sentence runs on", 0),
middle,
on_page(BlockKind::Paragraph, "A new one.", 2),
];
assert_eq!(
to_markdown(&blocks),
"The sentence runs on to its end.\n\n<!-- page 2 -->\n\nA new one.\n"
);
}
#[test]
fn the_deferred_marker_stands_after_the_joined_paragraph() {
let mut middle = on_page(BlockKind::Paragraph, "to its end.", 1);
middle.continues_previous = true;
let blocks = vec![
on_page(BlockKind::Paragraph, "The sentence runs on", 0),
middle,
on_page(BlockKind::Heading { level: 1 }, "Finance", 1),
on_page(BlockKind::Paragraph, "Revenue was ahead of budget.", 1),
];
assert_eq!(
to_markdown(&blocks),
"The sentence runs on to its end.\n\n<!-- page 1 -->\n\n# Finance\n\n\
Revenue was ahead of budget.\n"
);
}
#[test]
fn the_marker_closes_the_list_it_stands_after() {
let blocks = vec![
on_page(
BlockKind::ListItem {
level: 0,
ordinal: None,
},
"first",
0,
),
on_page(
BlockKind::ListItem {
level: 1,
ordinal: None,
},
"under it",
1,
),
];
assert_eq!(
to_markdown(&blocks),
"- first\n\n<!-- page 1 -->\n\n- under it\n"
);
}
#[test]
fn the_running_head_does_not_mark_a_page() {
let blocks = vec![
on_page(BlockKind::Paragraph, "Body one.", 0),
on_page(BlockKind::Paragraph, "Body two.", 1),
on_page(BlockKind::Header, "Annual Report 2026", 0),
on_page(BlockKind::Header, "Annual Report 2026", 1),
];
assert_eq!(
to_markdown(&blocks),
"Body one.\n\n<!-- page 1 -->\n\nBody two.\n"
);
}
#[test]
fn furniture_asked_for_is_plain_paragraphs_where_they_stand() {
let blocks = vec![
on_page(BlockKind::Paragraph, "Body one.", 0),
on_page(BlockKind::Paragraph, "Body two.", 1),
on_page(BlockKind::Header, "Annual Report 2026", 0),
on_page(BlockKind::Footer, "Page 1 of 2", 0),
];
let options = MarkdownOptions {
include_headers: true,
include_footers: true,
..Default::default()
};
let markdown = to_markdown_with_options(&blocks, &options);
assert_eq!(
markdown,
"Body one.\n\n<!-- page 1 -->\n\nBody two.\n\n<!-- page 0 -->\n\n\
Annual Report 2026\n\nPage 1 of 2\n"
);
assert!(
!markdown.lines().any(|line| line.starts_with('#')),
"no heading syntax for furniture:\n{markdown}"
);
}
#[test]
fn headers_and_footers_are_asked_for_separately() {
let blocks = vec![
on_page(BlockKind::Paragraph, "Body one.", 0),
on_page(BlockKind::Header, "Annual Report 2026", 0),
on_page(BlockKind::Footer, "Page 1 of 2", 0),
];
let heads = MarkdownOptions {
include_headers: true,
..Default::default()
};
let heads_only = to_markdown_with_options(&blocks, &heads);
assert!(
heads_only.contains("Annual Report 2026"),
"the head is written:\n{heads_only}"
);
assert!(
!heads_only.contains("Page 1 of 2"),
"the footer is not:\n{heads_only}"
);
let feet = MarkdownOptions {
include_footers: true,
..Default::default()
};
let feet_only = to_markdown_with_options(&blocks, &feet);
assert!(
!feet_only.contains("Annual Report 2026"),
"the head is not:\n{feet_only}"
);
assert!(
feet_only.contains("Page 1 of 2"),
"the footer is written:\n{feet_only}"
);
}
#[test]
fn without_separators_pages_stand_a_blank_line_apart() {
let blocks = vec![
on_page(BlockKind::Paragraph, "Body one.", 0),
on_page(BlockKind::Paragraph, "Body two.", 1),
];
let options = MarkdownOptions {
page_separators: PageSeparators::None,
..Default::default()
};
assert_eq!(
to_markdown_with_options(&blocks, &options),
"Body one.\n\nBody two.\n"
);
}
#[test]
fn a_table_the_page_broke_is_two_tables_with_the_marker_between_them() {
let mut blocks = vec![cell(0, 0, true, "Code"), cell(0, 1, true, "Means")];
blocks.push(on_page(
BlockKind::TableCell {
row: 0,
column: 0,
row_span: 1,
column_span: 1,
header: true,
},
"Code",
1,
));
blocks.push(on_page(
BlockKind::TableCell {
row: 0,
column: 1,
row_span: 1,
column_span: 1,
header: true,
},
"Means",
1,
));
assert_eq!(
to_markdown(&blocks),
"<table>\n<tr><th>Code</th><th>Means</th></tr>\n</table>\n\n<!-- page 1 -->\n\n<table>\n<tr><th>Code</th><th>Means</th></tr>\n</table>\n"
);
}
#[test]
fn nothing_in_means_nothing_out() {
assert_eq!(to_markdown(&[]), "");
assert_eq!(to_markdown(&[block(BlockKind::Paragraph, " ")]), "");
}
}