use super::NativeDocument;
use super::span_geometry::{
has_same_rotation, is_horizontal_ltr, is_ltr_writing_mode, is_unrotated, upright_advance_extent,
upright_cross_extent,
};
use crate::core::config::{ExtractionConfig, PageConfig};
use crate::pdf::error::{PdfError, Result};
use crate::pdf::metadata::PdfExtractionMetadata;
use crate::pdf::structure::constants::{COALESCE_THRESHOLD, MAX_GLYPH_JITTER_PT, MIN_DISORDER_COUNT};
use crate::pdf::text::{contains_html_markup, fix_pdf_control_chars};
use crate::types::{PageBoundary, PageContent};
use std::borrow::Cow;
use xberg_native_pdf::document::ReadingOrder;
type PdfTextExtractionResult = (String, Option<Vec<PageBoundary>>, Option<Vec<PageContent>>);
const DEFAULT_TOP_MARGIN_FRACTION: f32 = 0.0;
const DEFAULT_BOTTOM_MARGIN_FRACTION: f32 = 0.0;
#[derive(Debug, Clone, Copy)]
pub(crate) struct PageMarginFractions {
pub(crate) top: f32,
pub(crate) bottom: f32,
}
impl Default for PageMarginFractions {
fn default() -> Self {
Self {
top: DEFAULT_TOP_MARGIN_FRACTION,
bottom: DEFAULT_BOTTOM_MARGIN_FRACTION,
}
}
}
impl PageMarginFractions {
pub(crate) fn from_extraction_config(config: Option<&ExtractionConfig>) -> Self {
let defaults = Self::default();
let top = config
.and_then(|config| config.pdf_options.as_ref())
.and_then(|pdf| pdf.top_margin_fraction)
.unwrap_or(defaults.top);
let bottom = config
.and_then(|config| config.pdf_options.as_ref())
.and_then(|pdf| pdf.bottom_margin_fraction)
.unwrap_or(defaults.bottom);
let include_headers = config
.and_then(|config| config.content_filter.as_ref())
.is_some_and(|filter| filter.include_headers);
let include_footers = config
.and_then(|config| config.content_filter.as_ref())
.is_some_and(|filter| filter.include_footers);
Self {
top: if include_headers { 0.0 } else { top },
bottom: if include_footers { 0.0 } else { bottom },
}
}
}
pub type NativeUnifiedExtractionResult = (
String,
Option<Vec<PageBoundary>>,
Option<Vec<PageContent>>,
PdfExtractionMetadata,
);
pub(crate) fn extract_text_and_metadata(
doc: &mut NativeDocument,
extraction_config: Option<&ExtractionConfig>,
) -> Result<NativeUnifiedExtractionResult> {
let page_config = extraction_config.and_then(|c| c.pages.as_ref());
let margins = PageMarginFractions::from_extraction_config(extraction_config);
let (text, boundaries, page_contents) =
extract_text_from_native_document(doc, page_config, extraction_config, margins)?;
let scanned_min_confidence = extraction_config
.map(|c| c.ocr_strategy.effective_min_confidence())
.unwrap_or(crate::core::config::DEFAULT_SCANNED_MIN_CONFIDENCE);
let ocr_quality_thresholds = extraction_config
.and_then(|c| c.ocr.as_ref())
.and_then(|o| o.quality_thresholds.clone())
.unwrap_or_default();
let metadata = super::metadata::extract_metadata_from_native_document(
doc,
boundaries.as_deref(),
&text,
scanned_min_confidence,
&ocr_quality_thresholds,
)?;
Ok((text, boundaries, page_contents, metadata))
}
#[cfg(feature = "layout-detection")]
pub(crate) fn extract_spans_from_page(
doc: &mut xberg_native_pdf::PdfDocument,
page_index: usize,
margins: PageMarginFractions,
) -> Result<(Vec<crate::extractors::pdf::rotation::TextSpan>, bool)> {
use xberg_native_pdf::document::ReadingOrder;
let mut page_text_data = super::guard_native_panic(
|| {
doc.extract_page_text_with_options(page_index, ReadingOrder::ColumnAware)
.map_err(|e| PdfError::TextExtractionFailed(format!("Failed to extract page text: {}", e)))
},
|panic| PdfError::TextExtractionFailed(format!("Page text extraction panicked in xberg_native_pdf: {}", panic)),
)?;
let (page_bottom, page_top) = page_vertical_bounds(doc, page_index)?;
retain_spans_inside_page_margins(&mut page_text_data.spans, page_bottom, page_top, margins);
let reordered_sparse_columns = reorder_sparse_two_column_page(&mut page_text_data.spans, page_text_data.page_width);
let spans = page_text_data.spans.iter().map(rotation_span).collect();
Ok((spans, reordered_sparse_columns))
}
pub(crate) fn extract_text_from_native_document(
doc: &mut NativeDocument,
page_config: Option<&PageConfig>,
extraction_config: Option<&ExtractionConfig>,
margins: PageMarginFractions,
) -> Result<PdfTextExtractionResult> {
let needs_boundaries =
extraction_config.is_some_and(|c| c.force_ocr_pages.as_ref().is_some_and(|p| !p.is_empty()) || c.ocr.is_some());
if let Some(config) = page_config {
extract_text_with_tracking(doc, config, margins)
} else if needs_boundaries {
let default_config = PageConfig::default();
extract_text_with_tracking(doc, &default_config, margins)
} else {
extract_text_fast_path(doc, margins)
}
}
fn extract_text_fast_path(doc: &mut NativeDocument, margins: PageMarginFractions) -> Result<PdfTextExtractionResult> {
let page_count = doc
.doc
.page_count()
.map_err(|e| PdfError::TextExtractionFailed(format!("Failed to get page count: {}", e)))?;
let excluded_layers = xberg_native_pdf::optional_content::compute_default_off_ocgs(&doc.doc);
let mut content = String::new();
let mut total_sample_size = 0usize;
let mut sample_count = 0;
for page_idx in 0..page_count {
let page_text = extract_page_text_column_aware(&mut doc.doc, page_idx, &excluded_layers, margins)?;
let page_size = page_text.len();
if page_idx > 0 {
content.push_str("\n\n");
}
let cleaned = apply_text_cleanup(&page_text);
content.push_str(&cleaned);
if page_idx < 5 {
total_sample_size += page_size;
sample_count += 1;
}
if page_idx == 4 && sample_count > 0 && page_count > 5 {
let avg_page_size = total_sample_size / sample_count;
let estimated_remaining = avg_page_size * (page_count - 5);
content.reserve(estimated_remaining + (estimated_remaining / 10));
}
}
Ok((content, None, None))
}
fn extract_text_with_tracking(
doc: &mut NativeDocument,
config: &PageConfig,
margins: PageMarginFractions,
) -> Result<PdfTextExtractionResult> {
let page_count = doc
.doc
.page_count()
.map_err(|e| PdfError::TextExtractionFailed(format!("Failed to get page count: {}", e)))?;
let excluded_layers = xberg_native_pdf::optional_content::compute_default_off_ocgs(&doc.doc);
let mut content = String::new();
let mut boundaries = Vec::with_capacity(page_count);
let mut page_contents = if config.extract_pages {
Some(Vec::with_capacity(page_count))
} else {
None
};
let mut total_sample_size = 0usize;
let mut sample_count = 0;
for page_idx in 0..page_count {
let page_number = page_idx + 1;
let page_text = extract_page_text_column_aware(&mut doc.doc, page_idx, &excluded_layers, margins)?;
let page_size = page_text.len();
if page_idx < 5 {
total_sample_size += page_size;
sample_count += 1;
}
if config.insert_page_markers {
let marker = config.marker_format.replace("{page_num}", &page_number.to_string());
content.push_str(&marker);
} else if page_idx > 0 {
content.push_str("\n\n");
}
let cleaned = apply_text_cleanup(&page_text);
let byte_start = content.len();
content.push_str(&cleaned);
let byte_end = content.len();
boundaries.push(PageBoundary {
byte_start,
byte_end,
page_number: page_number as u32,
});
if let Some(ref mut pages) = page_contents {
let is_blank = Some(crate::extraction::blank_detection::is_page_text_blank(&cleaned));
pages.push(PageContent {
page_number: page_number as u32,
content: cleaned.into_owned(),
tables: Vec::new(),
image_indices: Vec::new(),
image_preprocessing: None,
hierarchy: None,
is_blank,
layout_regions: None,
speaker_notes: None,
section_name: None,
sheet_name: None,
ocr_confidence: None,
});
}
if page_idx == 4 && page_count > 5 && sample_count > 0 {
let avg_page_size = total_sample_size / sample_count;
let estimated_remaining = avg_page_size * (page_count - 5);
let separator_overhead = (page_count - 5) * 3;
content.reserve(estimated_remaining + separator_overhead + (estimated_remaining / 10));
}
}
Ok((content, Some(boundaries), page_contents))
}
fn collect_widget_field_values(doc: &xberg_native_pdf::PdfDocument, page_index: usize) -> Vec<(f64, String)> {
let annotations = match doc.get_annotations(page_index) {
Ok(a) => a,
Err(e) => {
tracing::debug!(
page = page_index,
"xberg_native_pdf: could not read annotations for widget values: {e}"
);
return Vec::new();
}
};
let mut widgets: Vec<(f64, String)> = annotations
.into_iter()
.filter(|a| a.subtype_enum == xberg_native_pdf::AnnotationSubtype::Widget)
.filter_map(|a| {
let value = a.field_value?.trim().to_string();
if value.is_empty() {
return None;
}
let mid_y = a.rect.map_or(f64::NEG_INFINITY, |r| (r[1] + r[3]) / 2.0);
Some((mid_y, value))
})
.collect();
widgets.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
widgets
}
fn append_missing_widget_values(text: &mut String, widgets: &[(f64, String)]) {
for (_, value) in widgets {
if !text.contains(value.as_str()) {
if !text.is_empty() && !text.ends_with('\n') {
text.push('\n');
}
text.push_str(value);
}
}
}
fn is_fragmented_span_list(spans: &[xberg_native_pdf::layout::TextSpan]) -> bool {
let mut disorder_count = 0;
for window in spans.windows(2) {
let prev = &window[0];
let cur = &window[1];
if prev.text.chars().count() > 3 || cur.text.chars().count() > 3 {
continue;
}
let y_gap = (prev.bbox.y - cur.bbox.y).abs();
let eff_height = prev.bbox.height.max(cur.bbox.height);
let same_line = if eff_height > 0.0 {
y_gap < eff_height * 0.5
} else {
y_gap <= MAX_GLYPH_JITTER_PT
};
if same_line && cur.bbox.x < prev.bbox.x - prev.font_size {
disorder_count += 1;
if disorder_count >= MIN_DISORDER_COUNT {
return true;
}
}
}
false
}
fn rebuild_text_from_fragmented_spans(spans: &[xberg_native_pdf::layout::TextSpan]) -> String {
if spans.is_empty() {
return String::new();
}
let mut sorted: Vec<&xberg_native_pdf::layout::TextSpan> = spans.iter().collect();
sorted.sort_by(|a, b| b.bbox.y.partial_cmp(&a.bbox.y).unwrap_or(std::cmp::Ordering::Equal));
let mut groups: Vec<Vec<&xberg_native_pdf::layout::TextSpan>> = Vec::new();
for span in sorted {
let belongs = groups.last().is_some_and(|g| {
let prev_y = g.last().unwrap().bbox.y;
(span.bbox.y - prev_y).abs() <= COALESCE_THRESHOLD
});
if belongs {
groups.last_mut().unwrap().push(span);
} else {
groups.push(vec![span]);
}
}
let mut result = String::new();
for (gi, group) in groups.iter_mut().enumerate() {
group.sort_by(|a, b| a.bbox.x.partial_cmp(&b.bbox.x).unwrap_or(std::cmp::Ordering::Equal));
if gi > 0 {
result.push('\n');
}
let font_size = group.iter().map(|s| s.font_size).fold(0.0_f32, f32::max);
let space_threshold = font_size * 0.5;
let mut prev_end_x = f32::NEG_INFINITY;
for span in group.iter() {
if prev_end_x.is_finite() && span.bbox.x - prev_end_x > space_threshold {
result.push(' ');
}
result.push_str(&span.text);
prev_end_x = span.bbox.x + span.bbox.width;
}
}
result
}
const INLINE_FRAGMENT_GAP_RATIO: f32 = 0.1;
const MAX_INLINE_FRAGMENT_ANCHOR_LOOKBACK: usize = 256;
const ROW_RESET_MIN_BACKTRACK_EMS: f32 = 4.0;
#[derive(Clone, Copy)]
struct OrderedSpan<'a> {
span: &'a xberg_native_pdf::layout::TextSpan,
glue_to_previous: bool,
}
fn spans_overlap_on_cross_axis(
first: &xberg_native_pdf::layout::TextSpan,
second: &xberg_native_pdf::layout::TextSpan,
) -> bool {
let (first_low, first_high) = upright_cross_extent(first);
let (second_low, second_high) = upright_cross_extent(second);
first_high.min(second_high) > first_low.max(second_low)
}
fn is_short_inline_fragment(span: &xberg_native_pdf::layout::TextSpan) -> bool {
let mut chars = span.text.chars();
let Some(first) = chars.next() else {
return false;
};
let char_count = 1 + chars.count();
if char_count > 3 || span.text.chars().all(char::is_whitespace) {
return false;
}
!(char_count == 1 && matches!(first, 'a' | 'A' | 'I'))
}
fn has_rtl_or_bidi_content(text: &str) -> bool {
text.chars()
.any(|character| xberg_native_pdf::text::is_rtl_text(character as u32))
}
fn find_inline_fragment_anchor(
index: usize,
spans: &[xberg_native_pdf::layout::TextSpan],
anchors: &[Option<usize>],
) -> Option<usize> {
let span = &spans[index];
if span.split_boundary_before
|| !is_short_inline_fragment(span)
|| !is_ltr_writing_mode(span)
|| has_rtl_or_bidi_content(&span.text)
{
return None;
}
let (span_start, _) = upright_advance_extent(span);
let search_start = index.saturating_sub(MAX_INLINE_FRAGMENT_ANCHOR_LOOKBACK);
(search_start..index)
.filter(|candidate_index| anchors[*candidate_index].is_none())
.filter_map(|candidate_index| {
let candidate = &spans[candidate_index];
if !is_ltr_writing_mode(candidate)
|| has_rtl_or_bidi_content(&candidate.text)
|| !has_same_rotation(candidate, span)
|| !spans_overlap_on_cross_axis(candidate, span)
{
return None;
}
let (_, candidate_end) = upright_advance_extent(candidate);
let gap = span_start - candidate_end;
let tolerance = candidate.font_size.max(span.font_size) * INLINE_FRAGMENT_GAP_RATIO;
(gap >= -tolerance && gap <= tolerance).then_some((candidate_index, gap.abs()))
})
.min_by(|(_, first_gap), (_, second_gap)| {
first_gap.partial_cmp(second_gap).unwrap_or(std::cmp::Ordering::Equal)
})
.map(|(candidate_index, _)| candidate_index)
}
fn order_spans_with_inline_fragments(spans: &[xberg_native_pdf::layout::TextSpan]) -> Vec<OrderedSpan<'_>> {
let mut anchors = vec![None; spans.len()];
for index in 0..spans.len() {
anchors[index] = find_inline_fragment_anchor(index, spans, &anchors);
}
let mut children = vec![Vec::new(); spans.len()];
for (index, anchor) in anchors.iter().enumerate() {
if let Some(anchor) = anchor {
children[*anchor].push(index);
}
}
for attached in &mut children {
attached.sort_by(|first, second| {
let (first_start, _) = upright_advance_extent(&spans[*first]);
let (second_start, _) = upright_advance_extent(&spans[*second]);
first_start
.partial_cmp(&second_start)
.unwrap_or(std::cmp::Ordering::Equal)
});
}
let mut ordered = Vec::with_capacity(spans.len());
for (index, span) in spans.iter().enumerate() {
if anchors[index].is_some() {
continue;
}
ordered.push(OrderedSpan {
span,
glue_to_previous: false,
});
ordered.extend(children[index].iter().map(|child| OrderedSpan {
span: &spans[*child],
glue_to_previous: true,
}));
}
ordered
}
fn append_span_separator(
text: &mut String,
previous: &xberg_native_pdf::layout::TextSpan,
current: OrderedSpan<'_>,
paragraph_gap_threshold: f32,
allow_ltr_row_resets: bool,
) {
if current.glue_to_previous {
return;
}
let span = current.span;
if !has_same_rotation(previous, span) {
text.push_str("\n\n");
return;
}
let (previous_start, previous_end) = upright_advance_extent(previous);
let (span_start, _) = upright_advance_extent(span);
let (previous_baseline, _) = upright_cross_extent(previous);
let (span_baseline, _) = upright_cross_extent(span);
let baseline_gap = (previous_baseline - span_baseline).abs();
let reset_threshold = previous.font_size.max(span.font_size) * ROW_RESET_MIN_BACKTRACK_EMS;
let is_ltr_pair = is_ltr_writing_mode(previous)
&& is_ltr_writing_mode(span)
&& !has_rtl_or_bidi_content(&previous.text)
&& !has_rtl_or_bidi_content(&span.text);
if allow_ltr_row_resets && is_ltr_pair && span_start < previous_start - reset_threshold {
if baseline_gap > paragraph_gap_threshold {
text.push_str("\n\n");
} else {
text.push('\n');
}
return;
}
if span.split_boundary_before {
if !previous.text.ends_with(char::is_whitespace) && !span.text.starts_with(char::is_whitespace) {
text.push(' ');
}
return;
}
let effective_height = span.bbox.height.max(previous.bbox.height).max(span.font_size * 0.5);
if baseline_gap < effective_height * 0.5 {
if span_start - previous_end > span.font_size * 0.15 {
text.push(' ');
}
} else if baseline_gap > paragraph_gap_threshold {
text.push_str("\n\n");
} else {
text.push('\n');
}
}
fn assemble_page_text(spans: &[xberg_native_pdf::layout::TextSpan]) -> String {
let mut heights: Vec<f32> = spans.iter().map(|span| span.bbox.height).collect();
heights.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let median_height = if heights.is_empty() {
1.0
} else {
heights[heights.len() / 2]
};
let paragraph_gap_threshold = median_height * 1.5;
tracing::debug!(
span_count = spans.len(),
median_height,
paragraph_gap_threshold,
"paragraph break detection initialized"
);
let ordered = order_spans_with_inline_fragments(spans);
let allow_ltr_row_resets = !spans
.iter()
.any(|span| span.rtl_draw_logical || has_rtl_or_bidi_content(&span.text));
let mut text = String::with_capacity(spans.len() * 20);
let mut prev_span: Option<&xberg_native_pdf::layout::TextSpan> = None;
for current in ordered {
let span = current.span;
if let Some(prev) = prev_span {
append_span_separator(&mut text, prev, current, paragraph_gap_threshold, allow_ltr_row_resets);
}
text.push_str(&span.text);
prev_span = Some(span);
}
text
}
const MIN_SPARSE_COLUMN_GUTTER_FRACTION: f32 = 0.05;
const MIN_SPARSE_COLUMN_GUTTER_PTS: f32 = 15.0;
const MIN_SPARSE_COLUMN_CONTENT_WIDTH_PTS: f32 = 144.0;
const MIN_SPARSE_COLUMN_WORDS: usize = 2;
const MIN_SPARSE_COLUMN_WORDS_PER_SIDE: usize = 6;
const MIN_SPARSE_COLUMN_ALPHA_CHARS: usize = 8;
const MIN_SPARSE_COLUMN_ALPHA_RATIO: f32 = 0.55;
const MIN_SPARSE_COLUMN_VERTICAL_OVERLAP: f32 = 0.5;
const XY_CUT_MIN_SPANS_FOR_SPLIT: usize = 5;
fn is_sparse_column_prose(span: &xberg_native_pdf::layout::TextSpan) -> bool {
let alpha_chars = span.text.chars().filter(|character| character.is_alphabetic()).count();
let non_whitespace_chars = span.text.chars().filter(|character| !character.is_whitespace()).count();
let word_count = span.text.split_whitespace().count();
let geometry_is_valid = span.bbox.x.is_finite()
&& span.bbox.y.is_finite()
&& span.bbox.width.is_finite()
&& span.bbox.height.is_finite()
&& span.bbox.width > 0.0;
geometry_is_valid
&& !span.is_monospace
&& is_horizontal_ltr(span)
&& !has_rtl_or_bidi_content(&span.text)
&& !span.text.contains(':')
&& word_count >= MIN_SPARSE_COLUMN_WORDS
&& alpha_chars >= MIN_SPARSE_COLUMN_ALPHA_CHARS
&& alpha_chars as f32 / non_whitespace_chars.max(1) as f32 >= MIN_SPARSE_COLUMN_ALPHA_RATIO
}
fn sparse_columns_overlap(
left: &[&xberg_native_pdf::layout::TextSpan],
right: &[&xberg_native_pdf::layout::TextSpan],
) -> bool {
let extent = |side: &[&xberg_native_pdf::layout::TextSpan]| {
side.iter()
.map(|span| span.bbox.y)
.fold((f32::INFINITY, f32::NEG_INFINITY), |(low, high), y| {
(low.min(y), high.max(y))
})
};
let (left_low, left_high) = extent(left);
let (right_low, right_high) = extent(right);
let overlap = (left_high.min(right_high) - left_low.max(right_low)).max(0.0);
let shorter_extent = (left_high - left_low).min(right_high - right_low);
shorter_extent > 0.0 && overlap / shorter_extent >= MIN_SPARSE_COLUMN_VERTICAL_OVERLAP
}
fn sparse_columns_continue_one_sentence(
left: &[&xberg_native_pdf::layout::TextSpan],
right: &[&xberg_native_pdf::layout::TextSpan],
) -> bool {
let mut left_by_y = left.to_vec();
let mut right_by_y = right.to_vec();
left_by_y.sort_by(|first, second| second.bbox.y.total_cmp(&first.bbox.y));
right_by_y.sort_by(|first, second| second.bbox.y.total_cmp(&first.bbox.y));
let starts_lowercase = |span: &&xberg_native_pdf::layout::TextSpan| {
span.text
.chars()
.find(|character| character.is_alphabetic())
.is_some_and(char::is_lowercase)
};
let starts_uppercase = |span: &&xberg_native_pdf::layout::TextSpan| {
span.text
.chars()
.find(|character| character.is_alphabetic())
.is_some_and(char::is_uppercase)
};
let has_terminal = |span: &&xberg_native_pdf::layout::TextSpan| span.text.trim_end().ends_with(['.', '!', '?']);
let continuations = [&left_by_y[1], &right_by_y[0], &right_by_y[1]];
let all_spans = left_by_y.iter().chain(&right_by_y);
starts_uppercase(&left_by_y[0])
&& continuations.into_iter().all(starts_lowercase)
&& all_spans.clone().filter(|span| has_terminal(span)).count() == 1
&& has_terminal(&right_by_y[1])
}
fn is_sparse_column_split(spans: &[xberg_native_pdf::layout::TextSpan], split_x: f32, min_gutter: f32) -> bool {
let left: Vec<_> = spans.iter().filter(|span| span.bbox.x < split_x).collect();
let right: Vec<_> = spans.iter().filter(|span| span.bbox.x >= split_x).collect();
if left.len() != 2 || right.len() != 2 {
return false;
}
let word_count = |side: &[&xberg_native_pdf::layout::TextSpan]| {
side.iter()
.map(|span| span.text.split_whitespace().count())
.sum::<usize>()
};
if word_count(&left) < MIN_SPARSE_COLUMN_WORDS_PER_SIDE || word_count(&right) < MIN_SPARSE_COLUMN_WORDS_PER_SIDE {
return false;
}
let left_right = left
.iter()
.map(|span| span.bbox.x + span.bbox.width)
.fold(f32::NEG_INFINITY, f32::max);
split_x - left_right >= min_gutter
&& sparse_columns_overlap(&left, &right)
&& sparse_columns_continue_one_sentence(&left, &right)
}
fn sparse_column_split(spans: &[xberg_native_pdf::layout::TextSpan], page_width: f32) -> Option<f32> {
let has_sparse_prose_shape =
spans.len() == XY_CUT_MIN_SPANS_FOR_SPLIT - 1 && spans.iter().all(is_sparse_column_prose);
let content_left = spans.iter().map(|span| span.bbox.x).fold(f32::INFINITY, f32::min);
let content_right = spans
.iter()
.map(|span| span.bbox.x + span.bbox.width)
.fold(f32::NEG_INFINITY, f32::max);
if !has_sparse_prose_shape || content_right - content_left < MIN_SPARSE_COLUMN_CONTENT_WIDTH_PTS {
return None;
}
let min_gutter = (page_width * MIN_SPARSE_COLUMN_GUTTER_FRACTION).max(MIN_SPARSE_COLUMN_GUTTER_PTS);
let mut starts: Vec<f32> = spans.iter().map(|span| span.bbox.x).collect();
starts.sort_by(f32::total_cmp);
starts.dedup_by(|left, right| (*left - *right).abs() <= f32::EPSILON);
starts
.into_iter()
.find(|&split_x| is_sparse_column_split(spans, split_x, min_gutter))
}
pub(crate) fn reorder_sparse_two_column_page(
spans: &mut [xberg_native_pdf::layout::TextSpan],
page_width: f32,
) -> bool {
let Some(split_x) = sparse_column_split(spans, page_width) else {
return false;
};
spans.sort_by(|left, right| {
let left_column = usize::from(left.bbox.x >= split_x);
let right_column = usize::from(right.bbox.x >= split_x);
left_column
.cmp(&right_column)
.then_with(|| right.bbox.y.total_cmp(&left.bbox.y))
.then_with(|| left.bbox.x.total_cmp(&right.bbox.x))
});
true
}
const MIN_DENSE_COLUMN_CONTENT_WIDTH_PTS: f32 = 200.0;
const MIN_DENSE_COLUMN_GUTTER_FRACTION: f32 = 0.02;
const MIN_DENSE_COLUMN_GUTTER_PTS: f32 = 10.0;
const MIN_DENSE_COLUMN_SPANS_PER_SIDE: usize = 6;
const MAX_DENSE_COLUMN_SPLIT_SNAP_SPAN_FRACTION: f32 = 0.06;
const DENSE_COLUMN_SPLIT_SNAP_X_TOLERANCE_PTS: f32 = 1.0;
const MAX_DENSE_COLUMN_SPLIT_SNAP_PASSES: usize = 4;
const FULL_WIDTH_FURNITURE_FRACTION: f32 = 0.55;
const LINE_Y_TOLERANCE_PTS: f32 = 0.5;
const MIN_DENSE_COLUMN_SPLIT_LINES: usize = MIN_DENSE_COLUMN_SPANS_PER_SIDE;
const MAX_CROSS_GUTTER_ROW_PAIRING_FRACTION: f32 = 0.5;
const MIN_PANEL_VALUE_COLUMN_NUMERIC_FRACTION: f32 = 0.8;
const MAX_PANEL_LABEL_COLUMN_NUMERIC_FRACTION: f32 = 0.2;
const MIN_PANEL_SPLIT_COLUMNS: usize = 4;
const MIN_COLUMNS_PER_PANEL: usize = 2;
const MIN_GRID_ROW_COLUMN_ALIGNMENT_FRACTION: f32 = 0.4;
type SpanLine = Vec<usize>;
fn spans_sorted_top_to_bottom(spans: &[xberg_native_pdf::layout::TextSpan]) -> Vec<usize> {
let mut order: Vec<usize> = (0..spans.len()).collect();
order.sort_by(|&a, &b| {
spans[b]
.bbox
.y
.total_cmp(&spans[a].bbox.y)
.then_with(|| spans[a].bbox.x.total_cmp(&spans[b].bbox.x))
});
order
}
fn group_into_lines(spans: &[xberg_native_pdf::layout::TextSpan], order: &[usize]) -> Vec<SpanLine> {
let mut lines: Vec<SpanLine> = Vec::new();
let mut anchor_y = f32::NAN;
for &index in order {
let y = spans[index].bbox.y;
if lines.is_empty() || (anchor_y - y).abs() > LINE_Y_TOLERANCE_PTS {
anchor_y = y;
lines.push(Vec::new());
}
lines.last_mut().expect("just pushed above").push(index);
}
for line in &mut lines {
line.sort_by(|&a, &b| spans[a].bbox.x.total_cmp(&spans[b].bbox.x));
}
lines
}
fn widest_gap_midpoint(mut edges: impl Iterator<Item = (f32, f32)>, min_gutter: f32) -> Option<f32> {
let (_, mut running_right) = edges.next()?;
let mut best_gap = 0.0_f32;
let mut best_split = None;
for (left, right) in edges {
let gap = left - running_right;
if gap > best_gap {
best_gap = gap;
best_split = Some((running_right + left) / 2.0);
}
running_right = running_right.max(right);
}
if best_gap < min_gutter { None } else { best_split }
}
fn line_has_width_furniture(
spans: &[xberg_native_pdf::layout::TextSpan],
line: &SpanLine,
furniture_width: f32,
) -> bool {
line.iter().any(|&index| spans[index].bbox.width >= furniture_width)
}
fn detect_split_x(spans: &[xberg_native_pdf::layout::TextSpan], lines: &[SpanLine], page_width: f32) -> Option<f32> {
let min_gutter = (page_width * MIN_DENSE_COLUMN_GUTTER_FRACTION).max(MIN_DENSE_COLUMN_GUTTER_PTS);
let furniture_width = page_width * FULL_WIDTH_FURNITURE_FRACTION;
let mut midpoints: Vec<f32> = lines
.iter()
.filter(|&line| !line_has_width_furniture(spans, line, furniture_width))
.filter_map(|line| {
let edges = line
.iter()
.map(|&index| (spans[index].bbox.left(), spans[index].bbox.right()));
widest_gap_midpoint(edges, min_gutter)
})
.collect();
if midpoints.len() < MIN_DENSE_COLUMN_SPLIT_LINES {
return None;
}
midpoints.sort_by(f32::total_cmp);
let mid = midpoints.len() / 2;
Some(if midpoints.len().is_multiple_of(2) {
(midpoints[mid - 1] + midpoints[mid]) / 2.0
} else {
midpoints[mid]
})
}
fn redirect_split_out_of_content(
spans: &[xberg_native_pdf::layout::TextSpan],
lines: &[SpanLine],
page_width: f32,
split_x: f32,
) -> f32 {
let cuts_a_span = spans
.iter()
.any(|span| span.bbox.left() < split_x && span.bbox.right() > split_x);
if !cuts_a_span {
return split_x;
}
let furniture_width = page_width * FULL_WIDTH_FURNITURE_FRACTION;
let min_gutter = (page_width * MIN_DENSE_COLUMN_GUTTER_FRACTION).max(MIN_DENSE_COLUMN_GUTTER_PTS);
page_whitespace_corridors(spans, lines, furniture_width, min_gutter)
.into_iter()
.max_by(|a, b| (a.1 - a.0).total_cmp(&(b.1 - b.0)))
.map_or(split_x, |(left, right)| (left + right) / 2.0)
}
fn page_whitespace_corridors(
spans: &[xberg_native_pdf::layout::TextSpan],
lines: &[SpanLine],
furniture_width: f32,
min_gutter: f32,
) -> Vec<(f32, f32)> {
let mut extents: Vec<(f32, f32)> = lines
.iter()
.filter(|&line| !line_has_width_furniture(spans, line, furniture_width))
.flat_map(|line| line.iter())
.map(|&index| (spans[index].bbox.left(), spans[index].bbox.right()))
.filter(|(left, right)| left.is_finite() && right.is_finite())
.collect();
extents.sort_by(|a, b| a.0.total_cmp(&b.0));
let mut corridors = Vec::new();
let mut running_right = match extents.first() {
Some(&(_, right)) => right,
None => return corridors,
};
for (left, right) in extents {
if left - running_right >= min_gutter {
corridors.push((running_right, left));
}
running_right = running_right.max(right);
}
corridors
}
fn snap_split_left_of_hanging_labels(
spans: &[xberg_native_pdf::layout::TextSpan],
lines: &[SpanLine],
page_width: f32,
mut split_x: f32,
) -> f32 {
let max_snap_width = page_width * MAX_DENSE_COLUMN_SPLIT_SNAP_SPAN_FRACTION;
for _ in 0..MAX_DENSE_COLUMN_SPLIT_SNAP_PASSES {
let Some(left_edge) = aligned_hanging_label_left_edge(spans, lines, max_snap_width, split_x) else {
break;
};
split_x = left_edge;
}
split_x
}
fn aligned_hanging_label_left_edge(
spans: &[xberg_native_pdf::layout::TextSpan],
lines: &[SpanLine],
max_snap_width: f32,
split_x: f32,
) -> Option<f32> {
let mut left_edges = lines
.iter()
.filter_map(|line| {
line.iter()
.filter_map(|&index| {
let bbox = &spans[index].bbox;
(bbox.width > 0.0
&& bbox.width <= max_snap_width
&& bbox.left() < split_x
&& bbox.right() > split_x)
.then_some(bbox.left())
})
.min_by(f32::total_cmp)
})
.collect::<Vec<_>>();
left_edges.sort_by(f32::total_cmp);
for (start, &left_edge) in left_edges.iter().enumerate() {
let aligned_count = left_edges[start..]
.iter()
.take_while(|&&candidate| candidate - left_edge <= DENSE_COLUMN_SPLIT_SNAP_X_TOLERANCE_PTS)
.count();
if aligned_count >= MIN_DENSE_COLUMN_SPLIT_LINES {
return Some(left_edge);
}
}
None
}
enum Band {
Content(Vec<usize>),
Boundary(SpanLine),
}
fn line_is_boundary(
spans: &[xberg_native_pdf::layout::TextSpan],
line: &SpanLine,
furniture_width: f32,
split_x: f32,
) -> bool {
line.iter().any(|&index| {
let bbox = &spans[index].bbox;
bbox.width >= furniture_width || (bbox.left() < split_x && bbox.right() > split_x)
})
}
fn build_bands(
spans: &[xberg_native_pdf::layout::TextSpan],
lines: &[SpanLine],
furniture_width: f32,
split_x: f32,
) -> Vec<Band> {
let mut bands = Vec::new();
let mut current: Vec<usize> = Vec::new();
for line in lines {
if !line_is_boundary(spans, line, furniture_width, split_x) {
current.extend(line.iter().copied());
continue;
}
if !current.is_empty() {
bands.push(Band::Content(std::mem::take(&mut current)));
}
bands.push(Band::Boundary(line.clone()));
}
if !current.is_empty() {
bands.push(Band::Content(current));
}
bands
}
fn reorder_band_columns(
spans: &[xberg_native_pdf::layout::TextSpan],
band: &[usize],
split_x: f32,
) -> Option<Vec<usize>> {
let (left, right): (Vec<usize>, Vec<usize>) =
band.iter().copied().partition(|&index| spans[index].bbox.x < split_x);
if left.len() < MIN_DENSE_COLUMN_SPANS_PER_SIDE || right.len() < MIN_DENSE_COLUMN_SPANS_PER_SIDE {
return None;
}
let left_reorderable = xberg_native_pdf::layout::classify_region(spans, &left).is_reorderable_column();
let right_reorderable = xberg_native_pdf::layout::classify_region(spans, &right).is_reorderable_column();
if !left_reorderable && !right_reorderable {
return None;
}
if !(left_reorderable && right_reorderable)
&& cross_gutter_row_pairing_fraction(spans, band, split_x) > MAX_CROSS_GUTTER_ROW_PAIRING_FRACTION
{
return None;
}
let left = if left_reorderable {
left
} else {
order_region_by_panels(spans, left)
};
let right = if right_reorderable {
right
} else {
order_region_by_panels(spans, right)
};
Some(left.into_iter().chain(right).collect())
}
fn region_rows(spans: &[xberg_native_pdf::layout::TextSpan], region: &[usize]) -> Vec<SpanLine> {
let mut order = region.to_vec();
order.sort_by(|&a, &b| {
spans[b]
.bbox
.y
.total_cmp(&spans[a].bbox.y)
.then_with(|| spans[a].bbox.x.total_cmp(&spans[b].bbox.x))
});
group_into_lines(spans, &order)
}
fn cross_gutter_row_pairing_fraction(
spans: &[xberg_native_pdf::layout::TextSpan],
band: &[usize],
split_x: f32,
) -> f32 {
let rows = region_rows(spans, band);
if rows.is_empty() {
return 0.0;
}
let paired = rows
.iter()
.filter(|row| {
row.iter().any(|&index| spans[index].bbox.x < split_x)
&& row.iter().any(|&index| spans[index].bbox.x >= split_x)
})
.count();
paired as f32 / rows.len() as f32
}
fn is_numeric_cell(text: &str) -> bool {
let trimmed = text.trim();
!trimmed.is_empty()
&& trimmed
.chars()
.all(|character| character.is_ascii_digit() || matches!(character, '.' | ',' | '%'))
}
fn strong_column_edges(spans: &[xberg_native_pdf::layout::TextSpan], rows: &[SpanLine]) -> Vec<f32> {
let mut edges: Vec<(f32, usize)> = rows
.iter()
.enumerate()
.flat_map(|(row_index, row)| row.iter().map(move |&index| (index, row_index)))
.map(|(index, row_index)| (spans[index].bbox.left(), row_index))
.collect();
edges.sort_by(|a, b| a.0.total_cmp(&b.0));
let mut columns = Vec::new();
let mut cluster: Vec<(f32, usize)> = Vec::new();
for edge in edges {
let split = cluster
.first()
.is_some_and(|&(first, _)| edge.0 - first > DENSE_COLUMN_SPLIT_SNAP_X_TOLERANCE_PTS);
if split {
push_supported_column(&mut columns, &cluster);
cluster.clear();
}
cluster.push(edge);
}
push_supported_column(&mut columns, &cluster);
columns
}
fn push_supported_column(columns: &mut Vec<f32>, cluster: &[(f32, usize)]) {
let Some(&(first, _)) = cluster.first() else {
return;
};
let mut supporting: Vec<usize> = cluster.iter().map(|&(_, row)| row).collect();
supporting.sort_unstable();
supporting.dedup();
if supporting.len() >= MIN_DENSE_COLUMN_SPLIT_LINES {
columns.push(first);
}
}
fn column_index_for_x(columns: &[f32], x: f32) -> Option<usize> {
columns
.iter()
.rposition(|&column| x >= column - DENSE_COLUMN_SPLIT_SNAP_X_TOLERANCE_PTS)
}
fn panel_boundary_columns(
spans: &[xberg_native_pdf::layout::TextSpan],
rows: &[SpanLine],
columns: &[f32],
) -> Vec<usize> {
let mut totals = vec![0usize; columns.len()];
let mut numeric = vec![0usize; columns.len()];
for &index in rows.iter().flatten() {
if let Some(column) = column_index_for_x(columns, spans[index].bbox.left()) {
totals[column] += 1;
numeric[column] += usize::from(is_numeric_cell(&spans[index].text));
}
}
let fraction = |column: usize| {
if totals[column] == 0 {
return None;
}
Some(numeric[column] as f32 / totals[column] as f32)
};
(0..columns.len().saturating_sub(1))
.filter(|&column| {
let (Some(value), Some(label)) = (fraction(column), fraction(column + 1)) else {
return false;
};
value >= MIN_PANEL_VALUE_COLUMN_NUMERIC_FRACTION && label <= MAX_PANEL_LABEL_COLUMN_NUMERIC_FRACTION
})
.map(|column| column + 1)
.collect()
}
fn panels_are_wide_enough(boundaries: &[usize], column_count: usize) -> bool {
let mut start = 0usize;
for &boundary in boundaries {
if boundary.saturating_sub(start) < MIN_COLUMNS_PER_PANEL {
return false;
}
start = boundary;
}
column_count.saturating_sub(start) >= MIN_COLUMNS_PER_PANEL
}
fn row_follows_column_grid(spans: &[xberg_native_pdf::layout::TextSpan], row: &SpanLine, columns: &[f32]) -> bool {
if row.is_empty() {
return false;
}
let aligned = row
.iter()
.filter(|&&index| {
let left = spans[index].bbox.left();
columns
.iter()
.any(|&column| (left - column).abs() <= DENSE_COLUMN_SPLIT_SNAP_X_TOLERANCE_PTS)
})
.count();
aligned as f32 / row.len() as f32 >= MIN_GRID_ROW_COLUMN_ALIGNMENT_FRACTION
}
fn order_region_by_panels(spans: &[xberg_native_pdf::layout::TextSpan], region: Vec<usize>) -> Vec<usize> {
let rows = region_rows(spans, ®ion);
let columns = strong_column_edges(spans, &rows);
if columns.len() < MIN_PANEL_SPLIT_COLUMNS {
return region;
}
let boundaries = panel_boundary_columns(spans, &rows, &columns);
if boundaries.is_empty() || !panels_are_wide_enough(&boundaries, columns.len()) {
return region;
}
let leading = rows
.iter()
.take_while(|row| !row_follows_column_grid(spans, row, &columns))
.count();
if rows[leading..]
.iter()
.any(|row| !row_follows_column_grid(spans, row, &columns))
{
return region;
}
let panel_of = |index: usize| {
let column = column_index_for_x(&columns, spans[index].bbox.left());
boundaries
.iter()
.filter(|&&boundary| column.is_some_and(|column| column >= boundary))
.count()
};
let mut ordered: Vec<usize> = rows[..leading].iter().flatten().copied().collect();
for panel in 0..=boundaries.len() {
for row in &rows[leading..] {
ordered.extend(row.iter().copied().filter(|&index| panel_of(index) == panel));
}
}
ordered
}
fn emit_band_order(spans: &[xberg_native_pdf::layout::TextSpan], bands: Vec<Band>, split_x: f32) -> Option<Vec<usize>> {
let mut any_reordered = false;
let mut order = Vec::new();
for band in bands {
match band {
Band::Boundary(line) => order.extend(line),
Band::Content(indices) => match reorder_band_columns(spans, &indices, split_x) {
Some(reordered) => {
any_reordered = true;
order.extend(reordered);
}
None => order.extend(indices),
},
}
}
any_reordered.then_some(order)
}
fn apply_span_order(spans: &mut [xberg_native_pdf::layout::TextSpan], order: &[usize]) {
let mut taken: Vec<Option<xberg_native_pdf::layout::TextSpan>> =
spans.iter_mut().map(|span| Some(std::mem::take(span))).collect();
for (slot, &source) in spans.iter_mut().zip(order) {
*slot = taken[source].take().expect("each source index is used exactly once");
}
}
pub(crate) fn reorder_dense_two_column_page(spans: &mut [xberg_native_pdf::layout::TextSpan], page_width: f32) -> bool {
let content_left = spans.iter().map(|span| span.bbox.x).fold(f32::INFINITY, f32::min);
let content_right = spans
.iter()
.map(|span| span.bbox.x + span.bbox.width)
.fold(f32::NEG_INFINITY, f32::max);
if spans.len() < 2 || content_right - content_left < MIN_DENSE_COLUMN_CONTENT_WIDTH_PTS {
return false;
}
let order = spans_sorted_top_to_bottom(spans);
let lines = group_into_lines(spans, &order);
let Some(detected_split_x) = detect_split_x(spans, &lines, page_width) else {
return false;
};
let split_x = snap_split_left_of_hanging_labels(spans, &lines, page_width, detected_split_x);
let split_x = redirect_split_out_of_content(spans, &lines, page_width, split_x);
let furniture_width = page_width * FULL_WIDTH_FURNITURE_FRACTION;
let bands = build_bands(spans, &lines, furniture_width, split_x);
let Some(final_order) = emit_band_order(spans, bands, split_x) else {
return false;
};
apply_span_order(spans, &final_order);
true
}
fn page_text_with_options_excluding_layers(
doc: &xberg_native_pdf::PdfDocument,
page_index: usize,
excluded_layers: &std::collections::HashSet<String>,
) -> xberg_native_pdf::error::Result<xberg_native_pdf::layout::PageText> {
if excluded_layers.is_empty() {
return doc.extract_page_text_with_options(page_index, ReadingOrder::ColumnAware);
}
let spans = doc.extract_spans_filtered_with_reading_order(
page_index,
ReadingOrder::ColumnAware,
excluded_layers.clone(),
Default::default(),
)?;
let chars: Vec<xberg_native_pdf::layout::TextChar> = spans.iter().flat_map(|s| s.to_chars()).collect();
let (_, _, page_width, page_height) = doc.get_page_media_box(page_index)?;
Ok(xberg_native_pdf::layout::PageText {
spans,
chars,
page_width,
page_height,
})
}
fn page_vertical_bounds(doc: &xberg_native_pdf::PdfDocument, page_index: usize) -> Result<(f32, f32)> {
let (_, lower_y, _, upper_y) = doc.get_page_media_box(page_index).map_err(|error| {
PdfError::TextExtractionFailed(format!(
"Failed to read page {} media box for margin filtering: {error}",
page_index + 1
))
})?;
Ok((lower_y.min(upper_y), lower_y.max(upper_y)))
}
pub(crate) fn baseline_is_inside_page_margins(
baseline_y: f32,
page_bottom: f32,
page_top: f32,
margins: PageMarginFractions,
) -> bool {
let page_height = page_top - page_bottom;
if !page_height.is_finite() || page_height <= 0.0 {
return true;
}
let bottom_cutoff = page_bottom + page_height * margins.bottom;
let top_cutoff = page_top - page_height * margins.top;
baseline_y >= bottom_cutoff && baseline_y <= top_cutoff
}
fn span_page_y_extent(span: &xberg_native_pdf::layout::TextSpan) -> (f32, f32) {
let (sin, cos) = span.rotation_degrees.to_radians().sin_cos();
let origin = span.bbox.y;
let advance = span.bbox.width * sin;
let cross = span.bbox.height * cos;
let corners = [origin, origin + advance, origin + cross, origin + advance + cross];
corners
.iter()
.fold((f32::INFINITY, f32::NEG_INFINITY), |(low, high), corner| {
(low.min(*corner), high.max(*corner))
})
}
fn span_is_inside_page_margins(
span: &xberg_native_pdf::layout::TextSpan,
page_bottom: f32,
page_top: f32,
margins: PageMarginFractions,
) -> bool {
if is_unrotated(span) {
return baseline_is_inside_page_margins(span.bbox.y, page_bottom, page_top, margins);
}
let (low, high) = span_page_y_extent(span);
baseline_is_inside_page_margins((low + high) / 2.0, page_bottom, page_top, margins)
}
fn retain_spans_inside_page_margins(
spans: &mut Vec<xberg_native_pdf::layout::TextSpan>,
page_bottom: f32,
page_top: f32,
margins: PageMarginFractions,
) {
spans.retain(|span| span_is_inside_page_margins(span, page_bottom, page_top, margins));
}
fn extract_page_text_column_aware(
doc: &mut xberg_native_pdf::PdfDocument,
page_index: usize,
excluded_layers: &std::collections::HashSet<String>,
margins: PageMarginFractions,
) -> Result<String> {
let (page_bottom, page_top) = page_vertical_bounds(doc, page_index)?;
let mut widgets = collect_widget_field_values(doc, page_index);
widgets
.retain(|(baseline_y, _)| baseline_is_inside_page_margins(*baseline_y as f32, page_bottom, page_top, margins));
let mut page_text_data = super::guard_native_panic(
|| {
page_text_with_options_excluding_layers(doc, page_index, excluded_layers).map_err(|e| {
PdfError::TextExtractionFailed(format!("Page {} text extraction failed: {}", page_index + 1, e))
})
},
|panic| {
PdfError::TextExtractionFailed(format!(
"Page {} text extraction panicked in xberg_native_pdf: {}",
page_index + 1,
panic
))
},
)?;
retain_spans_inside_page_margins(&mut page_text_data.spans, page_bottom, page_top, margins);
reorder_sparse_two_column_page(&mut page_text_data.spans, page_text_data.page_width);
reorder_dense_two_column_page(&mut page_text_data.spans, page_text_data.page_width);
let rotation_spans = page_text_data.spans.iter().map(rotation_span).collect::<Vec<_>>();
if let Some(mut text) = crate::extractors::pdf::rotation::repair_rotated_page_text(&rotation_spans) {
append_missing_widget_values(&mut text, &widgets);
return Ok(text);
}
if is_fragmented_span_list(&page_text_data.spans) {
tracing::debug!(
span_count = page_text_data.spans.len(),
"glyph fragmentation detected — rebuilding text from span positions (#962)"
);
let mut text = rebuild_text_from_fragmented_spans(&page_text_data.spans);
append_missing_widget_values(&mut text, &widgets);
return Ok(text);
}
let mut text = assemble_page_text(&page_text_data.spans);
append_missing_widget_values(&mut text, &widgets);
Ok(text)
}
fn rotation_span(span: &xberg_native_pdf::layout::TextSpan) -> crate::extractors::pdf::rotation::TextSpan {
crate::extractors::pdf::rotation::TextSpan {
text: span.text.clone(),
x: span.bbox.x,
y: span.bbox.y,
width: span.bbox.width,
height: span.bbox.height,
rotation_degrees: span.rotation_degrees,
}
}
fn apply_text_cleanup(text: &str) -> Cow<'_, str> {
let cleaned = fix_pdf_control_chars(text);
#[cfg(feature = "html")]
if contains_html_markup(&cleaned) {
return Cow::Owned(crate::pdf::text::convert_html_page_text(&cleaned));
}
#[cfg(not(feature = "html"))]
let _ = contains_html_markup(&cleaned);
cleaned
}
#[cfg(test)]
mod tests {
use super::*;
use xberg_native_pdf::geometry::Rect;
use xberg_native_pdf::layout::TextSpan;
fn span(text: &str, x: f32, y: f32, height: f32, font_size: f32) -> TextSpan {
span_with_width(text, x, y, font_size * 0.6, height, font_size)
}
fn span_with_width(text: &str, x: f32, y: f32, width: f32, height: f32, font_size: f32) -> TextSpan {
TextSpan {
text: text.to_string(),
bbox: Rect { x, y, width, height },
font_size,
..TextSpan::default()
}
}
#[test]
fn should_exclude_native_spans_by_configured_page_margins() {
let mut spans = vec![
span("header", 20.0, 950.0, 10.0, 10.0),
span("top boundary", 20.0, 900.0, 10.0, 10.0),
span("body", 20.0, 400.0, 10.0, 10.0),
span("bottom boundary", 20.0, 100.0, 10.0, 10.0),
span("footer", 20.0, 40.0, 10.0, 10.0),
];
retain_spans_inside_page_margins(
&mut spans,
0.0,
1000.0,
PageMarginFractions {
top: 0.10,
bottom: 0.10,
},
);
assert_eq!(
spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>(),
["top boundary", "body", "bottom boundary"]
);
}
#[test]
fn should_keep_a_rotated_side_stamp_that_reaches_out_of_the_footer_band() {
let mut stamp = span_with_width("side stamp", 60.0, 18.0, 112.0, 11.0, 9.0);
stamp.rotation_degrees = 90.0;
let mut confined = span_with_width("rotated footer", 300.0, 18.0, 12.0, 11.0, 9.0);
confined.rotation_degrees = 90.0;
let mut spans = vec![stamp, confined];
retain_spans_inside_page_margins(
&mut spans,
0.0,
792.0,
PageMarginFractions {
top: 0.06,
bottom: 0.05,
},
);
assert_eq!(
spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>(),
["side stamp"],
"a rotated run reaching into the body must survive while one confined to the band is dropped"
);
}
#[test]
fn should_not_filter_by_default_margins() {
let mut spans = vec![
span("header", 20.0, 860.0, 10.0, 10.0),
span("body", 20.0, 500.0, 10.0, 10.0),
span("footer", 20.0, 130.0, 10.0, 10.0),
];
retain_spans_inside_page_margins(&mut spans, 100.0, 900.0, PageMarginFractions::default());
assert_eq!(
spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>(),
["header", "body", "footer"]
);
}
#[test]
fn should_resolve_configured_margins_and_account_for_non_zero_page_origin() {
let mut spans = vec![
span("header", 20.0, 860.0, 10.0, 10.0),
span("body", 20.0, 500.0, 10.0, 10.0),
span("footer", 20.0, 130.0, 10.0, 10.0),
];
retain_spans_inside_page_margins(
&mut spans,
100.0,
900.0,
PageMarginFractions {
top: 0.06,
bottom: 0.05,
},
);
assert_eq!(
spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>(),
["body"]
);
}
#[test]
fn should_resolve_no_margins_for_a_default_config() {
let margins = PageMarginFractions::from_extraction_config(None);
assert_eq!(margins.top, 0.0);
assert_eq!(margins.bottom, 0.0);
let margins = PageMarginFractions::from_extraction_config(Some(&ExtractionConfig::default()));
assert_eq!(margins.top, 0.0);
assert_eq!(margins.bottom, 0.0);
let config = ExtractionConfig {
pdf_options: Some(crate::core::config::PdfConfig::default()),
..ExtractionConfig::default()
};
let margins = PageMarginFractions::from_extraction_config(Some(&config));
assert_eq!(margins.top, 0.0);
assert_eq!(margins.bottom, 0.0);
}
#[test]
fn should_disable_respective_pdf_margin_when_content_filter_includes_furniture() {
let mut config = ExtractionConfig {
pdf_options: Some(crate::core::config::PdfConfig {
top_margin_fraction: Some(0.25),
bottom_margin_fraction: Some(0.20),
..crate::core::config::PdfConfig::default()
}),
..ExtractionConfig::default()
};
config.content_filter = Some(crate::core::config::ContentFilterConfig {
include_headers: true,
..crate::core::config::ContentFilterConfig::default()
});
let margins = PageMarginFractions::from_extraction_config(Some(&config));
assert_eq!(margins.top, 0.0);
assert_eq!(margins.bottom, 0.20);
config.content_filter = Some(crate::core::config::ContentFilterConfig {
include_footers: true,
..crate::core::config::ContentFilterConfig::default()
});
let margins = PageMarginFractions::from_extraction_config(Some(&config));
assert_eq!(margins.top, 0.25);
assert_eq!(margins.bottom, 0.0);
}
fn disorder_spans(count: usize) -> Vec<TextSpan> {
let font_size = 12.0_f32;
let mut spans = Vec::with_capacity(count + 1);
let mut x = 300.0_f32;
for _i in 0..=count {
spans.push(span("A", x, 700.0, 0.0, font_size));
x = x - font_size - 1.0;
}
spans
}
#[test]
fn fragmentation_detected_at_threshold() {
let spans = disorder_spans(MIN_DISORDER_COUNT);
assert!(
is_fragmented_span_list(&spans),
"should detect fragmentation at exactly MIN_DISORDER_COUNT ({MIN_DISORDER_COUNT}) events"
);
}
#[test]
fn fragmentation_not_detected_below_threshold() {
let spans = disorder_spans(MIN_DISORDER_COUNT - 1);
assert!(
!is_fragmented_span_list(&spans),
"must NOT detect fragmentation with {} events (threshold is {MIN_DISORDER_COUNT})",
MIN_DISORDER_COUNT - 1
);
}
#[test]
fn long_spans_never_count_toward_disorder() {
let font_size = 12.0_f32;
let mut spans = Vec::new();
let mut x = 500.0_f32;
for _ in 0..20 {
spans.push(span("word", x, 700.0, 0.0, font_size));
x = x - font_size - 1.0;
}
assert!(
!is_fragmented_span_list(&spans),
"word-level spans (> 3 chars) must never trigger fragmentation detection"
);
}
#[test]
fn large_y_gap_not_classified_as_same_line() {
let spans = vec![span("A", 300.0, 700.0, 0.0, 12.0), span("B", 50.0, 686.0, 0.0, 12.0)];
assert!(
!is_fragmented_span_list(&spans),
"14 pt y-gap must not be classified as same-line (MAX_GLYPH_JITTER_PT={MAX_GLYPH_JITTER_PT})"
);
}
#[test]
fn empty_spans_returns_false() {
assert!(!is_fragmented_span_list(&[]));
}
#[test]
fn single_span_returns_false() {
assert!(!is_fragmented_span_list(&[span("A", 100.0, 700.0, 0.0, 12.0)]));
}
#[test]
fn detached_subscripts_are_reinserted_into_chemical_formula() {
let spans = vec![
span_with_width("H", 100.0, 100.0, 6.0, 10.0, 10.0),
span_with_width("SO", 108.0, 100.0, 12.0, 10.0, 10.0),
span_with_width("solution", 124.0, 100.0, 36.0, 10.0, 10.0),
span_with_width("2", 106.0, 96.0, 2.0, 6.0, 6.0),
span_with_width("4", 120.0, 96.0, 2.0, 6.0, 6.0),
];
assert_eq!(assemble_page_text(&spans), "H2SO4 solution");
}
#[test]
fn detached_phone_suffix_is_reinserted_without_space() {
let spans = vec![
span_with_width("273.879.750", 100.0, 100.0, 60.0, 10.0, 10.0),
span_with_width("Population", 100.0, 75.0, 45.0, 10.0, 10.0),
span_with_width("1", 160.0, 103.0, 3.0, 6.0, 6.0),
];
assert_eq!(assemble_page_text(&spans), "273.879.7501\n\nPopulation");
}
#[test]
fn detached_final_glyph_is_reinserted_into_word() {
let spans = vec![
span_with_width("eli", 100.0, 100.0, 15.0, 10.0, 10.0),
span_with_width("Table", 40.0, 75.0, 25.0, 10.0, 10.0),
span_with_width("t", 115.0, 100.0, 5.0, 10.0, 10.0),
];
assert_eq!(assemble_page_text(&spans), "elit\n\nTable");
}
#[test]
fn far_left_reset_starts_new_row_even_when_vertical_bands_overlap() {
let spans = vec![
span_with_width("1.000", 500.0, 100.0, 30.0, 10.0, 10.0),
span_with_width("002", 30.0, 99.0, 18.0, 10.0, 10.0),
];
assert_eq!(assemble_page_text(&spans), "1.000\n002");
}
#[test]
fn far_left_reset_does_not_split_rtl_text() {
let mut next = span_with_width("العالم", 430.0, 100.0, 35.0, 10.0, 10.0);
next.split_boundary_before = true;
let spans = vec![span_with_width("Ù…Ø±ØØ¨Ø§", 500.0, 100.0, 30.0, 10.0, 10.0), next];
assert_eq!(assemble_page_text(&spans), "Ù…Ø±ØØ¨Ø§ العالم");
}
#[test]
fn far_left_reset_respects_rtl_span_metadata_for_ascii_text() {
let mut previous = span_with_width("first", 500.0, 100.0, 30.0, 10.0, 10.0);
previous.rtl_draw_logical = true;
let mut next = span_with_width("second", 430.0, 100.0, 35.0, 10.0, 10.0);
next.rtl_draw_logical = true;
next.split_boundary_before = true;
assert_eq!(assemble_page_text(&[previous, next]), "first second");
}
#[test]
fn far_left_reset_does_not_split_ascii_numbers_on_rtl_page() {
let mut number = span_with_width("123", 500.0, 100.0, 20.0, 10.0, 10.0);
number.split_boundary_before = true;
let mut next_number = span_with_width("456", 430.0, 100.0, 20.0, 10.0, 10.0);
next_number.split_boundary_before = true;
let spans = vec![
span_with_width("Ù…Ø±ØØ¨Ø§", 570.0, 100.0, 30.0, 10.0, 10.0),
number,
next_number,
];
assert_eq!(assemble_page_text(&spans), "Ù…Ø±ØØ¨Ø§ 123 456");
}
#[test]
fn moderate_math_backtrack_does_not_start_new_row() {
let mut denominator = span_with_width("denominator", 65.0, 96.0, 55.0, 10.0, 10.0);
denominator.split_boundary_before = true;
let spans = vec![
span_with_width("numerator", 100.0, 104.0, 45.0, 10.0, 10.0),
denominator,
];
assert_eq!(assemble_page_text(&spans), "numerator denominator");
}
#[test]
fn far_left_reset_does_not_split_rotated_text() {
let mut previous = span_with_width("first", 500.0, 100.0, 30.0, 10.0, 10.0);
previous.rotation_degrees = 90.0;
let mut next = span_with_width("second", 430.0, 100.0, 35.0, 10.0, 10.0);
next.rotation_degrees = 90.0;
next.split_boundary_before = true;
assert_eq!(assemble_page_text(&[previous, next]), "first second");
}
fn rotated_span(text: &str, x: f32, y: f32, width: f32, height: f32, rotation_degrees: f32) -> TextSpan {
let mut span = span_with_width(text, x, y, width, height, height);
span.rotation_degrees = rotation_degrees;
span
}
#[test]
fn should_rejoin_detached_fragment_of_rotated_word_when_rotation_matches() {
let spans = vec![
rotated_span("Motorcraf", 400.0, 100.0, 45.0, 10.0, 90.0),
rotated_span("Premium", 400.0, 155.0, 40.0, 10.0, 90.0),
rotated_span("t", 400.0, 145.0, 5.0, 10.0, 90.0),
];
assert_eq!(assemble_page_text(&spans), "Motorcraft Premium");
}
#[test]
fn should_not_anchor_fragment_across_differing_rotations() {
let spans = vec![
span_with_width("Motorcraf", 400.0, 100.0, 45.0, 10.0, 10.0),
rotated_span("t", 445.0, 100.0, 5.0, 10.0, 90.0),
];
assert_eq!(find_inline_fragment_anchor(1, &spans, &[None, None]), None);
}
#[test]
fn should_read_rotated_table_rows_along_their_own_axis() {
let spans = vec![
rotated_span("Engine", 400.0, 100.0, 30.0, 10.0, 90.0),
rotated_span("coolant", 400.0, 132.0, 32.0, 10.0, 90.0),
rotated_span("18.6", 388.0, 100.0, 22.0, 10.0, 90.0),
rotated_span("quarts", 388.0, 124.0, 30.0, 10.0, 90.0),
];
assert_eq!(assemble_page_text(&spans), "Engine coolant\n18.6 quarts");
}
#[test]
fn should_read_rotated_body_and_upright_footer_on_same_page() {
let spans = vec![
rotated_span("Engine", 400.0, 100.0, 30.0, 10.0, 90.0),
rotated_span("coolant", 400.0, 132.0, 32.0, 10.0, 90.0),
rotated_span("18.6", 388.0, 100.0, 22.0, 10.0, 90.0),
rotated_span("quarts", 388.0, 124.0, 30.0, 10.0, 90.0),
span_with_width("Page", 60.0, 40.0, 25.0, 10.0, 10.0),
span_with_width("264", 88.0, 40.0, 15.0, 10.0, 10.0),
];
assert_eq!(assemble_page_text(&spans), "Engine coolant\n18.6 quarts\n\nPage 264");
}
#[test]
fn should_not_change_upright_page_assembly() {
let spans = vec![
span_with_width("Engine", 60.0, 700.0, 30.0, 10.0, 10.0),
span_with_width("coolant", 92.0, 700.0, 32.0, 10.0, 10.0),
span_with_width("18.6", 60.0, 688.0, 22.0, 10.0, 10.0),
span_with_width("quarts", 84.0, 688.0, 30.0, 10.0, 10.0),
span_with_width("Next", 60.0, 640.0, 25.0, 10.0, 10.0),
];
assert_eq!(assemble_page_text(&spans), "Engine coolant\n18.6 quarts\n\nNext");
}
#[test]
fn inline_fragment_anchor_rejects_non_ltr_geometry() {
let mut anchor = span_with_width("word", 100.0, 100.0, 30.0, 10.0, 10.0);
anchor.rtl_draw_logical = true;
let mut fragment = span_with_width("2", 130.0, 100.0, 3.0, 6.0, 6.0);
fragment.rtl_draw_logical = true;
let spans = vec![anchor, fragment];
assert_eq!(find_inline_fragment_anchor(1, &spans, &[None, None]), None);
}
#[test]
fn inline_fragment_anchor_search_is_local() {
let mut spans = vec![span_with_width("anchor", 100.0, 100.0, 30.0, 10.0, 10.0)];
spans.extend(
(0..=MAX_INLINE_FRAGMENT_ANCHOR_LOOKBACK)
.map(|index| span_with_width("filler", 300.0, index as f32, 30.0, 10.0, 10.0)),
);
spans.push(span_with_width("2", 130.0, 100.0, 3.0, 6.0, 6.0));
let anchors = vec![None; spans.len()];
assert_eq!(find_inline_fragment_anchor(spans.len() - 1, &spans, &anchors), None);
}
#[test]
fn split_boundary_before_forces_space_between_adjacent_spans() {
let mut next = span_with_width("002", 130.0, 100.0, 18.0, 10.0, 10.0);
next.split_boundary_before = true;
let spans = vec![span_with_width("1.000", 100.0, 100.0, 30.0, 10.0, 10.0), next];
assert_eq!(assemble_page_text(&spans), "1.000 002");
}
#[test]
fn line_local_repair_preserves_column_aware_order() {
let spans = vec![
span_with_width("left-top", 40.0, 100.0, 40.0, 10.0, 10.0),
span_with_width("left-bottom", 40.0, 80.0, 50.0, 10.0, 10.0),
span_with_width("right-top", 300.0, 100.0, 45.0, 10.0, 10.0),
span_with_width("right-bottom", 300.0, 80.0, 55.0, 10.0, 10.0),
];
assert_eq!(
assemble_page_text(&spans),
"left-top\n\nleft-bottom\n\nright-top\n\nright-bottom"
);
}
#[test]
fn sparse_two_column_prose_reorders_by_column() {
let mut spans = vec![
span_with_width("The committee reviewed the annual", 60.0, 712.0, 175.0, 11.0, 11.0),
span_with_width("approved the budget for the", 330.0, 712.0, 145.0, 11.0, 11.0),
span_with_width("report and", 60.0, 698.0, 52.0, 11.0, 11.0),
span_with_width("coming fiscal year.", 330.0, 698.0, 92.0, 11.0, 11.0),
];
assert!(reorder_sparse_two_column_page(&mut spans, 612.0));
assert_eq!(
spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>(),
[
"The committee reviewed the annual",
"report and",
"approved the budget for the",
"coming fiscal year."
]
);
}
#[test]
fn sparse_two_column_table_keeps_row_order() {
let mut spans = vec![
span_with_width(
"Regional revenue for the northern market.",
60.0,
712.0,
210.0,
11.0,
11.0,
),
span_with_width("Annual total for the current period.", 330.0, 712.0, 190.0, 11.0, 11.0),
span_with_width(
"Operating expense for the northern market.",
60.0,
698.0,
220.0,
11.0,
11.0,
),
span_with_width(
"Quarterly total for the current period.",
330.0,
698.0,
200.0,
11.0,
11.0,
),
];
let original = spans.iter().map(|span| span.text.clone()).collect::<Vec<_>>();
assert!(!reorder_sparse_two_column_page(&mut spans, 612.0));
assert_eq!(
spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>(),
original.iter().map(String::as_str).collect::<Vec<_>>()
);
}
#[test]
fn sparse_verbose_form_keeps_row_order() {
let mut spans = vec![
span_with_width(
"Account holder full legal name appears here:",
60.0,
712.0,
215.0,
11.0,
11.0,
),
span_with_width(
"Mailing address for all official correspondence:",
330.0,
712.0,
225.0,
11.0,
11.0,
),
span_with_width(
"Emergency contact relationship and telephone number:",
60.0,
698.0,
235.0,
11.0,
11.0,
),
span_with_width(
"Preferred delivery method for annual notices:",
330.0,
698.0,
215.0,
11.0,
11.0,
),
];
let original = spans.iter().map(|span| span.text.clone()).collect::<Vec<_>>();
assert!(!reorder_sparse_two_column_page(&mut spans, 612.0));
assert_eq!(
spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>(),
original.iter().map(String::as_str).collect::<Vec<_>>()
);
}
#[test]
fn sparse_lowercase_table_keeps_row_order() {
let mut spans = vec![
span_with_width(
"regional revenue for the northern market",
60.0,
712.0,
210.0,
11.0,
11.0,
),
span_with_width("annual total for the current period", 330.0, 712.0, 190.0, 11.0, 11.0),
span_with_width(
"operating expense for the northern market",
60.0,
698.0,
220.0,
11.0,
11.0,
),
span_with_width(
"quarterly total for the current period.",
330.0,
698.0,
200.0,
11.0,
11.0,
),
];
let original = spans.iter().map(|span| span.text.clone()).collect::<Vec<_>>();
assert!(!reorder_sparse_two_column_page(&mut spans, 612.0));
assert_eq!(
spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>(),
original.iter().map(String::as_str).collect::<Vec<_>>()
);
}
fn dense_two_column_spans() -> Vec<TextSpan> {
const LEFT_X: f32 = 60.0;
const RIGHT_X: f32 = 320.0;
let left_heading = span_with_width("Funding", LEFT_X, 830.0, 70.0, 11.0, 11.0);
let right_heading = span_with_width("References", RIGHT_X, 830.0, 90.0, 11.0, 11.0);
let left_body = [
"The committee reviewed annual budget totals",
"and approved new funding for the coming year",
"after several rounds of careful review by",
"senior staff members from every department",
"who evaluated priorities across the whole",
"organization before reaching a final decision",
"that reflected both short and long term goals",
"for sustainable growth across all programs",
];
let right_body = [
"Numerous studies have examined similar",
"programs across comparable institutions",
"using consistent methodology and controls",
"for measuring outcomes over multiple years",
"researchers found consistent positive trends",
"supporting continued investment going forward",
"additional citations appear in the appendix",
"for readers seeking further detail here",
];
let mut spans = vec![left_heading, right_heading];
for (row, (left_line, right_line)) in left_body.iter().copied().zip(right_body.iter().copied()).enumerate() {
let y = 816.0 - row as f32 * 14.0;
spans.push(span_with_width(left_line, LEFT_X, y, 200.0, 11.0, 11.0));
spans.push(span_with_width(right_line, RIGHT_X, y, 190.0, 11.0, 11.0));
}
spans
}
#[test]
fn dense_two_column_prose_reorders_by_column() {
let mut spans = dense_two_column_spans();
assert!(reorder_dense_two_column_page(&mut spans, 612.0));
let texts = spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>();
assert_eq!(
texts,
[
"Funding",
"The committee reviewed annual budget totals",
"and approved new funding for the coming year",
"after several rounds of careful review by",
"senior staff members from every department",
"who evaluated priorities across the whole",
"organization before reaching a final decision",
"that reflected both short and long term goals",
"for sustainable growth across all programs",
"References",
"Numerous studies have examined similar",
"programs across comparable institutions",
"using consistent methodology and controls",
"for measuring outcomes over multiple years",
"researchers found consistent positive trends",
"supporting continued investment going forward",
"additional citations appear in the appendix",
"for readers seeking further detail here",
]
);
}
const GH1484_PAGE_WIDTH: f32 = 595.0;
const GH1484_RIGHT_NUMBER_X: f32 = 304.87;
fn dense_two_column_hanging_number_spans() -> Vec<TextSpan> {
const LEFT_NUMBER_X: f32 = 36.0;
const LEFT_TEXT_X: f32 = 64.34;
const LEFT_TEXT_WIDTH: f32 = 226.8;
const RIGHT_TEXT_X: f32 = 333.19;
const ROW_COUNT: usize = 12;
let mut spans = Vec::new();
for row in 0..ROW_COUNT {
let y = 816.0 - row as f32 * 14.0;
if row.is_multiple_of(2) {
spans.push(span_with_width(
&format!("15.{}", row / 2 + 1),
LEFT_NUMBER_X,
y,
17.84,
11.0,
11.0,
));
}
spans.push(span_with_width(
&format!("The left clause line {row} continues with ordinary agreement terms"),
LEFT_TEXT_X,
y,
LEFT_TEXT_WIDTH,
11.0,
11.0,
));
if row.is_multiple_of(2) {
spans.push(span_with_width(
&format!("16.{}", row / 2 + 5),
GH1484_RIGHT_NUMBER_X,
y,
17.84,
11.0,
11.0,
));
}
spans.push(span_with_width(
&format!("The right clause line {row} continues with ordinary agreement terms"),
RIGHT_TEXT_X,
y,
220.0,
11.0,
11.0,
));
}
spans
}
#[test]
fn dense_two_column_hanging_numbers_reorder_by_column() {
let mut spans = dense_two_column_hanging_number_spans();
let expected = spans
.iter()
.filter(|span| span.bbox.x < GH1484_RIGHT_NUMBER_X)
.chain(spans.iter().filter(|span| span.bbox.x >= GH1484_RIGHT_NUMBER_X))
.map(|span| span.text.clone())
.collect::<Vec<_>>();
let order = spans_sorted_top_to_bottom(&spans);
let lines = group_into_lines(&spans, &order);
let detected_split = detect_split_x(&spans, &lines, GH1484_PAGE_WIDTH).expect("numbered page has a gutter");
assert!(detected_split > GH1484_RIGHT_NUMBER_X && detected_split < GH1484_RIGHT_NUMBER_X + 17.84);
assert_eq!(
snap_split_left_of_hanging_labels(&spans, &lines, GH1484_PAGE_WIDTH, detected_split),
GH1484_RIGHT_NUMBER_X
);
assert!(reorder_dense_two_column_page(&mut spans, GH1484_PAGE_WIDTH));
assert_eq!(
spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>(),
expected.iter().map(String::as_str).collect::<Vec<_>>()
);
}
#[test]
fn dense_two_column_unnumbered_control_keeps_detected_split() {
const PAGE_WIDTH: f32 = 612.0;
let spans = dense_two_column_spans();
let order = spans_sorted_top_to_bottom(&spans);
let lines = group_into_lines(&spans, &order);
let detected_split = detect_split_x(&spans, &lines, PAGE_WIDTH).expect("control has a gutter");
assert_eq!(
snap_split_left_of_hanging_labels(&spans, &lines, PAGE_WIDTH, detected_split),
detected_split
);
}
#[test]
fn dense_column_split_does_not_snap_to_furniture_or_one_off_label() {
const PAGE_WIDTH: f32 = 595.0;
const SPLIT_X: f32 = 300.0;
let mut furniture = (0..MIN_DENSE_COLUMN_SPLIT_LINES)
.map(|row| span_with_width("centred furniture", 250.0, 800.0 - row as f32 * 14.0, 100.0, 11.0, 11.0))
.collect::<Vec<_>>();
furniture.push(span_with_width("note", 295.0, 700.0, 20.0, 11.0, 11.0));
let order = spans_sorted_top_to_bottom(&furniture);
let lines = group_into_lines(&furniture, &order);
assert_eq!(
snap_split_left_of_hanging_labels(&furniture, &lines, PAGE_WIDTH, SPLIT_X),
SPLIT_X
);
}
#[test]
fn dense_column_split_repeats_when_first_snap_reveals_another_fragment() {
const PAGE_WIDTH: f32 = 595.0;
const INITIAL_SPLIT_X: f32 = 305.64;
const FIRST_FRAGMENT_LEFT: f32 = 304.87;
const SECOND_FRAGMENT_LEFT: f32 = 300.0;
let mut spans = Vec::new();
for row in 0..MIN_DENSE_COLUMN_SPLIT_LINES {
let y = 800.0 - row as f32 * 14.0;
spans.push(span_with_width("prefix", SECOND_FRAGMENT_LEFT, y, 5.2, 11.0, 11.0));
spans.push(span_with_width("number", FIRST_FRAGMENT_LEFT, y, 17.84, 11.0, 11.0));
}
let order = spans_sorted_top_to_bottom(&spans);
let lines = group_into_lines(&spans, &order);
let max_snap_width = PAGE_WIDTH * MAX_DENSE_COLUMN_SPLIT_SNAP_SPAN_FRACTION;
assert_eq!(
aligned_hanging_label_left_edge(&spans, &lines, max_snap_width, INITIAL_SPLIT_X),
Some(FIRST_FRAGMENT_LEFT)
);
assert_eq!(
aligned_hanging_label_left_edge(&spans, &lines, max_snap_width, FIRST_FRAGMENT_LEFT),
Some(SECOND_FRAGMENT_LEFT)
);
assert_eq!(
snap_split_left_of_hanging_labels(&spans, &lines, PAGE_WIDTH, INITIAL_SPLIT_X),
SECOND_FRAGMENT_LEFT
);
}
#[test]
fn dense_two_column_prose_assembles_without_interleaving_or_heading_weld() {
let mut spans = dense_two_column_spans();
assert!(reorder_dense_two_column_page(&mut spans, 612.0));
assert_eq!(
assemble_page_text(&spans),
"Funding\n\
The committee reviewed annual budget totals\n\
and approved new funding for the coming year\n\
after several rounds of careful review by\n\
senior staff members from every department\n\
who evaluated priorities across the whole\n\
organization before reaching a final decision\n\
that reflected both short and long term goals\n\
for sustainable growth across all programs\n\n\
References\n\
Numerous studies have examined similar\n\
programs across comparable institutions\n\
using consistent methodology and controls\n\
for measuring outcomes over multiple years\n\
researchers found consistent positive trends\n\
supporting continued investment going forward\n\
additional citations appear in the appendix\n\
for readers seeking further detail here"
);
}
#[test]
fn dense_two_column_table_keeps_row_order() {
const LEFT_X: f32 = 60.0;
const RIGHT_X: f32 = 320.0;
let left_body = [
"The committee reviewed annual budget totals",
"and approved new funding for the coming year",
"after several rounds of careful review by",
"senior staff members from every department",
"who evaluated priorities across the whole",
"organization before reaching a final decision",
"that reflected both short and long term goals",
"for sustainable growth across all programs",
];
let right_cells = ["12.3", "45.6", "78.9", "10.1", "21.2", "33.4", "45.5", "67.8"];
let mut spans = Vec::new();
for (row, (left_line, right_cell)) in left_body.iter().copied().zip(right_cells.iter().copied()).enumerate() {
let y = 816.0 - row as f32 * 14.0;
spans.push(span_with_width(left_line, LEFT_X, y, 200.0, 11.0, 11.0));
spans.push(span_with_width(right_cell, RIGHT_X, y, 30.0, 11.0, 11.0));
}
let original = spans.iter().map(|span| span.text.clone()).collect::<Vec<_>>();
assert!(!reorder_dense_two_column_page(&mut spans, 612.0));
assert_eq!(
spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>(),
original.iter().map(String::as_str).collect::<Vec<_>>()
);
}
#[test]
fn dense_two_column_prose_reorders_around_header_and_footer() {
let mut spans = vec![span_with_width(
"Quarterly Report - Internal Distribution Only",
60.0,
850.0,
497.0,
11.0,
11.0,
)];
spans.extend(dense_two_column_spans());
spans.push(span_with_width("Page 1 of 12", 60.0, 700.0, 497.0, 11.0, 11.0));
assert!(reorder_dense_two_column_page(&mut spans, 612.0));
let texts = spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>();
assert_eq!(
texts,
[
"Quarterly Report - Internal Distribution Only",
"Funding",
"The committee reviewed annual budget totals",
"and approved new funding for the coming year",
"after several rounds of careful review by",
"senior staff members from every department",
"who evaluated priorities across the whole",
"organization before reaching a final decision",
"that reflected both short and long term goals",
"for sustainable growth across all programs",
"References",
"Numerous studies have examined similar",
"programs across comparable institutions",
"using consistent methodology and controls",
"for measuring outcomes over multiple years",
"researchers found consistent positive trends",
"supporting continued investment going forward",
"additional citations appear in the appendix",
"for readers seeking further detail here",
"Page 1 of 12",
]
);
}
#[test]
fn dense_two_column_prose_keeps_midpage_heading_above_both_columns() {
let mut spans = vec![span_with_width(
"Annual Committee Findings",
60.0,
840.0,
497.0,
11.0,
11.0,
)];
spans.extend(dense_two_column_spans());
assert!(reorder_dense_two_column_page(&mut spans, 612.0));
let texts = spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>();
assert_eq!(texts[0], "Annual Committee Findings");
let heading_index = 0;
let funding_index = texts.iter().position(|&text| text == "Funding").unwrap();
let references_index = texts.iter().position(|&text| text == "References").unwrap();
assert!(heading_index < funding_index && heading_index < references_index);
}
fn two_column_band(row_count: usize, y_start: f32, label: &str) -> Vec<TextSpan> {
const LEFT_X: f32 = 60.0;
const RIGHT_X: f32 = 320.0;
let mut spans = Vec::with_capacity(row_count * 2);
for row in 0..row_count {
let y = y_start - row as f32 * 14.0;
let left_text = format!("The {label} left column continues with sentence number {row} of the report");
let right_text = format!("The {label} right column continues with sentence number {row} of the report");
spans.push(span_with_width(&left_text, LEFT_X, y, 200.0, 11.0, 11.0));
spans.push(span_with_width(&right_text, RIGHT_X, y, 190.0, 11.0, 11.0));
}
spans
}
fn assert_bands_reordered_around_furniture(spans: &mut [TextSpan], furniture_text: &str, rows_per_band: usize) {
assert!(reorder_dense_two_column_page(spans, 612.0));
let texts = spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>();
let furniture_index = texts.iter().position(|&text| text == furniture_text).unwrap();
assert_eq!(
furniture_index,
rows_per_band * 2,
"furniture must land strictly after the whole band above it"
);
assert_eq!(texts.len(), rows_per_band * 4 + 1);
for row in 0..rows_per_band {
assert_eq!(
texts[row],
format!("The first left column continues with sentence number {row} of the report")
);
assert_eq!(
texts[rows_per_band + row],
format!("The first right column continues with sentence number {row} of the report")
);
}
let below_start = furniture_index + 1;
for row in 0..rows_per_band {
assert_eq!(
texts[below_start + row],
format!("The second left column continues with sentence number {row} of the report")
);
assert_eq!(
texts[below_start + rows_per_band + row],
format!("The second right column continues with sentence number {row} of the report")
);
}
}
#[test]
fn dense_two_column_prose_reorders_around_midpage_banner() {
const ROWS_PER_BAND: usize = 7;
const PAGE_WIDTH: f32 = 612.0;
const BANNER_TEXT: &str = "Quarterly Report - Company Wide Distribution Banner";
let band_above = two_column_band(ROWS_PER_BAND, 830.0, "first");
let band_below = two_column_band(ROWS_PER_BAND, 830.0 - ROWS_PER_BAND as f32 * 14.0 - 20.0, "second");
let banner_y = 830.0 - (ROWS_PER_BAND as f32 - 1.0) * 14.0 - 10.0;
let banner_width = PAGE_WIDTH * 0.66;
let banner_x = (PAGE_WIDTH - banner_width) / 2.0;
let mut spans = band_above;
spans.push(span_with_width(
BANNER_TEXT,
banner_x,
banner_y,
banner_width,
11.0,
11.0,
));
spans.extend(band_below);
assert_bands_reordered_around_furniture(&mut spans, BANNER_TEXT, ROWS_PER_BAND);
}
#[test]
fn dense_two_column_prose_reorders_around_narrow_gutter_crossing_rule() {
const ROWS_PER_BAND: usize = 7;
const PAGE_WIDTH: f32 = 612.0;
const RULE_TEXT: &str = "----------";
let band_above = two_column_band(ROWS_PER_BAND, 830.0, "first");
let band_below = two_column_band(ROWS_PER_BAND, 830.0 - ROWS_PER_BAND as f32 * 14.0 - 20.0, "second");
let rule_y = 830.0 - (ROWS_PER_BAND as f32 - 1.0) * 14.0 - 10.0;
let rule_width = PAGE_WIDTH * 0.30;
let mut spans = band_above;
spans.push(span_with_width(RULE_TEXT, 200.0, rule_y, rule_width, 2.0, 2.0));
spans.extend(band_below);
assert_bands_reordered_around_furniture(&mut spans, RULE_TEXT, ROWS_PER_BAND);
}
#[test]
fn single_column_page_with_wide_and_narrow_lines_is_not_split() {
const COLUMN_X: f32 = 60.0;
let lines: [(&str, f32); 8] = [
("This is a long justified line of body text filling", 470.0),
("the page width almost completely from margin", 470.0),
("to margin, as ordinary single-column prose does", 470.0),
("Short line.", 90.0),
("Another full-width line of ordinary body text here", 470.0),
("Brief.", 90.0),
("A further wide line completing this single paragraph", 470.0),
("End.", 90.0),
];
let mut spans = Vec::new();
for (row, (text, width)) in lines.iter().enumerate() {
let y = 800.0 - row as f32 * 14.0;
spans.push(span_with_width(text, COLUMN_X, y, *width, 11.0, 11.0));
}
let original = spans.iter().map(|span| span.text.clone()).collect::<Vec<_>>();
assert!(!reorder_dense_two_column_page(&mut spans, 612.0));
assert_eq!(
spans.iter().map(|span| span.text.as_str()).collect::<Vec<_>>(),
original.iter().map(String::as_str).collect::<Vec<_>>()
);
}
#[test]
#[expect(
clippy::too_many_lines,
reason = "one span literal per pdftotext word, transcribed verbatim for fidelity"
)]
fn gh1545_table_beside_prose_emits_table_then_prose() {
const PAGE_WIDTH: f32 = 595.0;
const IDEAL_TABLE_PROSE_SPLIT_X: f32 = 295.0;
const TABLE_RIGHT_EDGE_X: f32 = 285.622;
const PROSE_LEFT_EDGE_X: f32 = 303.600;
const PANEL_B_LEFT_EDGE_X: f32 = 164.803;
const TITLE_ROW_Y: f32 = 730.0;
#[rustfmt::skip]
let mut spans = vec![
span_with_width("Table", 47.700, 735.026, 17.507, 6.475, 6.475),
span_with_width("1", 67.153, 735.026, 3.892, 6.475, 6.475),
span_with_width("Sample", 72.991, 735.026, 23.730, 6.475, 6.475),
span_with_width("characteristics", 98.667, 735.026, 44.730, 6.475, 6.475),
span_with_width("of", 145.343, 735.026, 5.838, 6.475, 6.475),
span_with_width("the", 153.127, 735.026, 9.730, 6.475, 6.475),
span_with_width("Northfield", 164.803, 735.026, 29.953, 6.475, 6.475),
span_with_width("and", 196.702, 735.026, 11.676, 6.475, 6.475),
span_with_width("Eastgate", 210.324, 735.026, 27.629, 6.475, 6.475),
span_with_width("cohorts.", 239.899, 735.026, 24.899, 6.475, 6.475),
span_with_width("Sex", 47.700, 718.926, 12.061, 6.475, 6.475),
span_with_width("Female", 47.700, 710.876, 23.338, 6.475, 6.475),
span_with_width("Male", 47.700, 702.826, 15.169, 6.475, 6.475),
span_with_width("Age", 47.700, 694.776, 12.453, 6.475, 6.475),
span_with_width("18-24", 62.099, 694.776, 17.899, 6.475, 6.475),
span_with_width("25-34", 47.700, 686.726, 17.899, 6.475, 6.475),
span_with_width("35-44", 47.700, 678.676, 17.899, 6.475, 6.475),
span_with_width("45-54", 47.700, 670.626, 17.899, 6.475, 6.475),
span_with_width("55-64", 47.700, 662.576, 17.899, 6.475, 6.475),
span_with_width("Ethnicity", 47.700, 654.526, 26.453, 6.475, 6.475),
span_with_width("Group", 47.700, 646.476, 19.453, 6.475, 6.475),
span_with_width("one", 69.099, 646.476, 11.676, 6.475, 6.475),
span_with_width("Group", 47.700, 638.426, 19.453, 6.475, 6.475),
span_with_width("two", 69.099, 638.426, 10.892, 6.475, 6.475),
span_with_width("Group", 47.700, 630.376, 19.453, 6.475, 6.475),
span_with_width("three", 69.099, 630.376, 15.953, 6.475, 6.475),
span_with_width("Group", 47.700, 622.326, 19.453, 6.475, 6.475),
span_with_width("four", 69.099, 622.326, 12.061, 6.475, 6.475),
span_with_width("Group", 47.700, 614.276, 19.453, 6.475, 6.475),
span_with_width("five", 69.099, 614.276, 10.892, 6.475, 6.475),
span_with_width("Living", 47.700, 606.226, 18.284, 6.475, 6.475),
span_with_width("location", 67.930, 606.226, 24.122, 6.475, 6.475),
span_with_width("City", 47.700, 598.176, 12.054, 6.475, 6.475),
span_with_width("Suburb", 47.700, 590.126, 22.568, 6.475, 6.475),
span_with_width("Town", 47.700, 582.076, 17.115, 6.475, 6.475),
span_with_width("Rural", 47.700, 574.026, 16.723, 6.475, 6.475),
span_with_width("Highest", 47.700, 565.976, 23.730, 6.475, 6.475),
span_with_width("education", 73.376, 565.976, 30.352, 6.475, 6.475),
span_with_width("No", 47.700, 557.926, 8.946, 6.475, 6.475),
span_with_width("qualifications", 58.592, 557.926, 40.460, 6.475, 6.475),
span_with_width("Secondary", 47.700, 549.876, 33.460, 6.475, 6.475),
span_with_width("school", 83.106, 549.876, 20.230, 6.475, 6.475),
span_with_width("Diploma", 47.700, 541.826, 25.669, 6.475, 6.475),
span_with_width("Undergraduate", 47.700, 533.776, 46.690, 6.475, 6.475),
span_with_width("degree", 96.336, 533.776, 21.791, 6.475, 6.475),
span_with_width("Postgraduate", 47.700, 525.726, 41.636, 6.475, 6.475),
span_with_width("degree", 91.282, 525.726, 21.791, 6.475, 6.475),
span_with_width("Employment", 47.700, 517.676, 38.899, 6.475, 6.475),
span_with_width("status", 88.545, 517.676, 18.676, 6.475, 6.475),
span_with_width("Full-time", 47.700, 509.626, 26.831, 6.475, 6.475),
span_with_width("employed", 76.477, 509.626, 30.345, 6.475, 6.475),
span_with_width("Part-time", 47.700, 501.576, 28.392, 6.475, 6.475),
span_with_width("employed", 78.038, 501.576, 30.345, 6.475, 6.475),
span_with_width("Retired", 47.700, 493.526, 22.561, 6.475, 6.475),
span_with_width("Not", 47.700, 485.476, 10.892, 6.475, 6.475),
span_with_width("employed", 60.538, 485.476, 30.345, 6.475, 6.475),
span_with_width("%", 148.100, 718.926, 6.223, 6.475, 6.475),
span_with_width("Sex", 165.000, 718.926, 12.061, 6.475, 6.475),
span_with_width("51.5", 148.100, 710.876, 13.622, 6.475, 6.475),
span_with_width("Female", 165.000, 710.876, 23.338, 6.475, 6.475),
span_with_width("48.2", 148.100, 702.826, 13.622, 6.475, 6.475),
span_with_width("Male", 165.000, 702.826, 15.169, 6.475, 6.475),
span_with_width("11.1", 148.100, 694.776, 13.622, 6.475, 6.475),
span_with_width("Age", 165.000, 694.776, 12.453, 6.475, 6.475),
span_with_width("18-24", 179.399, 694.776, 17.899, 6.475, 6.475),
span_with_width("19.2", 148.100, 686.726, 13.622, 6.475, 6.475),
span_with_width("25-34", 165.000, 686.726, 17.899, 6.475, 6.475),
span_with_width("20.6", 148.100, 678.676, 13.622, 6.475, 6.475),
span_with_width("35-44", 165.000, 678.676, 17.899, 6.475, 6.475),
span_with_width("15.9", 148.100, 670.626, 13.622, 6.475, 6.475),
span_with_width("45-54", 165.000, 670.626, 17.899, 6.475, 6.475),
span_with_width("21.0", 148.100, 662.576, 13.622, 6.475, 6.475),
span_with_width("55-64", 165.000, 662.576, 17.899, 6.475, 6.475),
span_with_width("%", 272.000, 718.926, 6.223, 6.475, 6.475),
span_with_width("51.7", 272.000, 710.876, 13.622, 6.475, 6.475),
span_with_width("48.3", 272.000, 702.826, 13.622, 6.475, 6.475),
span_with_width("12.1", 272.000, 694.776, 13.622, 6.475, 6.475),
span_with_width("18.8", 272.000, 686.726, 13.622, 6.475, 6.475),
span_with_width("17.4", 272.000, 678.676, 13.622, 6.475, 6.475),
span_with_width("20.2", 272.000, 670.626, 13.622, 6.475, 6.475),
span_with_width("17.2", 272.000, 662.576, 13.622, 6.475, 6.475),
span_with_width("17.3", 148.100, 646.476, 13.622, 6.475, 6.475),
span_with_width("Group", 165.000, 646.476, 19.453, 6.475, 6.475),
span_with_width("one", 186.399, 646.476, 11.676, 6.475, 6.475),
span_with_width("1.9", 148.100, 638.426, 9.730, 6.475, 6.475),
span_with_width("Group", 165.000, 638.426, 19.453, 6.475, 6.475),
span_with_width("two", 186.399, 638.426, 10.892, 6.475, 6.475),
span_with_width("0.3", 148.100, 630.376, 9.730, 6.475, 6.475),
span_with_width("Group", 165.000, 630.376, 19.453, 6.475, 6.475),
span_with_width("three", 186.399, 630.376, 15.953, 6.475, 6.475),
span_with_width("0.4", 148.100, 622.326, 9.730, 6.475, 6.475),
span_with_width("Group", 165.000, 622.326, 19.453, 6.475, 6.475),
span_with_width("four", 186.399, 622.326, 12.061, 6.475, 6.475),
span_with_width("3.2", 148.100, 614.276, 9.730, 6.475, 6.475),
span_with_width("Group", 165.000, 614.276, 19.453, 6.475, 6.475),
span_with_width("five", 186.399, 614.276, 10.892, 6.475, 6.475),
span_with_width("14.2", 272.000, 646.476, 13.622, 6.475, 6.475),
span_with_width("2.4", 272.000, 638.426, 9.730, 6.475, 6.475),
span_with_width("0.6", 272.000, 630.376, 9.730, 6.475, 6.475),
span_with_width("1.1", 272.000, 622.326, 9.730, 6.475, 6.475),
span_with_width("2.8", 272.000, 614.276, 9.730, 6.475, 6.475),
span_with_width("24.5", 148.100, 598.176, 13.622, 6.475, 6.475),
span_with_width("City", 165.000, 598.176, 12.054, 6.475, 6.475),
span_with_width("18.1", 148.100, 590.126, 13.622, 6.475, 6.475),
span_with_width("Suburb", 165.000, 590.126, 22.568, 6.475, 6.475),
span_with_width("26.8", 148.100, 582.076, 13.622, 6.475, 6.475),
span_with_width("Town", 165.000, 582.076, 17.115, 6.475, 6.475),
span_with_width("28.8", 148.100, 574.026, 13.622, 6.475, 6.475),
span_with_width("Rural", 165.000, 574.026, 16.723, 6.475, 6.475),
span_with_width("26.1", 272.000, 598.176, 13.622, 6.475, 6.475),
span_with_width("19.4", 272.000, 590.126, 13.622, 6.475, 6.475),
span_with_width("24.9", 272.000, 582.076, 13.622, 6.475, 6.475),
span_with_width("29.6", 272.000, 574.026, 13.622, 6.475, 6.475),
span_with_width("1.2", 148.100, 557.926, 9.730, 6.475, 6.475),
span_with_width("No", 165.000, 557.926, 8.946, 6.475, 6.475),
span_with_width("qualifications", 175.892, 557.926, 40.460, 6.475, 6.475),
span_with_width("6.4", 148.100, 549.876, 9.730, 6.475, 6.475),
span_with_width("Secondary", 165.000, 549.876, 33.460, 6.475, 6.475),
span_with_width("school", 200.406, 549.876, 20.230, 6.475, 6.475),
span_with_width("22.5", 148.100, 541.826, 13.622, 6.475, 6.475),
span_with_width("Diploma", 165.000, 541.826, 25.669, 6.475, 6.475),
span_with_width("19.8", 148.100, 533.776, 13.622, 6.475, 6.475),
span_with_width("Undergraduate", 165.000, 533.776, 46.690, 6.475, 6.475),
span_with_width("degree", 213.636, 533.776, 21.791, 6.475, 6.475),
span_with_width("27.9", 148.100, 525.726, 13.622, 6.475, 6.475),
span_with_width("Postgraduate", 165.000, 525.726, 41.636, 6.475, 6.475),
span_with_width("degree", 208.582, 525.726, 21.791, 6.475, 6.475),
span_with_width("1.8", 272.000, 557.926, 9.730, 6.475, 6.475),
span_with_width("7.1", 272.000, 549.876, 9.730, 6.475, 6.475),
span_with_width("21.8", 272.000, 541.826, 13.622, 6.475, 6.475),
span_with_width("20.4", 272.000, 533.776, 13.622, 6.475, 6.475),
span_with_width("26.3", 272.000, 525.726, 13.622, 6.475, 6.475),
span_with_width("43.3", 148.100, 509.626, 13.622, 6.475, 6.475),
span_with_width("Full-time", 165.000, 509.626, 26.831, 6.475, 6.475),
span_with_width("employed", 193.777, 509.626, 30.345, 6.475, 6.475),
span_with_width("15.7", 148.100, 501.576, 13.622, 6.475, 6.475),
span_with_width("Part-time", 165.000, 501.576, 28.392, 6.475, 6.475),
span_with_width("employed", 195.338, 501.576, 30.345, 6.475, 6.475),
span_with_width("15.0", 148.100, 493.526, 13.622, 6.475, 6.475),
span_with_width("Retired", 165.000, 493.526, 22.561, 6.475, 6.475),
span_with_width("8.4", 148.100, 485.476, 9.730, 6.475, 6.475),
span_with_width("Not", 165.000, 485.476, 10.892, 6.475, 6.475),
span_with_width("employed", 177.838, 485.476, 30.345, 6.475, 6.475),
span_with_width("41.9", 272.000, 509.626, 13.622, 6.475, 6.475),
span_with_width("16.4", 272.000, 501.576, 13.622, 6.475, 6.475),
span_with_width("14.6", 272.000, 493.526, 13.622, 6.475, 6.475),
span_with_width("9.2", 272.000, 485.476, 9.730, 6.475, 6.475),
span_with_width("Participants", 303.600, 736.103, 44.404, 7.862, 7.862),
span_with_width("in", 350.367, 736.103, 6.613, 7.862, 7.862),
span_with_width("the", 359.343, 736.103, 11.815, 7.862, 7.862),
span_with_width("Northfield", 373.521, 736.103, 36.371, 7.862, 7.862),
span_with_width("cohort", 412.255, 736.103, 23.622, 7.862, 7.862),
span_with_width("who", 438.240, 736.103, 15.589, 7.862, 7.862),
span_with_width("reported", 456.192, 736.103, 31.654, 7.862, 7.862),
span_with_width("low", 490.209, 736.103, 12.750, 7.862, 7.862),
span_with_width("confidence", 303.600, 725.653, 41.106, 7.863, 7.863),
span_with_width("in", 347.069, 725.653, 6.613, 7.863, 7.863),
span_with_width("the", 356.045, 725.653, 11.815, 7.863, 7.863),
span_with_width("programme", 370.223, 725.653, 43.452, 7.863, 7.863),
span_with_width("were,", 416.038, 725.653, 20.782, 7.863, 7.863),
span_with_width("compared", 439.183, 725.653, 37.791, 7.863, 7.863),
span_with_width("with", 479.337, 725.653, 15.113, 7.863, 7.863),
span_with_width("those", 496.813, 725.653, 20.791, 7.863, 7.863),
span_with_width("who", 303.600, 715.203, 15.589, 7.863, 7.863),
span_with_width("reported", 321.552, 715.203, 31.654, 7.863, 7.863),
span_with_width("high", 355.569, 715.203, 16.065, 7.863, 7.863),
span_with_width("confidence,", 373.997, 715.203, 43.469, 7.863, 7.863),
span_with_width("more", 419.829, 715.203, 19.363, 7.863, 7.863),
span_with_width("likely", 441.555, 715.203, 18.887, 7.863, 7.863),
span_with_width("to", 462.805, 715.203, 7.089, 7.863, 7.863),
span_with_width("be", 472.257, 715.203, 9.452, 7.863, 7.863),
span_with_width("aged", 484.072, 715.203, 18.904, 7.863, 7.863),
span_with_width("35", 505.339, 715.203, 9.452, 7.863, 7.863),
span_with_width("to", 303.600, 704.753, 7.089, 7.862, 7.862),
span_with_width("44", 313.052, 704.753, 9.452, 7.862, 7.862),
span_with_width("years,", 324.867, 704.753, 23.145, 7.862, 7.862),
span_with_width("to", 350.375, 704.753, 7.089, 7.862, 7.862),
span_with_width("live", 359.827, 704.753, 12.750, 7.862, 7.862),
span_with_width("in", 374.940, 704.753, 6.613, 7.862, 7.862),
span_with_width("a", 383.916, 704.753, 4.726, 7.862, 7.862),
span_with_width("city,", 391.005, 704.753, 15.113, 7.862, 7.862),
span_with_width("to", 408.481, 704.753, 7.089, 7.862, 7.862),
span_with_width("hold", 417.933, 704.753, 16.065, 7.862, 7.862),
span_with_width("no", 436.361, 704.753, 9.452, 7.862, 7.862),
span_with_width("post-school", 448.176, 704.753, 43.461, 7.862, 7.862),
span_with_width("qualification,", 303.600, 694.303, 47.243, 7.863, 7.863),
span_with_width("and", 353.206, 694.303, 14.178, 7.863, 7.863),
span_with_width("to", 369.747, 694.303, 7.089, 7.863, 7.863),
span_with_width("report", 379.199, 694.303, 22.202, 7.863, 7.863),
span_with_width("that", 403.764, 694.303, 14.178, 7.863, 7.863),
span_with_width("they", 420.305, 694.303, 16.065, 7.863, 7.863),
span_with_width("had", 438.733, 694.303, 14.178, 7.863, 7.863),
span_with_width("not", 455.274, 694.303, 11.815, 7.863, 7.863),
span_with_width("voted", 469.452, 694.303, 20.791, 7.863, 7.863),
span_with_width("at", 492.606, 694.303, 7.089, 7.863, 7.863),
span_with_width("the", 303.600, 683.853, 11.815, 7.863, 7.863),
span_with_width("most", 317.778, 683.853, 18.419, 7.863, 7.863),
span_with_width("recent", 338.560, 683.853, 23.622, 7.863, 7.863),
span_with_width("municipal", 364.545, 683.853, 35.895, 7.863, 7.863),
span_with_width("election.", 402.803, 683.853, 31.654, 7.863, 7.863),
span_with_width("The", 436.820, 683.853, 14.646, 7.863, 7.863),
span_with_width("same", 453.829, 683.853, 20.782, 7.863, 7.863),
span_with_width("pattern", 476.974, 683.853, 26.461, 7.863, 7.863),
span_with_width("was", 505.798, 683.853, 15.113, 7.863, 7.863),
span_with_width("not", 303.600, 673.403, 11.815, 7.862, 7.862),
span_with_width("observed", 317.778, 673.403, 34.960, 7.862, 7.862),
span_with_width("in", 355.101, 673.403, 6.613, 7.862, 7.862),
span_with_width("the", 364.077, 673.403, 11.815, 7.862, 7.862),
span_with_width("Eastgate", 378.255, 673.403, 33.550, 7.862, 7.862),
span_with_width("cohort,", 414.168, 673.403, 25.984, 7.862, 7.862),
span_with_width("where", 442.515, 673.403, 23.146, 7.862, 7.862),
span_with_width("the", 468.024, 673.403, 11.815, 7.862, 7.862),
span_with_width("strongest", 482.202, 673.403, 34.961, 7.862, 7.862),
span_with_width("association", 303.600, 662.953, 42.517, 7.863, 7.863),
span_with_width("was", 348.480, 662.953, 15.113, 7.863, 7.863),
span_with_width("with", 365.956, 662.953, 15.113, 7.863, 7.863),
span_with_width("employment", 383.432, 662.953, 46.291, 7.863, 7.863),
span_with_width("status", 432.086, 662.953, 22.678, 7.863, 7.863),
span_with_width("rather", 457.127, 662.953, 22.202, 7.863, 7.863),
span_with_width("than", 481.692, 662.953, 16.541, 7.863, 7.863),
span_with_width("with", 500.596, 662.953, 15.113, 7.863, 7.863),
span_with_width("age", 303.600, 652.503, 14.178, 7.862, 7.862),
span_with_width("or", 320.141, 652.503, 7.556, 7.862, 7.862),
span_with_width("education.", 330.060, 652.503, 39.219, 7.862, 7.862),
span_with_width("Full", 371.642, 652.503, 13.694, 7.862, 7.862),
span_with_width("model", 387.699, 652.503, 23.145, 7.862, 7.862),
span_with_width("output", 413.207, 652.503, 23.630, 7.862, 7.862),
span_with_width("for", 439.200, 652.503, 9.920, 7.862, 7.862),
span_with_width("both", 451.483, 652.503, 16.541, 7.862, 7.862),
span_with_width("cohorts", 470.387, 652.503, 27.872, 7.862, 7.862),
span_with_width("is", 500.622, 652.503, 6.137, 7.862, 7.862),
span_with_width("given", 303.600, 642.053, 20.315, 7.863, 7.863),
span_with_width("in", 326.278, 642.053, 6.613, 7.863, 7.863),
span_with_width("Tables", 335.254, 642.053, 25.508, 7.863, 7.863),
span_with_width("2", 363.125, 642.053, 4.726, 7.863, 7.863),
span_with_width("and", 370.214, 642.053, 14.178, 7.863, 7.863),
span_with_width("3.", 386.755, 642.053, 7.089, 7.863, 7.863),
span_with_width("Percentages", 396.207, 642.053, 47.719, 7.863, 7.863),
span_with_width("in", 446.289, 642.053, 6.613, 7.863, 7.863),
span_with_width("Table", 455.265, 642.053, 21.259, 7.863, 7.863),
span_with_width("1", 478.887, 642.053, 4.726, 7.863, 7.863),
span_with_width("are", 485.976, 642.053, 12.283, 7.863, 7.863),
span_with_width("column", 303.600, 631.603, 27.395, 7.863, 7.863),
span_with_width("percentages", 333.358, 631.603, 46.776, 7.863, 7.863),
span_with_width("and", 382.497, 631.603, 14.178, 7.863, 7.863),
span_with_width("may", 399.038, 631.603, 16.056, 7.863, 7.863),
span_with_width("not", 417.457, 631.603, 11.815, 7.863, 7.863),
span_with_width("sum", 431.635, 631.603, 16.057, 7.863, 7.863),
span_with_width("to", 450.055, 631.603, 7.089, 7.863, 7.863),
span_with_width("one", 459.507, 631.603, 14.178, 7.863, 7.863),
span_with_width("hundred", 476.048, 631.603, 31.187, 7.863, 7.863),
span_with_width("where", 509.598, 631.603, 23.146, 7.863, 7.863),
span_with_width("a", 303.600, 621.153, 4.726, 7.862, 7.862),
span_with_width("category", 310.689, 621.153, 32.597, 7.862, 7.862),
span_with_width("was", 345.649, 621.153, 15.113, 7.862, 7.862),
span_with_width("left", 363.125, 621.153, 11.339, 7.862, 7.862),
span_with_width("blank", 376.827, 621.153, 20.315, 7.862, 7.862),
span_with_width("by", 399.505, 621.153, 8.976, 7.862, 7.862),
span_with_width("the", 410.844, 621.153, 11.815, 7.862, 7.862),
span_with_width("respondent.", 425.022, 621.153, 44.889, 7.862, 7.862),
span_with_width("Weighting", 472.274, 621.153, 37.791, 7.862, 7.862),
span_with_width("was", 303.600, 610.703, 15.113, 7.863, 7.863),
span_with_width("applied", 321.076, 610.703, 27.404, 7.863, 7.863),
span_with_width("to", 350.843, 610.703, 7.089, 7.863, 7.863),
span_with_width("the", 360.295, 610.703, 11.815, 7.863, 7.863),
span_with_width("age", 374.473, 610.703, 14.178, 7.863, 7.863),
span_with_width("and", 391.014, 610.703, 14.178, 7.863, 7.863),
span_with_width("sex", 407.555, 610.703, 13.226, 7.863, 7.863),
span_with_width("margins", 423.144, 610.703, 30.226, 7.863, 7.863),
span_with_width("of", 455.733, 610.703, 7.089, 7.863, 7.863),
span_with_width("each", 465.185, 610.703, 18.428, 7.863, 7.863),
span_with_width("cohort", 485.976, 610.703, 23.622, 7.863, 7.863),
span_with_width("separately,", 303.600, 600.253, 41.573, 7.862, 7.862),
span_with_width("using", 347.536, 600.253, 20.315, 7.862, 7.862),
span_with_width("the", 370.214, 600.253, 11.815, 7.862, 7.862),
span_with_width("published", 384.392, 600.253, 36.380, 7.862, 7.862),
span_with_width("municipal", 423.135, 600.253, 35.896, 7.862, 7.862),
span_with_width("register", 461.394, 600.253, 28.339, 7.862, 7.862),
span_with_width("as", 492.096, 600.253, 8.976, 7.862, 7.862),
span_with_width("the", 303.600, 589.803, 11.815, 7.863, 7.863),
span_with_width("reference", 317.778, 589.803, 35.904, 7.863, 7.863),
span_with_width("distribution", 356.045, 589.803, 41.097, 7.863, 7.863),
span_with_width("for", 399.505, 589.803, 9.920, 7.863, 7.863),
span_with_width("both.", 411.788, 589.803, 18.904, 7.863, 7.863),
span_with_width("Respondents", 433.055, 589.803, 50.082, 7.863, 7.863),
span_with_width("who", 485.500, 589.803, 15.589, 7.863, 7.863),
span_with_width("completed", 303.600, 579.353, 39.210, 7.863, 7.863),
span_with_width("fewer", 345.173, 579.353, 20.783, 7.863, 7.863),
span_with_width("than", 368.319, 579.353, 16.541, 7.863, 7.863),
span_with_width("half", 387.223, 579.353, 13.702, 7.863, 7.863),
span_with_width("of", 403.288, 579.353, 7.089, 7.863, 7.863),
span_with_width("the", 412.740, 579.353, 11.815, 7.863, 7.863),
span_with_width("items", 426.918, 579.353, 20.306, 7.863, 7.863),
span_with_width("were", 449.587, 579.353, 18.420, 7.863, 7.863),
span_with_width("excluded", 470.370, 579.353, 34.017, 7.863, 7.863),
span_with_width("before", 303.600, 568.903, 24.097, 7.863, 7.863),
span_with_width("weighting,", 330.060, 568.903, 38.267, 7.863, 7.863),
span_with_width("which", 370.690, 568.903, 21.726, 7.863, 7.863),
span_with_width("removed", 394.779, 568.903, 33.065, 7.863, 7.863),
span_with_width("a", 430.207, 568.903, 4.726, 7.863, 7.863),
span_with_width("small", 437.296, 568.903, 19.831, 7.863, 7.863),
span_with_width("number", 459.490, 568.903, 28.815, 7.863, 7.863),
span_with_width("of", 490.668, 568.903, 7.089, 7.863, 7.863),
span_with_width("cases", 500.120, 568.903, 22.202, 7.863, 7.863),
span_with_width("from", 303.600, 558.453, 17.000, 7.862, 7.862),
span_with_width("each", 322.963, 558.453, 18.428, 7.862, 7.862),
span_with_width("cohort", 343.754, 558.453, 23.621, 7.862, 7.862),
span_with_width("and", 369.738, 558.453, 14.178, 7.862, 7.862),
span_with_width("did", 386.279, 558.453, 11.339, 7.862, 7.862),
span_with_width("not", 399.981, 558.453, 11.815, 7.862, 7.862),
span_with_width("change", 414.159, 558.453, 27.880, 7.862, 7.862),
span_with_width("the", 444.402, 558.453, 11.815, 7.862, 7.862),
span_with_width("direction", 458.580, 558.453, 32.122, 7.862, 7.862),
span_with_width("of", 493.065, 558.453, 7.089, 7.862, 7.862),
span_with_width("any", 502.517, 558.453, 13.702, 7.862, 7.862),
span_with_width("reported", 303.600, 548.003, 31.654, 7.863, 7.863),
span_with_width("association.", 337.617, 548.003, 44.880, 7.863, 7.863),
span_with_width("The", 384.860, 548.003, 14.645, 7.863, 7.863),
span_with_width("analysis", 401.868, 548.003, 30.702, 7.863, 7.863),
span_with_width("was", 434.933, 548.003, 15.113, 7.863, 7.863),
span_with_width("pre-registered.", 452.409, 548.003, 55.267, 7.863, 7.863),
];
let prose_order_before: Vec<String> = spans
.iter()
.filter(|span| span.bbox.x >= IDEAL_TABLE_PROSE_SPLIT_X)
.map(|span| span.text.clone())
.collect();
let order = spans_sorted_top_to_bottom(&spans);
let lines = group_into_lines(&spans, &order);
let detected = detect_split_x(&spans, &lines, PAGE_WIDTH).expect("a split is detectable");
let snapped = snap_split_left_of_hanging_labels(&spans, &lines, PAGE_WIDTH, detected);
assert!(
spans
.iter()
.any(|span| span.bbox.left() < snapped && span.bbox.right() > snapped),
"the per-line median is expected to still cut a word here ({snapped}); if it no longer \
does, this fixture has stopped exercising the redirect"
);
let split_x = redirect_split_out_of_content(&spans, &lines, PAGE_WIDTH, snapped);
assert!(
split_x > TABLE_RIGHT_EDGE_X && split_x < PROSE_LEFT_EDGE_X,
"split must land in the real table/prose gutter, got {split_x}"
);
assert!(
reorder_dense_two_column_page(&mut spans, PAGE_WIDTH),
"a table beside a prose column must be reordered, not left in full-width Y order"
);
let table_spans = spans
.iter()
.filter(|span| span.bbox.x < IDEAL_TABLE_PROSE_SPLIT_X)
.count();
let first_prose = spans
.iter()
.position(|span| span.bbox.x >= IDEAL_TABLE_PROSE_SPLIT_X)
.expect("the prose column survives the reorder");
assert_eq!(
first_prose, table_spans,
"every table span must be emitted before every prose span"
);
let prose_order_after: Vec<&str> = spans
.iter()
.filter(|span| span.bbox.x >= IDEAL_TABLE_PROSE_SPLIT_X)
.map(|span| span.text.as_str())
.collect();
assert_eq!(
prose_order_after, prose_order_before,
"the prose column's own reading order must be untouched"
);
let prose = prose_order_after.join(" ");
assert!(
prose.contains("more likely to be aged 35 to 44 years"),
"the sentence must not be spliced by table rows, got: {prose}"
);
let panel_of_row = |span: &xberg_native_pdf::layout::TextSpan| {
(span.bbox.y < TITLE_ROW_Y && span.bbox.x < IDEAL_TABLE_PROSE_SPLIT_X)
.then(|| usize::from(span.bbox.x >= PANEL_B_LEFT_EDGE_X))
};
let last_panel_a = spans.iter().rposition(|span| panel_of_row(span) == Some(0));
let first_panel_b = spans.iter().position(|span| panel_of_row(span) == Some(1));
let (Some(last_panel_a), Some(first_panel_b)) = (last_panel_a, first_panel_b) else {
panic!("both table panels must survive the reorder");
};
assert!(
last_panel_a < first_panel_b,
"panel A must be emitted whole before panel B, but panel A's last span sits at \
{last_panel_a} and panel B's first at {first_panel_b}"
);
}
}