pdf-inspector 1.22.0

//! Markdown conversion with structure detection.
//!
//! Converts extracted text to markdown, detecting:
//! - Headers (by font size)
//! - Lists (bullet points, numbered lists)
//! - Code blocks (monospace fonts, indentation)
//! - Paragraphs

pub(crate) mod analysis;
mod classify;
mod convert;
mod furniture;
mod heading;
mod postprocess;
mod preprocess;

pub use convert::to_markdown_from_lines;

use std::collections::{HashMap, HashSet};

use crate::types::{PdfLine, PdfRect, TextItem};

use analysis::calculate_font_stats_from_items;
use classify::{format_list_item, is_caption_line, is_code_like, is_list_item};
use convert::{
    merge_continuation_tables, to_markdown_from_lines_with_tables_and_images, ChartProseOrder,
    PositionedMarkdown,
};

const CHART_REGION_PAD: f32 = 20.0;
const CHART_SEPARATOR_PAD: f32 = 8.0;

fn is_chart_adjacent_label(item: &TextItem, region: (f32, f32, f32, f32)) -> bool {
    let text = item.text.trim();
    let is_bare_bullet = matches!(text, "•" | "●" | "○" | "◦" | "-" | "*");
    if text.is_empty() || is_list_item(text) || is_bare_bullet {
        return false;
    }

    let (x0, y0, x1, y1) = region;
    let (left, right) = (x0.min(x1), x0.max(x1));
    let (bottom, top) = (y0.min(y1), y0.max(y1));
    let item_left = item.x.min(item.x + item.width);
    let item_right = item.x.max(item.x + item.width);
    let item_width = (item_right - item_left).max(1.0);
    let chart_width = (right - left).max(1.0);
    let horizontal_overlap = (item_right.min(right) - item_left.max(left)).max(0.0);
    let mostly_inside_chart_width = horizontal_overlap >= item_width * 0.8;
    let vertical_gap = if item.y < bottom {
        bottom - item.y
    } else if item.y > top {
        item.y - top
    } else {
        0.0
    };
    let is_caption = is_caption_line(text);
    let em = item.cross_extent().max(item.font_size).max(1.0);
    let compact_label = item_width <= em * 18.5;
    let category_band = (em * 1.85).clamp(6.0, CHART_REGION_PAD);
    let close_to_chart_edge = if is_caption {
        vertical_gap <= CHART_REGION_PAD
    } else {
        vertical_gap <= category_band
    };
    let category_sized = item_width <= chart_width * 0.75;

    vertical_gap <= CHART_REGION_PAD
        && (compact_label
            || is_caption
            || (mostly_inside_chart_width && close_to_chart_edge && category_sized))
}

pub(crate) fn item_is_in_chart_region(item: &TextItem, regions: &[(f32, f32, f32, f32)]) -> bool {
    regions.iter().any(|&(x0, y0, x1, y1)| {
        let cx = item.x + item.width / 2.0;
        let within_padded_x = cx >= x0 - CHART_REGION_PAD && cx <= x1 + CHART_REGION_PAD;
        let within_core_y = item.y >= y0 && item.y <= y1;
        let within_padded_y = item.y >= y0 - CHART_REGION_PAD
            && item.y <= y1 + CHART_REGION_PAD
            && is_chart_adjacent_label(item, (x0, y0, x1, y1));
        within_padded_x && (within_core_y || within_padded_y)
    })
}

fn items_outside_chart_regions(
    items: &[TextItem],
    regions: &[(f32, f32, f32, f32)],
) -> Vec<TextItem> {
    items
        .iter()
        .filter(|item| !item_is_in_chart_region(item, regions))
        .cloned()
        .collect()
}

pub(crate) fn merge_chart_regions(
    regions: impl IntoIterator<Item = (f32, f32, f32, f32)>,
) -> Vec<(f32, f32, f32, f32)> {
    const MERGE_TOLERANCE: f32 = 3.0;

    let mut merged: Vec<(f32, f32, f32, f32)> = Vec::new();
    for (x0, y0, x1, y1) in regions {
        let mut current = (x0.min(x1), y0.min(y1), x0.max(x1), y0.max(y1));
        let mut index = 0;
        while index < merged.len() {
            let candidate = merged[index];
            let overlaps = current.2 + MERGE_TOLERANCE >= candidate.0
                && candidate.2 + MERGE_TOLERANCE >= current.0
                && current.3 + MERGE_TOLERANCE >= candidate.1
                && candidate.3 + MERGE_TOLERANCE >= current.1;
            if overlaps {
                current = (
                    current.0.min(candidate.0),
                    current.1.min(candidate.1),
                    current.2.max(candidate.2),
                    current.3.max(candidate.3),
                );
                merged.swap_remove(index);
            } else {
                index += 1;
            }
        }
        merged.push(current);
    }
    merged
}

pub(crate) type PageChartRegions = HashMap<u32, Vec<(f32, f32, f32, f32)>>;

/// Compute the chart masks used by both layout analysis and Markdown output.
///
/// Keeping the rect-backed and dense-line heuristics behind one entry point
/// ensures metadata and extraction cannot drift when either detector changes.
pub(crate) fn chart_regions_by_page(
    items: &[TextItem],
    rects: &[PdfRect],
    lines: &[PdfLine],
) -> PageChartRegions {
    let mut page_items: HashMap<u32, Vec<TextItem>> = HashMap::new();
    for item in items.iter().filter(|item| {
        matches!(
            &item.item_type,
            crate::types::ItemType::Text | crate::types::ItemType::FormField
        )
    }) {
        page_items.entry(item.page).or_default().push(item.clone());
    }

    page_items
        .into_iter()
        .filter_map(|(page, items)| {
            let rect_regions = crate::tables::detect_chart_regions(&items, rects, page);
            let line_regions = crate::tables::detect_dense_line_chart_regions(lines, rects, page)
                .into_iter()
                .filter(|&region| chart_region_separates_prose_columns(&items, region));
            let regions = merge_chart_regions(rect_regions.into_iter().chain(line_regions));
            (!regions.is_empty()).then_some((page, regions))
        })
        .collect()
}

/// Detect side-by-side table layout by finding a significant X-position gap.
///
/// Returns X-band boundaries `[(x_min, split_x), (split_x, x_max)]` when a
/// clear vertical gap separates two groups of items, or an empty vec if the
/// page has a single-region layout.
///
/// Candidate gaps must be ≥30pt and in the middle 60% of the page's X range.
/// Items are counted by center position for accurate balance (each side ≥20%).
/// The candidate with the fewest bounding-box crossings is chosen (must be
/// under 5% of total items). To reject single wide tables with multiple
/// column gaps, only pages with one balanced-candidate cluster (within 50pt)
/// are accepted.
pub(crate) fn split_side_by_side(items: &[TextItem]) -> Vec<(f32, f32)> {
    if items.len() < 40 {
        return vec![];
    }

    // Sort items by left edge
    let mut xs: Vec<f32> = items.iter().map(|i| i.x).collect();
    xs.sort_by(|a, b| a.total_cmp(b));

    // Find all candidate gaps: ≥30pt, in the middle 60% of the X range,
    // with ≥20 items on each side.
    let x_min = xs[0];
    let x_max = *xs.last().unwrap();
    let x_range = x_max - x_min;
    let center_lo = x_min + x_range * 0.2;
    let center_hi = x_min + x_range * 0.8;
    let mut candidates: Vec<f32> = Vec::new();
    for i in 1..xs.len() {
        let gap = xs[i] - xs[i - 1];
        let split_x = (xs[i - 1] + xs[i]) / 2.0;
        if gap >= 30.0
            && i >= 20
            && (xs.len() - i) >= 20
            && split_x >= center_lo
            && split_x <= center_hi
        {
            candidates.push(split_x);
        }
    }

    if candidates.is_empty() {
        return vec![];
    }

    // Pick the candidate with the fewest bounding-box crossings,
    // but only consider balanced splits (each side ≥ 20% of total items
    // by center position, which is more accurate than left-edge counting).
    let min_side = items.len() / 5;
    let mut best_split = 0.0f32;
    let mut best_crossing = usize::MAX;
    for &split_x in &candidates {
        // Count items by center position for accurate balance check
        let left_count = items
            .iter()
            .filter(|i| i.x + i.width / 2.0 < split_x)
            .count();
        let right_count = items.len() - left_count;
        if left_count.min(right_count) < min_side {
            continue;
        }
        let crossing = items
            .iter()
            .filter(|item| item.x < split_x && (item.x + item.width) > split_x)
            .count();
        if crossing < best_crossing {
            best_crossing = crossing;
            best_split = split_x;
        }
    }

    if best_crossing == usize::MAX {
        return vec![];
    }

    // Crossing items must be < 5% of total (allows spanning headers/labels)
    let max_crossing = (items.len() / 20).max(2);
    if best_crossing > max_crossing {
        return vec![];
    }

    // Multiple balanced split candidates that are far apart indicate a
    // multi-column single table. Adjacent candidates (within 20pt) are
    // treated as the same split point. Side-by-side tables have exactly
    // one cluster of candidates near the inter-table gap.
    let mut balanced_positions: Vec<f32> = candidates
        .iter()
        .filter(|&&sx| {
            let lc = items.iter().filter(|i| i.x + i.width / 2.0 < sx).count();
            let rc = items.len() - lc;
            lc.min(rc) >= min_side
        })
        .copied()
        .collect();
    balanced_positions.sort_by(|a, b| a.total_cmp(b));
    balanced_positions.dedup_by(|a, b| (*a - *b).abs() < 50.0);
    if balanced_positions.len() > 1 {
        return vec![];
    }

    // Don't split when the left side is text labels and the right side is numeric
    // data at matching Y positions — this is a single table (labels + numbers),
    // not two independent side-by-side regions.
    // Requires ALL THREE: left side is mostly non-numeric, right side is mostly
    // numeric, AND high Y-correlation between the two sides.
    let is_numeric_item = |item: &&&TextItem| -> bool {
        let text = item.text.trim();
        if text.is_empty() {
            return false;
        }
        let data_chars = text
            .chars()
            .filter(|c| c.is_ascii_digit() || ",.-+%€$£¥()".contains(*c))
            .count();
        data_chars as f32 / text.chars().count() as f32 >= 0.6
    };

    let left_items: Vec<&TextItem> = items
        .iter()
        .filter(|i| i.x + i.width / 2.0 < best_split)
        .collect();
    let right_items: Vec<&TextItem> = items
        .iter()
        .filter(|i| i.x + i.width / 2.0 >= best_split)
        .collect();

    if !left_items.is_empty() && !right_items.is_empty() {
        let left_numeric_ratio =
            left_items.iter().filter(is_numeric_item).count() as f32 / left_items.len() as f32;
        let right_numeric_ratio =
            right_items.iter().filter(is_numeric_item).count() as f32 / right_items.len() as f32;

        // Left side is mostly text (< 30% numeric) AND right side is mostly numbers (≥ 70%)
        if left_numeric_ratio < 0.30 && right_numeric_ratio >= 0.70 {
            let y_tol = 5.0;
            let y_matches = right_items
                .iter()
                .filter(|ri| left_items.iter().any(|li| (li.y - ri.y).abs() < y_tol))
                .count();
            if y_matches as f32 / right_items.len() as f32 >= 0.5 {
                return vec![];
            }
        }
    }

    vec![(x_min, best_split), (best_split, x_max)]
}

/// Detect two short prose columns on a chart page from repeated left anchors.
///
/// Chart masking can leave fewer than 20 lines per column, while justified text
/// can reduce the physical gutter below the projection detector's 8pt minimum.
/// Repeated prose left edges remain reliable in that case. The caller scopes
/// this signal to pages with a confirmed chart region.
fn chart_page_prose_column_split(items: &[TextItem]) -> Option<f32> {
    const X_TOLERANCE: f32 = 12.0;
    const MIN_LINES_PER_COLUMN: usize = 6;
    const MIN_ANCHOR_SEPARATION: f32 = 120.0;
    const MIN_VERTICAL_SPAN: f32 = 60.0;

    let mut prose: Vec<&TextItem> = items
        .iter()
        .filter(|item| {
            let words = item.text.split_whitespace().count();
            let chars = item.text.chars().count().max(1);
            let alphabetic = item.text.chars().filter(|c| c.is_alphabetic()).count();
            words >= 4 && item.width >= 80.0 && alphabetic * 2 >= chars
        })
        .collect();
    if prose.len() < MIN_LINES_PER_COLUMN * 2 {
        return None;
    }
    prose.sort_by(|a, b| a.x.total_cmp(&b.x));

    let mut clusters: Vec<(f32, Vec<&TextItem>)> = Vec::new();
    for item in prose {
        if let Some((anchor, members)) = clusters
            .iter_mut()
            .find(|(anchor, _)| (item.x - *anchor).abs() <= X_TOLERANCE)
        {
            members.push(item);
            *anchor = members.iter().map(|member| member.x).sum::<f32>() / members.len() as f32;
        } else {
            clusters.push((item.x, vec![item]));
        }
    }

    let mut dominant: Vec<(f32, Vec<&TextItem>)> = clusters
        .into_iter()
        .filter(|(_, members)| members.len() >= MIN_LINES_PER_COLUMN)
        .collect();
    if dominant.len() != 2 {
        return None;
    }
    dominant.sort_by(|a, b| a.0.total_cmp(&b.0));
    if dominant[1].0 - dominant[0].0 < MIN_ANCHOR_SEPARATION {
        return None;
    }

    let vertical_range = |members: &[&TextItem]| {
        let y_min = members
            .iter()
            .map(|item| item.y)
            .fold(f32::INFINITY, f32::min);
        let y_max = members
            .iter()
            .map(|item| item.y)
            .fold(f32::NEG_INFINITY, f32::max);
        (y_min, y_max)
    };
    let left_y = vertical_range(&dominant[0].1);
    let right_y = vertical_range(&dominant[1].1);
    if left_y.1 - left_y.0 < MIN_VERTICAL_SPAN || right_y.1 - right_y.0 < MIN_VERTICAL_SPAN {
        return None;
    }
    let overlap = (left_y.1.min(right_y.1) - left_y.0.max(right_y.0)).max(0.0);
    let shorter_span = (left_y.1 - left_y.0).min(right_y.1 - right_y.0);
    if overlap < shorter_span * 0.4 {
        return None;
    }

    Some((dominant[0].0 + dominant[1].0) / 2.0)
}

/// True when a chart crosses the inferred prose gutter by enough to act as a
/// page-wide separator. A chart confined to one column must stay in that
/// column's local reading order instead of reordering the entire page.
fn chart_spans_prose_split(region: (f32, f32, f32, f32), split_x: f32) -> bool {
    const MIN_CHART_WIDTH_PER_SIDE: f32 = 40.0;

    let (x0, _, x1, _) = region;
    let left = x0.min(x1);
    let right = x0.max(x1);
    split_x - left >= MIN_CHART_WIDTH_PER_SIDE && right - split_x >= MIN_CHART_WIDTH_PER_SIDE
}

pub(crate) fn chart_region_separates_prose_columns(
    items: &[TextItem],
    region: (f32, f32, f32, f32),
) -> bool {
    let outside = items_outside_chart_regions(items, &[region]);
    chart_page_prose_column_split(&outside)
        .is_some_and(|split_x| chart_spans_prose_split(region, split_x))
}

/// True when adjacent physical rows form an unterminated, lowercase prose
/// continuation in the same projected column.
fn is_cross_row_prose_continuation(previous: &str, current: &str) -> bool {
    let previous = previous.trim();
    let current = current.trim();
    if previous.is_empty() || current.is_empty() {
        return false;
    }

    let previous_without_closers = previous.trim_end_matches(['"', '\'', '”', ')', ']']);
    let previous_is_open = previous_without_closers
        .chars()
        .next_back()
        .is_some_and(|ch| !matches!(ch, '.' | '!' | '?' | ':' | ';'));
    let current_starts_as_continuation = current
        .chars()
        .find(|ch| ch.is_alphabetic())
        .is_some_and(|ch| ch.is_lowercase());

    previous_is_open && current_starts_as_continuation
}

/// Section-numbered headings embedded in a candidate are strong evidence that
/// a heuristic grid has captured page prose rather than a real table.
fn looks_like_numbered_section_heading(text: &str) -> bool {
    let Some((prefix, title)) = text.trim().split_once(char::is_whitespace) else {
        return false;
    };
    let prefix = prefix.trim_end_matches('.');
    let mut group_count = 0;
    for group in prefix.split('.') {
        if group.is_empty() || group.len() > 3 || !group.chars().all(|ch| ch.is_ascii_digit()) {
            return false;
        }
        group_count += 1;
    }
    let title = title.trim();
    (1..=4).contains(&group_count)
        && title.split_whitespace().count() >= 3
        && title
            .chars()
            .find(|ch| ch.is_alphabetic())
            .is_some_and(|ch| ch.is_uppercase())
}

fn merged_retry_skips_body_font(detected_columns: bool, has_chart_regions: bool) -> bool {
    detected_columns && !has_chart_regions
}

/// Identity of a piece of page furniture: the same trimmed text drawn at the
/// same position (quantized to 0.5pt) — page numbers excluded by construction
/// because their text differs per page.
type FurnitureKey = (String, i32, i32);

fn furniture_key(item: &TextItem) -> FurnitureKey {
    (
        item.text.trim().to_string(),
        (item.x * 2.0).round() as i32,
        (item.y * 2.0).round() as i32,
    )
}

/// Minimum distinct pages an identical (text, position) must appear on before
/// it counts as a running header/footer rather than coincidence.
const RUNNING_FURNITURE_MIN_PAGES: usize = 3;

/// Fraction of each page's vertical content extent, at the top and at the
/// bottom, where running furniture may live. Repetition alone is not enough:
/// a form template repeated per record carries identical labels at identical
/// mid-page coordinates on every page, and those are real table cells. What
/// makes a header/footer is repetition *at the page edge*.
const RUNNING_FURNITURE_BAND: f32 = 0.2;

/// Collect the keys of items that repeat verbatim at the same position on at
/// least [`RUNNING_FURNITURE_MIN_PAGES`] distinct pages, restricted to the
/// top/bottom [`RUNNING_FURNITURE_BAND`] of each page's content extent —
/// running headers and footers. Single- and two-page documents produce an
/// empty set.
fn running_furniture_keys(items: &[TextItem]) -> HashSet<FurnitureKey> {
    // Vertical content extent per page, so the edge bands adapt to the
    // document's real margins instead of assuming a media box.
    let mut page_extent: HashMap<u32, (f32, f32)> = HashMap::new();
    for item in items {
        if item.text.trim().is_empty() {
            continue;
        }
        let entry = page_extent.entry(item.page).or_insert((item.y, item.y));
        entry.0 = entry.0.min(item.y);
        entry.1 = entry.1.max(item.y);
    }

    let mut pages_by_key: HashMap<FurnitureKey, HashSet<u32>> = HashMap::new();
    for item in items {
        if item.text.trim().is_empty() {
            continue;
        }
        let Some(&(min_y, max_y)) = page_extent.get(&item.page) else {
            continue;
        };
        // A page whose text has no vertical span gives no evidence of where
        // its edges are — without this guard, a zero band would classify its
        // every item as edge furniture.
        let extent = max_y - min_y;
        if extent <= 0.0 {
            continue;
        }
        let band = extent * RUNNING_FURNITURE_BAND;
        if item.y > min_y + band && item.y < max_y - band {
            continue; // mid-page: never furniture, however often it repeats
        }
        pages_by_key
            .entry(furniture_key(item))
            .or_default()
            .insert(item.page);
    }
    pages_by_key
        .into_iter()
        .filter(|(_, pages)| pages.len() >= RUNNING_FURNITURE_MIN_PAGES)
        .map(|(key, _)| key)
        .collect()
}

/// Reject a heuristic table whose items are almost entirely running
/// headers/footers. A wrapped document title repeated at the bottom of every
/// page aligns well enough to read as a grid, but it is page furniture, not
/// data — vetoing the table lets the text flow as prose instead. Real tables
/// carry per-page content, so even a repeated *header row* stays under the
/// threshold once its body rows differ.
fn is_running_furniture_table(
    detection_items: &[TextItem],
    table: &crate::tables::Table,
    running: &HashSet<FurnitureKey>,
) -> bool {
    if running.is_empty() {
        return false;
    }
    let mut total = 0usize;
    let mut furniture = 0usize;
    for &idx in &table.item_indices {
        let Some(item) = detection_items.get(idx) else {
            continue;
        };
        if item.text.trim().is_empty() {
            continue;
        }
        total += 1;
        if running.contains(&furniture_key(item)) {
            furniture += 1;
        }
    }
    total > 0 && (furniture as f32) >= (total as f32) * 0.8
}

/// Reject a heuristic table only when its cells are overwhelmingly parallel
/// prose fragments. This is deliberately narrower than disabling body-font
/// detection for the whole page: numeric, compact, headed, and otherwise
/// table-shaped candidates remain eligible on chart pages.
fn is_parallel_prose_table(table: &crate::tables::Table) -> bool {
    if table.kind != crate::tables::TableKind::Data
        || !(2..=6).contains(&table.columns.len())
        || table.rows.len() < 3
    {
        return false;
    }

    let mut non_empty = 0;
    let mut long_prose = 0;
    let mut rows_with_parallel_prose = 0;
    let mut occupied_rows = 0;
    let has_numbered_section_heading = table
        .cells
        .iter()
        .flatten()
        .any(|cell| looks_like_numbered_section_heading(cell));
    let has_compact_header = table
        .cells
        .iter()
        .find(|row| row.iter().any(|cell| !cell.trim().is_empty()))
        .is_some_and(|row| {
            let filled: Vec<&String> = row.iter().filter(|cell| !cell.trim().is_empty()).collect();
            filled.len() >= 2
                && filled.iter().all(|cell| {
                    cell.split_whitespace().count() <= 4 && cell.trim().chars().count() <= 28
                })
        });

    for row in &table.cells {
        let mut row_long_prose = 0;
        let mut row_non_empty = 0;
        for cell in row {
            let text = cell.trim();
            if text.is_empty() {
                continue;
            }
            non_empty += 1;
            row_non_empty += 1;
            let chars = text.chars().filter(|ch| !ch.is_whitespace()).count();
            let alphabetic = text.chars().filter(|ch| ch.is_alphabetic()).count();
            let words = text.split_whitespace().count();
            if chars >= 28 && words >= 5 && alphabetic * 5 >= chars * 3 {
                long_prose += 1;
                row_long_prose += 1;
            }
        }
        if row_long_prose >= 2 {
            rows_with_parallel_prose += 1;
        }
        if row_non_empty > 0 {
            occupied_rows += 1;
        }
    }

    // A lowercase cell is not continuation evidence by itself: legitimate
    // headerless tables often use sentence fragments as row values. Require a
    // direct physical-row transition from an unterminated cell in the same
    // column. This is the shape produced when independent prose columns are
    // accidentally projected onto one table grid.
    // Prose flowing through a grid is often interrupted by empty cells
    // (the interleaved fragment occupies the other column on that row), so
    // each cell is compared against the last non-empty cell above it in the
    // same column, not just the immediately preceding row.
    let mut continuation_fragments = 0;
    let mut continuation_columns = vec![false; table.columns.len()];
    for (column, has_continuation) in continuation_columns.iter_mut().enumerate() {
        let mut previous_non_empty: Option<&str> = None;
        for row in &table.cells {
            let current = row.get(column).map(String::as_str).unwrap_or("");
            if current.trim().is_empty() {
                continue;
            }
            if let Some(previous) = previous_non_empty {
                if is_cross_row_prose_continuation(previous, current) {
                    continuation_fragments += 1;
                    *has_continuation = true;
                }
            }
            previous_non_empty = Some(current);
        }
    }

    // A compact header row is evidence for a real table — unless cross-row
    // prose continuations outnumber the rows, which no genuine table
    // produces: the "header" is then just two short line fragments at the
    // top of parallel prose columns.
    let header_blocks = has_compact_header && continuation_fragments <= table.cells.len();

    // Citation blocks over-fragmented into wide grids: nearly every cell is
    // a bare word and the rows read on as prose. Real wide tables carry
    // numbers, units, or multi-word values.
    let single_token_alpha = table
        .cells
        .iter()
        .flatten()
        .filter(|cell| {
            let t = cell.trim();
            !t.is_empty()
                && t.split_whitespace().count() == 1
                && t.chars().filter(|c| c.is_alphabetic()).count() * 2 >= t.chars().count().max(1)
        })
        .count();
    // Categorical grids repeat a small value vocabulary down their
    // columns; flowing text almost never repeats a cell. A fully
    // populated word grid is only rejected when its cells are almost all
    // distinct AND the continuations strongly outnumber the rows.
    let distinct_cells: std::collections::HashSet<String> = table
        .cells
        .iter()
        .flatten()
        .map(|c| c.trim().to_ascii_lowercase())
        .filter(|c| !c.is_empty())
        .collect();
    let mostly_distinct = distinct_cells.len() * 4 >= non_empty * 3;
    let word_fragment_grid = !header_blocks
        && table.columns.len() >= 4
        && non_empty >= 8
        && (non_empty < table.cells.len() * table.columns.len()
            || (mostly_distinct && continuation_fragments > table.cells.len() * 2))
        && single_token_alpha * 4 >= non_empty * 3
        && continuation_fragments >= 4
        && continuation_columns.iter().filter(|&&value| value).count() >= 2;

    // Numbered lists projected onto a two-column grid: the first column is
    // list markers ("1.", "2)", …), the second the item text. Rendering
    // these as tables loses the list; the page flow keeps it.
    // A leading "1. | <text>" row is the list's own first item, not a
    // table header — but only when the whole first column actually has
    // the list shape (mostly ordinal markers); a lone punctuation-styled
    // rank must not disable the compact-header protection.
    let is_ordinal_marker = |cell: &str| {
        let t = cell.trim();
        t.len() <= 4
            && t.ends_with(['.', ')'])
            && !t[..t.len() - 1].is_empty()
            && t[..t.len() - 1].chars().all(|c| c.is_ascii_digit())
    };
    let first_column_list_shape = {
        let mut markers = 0usize;
        let mut filled = 0usize;
        for row in &table.cells {
            let first = row.first().map(|s| s.trim()).unwrap_or("");
            if first.is_empty() {
                continue;
            }
            filled += 1;
            if is_ordinal_marker(first)
                || first
                    .split_whitespace()
                    .next()
                    .is_some_and(is_ordinal_marker)
            {
                markers += 1;
            }
        }
        filled >= 4 && markers * 10 >= filled * 7
    };
    let ordinal_header_blocks = header_blocks && !first_column_list_shape;
    let ordinal_list =
        !ordinal_header_blocks && table.columns.len() == 2 && table.cells.len() >= 4 && {
            let mut markers = 0;
            let mut filled_first = 0;
            for row in &table.cells {
                let first = row.first().map(|s| s.trim()).unwrap_or("");
                if first.is_empty() {
                    continue;
                }
                filled_first += 1;
                let is_marker = first.len() <= 4
                    && first.ends_with(['.', ')'])
                    && first[..first.len() - 1].chars().all(|c| c.is_ascii_digit())
                    && !first[..first.len() - 1].is_empty();
                if is_marker {
                    markers += 1;
                }
            }
            filled_first >= 4 && markers * 10 >= filled_first * 7
        };

    let is_parallel = !header_blocks
        && non_empty >= 5
        // Independent prose columns have asynchronous line/paragraph breaks;
        // a fully populated grid is positive evidence for a real descriptive
        // table even when every value is a lowercase sentence fragment.
        && non_empty < table.cells.len() * table.columns.len()
        && long_prose >= 4
        && long_prose * 5 >= non_empty * 3
        // Row-spanning blanks are common in real headerless description
        // tables. Require long text in parallel on at least half of occupied
        // rows, unless a section heading was swallowed into the grid: that is
        // direct evidence that this candidate is page prose.
        && ((rows_with_parallel_prose >= 2
            && rows_with_parallel_prose * 2 >= occupied_rows)
            || (rows_with_parallel_prose >= 1 && has_numbered_section_heading))
        && continuation_fragments >= 3
        && continuation_columns.iter().filter(|&&value| value).count() >= 2;
    let not_fully_populated = non_empty < table.cells.len() * table.columns.len();
    // Small 2-column weaves rarely accumulate 3 cross-row continuations —
    // there aren't enough rows — but long prose in both columns plus a
    // continuation in each is already the projection signature.
    let tiny_grid_weave = !header_blocks
        && table.cells.len() <= 4
        && not_fully_populated
        && long_prose >= 4
        && continuation_fragments >= 2
        && continuation_columns.iter().filter(|&&value| value).count() >= 2;
    // Wide multi-column projections of page text: continuations outnumber
    // the rows and appear in 3+ columns. Genuine wide tables wrap inside
    // one or two description columns; they never flow everywhere at once.
    // Long prose cells are required: reference tables whose wrapped
    // entries continue in every column (short citation fragments) look
    // continuation-dominated too, but they never read as running text.
    let continuation_dominated = !header_blocks
        && not_fully_populated
        && long_prose >= 4
        // ...and long prose must be a real share of the grid: a large data
        // table with a handful of wordy cells and many wrapped-cell
        // continuations is not a weave.
        && long_prose * 10 >= non_empty
        // Continuations must strictly outnumber the rows (with an absolute
        // floor), preserving the compact-header rule's spirit for every
        // rejection branch.
        && continuation_fragments > table.cells.len().max(7)
        && continuation_columns.iter().filter(|&&value| value).count() >= 3;
    // Numbered list items ("1. Restructuring ...") woven beside sidebar
    // fragments: the first column is a monotone ordinal-prefixed list.
    let ordinal_prefix_list =
        !ordinal_header_blocks && table.columns.len() == 2 && table.cells.len() >= 4 && {
            let mut values: Vec<u32> = Vec::new();
            let mut filled_first = 0;
            for row in &table.cells {
                let first = row.first().map(|s| s.trim()).unwrap_or("");
                if first.is_empty() {
                    continue;
                }
                filled_first += 1;
                let mut parts = first.splitn(2, char::is_whitespace);
                let marker = parts.next().unwrap_or("");
                let rest = parts.next().unwrap_or("").trim();
                if marker.len() <= 4
                    && marker.ends_with(['.', ')'])
                    && marker[..marker.len() - 1]
                        .chars()
                        .all(|c| c.is_ascii_digit())
                    && !marker[..marker.len() - 1].is_empty()
                    && rest.split_whitespace().count() >= 2
                {
                    if let Ok(v) = marker[..marker.len() - 1].parse() {
                        values.push(v);
                    }
                }
            }
            filled_first >= 4
                && values.len() * 10 >= filled_first * 7
                && values.windows(2).all(|w| w[1] >= w[0])
        };
    let is_parallel = is_parallel
        || word_fragment_grid
        || ordinal_list
        || ordinal_prefix_list
        || tiny_grid_weave
        || continuation_dominated;
    log::debug!(
        "chart table hypothesis: {}x{}, non_empty={}, long_prose={}, parallel_rows={}/{}, section_heading={}, continuation_fragments={}, continuation_columns={}, reject={}",
        table.rows.len(),
        table.columns.len(),
        non_empty,
        long_prose,
        rows_with_parallel_prose,
        occupied_rows,
        has_numbered_section_heading,
        continuation_fragments,
        continuation_columns.iter().filter(|&&value| value).count(),
        is_parallel
    );
    is_parallel
}

#[derive(Clone, Copy)]
enum TableOutputMode {
    Markdown,
    #[cfg(feature = "ocr")]
    CompleteTables,
}

struct TableDetectionOutput {
    mode: TableOutputMode,
    pages_with_detected_tables: HashSet<u32>,
    pages_with_tables: HashSet<u32>,
    markdown_by_page: HashMap<u32, Vec<PositionedMarkdown>>,
    #[cfg(feature = "ocr")]
    complete_tables: Vec<(u32, crate::tables::Table)>,
}

impl TableDetectionOutput {
    fn new(mode: TableOutputMode) -> Self {
        Self {
            mode,
            pages_with_detected_tables: HashSet::new(),
            pages_with_tables: HashSet::new(),
            markdown_by_page: HashMap::new(),
            #[cfg(feature = "ocr")]
            complete_tables: Vec::new(),
        }
    }

    fn record(
        &mut self,
        page: u32,
        table: &crate::tables::Table,
        chart_order: Option<ChartProseOrder>,
    ) {
        self.pages_with_detected_tables.insert(page);
        match self.mode {
            TableOutputMode::Markdown => {
                self.pages_with_tables.insert(page);
                self.markdown_by_page
                    .entry(page)
                    .or_default()
                    .push(PositionedMarkdown::new(
                        table.rows.first().copied().unwrap_or(0.0),
                        table.columns.first().copied().unwrap_or(0.0),
                        crate::tables::table_to_markdown(table),
                        chart_order,
                    ));
            }
            #[cfg(feature = "ocr")]
            TableOutputMode::CompleteTables => {
                if crate::tables::is_complete_data_table(table) {
                    self.pages_with_tables.insert(page);
                    self.complete_tables.push((page, table.clone()));
                }
            }
        }
    }

    fn has_tables_on_page(&self, page: u32) -> bool {
        self.pages_with_tables.contains(&page)
    }

    fn has_detected_tables_on_page(&self, page: u32) -> bool {
        self.pages_with_detected_tables.contains(&page)
    }
}

#[derive(Default)]
struct MarkdownConversionOutput {
    markdown: String,
    #[cfg(feature = "ocr")]
    detected_tables: Vec<(u32, crate::tables::Table)>,
}

/// Derive a side-by-side split from rect hint regions.
///
/// When `split_side_by_side` doesn't detect a gap (e.g. the text gap is too
/// small), hint regions from large rect clusters can still reveal a left/right
/// zone layout (calendar months, form sections).  This function checks if hint
/// regions pair up at the same Y bands and returns `[(x_min, split), (split,
/// x_max)]` if a consistent split exists.
/// True when a table-shaped rect cluster (≥6 rects) ends at an interior band
/// boundary and its rows visibly continue on the far side: cell-like text
/// across the boundary is y-aligned with most cluster rows, and nearly all
/// far-side text in the cluster's y-range participates in that alignment.
/// Tables often rule only their leading columns, so the text gap before the
/// borderless columns masquerades as a page-layout gutter — a real second
/// layout column would instead be dense prose that doesn't track table rows.
fn rect_cluster_spans_band_boundary(
    items: &[TextItem],
    rects: &[PdfRect],
    page: u32,
    bands: &[(f32, f32)],
) -> bool {
    if bands.len() < 2 {
        return false;
    }
    // Normalize: raw PDF rects can carry negative extents.
    let page_rects: Vec<(f32, f32, f32, f32)> = rects
        .iter()
        .filter(|r| r.page == page)
        .map(|r| {
            let (x, w) = if r.width < 0.0 {
                (r.x + r.width, -r.width)
            } else {
                (r.x, r.width)
            };
            let (y, h) = if r.height < 0.0 {
                (r.y + r.height, -r.height)
            } else {
                (r.y, r.height)
            };
            (x, y, w, h)
        })
        .collect();
    if page_rects.len() < 6 {
        return false;
    }
    let clusters = crate::tables::detect_rects::cluster_rects(&page_rects, 3.0, 6);
    let boundaries: Vec<f32> = bands[..bands.len() - 1].iter().map(|&(_, hi)| hi).collect();

    boundaries.iter().any(|&b| {
        // Y-ranges of clusters that individually indicate the split cuts a
        // table: either ruled on both sides of the boundary, or ending at
        // the boundary with cell-like text row-aligned beyond it.
        let mut table_y_ranges: Vec<(f32, f32)> = Vec::new();
        for cluster in &clusters {
            let bbox = cluster.iter().fold(
                (
                    f32::INFINITY,
                    f32::INFINITY,
                    f32::NEG_INFINITY,
                    f32::NEG_INFINITY,
                ),
                |(x0, y0, x1, y1), &i| {
                    let (x, y, w, h) = page_rects[i];
                    (x0.min(x), y0.min(y), x1.max(x + w), y1.max(y + h))
                },
            );
            let spans = bbox.0 < b - 20.0 && bbox.2 > b + 20.0;
            let ends_at = bbox.2 >= b - 60.0 && bbox.2 <= b + 10.0 && bbox.0 <= b;
            if !spans && !ends_at {
                continue;
            }
            // Distinct row baselines of items inside the cluster bbox.
            let mut row_ys: Vec<f32> = Vec::new();
            for it in items {
                let cx = it.x + it.width / 2.0;
                if it.page == page
                    && cx > bbox.0
                    && cx < bbox.2
                    && it.y >= bbox.1 - 2.0
                    && it.y <= bbox.3 + 2.0
                    && !row_ys.iter().any(|&y| (y - it.y).abs() <= 2.0)
                {
                    row_ys.push(it.y);
                }
            }
            if row_ys.len() < 2 {
                continue;
            }
            // Cell-like far-side items row-aligned with the cluster.
            let cell_like = |it: &&TextItem| it.width <= 150.0;
            let far_aligned_rows = row_ys
                .iter()
                .filter(|&&y| {
                    items.iter().any(|it| {
                        it.page == page
                            && it.x + it.width / 2.0 > b
                            && cell_like(&it)
                            && (it.y - y).abs() <= 2.0
                    })
                })
                .count();
            if far_aligned_rows >= 2 && far_aligned_rows * 2 >= row_ys.len() {
                table_y_ranges.push((bbox.1, bbox.3));
            }
        }
        if table_y_ranges.is_empty() {
            return false;
        }
        // The split is only wrong if the table rows account for most of the
        // far side. A figure legitimately spanning two text columns leaves
        // the majority of far-side text (column prose) outside its y-range.
        let far: Vec<&TextItem> = items
            .iter()
            .filter(|it| it.page == page && it.x + it.width / 2.0 > b)
            .collect();
        if far.is_empty() {
            return false;
        }
        let inside = far
            .iter()
            .filter(|it| {
                table_y_ranges
                    .iter()
                    .any(|&(lo, hi)| it.y >= lo - 2.0 && it.y <= hi + 2.0)
            })
            .count();
        inside * 10 >= far.len() * 6
    })
}

fn split_from_hint_regions(items: &[TextItem], rects: &[PdfRect], page: u32) -> Vec<(f32, f32)> {
    use crate::tables::{cluster_rects, RectHintRegion};

    // Quick hint region computation (same logic as detect_tables_from_rects
    // but without table detection).
    let mut page_rects: Vec<(f32, f32, f32, f32)> = Vec::new();
    for r in rects {
        if r.page != page {
            continue;
        }
        let (mut x, mut y, mut w, mut h) = (r.x, r.y, r.width, r.height);
        if w < 0.0 {
            x += w;
            w = -w;
        }
        if h < 0.0 {
            y += h;
            h = -h;
        }
        if w < 5.0 || h < 5.0 {
            continue;
        }
        page_rects.push((x, y, w, h));
    }
    if page_rects.len() < 60 {
        return vec![];
    }

    // Width outlier filter (same as detect_tables_from_rects)
    let mut widths: Vec<f32> = page_rects.iter().map(|&(_, _, w, _)| w).collect();
    widths.sort_by(|a, b| a.total_cmp(b));
    let median_width = widths[widths.len() / 2];
    page_rects.retain(|&(_, _, w, _)| w <= median_width * 10.0);

    let clusters = cluster_rects(&page_rects, 3.0, 6);
    if clusters.len() < 4 {
        return vec![];
    }

    // Build hint regions from large clusters
    let mut hints: Vec<RectHintRegion> = Vec::new();
    for cluster_indices in &clusters {
        let group_rects: Vec<(f32, f32, f32, f32)> =
            cluster_indices.iter().map(|&i| page_rects[i]).collect();
        if group_rects.len() < 30 {
            continue;
        }
        let x_left = group_rects.iter().map(|r| r.0).reduce(f32::min).unwrap();
        let x_right = group_rects
            .iter()
            .map(|r| r.0 + r.2)
            .reduce(f32::max)
            .unwrap();
        let y_bottom = group_rects.iter().map(|r| r.1).reduce(f32::min).unwrap();
        let y_top = group_rects
            .iter()
            .map(|r| r.1 + r.3)
            .reduce(f32::max)
            .unwrap();
        let w = x_right - x_left;
        let h = y_top - y_bottom;
        if (30.0..=400.0).contains(&w) && (10.0..=400.0).contains(&h) {
            hints.push(RectHintRegion {
                y_top,
                y_bottom,
                x_left,
                x_right,
                cluster_rects: Vec::new(),
            });
        }
    }
    if hints.len() < 4 {
        return vec![];
    }

    // Check for left/right pairing: hints at the same Y band should split
    // into distinct X groups.  Count pairs where two hints share a Y band
    // (>50% overlap) but occupy different X halves.
    let page_x_mid = {
        let x_min = items.iter().map(|i| i.x).reduce(f32::min).unwrap_or(0.0);
        let x_max = items
            .iter()
            .map(|i| i.x + i.width)
            .reduce(f32::max)
            .unwrap_or(800.0);
        (x_min + x_max) / 2.0
    };

    let mut pair_count = 0;
    for (i, a) in hints.iter().enumerate() {
        for b in hints.iter().skip(i + 1) {
            let y_overlap = a.y_top.min(b.y_top) - a.y_bottom.max(b.y_bottom);
            let y_min_span = (a.y_top - a.y_bottom).min(b.y_top - b.y_bottom);
            if y_overlap > y_min_span * 0.5 {
                let a_center = (a.x_left + a.x_right) / 2.0;
                let b_center = (b.x_left + b.x_right) / 2.0;
                if (a_center < page_x_mid) != (b_center < page_x_mid) {
                    pair_count += 1;
                }
            }
        }
    }

    // Require at least 3 left/right pairs to confirm the layout
    if pair_count < 3 {
        return vec![];
    }

    // Find the split X: midpoint between the rightmost left-zone hint
    // and the leftmost right-zone hint
    let max_left_x = hints
        .iter()
        .filter(|h| (h.x_left + h.x_right) / 2.0 < page_x_mid)
        .map(|h| h.x_right)
        .reduce(f32::max);
    let min_right_x = hints
        .iter()
        .filter(|h| (h.x_left + h.x_right) / 2.0 >= page_x_mid)
        .map(|h| h.x_left)
        .reduce(f32::min);

    if let (Some(left_edge), Some(right_edge)) = (max_left_x, min_right_x) {
        let split_x = (left_edge + right_edge) / 2.0;
        let x_min = items.iter().map(|i| i.x).reduce(f32::min).unwrap_or(0.0);
        let x_max = items
            .iter()
            .map(|i| i.x + i.width)
            .reduce(f32::max)
            .unwrap_or(800.0);
        log::debug!(
            "page {}: hint-derived side-by-side split at x={:.1}",
            page,
            split_x
        );
        vec![(x_min, split_x), (split_x, x_max)]
    } else {
        vec![]
    }
}

/// Filter rects to those mostly contained within an X band.
///
/// Excludes rects that extend significantly beyond the band (e.g. page-wide
/// background stripes spanning both side-by-side tables). A rect must have
/// at least 70% of its width inside the band to be included.
pub(crate) fn filter_rects_to_band(
    rects: &[PdfRect],
    page: u32,
    x_lo: f32,
    x_hi: f32,
) -> Vec<PdfRect> {
    let band_width = x_hi - x_lo;
    rects
        .iter()
        .filter(|r| {
            r.page == page && {
                let rx_min = if r.width >= 0.0 { r.x } else { r.x + r.width };
                let rx_max = if r.width >= 0.0 { r.x + r.width } else { r.x };
                let rw = rx_max - rx_min;
                // Overlap region
                let overlap = rx_max.min(x_hi) - rx_min.max(x_lo);
                if overlap <= 0.0 {
                    return false;
                }
                // Small rects (< 70% of band): require any overlap (cell borders, etc.)
                // Large rects (≥ 70% of band): require ≥70% of rect inside band
                if rw < band_width * 0.7 {
                    true
                } else {
                    overlap >= rw * 0.7
                }
            }
        })
        .cloned()
        .collect()
}

/// A band of items/indices/rects/lines for side-by-side table detection.
type BandSpec = (Vec<TextItem>, Vec<usize>, Vec<PdfRect>, Vec<PdfLine>);

/// Filter PDF lines to those overlapping an X band.
pub(crate) fn filter_lines_to_band(
    lines: &[PdfLine],
    page: u32,
    x_lo: f32,
    x_hi: f32,
) -> Vec<PdfLine> {
    lines
        .iter()
        .filter(|l| {
            l.page == page && {
                let lx_min = l.x1.min(l.x2);
                let lx_max = l.x1.max(l.x2);
                lx_max > x_lo && lx_min < x_hi
            }
        })
        .cloned()
        .collect()
}

/// Output policy for Markdown post-processing.
///
/// [`MarkdownProfile::Fidelity`] preserves source characters wherever possible.
/// [`MarkdownProfile::Compact`] enables optional token-saving rewrites that may
/// be useful for agent context windows but are not byte-faithful to the PDF.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum MarkdownProfile {
    /// Preserve source text fidelity. This is the default.
    #[default]
    Fidelity,
    /// Prefer token-efficient output, including collapsing long dot leaders.
    Compact,
}

/// Options for markdown conversion
#[derive(Debug, Clone)]
pub struct MarkdownOptions {
    /// Source-fidelity versus token-efficient post-processing policy.
    pub profile: MarkdownProfile,
    /// Detect headers by font size
    pub detect_headers: bool,
    /// Detect list items
    pub detect_lists: bool,
    /// Detect code blocks
    pub detect_code: bool,
    /// Base font size for comparison
    pub base_font_size: Option<f32>,
    /// Remove standalone page numbers
    pub remove_page_numbers: bool,
    /// Convert URLs to markdown links
    pub format_urls: bool,
    /// Fix hyphenation (broken words across lines)
    pub fix_hyphenation: bool,
    /// Detect and format bold text from font names
    pub detect_bold: bool,
    /// Detect and format italic text from font names
    pub detect_italic: bool,
    /// Emit `<u>` runs for text with a geometrically-detected underline
    pub detect_underline: bool,
    /// Include image placeholders in output
    pub include_images: bool,
    /// Include extracted hyperlinks
    pub include_links: bool,
    /// Insert page break markers (<!-- Page N -->) between pages
    pub include_page_numbers: bool,
    /// Strip repeated headers/footers that appear on many pages
    pub strip_headers_footers: bool,
}

impl Default for MarkdownOptions {
    fn default() -> Self {
        Self {
            profile: MarkdownProfile::default(),
            detect_headers: true,
            detect_lists: true,
            detect_code: true,
            base_font_size: None,
            remove_page_numbers: true,
            format_urls: true,
            fix_hyphenation: true,
            detect_bold: true,
            detect_italic: true,
            detect_underline: true,
            // `include_images: false` is intentional. The content-stream walker
            // now emits `ItemType::Image` `TextItem`s for every Image XObject
            // it encounters (see `extractor/content_stream.rs`). If we rendered
            // those into markdown by default, every existing caller would
            // suddenly see `![Image: Im0](image)` placeholders inserted
            // throughout their output — a silent regression for anyone who
            // upgrades. Image bboxes are still available via
            // `extract_text_with_positions` for callers (e.g. layout-aware
            // pipelines) that want to crop + caption figures themselves.
            include_images: false,
            include_links: true,
            include_page_numbers: false,
            strip_headers_footers: true,
        }
    }
}

/// Convert plain text to markdown (basic conversion)
pub fn to_markdown(text: &str, options: MarkdownOptions) -> String {
    let mut output = String::new();
    let mut in_list = false;
    let mut in_code_block = false;

    for line in text.lines() {
        let trimmed = line.trim();

        if trimmed.is_empty() {
            if in_list {
                in_list = false;
            }
            if in_code_block {
                output.push_str("```\n");
                in_code_block = false;
            }
            output.push('\n');
            continue;
        }

        // Detect list items
        if options.detect_lists && is_list_item(trimmed) {
            let formatted = format_list_item(trimmed);
            output.push_str(&formatted);
            output.push('\n');
            in_list = true;
            continue;
        }

        // Detect code blocks (indented lines)
        if options.detect_code && is_code_like(trimmed) {
            if !in_code_block {
                output.push_str("```\n");
                in_code_block = true;
            }
            output.push_str(trimmed);
            output.push('\n');
            continue;
        } else if in_code_block {
            output.push_str("```\n");
            in_code_block = false;
        }

        // Regular paragraph text
        output.push_str(trimmed);
        output.push('\n');
    }

    if in_code_block {
        output.push_str("```\n");
    }

    output
}

/// Applies the document-wide repeated header/footer classifier to grouped lines.
pub(crate) fn strip_repeated_header_footer_lines(
    lines: Vec<crate::types::TextLine>,
    page_count: u32,
) -> Vec<crate::types::TextLine> {
    furniture::strip_header_footer_lines(lines, page_count)
}

/// Convert positioned text items to markdown with structure detection
pub fn to_markdown_from_items(items: Vec<TextItem>, options: MarkdownOptions) -> String {
    to_markdown_from_items_with_rects(items, options, &[])
}

/// Convert positioned text items to markdown, using rectangle data for table detection
pub fn to_markdown_from_items_with_rects(
    items: Vec<TextItem>,
    options: MarkdownOptions,
    rects: &[crate::types::PdfRect],
) -> String {
    let document_page_count = items.iter().map(|item| item.page).max().unwrap_or(0);
    to_markdown_from_items_with_rects_and_page_count(items, options, rects, document_page_count)
}

/// Convert positioned text items to Markdown with an authoritative PDF page count.
///
/// Use this overload when the owning PDF is available so trailing blank or
/// unextracted pages are included in document-level header and folio coverage.
/// Item-only callers can continue using [`to_markdown_from_items_with_rects`],
/// which falls back to the highest observed item page.
pub fn to_markdown_from_items_with_rects_and_page_count(
    items: Vec<TextItem>,
    options: MarkdownOptions,
    rects: &[crate::types::PdfRect],
    document_page_count: u32,
) -> String {
    to_markdown_from_items_with_rects_and_lines(
        items,
        options,
        rects,
        &[],
        MarkdownDocumentContext {
            page_thresholds: &HashMap::new(),
            struct_roles: None,
            struct_tables: &[],
            page_count: document_page_count,
            prefiltered_page_number_pages: None,
            prefiltered_page_number_mask: None,
            precomputed_chart_regions: None,
        },
    )
}

pub(crate) struct MarkdownDocumentContext<'a> {
    pub(crate) page_thresholds: &'a HashMap<u32, f32>,
    pub(crate) struct_roles:
        Option<&'a HashMap<u32, HashMap<i64, crate::structure_tree::StructRole>>>,
    pub(crate) struct_tables: &'a [crate::structure_tree::StructTable],
    pub(crate) page_count: u32,
    /// Pages where an upstream document-level pass removed folios. This keeps
    /// table-continuation classification consistent after masked items drop.
    pub(crate) prefiltered_page_number_pages: Option<&'a HashSet<u32>>,
    /// Document-level removal decisions aligned with this call's input items.
    /// Table detection consumes the original items; the mask is applied only
    /// after table claims have been established.
    pub(crate) prefiltered_page_number_mask: Option<&'a [bool]>,
    /// Optional chart masks shared with layout analysis so the geometry is
    /// detected once and interpreted identically by both pipelines.
    pub(crate) precomputed_chart_regions: Option<&'a PageChartRegions>,
}

/// Convert positioned text items to markdown, using rectangles and line segments for table detection.
///
/// Line-based detection runs first (strongest structural evidence), then rect-based,
/// then heuristic fallback on unclaimed items.
pub(crate) fn to_markdown_from_items_with_rects_and_lines(
    items: Vec<TextItem>,
    options: MarkdownOptions,
    rects: &[crate::types::PdfRect],
    pdf_lines: &[crate::types::PdfLine],
    context: MarkdownDocumentContext<'_>,
) -> String {
    convert_items_with_rects_lines_and_table_output(
        items,
        options,
        rects,
        pdf_lines,
        context,
        TableOutputMode::Markdown,
    )
    .markdown
}

/// Run the ordinary Markdown table pipeline but return only structurally
/// complete data tables. Supplemental OCR uses this to keep detector behavior
/// identical without parsing the serialized Markdown back into tables.
#[cfg(feature = "ocr")]
pub(crate) fn complete_table_markdown_from_items(
    items: Vec<TextItem>,
    options: MarkdownOptions,
    document_page_count: u32,
) -> String {
    let conversion = convert_items_with_rects_lines_and_table_output(
        items,
        options,
        &[],
        &[],
        MarkdownDocumentContext {
            page_thresholds: &HashMap::new(),
            struct_roles: None,
            struct_tables: &[],
            page_count: document_page_count,
            prefiltered_page_number_pages: None,
            prefiltered_page_number_mask: None,
            precomputed_chart_regions: None,
        },
        TableOutputMode::CompleteTables,
    );
    let mut output = String::new();
    for (_, table) in conversion.detected_tables {
        let markdown = crate::tables::table_to_markdown(&table);
        if markdown.is_empty() {
            continue;
        }
        if !output.is_empty() && !output.ends_with("\n\n") {
            output.push('\n');
        }
        output.push_str(&markdown);
    }
    output
}

fn convert_items_with_rects_lines_and_table_output(
    items: Vec<TextItem>,
    options: MarkdownOptions,
    rects: &[crate::types::PdfRect],
    pdf_lines: &[crate::types::PdfLine],
    context: MarkdownDocumentContext<'_>,
    table_output_mode: TableOutputMode,
) -> MarkdownConversionOutput {
    use crate::tables::{
        content_width, detect_tables_from_lines, detect_tables_from_rects,
        detect_tables_from_struct_tree, detect_tables_with_page_width, try_build_rect_guided_table,
    };
    use crate::types::ItemType;

    let MarkdownDocumentContext {
        page_thresholds,
        struct_roles,
        struct_tables,
        page_count: document_page_count,
        prefiltered_page_number_pages,
        prefiltered_page_number_mask,
        precomputed_chart_regions,
    } = context;

    if items.is_empty() {
        return MarkdownConversionOutput::default();
    }

    // Table detection must retain the original collection because short
    // numeric table cells can be indistinguishable from folios until
    // structural context is available. A precomputed mask carries the
    // document-wide decision without removing items before table claims.
    debug_assert!(prefiltered_page_number_mask.is_none_or(|mask| mask.len() == items.len()));
    let has_precomputed_page_number_mask = prefiltered_page_number_mask.is_some();
    let removed_page_number_pages = prefiltered_page_number_pages.cloned().unwrap_or_default();

    // Separate images and links from text items
    let mut images: Vec<TextItem> = Vec::new();
    let mut page_image_regions: HashMap<u32, Vec<(f32, f32, f32, f32)>> = HashMap::new();
    let mut links: Vec<TextItem> = Vec::new();
    let mut text_items: Vec<TextItem> = Vec::new();
    let mut text_item_page_number_mask: Vec<bool> = Vec::new();

    for (input_index, item) in items.into_iter().enumerate() {
        match &item.item_type {
            ItemType::Image => {
                page_image_regions.entry(item.page).or_default().push((
                    item.x,
                    item.y,
                    item.x + item.width,
                    item.y + item.height,
                ));
                if options.include_images {
                    images.push(item);
                }
            }
            ItemType::Link(_) => {
                if options.include_links {
                    links.push(item);
                }
            }
            ItemType::Text | ItemType::FormField => {
                text_item_page_number_mask.push(
                    prefiltered_page_number_mask
                        .and_then(|mask| mask.get(input_index))
                        .copied()
                        .unwrap_or(false),
                );
                text_items.push(item);
            }
        }
    }

    // Calculate base font size for table detection
    let font_stats = calculate_font_stats_from_items(&text_items);
    let base_size = options
        .base_font_size
        .unwrap_or(font_stats.most_common_size);

    // Detect tables on each page
    let mut table_items: HashSet<usize> = HashSet::new();
    let mut table_output = TableDetectionOutput::new(table_output_mode);

    // Running headers/footers repeat verbatim at the same position on many
    // pages. When such a block wraps a long title over aligned lines, the
    // heuristic detector reads it as a table. Knowing which items are page
    // furniture is a document-wide question, so answer it once here.
    let running_furniture = running_furniture_keys(&text_items);

    // Pre-group items by page with their global indices (O(n) instead of O(pages*n))
    let mut page_groups: HashMap<u32, Vec<(usize, &TextItem)>> = HashMap::new();
    for (global_idx, item) in text_items.iter().enumerate() {
        page_groups
            .entry(item.page)
            .or_default()
            .push((global_idx, item));
    }

    // Chart regions per page: their text must not steer column detection
    // during line grouping (it fills the gutter and fuses two-column lines).
    let page_chart_map = precomputed_chart_regions
        .cloned()
        .unwrap_or_else(|| chart_regions_by_page(&text_items, rects, pdf_lines));

    let mut pages: Vec<u32> = page_groups.keys().copied().collect();
    pages.sort();

    // Track band splits per page so we can split non-table items later
    let mut page_band_splits: HashMap<u32, Vec<(f32, f32)>> = HashMap::new();
    // Chart pages with two prose columns use the chart's vertical span as a
    // full-width separator and read each surrounding prose zone by column.
    let mut page_chart_prose_splits: HashMap<u32, f32> = HashMap::new();
    let mut page_chart_prose_orders: HashMap<u32, ChartProseOrder> = HashMap::new();

    for page in pages {
        let group = page_groups.get(&page).unwrap();
        let page_items: Vec<TextItem> = group.iter().map(|(_, item)| (*item).clone()).collect();
        let page_content_width = content_width(&page_items);

        // Chart-bar regions: bar charts drawn as filled rects read as cell
        // rects or aligned text and get gridded into phantom tables. Their
        // items are excluded from every table detector below and flow through
        // as plain text instead.
        let chart_regions: Vec<(f32, f32, f32, f32)> =
            page_chart_map.get(&page).cloned().unwrap_or_default();
        let in_chart = |item: &TextItem| item_is_in_chart_region(item, &chart_regions);
        let page_layout_items = items_outside_chart_regions(&page_items, &chart_regions);

        // Detect columns on chart-free text. Chart labels and values often fill
        // the prose gutter, hiding real columns and allowing body text to reach
        // heuristic table detection as one page-wide region.
        let detected_columns = {
            let cols = crate::extractor::detect_columns(&page_layout_items, page, false);
            cols.len() >= 2
        };
        if !chart_regions.is_empty() {
            log::debug!(
                "page {}: {} chart region(s) masked from table detection",
                page,
                chart_regions.len()
            );
        }

        // Repeated prose anchors provide a second, chart-scoped column signal.
        // It does not partition table detection: a narrow or partly spanning
        // gutter is too ambiguous for that. It can reject a body-font table
        // hypothesis and later order prose within chart-separated zones.
        // Multiple charts create several narrow vertical zones whose local
        // column structure needs stronger region-graph reasoning. Keep those
        // pages on the conservative full-page grouping path for now.
        let chart_prose_split = chart_regions.first().and_then(|&region| {
            if chart_regions.len() != 1 {
                return None;
            }
            chart_page_prose_column_split(&page_layout_items)
                .filter(|&split_x| chart_spans_prose_split(region, split_x))
        });

        // Check for side-by-side table layout using the original items. Sparse
        // numeric cells need table context before they can be distinguished
        // safely from folios; cleaned evidence is reserved for column and
        // final non-table layout decisions.
        let mut bands = split_side_by_side(&page_items);
        // A rect table crossing a proposed split boundary means the "gutter"
        // is really the gap between ruled and borderless table columns —
        // splitting there cleaves the table in half. Veto the split.
        if !bands.is_empty() && rect_cluster_spans_band_boundary(&page_items, rects, page, &bands) {
            log::debug!(
                "page {}: side-by-side split vetoed by spanning rect cluster",
                page
            );
            bands.clear();
        }
        // Fallback: use rect hint regions to detect side-by-side layout
        // when the text gap is too narrow for split_side_by_side to detect
        // (e.g. calendars with left/right month columns ~10pt apart).
        if bands.is_empty() {
            bands = split_from_hint_regions(&page_items, rects, page);
            // Only track hint-derived splits for non-table line grouping.
            // split_side_by_side splits already scope table detection and
            // their non-table items should flow through normal line grouping.
            if !bands.is_empty() {
                page_band_splits.insert(page, bands.clone());
            }
        }
        // Anchor-derived prose splits are lower-confidence than physical
        // gutters, so they do not partition table detection. They are applied
        // later only to vertical zones outside the chart. Physical bands can
        // still scope table detection without disabling this page-level prose
        // and positioned-block order.
        let chart_prose_order = chart_prose_split.and_then(|split_x| {
            chart_regions.first().copied().map(|region| {
                page_chart_prose_splits.insert(page, split_x);
                let order = ChartProseOrder::new(split_x, region);
                page_chart_prose_orders.insert(page, order);
                order
            })
        });

        // Build list of (band_items, band_index_map, band_rects, band_lines).
        // band_index_map[local_band_idx] → page_items index.
        let band_specs: Vec<BandSpec> = if bands.is_empty() {
            // Single-region page — use all items/rects/lines as-is
            let identity: Vec<usize> = (0..page_items.len()).collect();
            vec![(
                page_items.clone(),
                identity,
                rects.iter().filter(|r| r.page == page).cloned().collect(),
                pdf_lines
                    .iter()
                    .filter(|l| l.page == page)
                    .cloned()
                    .collect(),
            )]
        } else {
            bands
                .iter()
                .map(|&(x_lo, x_hi)| {
                    let margin = 2.0; // small margin to avoid clipping edge items
                    let (items_in_band, idx_map): (Vec<TextItem>, Vec<usize>) = page_items
                        .iter()
                        .enumerate()
                        .filter(|(_, item)| item.x >= x_lo - margin && item.x < x_hi + margin)
                        .map(|(idx, item)| (item.clone(), idx))
                        .unzip();
                    let band_rects = filter_rects_to_band(rects, page, x_lo, x_hi);
                    let band_lines = filter_lines_to_band(pdf_lines, page, x_lo, x_hi);
                    (items_in_band, idx_map, band_rects, band_lines)
                })
                .collect()
        };

        // When the page is split into bands but no band produces a table,
        // retry with all items merged as a single band.  This handles
        // borderless tables whose column alignment is misclassified as
        // page-layout columns by split_side_by_side.
        let was_split = band_specs.len() > 1;
        log::debug!(
            "page {}: {} bands (was_split={})",
            page,
            band_specs.len(),
            was_split
        );
        let merged_band: BandSpec = if was_split {
            let identity: Vec<usize> = (0..page_items.len()).collect();
            (
                page_items.clone(),
                identity,
                rects.iter().filter(|r| r.page == page).cloned().collect(),
                pdf_lines
                    .iter()
                    .filter(|l| l.page == page)
                    .cloned()
                    .collect(),
            )
        } else {
            (Vec::new(), Vec::new(), Vec::new(), Vec::new())
        };

        for (band_items, band_index_map, band_rects, band_lines) in &band_specs {
            if band_items.is_empty() {
                continue;
            }

            // Track which band-local indices are claimed by structural detection
            let mut rect_claimed: HashSet<usize> = HashSet::new();

            // Pre-claim chart items: every detector below skips claimed
            // indices, and unclaimed-by-tables text flows out as plain lines.
            if !chart_regions.is_empty() {
                for (idx, item) in band_items.iter().enumerate() {
                    if in_chart(item) {
                        rect_claimed.insert(idx);
                    }
                }
            }

            // 0. Structure-tree detection (highest priority — semantic PDF tagging)
            //    Only use struct-tree tables when they capture a majority (≥50%) of
            //    band items.  Incomplete struct trees (partial tagging) should fall
            //    through to geometry detection which sees all items.
            if !struct_tables.is_empty() {
                let st_tables = detect_tables_from_struct_tree(band_items, struct_tables, page);
                for table in &st_tables {
                    let coverage = table.item_indices.len() as f32 / band_items.len().max(1) as f32;
                    if coverage < 0.5 {
                        continue;
                    }
                    for &idx in &table.item_indices {
                        rect_claimed.insert(idx);
                        if let Some(&page_idx) = band_index_map.get(idx) {
                            if let Some(&(global_idx, _)) = group.get(page_idx) {
                                table_items.insert(global_idx);
                            }
                        }
                    }
                    table_output.record(page, table, chart_prose_order);
                }
            }

            // 1. Rect-based detection (skips tables overlapping struct-tree claims)
            let (rect_tables, hint_regions) =
                detect_tables_from_rects(band_items, band_rects, page);
            for table in &rect_tables {
                if !rect_claimed.is_empty()
                    && table
                        .item_indices
                        .iter()
                        .any(|idx| rect_claimed.contains(idx))
                {
                    continue;
                }
                for &idx in &table.item_indices {
                    rect_claimed.insert(idx);
                    if let Some(&page_idx) = band_index_map.get(idx) {
                        if let Some(&(global_idx, _)) = group.get(page_idx) {
                            table_items.insert(global_idx);
                        }
                    }
                }
                table_output.record(page, table, chart_prose_order);
            }

            // 2. Line-based detection on unclaimed items (when rects didn't find tables)
            if rect_claimed.is_empty() {
                let line_tables = detect_tables_from_lines(band_items, band_lines, page);
                for table in &line_tables {
                    for &idx in &table.item_indices {
                        rect_claimed.insert(idx);
                        if let Some(&page_idx) = band_index_map.get(idx) {
                            if let Some(&(global_idx, _)) = group.get(page_idx) {
                                table_items.insert(global_idx);
                            }
                        }
                    }
                    table_output.record(page, table, chart_prose_order);
                }
            }

            // 3a. Try rect-guided table construction on hint regions before
            //     creating the heuristic closure (avoids borrow conflicts).
            if rect_claimed.is_empty() && !hint_regions.is_empty() {
                let padding = 15.0;
                for hint in &hint_regions {
                    if hint.cluster_rects.is_empty() {
                        continue;
                    }
                    let (inside_items, inside_map): (Vec<TextItem>, Vec<usize>) = band_items
                        .iter()
                        .enumerate()
                        .filter(|(_, item)| {
                            item.y >= hint.y_bottom - padding
                                && item.y <= hint.y_top + padding
                                && item.x >= hint.x_left - padding
                                && item.x <= hint.x_right + padding
                        })
                        .map(|(idx, item)| (item.clone(), idx))
                        .unzip();

                    if let Some(table) =
                        try_build_rect_guided_table(&inside_items, &hint.cluster_rects)
                    {
                        for &idx in &table.item_indices {
                            if let Some(&band_idx) = inside_map.get(idx) {
                                if let Some(&page_idx) = band_index_map.get(band_idx) {
                                    if let Some(&(global_idx, _)) = group.get(page_idx) {
                                        table_items.insert(global_idx);
                                    }
                                }
                            }
                        }
                        table_output.record(page, &table, chart_prose_order);
                        for &band_idx in &inside_map {
                            rect_claimed.insert(band_idx);
                        }
                    }
                }
            }

            // 3b. Heuristic fallback on unclaimed items
            let mut run_heuristic =
                |subset_items: &[TextItem], index_map: &[usize], min_items: usize| {
                    if subset_items.len() < min_items {
                        return;
                    }
                    // Reject candidates whose cells prove they are parallel
                    // prose fragments — the shape produced when the body-font
                    // pass projects a multi-column text page onto one table
                    // grid (two-column reference sections are the classic
                    // case). The check needs internal transition evidence
                    // (unterminated cells flowing into lowercase starts in
                    // the same column), so genuine tables with long cells
                    // pass. Band-split retries stay exempt: they exist for
                    // tables that only assemble after recombining bands.
                    let reject_parallel_prose = !was_split;
                    let tables = detect_tables_with_page_width(
                        subset_items,
                        base_size,
                        false,
                        page_content_width,
                    );
                    for table in tables {
                        if reject_parallel_prose && is_parallel_prose_table(&table) {
                            log::debug!(
                                "page {}: rejected {}x{} parallel-prose table hypothesis",
                                page,
                                table.rows.len(),
                                table.columns.len()
                            );
                            continue;
                        }
                        if is_running_furniture_table(subset_items, &table, &running_furniture) {
                            log::debug!(
                                "page {}: rejected {}x{} running header/footer table hypothesis",
                                page,
                                table.rows.len(),
                                table.columns.len()
                            );
                            continue;
                        }
                        for &idx in &table.item_indices {
                            if let Some(&band_idx) = index_map.get(idx) {
                                if let Some(&page_idx) = band_index_map.get(band_idx) {
                                    if let Some(&(global_idx, _)) = group.get(page_idx) {
                                        table_items.insert(global_idx);
                                    }
                                }
                            }
                        }
                        table_output.record(page, &table, chart_prose_order);
                    }
                };

            // Run heuristic detection on unclaimed items
            if rect_claimed.is_empty() && hint_regions.is_empty() {
                // No rect tables or hints — run heuristic on all band items
                let identity_map: Vec<usize> = (0..band_items.len()).collect();
                run_heuristic(band_items, &identity_map, 6);
            } else if rect_claimed.is_empty() && !hint_regions.is_empty() {
                // No rect tables but hint regions exist — run heuristic separately
                // on items inside each hint region and on items outside all hints.
                let padding = 15.0;
                for hint in &hint_regions {
                    let (inside_items, inside_map): (Vec<TextItem>, Vec<usize>) = band_items
                        .iter()
                        .enumerate()
                        .filter(|(_, item)| {
                            item.y >= hint.y_bottom - padding && item.y <= hint.y_top + padding
                        })
                        .map(|(idx, item)| (item.clone(), idx))
                        .unzip();
                    run_heuristic(&inside_items, &inside_map, 6);
                    for &band_idx in &inside_map {
                        rect_claimed.insert(band_idx);
                    }
                }
                let (outside_items, outside_map): (Vec<TextItem>, Vec<usize>) = band_items
                    .iter()
                    .enumerate()
                    .filter(|(idx, _)| !rect_claimed.contains(idx))
                    .map(|(idx, item)| (item.clone(), idx))
                    .unzip();
                run_heuristic(&outside_items, &outside_map, 6);
            } else {
                // Rect tables found — run heuristic on unclaimed items
                let (unclaimed_items, unclaimed_map): (Vec<TextItem>, Vec<usize>) = band_items
                    .iter()
                    .enumerate()
                    .filter(|(idx, _)| !rect_claimed.contains(idx))
                    .map(|(idx, item)| (item.clone(), idx))
                    .unzip();
                run_heuristic(&unclaimed_items, &unclaimed_map, 6);
            }

            // 4. Column-based table detection for borderless tabular layouts.
            let band_has_tables = band_items.iter().enumerate().any(|(idx, _)| {
                band_index_map
                    .get(idx)
                    .and_then(|&page_idx| group.get(page_idx))
                    .is_some_and(|&(global_idx, _)| table_items.contains(&global_idx))
            });
            let has_structural_elements = band_rects.len() >= 6 || band_lines.len() >= 4;
            if !band_has_tables && !has_structural_elements {
                if let Some(table) = crate::tables::try_build_table_from_columns(band_items, page) {
                    for &idx in &table.item_indices {
                        if let Some(&page_idx) = band_index_map.get(idx) {
                            if let Some(&(global_idx, _)) = group.get(page_idx) {
                                table_items.insert(global_idx);
                            }
                        }
                    }
                    table_output.record(page, &table, chart_prose_order);
                }
            }
        }

        // 5. Thin-rect border synthesis: last resort for PDFs that draw table
        //    borders as thin filled rectangles (common in spreadsheet exports).
        //    Only runs when ALL other methods found nothing on this page.
        if !table_output.has_detected_tables_on_page(page) {
            let page_rects: Vec<&crate::types::PdfRect> =
                rects.iter().filter(|r| r.page == page).collect();
            let mut synth_lines: Vec<crate::types::PdfLine> = Vec::new();
            for r in &page_rects {
                let (mut w, mut h) = (r.width, r.height);
                let (mut x, mut y) = (r.x, r.y);
                if w < 0.0 {
                    x += w;
                    w = -w;
                }
                if h < 0.0 {
                    y += h;
                    h = -h;
                }
                if h < 2.0 && w >= 10.0 {
                    let mid_y = y + h / 2.0;
                    synth_lines.push(crate::types::PdfLine {
                        x1: x,
                        y1: mid_y,
                        x2: x + w,
                        y2: mid_y,
                        page,
                    });
                } else if w < 2.0 && h >= 10.0 {
                    let mid_x = x + w / 2.0;
                    synth_lines.push(crate::types::PdfLine {
                        x1: mid_x,
                        y1: y,
                        x2: mid_x,
                        y2: y + h,
                        page,
                    });
                }
            }
            if synth_lines.len() >= 10 {
                // Chart text stays out of the thin-rect fallback too — a
                // chart's thin grid rules would otherwise re-grid it.
                let (page_text, page_text_map): (Vec<TextItem>, Vec<usize>) = text_items
                    .iter()
                    .enumerate()
                    .filter(|(_, i)| i.page == page && !in_chart(i))
                    .map(|(idx, i)| (i.clone(), idx))
                    .unzip();
                let line_tables = detect_tables_from_lines(&page_text, &synth_lines, page);
                for table in &line_tables {
                    for &idx in &table.item_indices {
                        if let Some(&global_idx) = page_text_map.get(idx) {
                            table_items.insert(global_idx);
                        }
                    }
                    table_output.record(page, table, chart_prose_order);
                }
            }
        }

        // Merged-band retry: if we split into bands but found no tables in
        // any band, retry heuristic detection with all items as a single band.
        // This catches borderless tables whose text-column alignment was
        // misclassified as page-layout columns.
        if was_split && !table_output.has_tables_on_page(page) && !merged_band.0.is_empty() {
            let (ref band_items, ref band_index_map, _, _) = merged_band;
            log::debug!(
                "page {}: merged-band retry ({} items, was_split={})",
                page,
                band_items.len(),
                was_split
            );
            // Chart text stays out of the retry as well.
            let (chart_free, chart_free_map): (Vec<TextItem>, Vec<usize>) = band_items
                .iter()
                .enumerate()
                .filter(|(_, it)| !in_chart(it))
                .map(|(i, it)| (it.clone(), i))
                .unzip();
            // A chart-derived column signal must not disable body-font table
            // detection during the merged-band retry: this retry exists for
            // tables that only become visible after recombining false layout
            // bands. Keep the legacy skip on ordinary detected-column pages,
            // and reject chart-page prose candidates individually below.
            let skip_body_font =
                merged_retry_skips_body_font(detected_columns, !chart_regions.is_empty());
            let heuristic_tables = detect_tables_with_page_width(
                &chart_free,
                base_size,
                skip_body_font,
                page_content_width,
            );
            for table in &heuristic_tables {
                if !chart_regions.is_empty() && is_parallel_prose_table(table) {
                    log::debug!(
                        "page {}: rejected {}x{} merged-band parallel-prose table hypothesis",
                        page,
                        table.rows.len(),
                        table.columns.len()
                    );
                    continue;
                }
                if is_running_furniture_table(&chart_free, table, &running_furniture) {
                    log::debug!(
                        "page {}: rejected {}x{} merged-band running header/footer table hypothesis",
                        page,
                        table.rows.len(),
                        table.columns.len()
                    );
                    continue;
                }
                for &idx in &table.item_indices {
                    if let Some(&page_idx) = chart_free_map
                        .get(idx)
                        .and_then(|&band_idx| band_index_map.get(band_idx))
                    {
                        if let Some(&(global_idx, _)) = group.get(page_idx) {
                            table_items.insert(global_idx);
                        }
                    }
                }
                table_output.record(page, table, chart_prose_order);
            }
        }
    }

    #[cfg(feature = "ocr")]
    let mut detected_tables = table_output.complete_tables;
    #[cfg(feature = "ocr")]
    detected_tables.sort_by_key(|(page, _)| *page);
    let mut page_tables = table_output.markdown_by_page;

    // Images are also removed before line grouping, so give them the same
    // logical chart-page position as tables before reinsertion.
    let mut page_images: HashMap<u32, Vec<PositionedMarkdown>> = HashMap::new();
    for img in &images {
        let img_name = img
            .text
            .strip_prefix("[Image: ")
            .and_then(|s| s.strip_suffix(']'))
            .unwrap_or(&img.text);
        let img_md = format!("![Image: {}](image)\n", img_name);
        page_images
            .entry(img.page)
            .or_default()
            .push(PositionedMarkdown::new(
                img.y,
                img.x,
                img_md,
                page_chart_prose_orders.get(&img.page).copied(),
            ));
    }

    // Check structure tree coverage on ALL text items (before table filtering)
    // to decide whether to use structure-aware markdown generation.
    let struct_roles_coverage_ok = struct_roles.is_some_and(|roles| {
        let total = text_items.len();
        if total == 0 {
            return false;
        }
        let tagged = text_items
            .iter()
            .filter(|item| {
                item.mcid
                    .and_then(|mcid| {
                        roles
                            .get(&item.page)
                            .and_then(|page_roles| page_roles.get(&mcid))
                    })
                    .is_some()
            })
            .count();
        let coverage = tagged as f32 / total as f32;
        log::debug!(
            "structure tree coverage: {}/{} items ({:.0}%)",
            tagged,
            total,
            coverage * 100.0
        );
        coverage >= 0.5
    });
    let effective_struct_roles = if struct_roles_coverage_ok {
        struct_roles
    } else {
        None
    };

    // Filter out table items and process the rest
    let non_table_items: Vec<(usize, TextItem)> = text_items
        .into_iter()
        .enumerate()
        .filter(|(idx, _)| !table_items.contains(idx))
        .collect();

    // Find pages that are table-only (no remaining non-table text)
    let table_only_pages: HashSet<u32> = {
        let mut pages_with_text: HashSet<u32> =
            non_table_items.iter().map(|(_, item)| item.page).collect();
        // Preserve the pre-filter continuation classification: a page that
        // originally also contained a folio does not become table-only merely
        // because an upstream document-level pass removed it.
        pages_with_text.extend(removed_page_number_pages);
        page_tables
            .keys()
            .filter(|p| !pages_with_text.contains(p))
            .copied()
            .collect()
    };

    // Merge continuation tables across page breaks, but only for table-only pages
    merge_continuation_tables(&mut page_tables, &table_only_pages);

    // Collect pages that have detected tables — used to suppress relative valley
    // column detection on pages where table column gaps would be misidentified.
    let table_page_set: HashSet<u32> = page_tables.keys().copied().collect();

    let non_table_items = if has_precomputed_page_number_mask {
        non_table_items
            .into_iter()
            .filter(|(index, _)| !text_item_page_number_mask[*index])
            .map(|(_, item)| item)
            .collect()
    } else {
        crate::extractor::filter_markdown_page_numbers_with_removed_pages(
            non_table_items.into_iter().map(|(_, item)| item).collect(),
            document_page_count,
        )
        .0
    };

    // Split non-table items by band boundaries before line grouping so that
    // items from different side-by-side zones (e.g. left/right month columns
    // in a calendar) don't merge into the same line.
    let lines = if page_band_splits.is_empty() && page_chart_prose_splits.is_empty() {
        crate::extractor::group_prefiltered_items_into_lines_with_thresholds_and_regions(
            non_table_items,
            page_thresholds,
            &table_page_set,
            &page_chart_map,
            &page_image_regions,
        )
    } else {
        // Separate items into physical-band pages, chart/prose pages, and
        // ordinary pages. Chart/prose pages need a different reading order:
        // each chart is a full-width separator, while prose above and below
        // it reads down the left column and then down the right column.
        let mut split_page_items: HashMap<u32, Vec<TextItem>> = HashMap::new();
        let mut chart_prose_page_items: HashMap<u32, Vec<TextItem>> = HashMap::new();
        let mut unsplit_items: Vec<TextItem> = Vec::new();
        for item in non_table_items {
            if page_chart_prose_splits.contains_key(&item.page) {
                chart_prose_page_items
                    .entry(item.page)
                    .or_default()
                    .push(item);
            } else if page_band_splits.contains_key(&item.page) {
                split_page_items.entry(item.page).or_default().push(item);
            } else {
                unsplit_items.push(item);
            }
        }
        // Process unsplit pages normally
        let mut all_lines =
            crate::extractor::group_prefiltered_items_into_lines_with_thresholds_and_regions(
                unsplit_items,
                page_thresholds,
                &table_page_set,
                &page_chart_map,
                &page_image_regions,
            );
        // Process each split page's bands independently, then interleave
        // by Y position so paired zones (e.g. left/right months) appear together.
        let mut split_pages: Vec<u32> = split_page_items.keys().copied().collect();
        split_pages.sort();
        for page in split_pages {
            let items = split_page_items.remove(&page).unwrap();
            let bands = &page_band_splits[&page];
            let mut page_lines: Vec<crate::types::TextLine> = Vec::new();
            for &(x_lo, x_hi) in bands {
                let margin = 2.0;
                let band_items: Vec<TextItem> = items
                    .iter()
                    .filter(|i| i.x >= x_lo - margin && i.x < x_hi + margin)
                    .cloned()
                    .collect();
                if !band_items.is_empty() {
                    page_lines.extend(
                        crate::extractor::group_prefiltered_items_into_lines_with_thresholds_and_charts(
                            band_items,
                            page_thresholds,
                            &table_page_set,
                            &page_chart_map,
                        ),
                    );
                }
            }
            // Sort by Y descending (top to bottom) so left and right
            // band lines interleave in visual reading order.
            page_lines.sort_by(|a, b| b.y.total_cmp(&a.y));
            all_lines.extend(page_lines);
        }

        // Process chart/prose pages as alternating vertical zones. Within a
        // prose zone, group each column independently and append columns in
        // newspaper order. Within a chart zone, group the full width normally.
        let mut chart_prose_pages: Vec<u32> = chart_prose_page_items.keys().copied().collect();
        chart_prose_pages.sort();
        for page in chart_prose_pages {
            let mut remaining = chart_prose_page_items.remove(&page).unwrap();
            let split_x = page_chart_prose_splits[&page];
            let chart_regions = &page_chart_map[&page];
            let group_prose_zone = |zone_items: Vec<TextItem>| {
                let mut zone_lines = Vec::new();
                for right_column in [false, true] {
                    let column_items: Vec<TextItem> = zone_items
                        .iter()
                        .filter(|item| (item.x >= split_x) == right_column)
                        .cloned()
                        .collect();
                    if !column_items.is_empty() {
                        zone_lines.extend(
                            crate::extractor::group_prefiltered_items_into_lines_with_thresholds_and_charts(
                                column_items,
                                page_thresholds,
                                &table_page_set,
                                &page_chart_map,
                            ),
                        );
                    }
                }
                zone_lines
            };

            let mut chart_y_bands: Vec<(f32, f32)> = page_chart_map[&page]
                .iter()
                .map(|&(_, y0, _, y1)| (y0 - CHART_SEPARATOR_PAD, y1 + CHART_SEPARATOR_PAD))
                .collect();
            chart_y_bands.sort_by(|a, b| b.1.total_cmp(&a.1));
            let mut merged_chart_y_bands: Vec<(f32, f32)> = Vec::new();
            for (low, high) in chart_y_bands {
                if let Some(last) = merged_chart_y_bands.last_mut() {
                    if high >= last.0 {
                        last.0 = last.0.min(low);
                        last.1 = last.1.max(high);
                        continue;
                    }
                }
                merged_chart_y_bands.push((low, high));
            }

            for (low, high) in merged_chart_y_bands {
                let (above, at_or_below): (Vec<TextItem>, Vec<TextItem>) =
                    remaining.into_iter().partition(|item| {
                        item.y > high && !item_is_in_chart_region(item, chart_regions)
                    });
                all_lines.extend(group_prose_zone(above));

                let (chart_zone, below): (Vec<TextItem>, Vec<TextItem>) =
                    at_or_below.into_iter().partition(|item| {
                        item.y >= low || item_is_in_chart_region(item, chart_regions)
                    });
                all_lines.extend(
                    crate::extractor::group_prefiltered_items_into_lines_with_thresholds_and_charts(
                        chart_zone,
                        page_thresholds,
                        &table_page_set,
                        &page_chart_map,
                    ),
                );
                remaining = below;
            }
            all_lines.extend(group_prose_zone(remaining));
        }
        // The three processing paths above are accumulated separately. Restore
        // document page order while preserving each page's chosen line order.
        all_lines.sort_by_key(|line| line.page);
        all_lines
    };

    // Strip repeated headers/footers before conversion
    let lines = if options.strip_headers_footers {
        furniture::strip_header_footer_lines(lines, document_page_count)
    } else {
        lines
    };

    // Convert to markdown, inserting tables and images at appropriate positions
    let mut band_split_page_set: HashSet<u32> = page_band_splits.keys().copied().collect();
    band_split_page_set.extend(page_chart_prose_splits.keys().copied());
    let markdown = to_markdown_from_lines_with_tables_and_images(
        lines,
        options,
        page_tables,
        page_images,
        &page_chart_map,
        &band_split_page_set,
        effective_struct_roles,
    );
    MarkdownConversionOutput {
        markdown,
        #[cfg(feature = "ocr")]
        detected_tables,
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use analysis::detect_header_level;
    use classify::{is_code_like, is_list_item};

    #[cfg(feature = "ocr")]
    #[test]
    fn complete_table_output_marks_only_pages_with_emitted_tables() {
        let mut output = TableDetectionOutput::new(TableOutputMode::CompleteTables);
        let incomplete = crate::tables::Table::new(
            vec![100.0, 200.0],
            vec![300.0],
            vec![vec!["header a".into(), "header b".into()]],
            vec![0, 1],
        );
        output.record(1, &incomplete, None);
        assert!(output.has_detected_tables_on_page(1));
        assert!(!output.has_tables_on_page(1));
        assert!(output.complete_tables.is_empty());

        let complete = crate::tables::Table::new(
            vec![100.0, 200.0],
            vec![300.0, 280.0],
            vec![
                vec!["header a".into(), "header b".into()],
                vec!["value a".into(), "value b".into()],
            ],
            vec![0, 1, 2, 3],
        );
        output.record(1, &complete, None);
        assert!(output.has_tables_on_page(1));
        assert_eq!(output.complete_tables.len(), 1);
    }

    #[test]
    fn test_is_list_item() {
        assert!(is_list_item("• Item one"));
        assert!(is_list_item("- Item two"));
        assert!(is_list_item("* Item three"));
        assert!(is_list_item("1. First"));
        assert!(is_list_item("2) Second"));
        assert!(is_list_item("a. Letter item"));
        assert!(!is_list_item("Regular text"));
    }

    #[test]
    fn test_format_list_item() {
        assert_eq!(format_list_item("• Item"), "- Item");
        assert_eq!(format_list_item("- Item"), "- Item");
        assert_eq!(format_list_item("1. First"), "1. First");
    }

    #[test]
    fn test_is_code_like() {
        assert!(is_code_like("const x = 5;"));
        assert!(is_code_like("function foo() {"));
        assert!(is_code_like("import React from 'react'"));
        assert!(!is_code_like("This is regular text."));
    }

    #[test]
    fn test_detect_header_level() {
        // With three tiers: 24→H1, 18→H2, 15→H3, 12→None
        let tiers = vec![24.0, 18.0, 15.0];
        assert_eq!(detect_header_level(24.0, 12.0, &tiers, false), Some(1));
        assert_eq!(detect_header_level(18.0, 12.0, &tiers, false), Some(2));
        assert_eq!(detect_header_level(15.0, 12.0, &tiers, false), Some(3));
        assert_eq!(detect_header_level(12.0, 12.0, &tiers, false), None);

        // Single tier: 15→H1 (ratio 1.25 ≥ 1.2), 14→None (ratio 1.17 < 1.2)
        let tiers = vec![15.0];
        assert_eq!(detect_header_level(15.0, 12.0, &tiers, false), Some(1));
        assert_eq!(detect_header_level(14.0, 12.0, &tiers, false), None);
        assert_eq!(detect_header_level(12.0, 12.0, &tiers, false), None);

        // No tiers (empty): falls back to ratio thresholds
        let tiers: Vec<f32> = vec![];
        assert_eq!(detect_header_level(24.0, 12.0, &tiers, false), Some(1));
        assert_eq!(detect_header_level(18.0, 12.0, &tiers, false), Some(2));
        assert_eq!(detect_header_level(15.0, 12.0, &tiers, false), Some(3));
        assert_eq!(detect_header_level(14.5, 12.0, &tiers, false), Some(4));
        assert_eq!(detect_header_level(14.0, 12.0, &tiers, false), None);
        assert_eq!(detect_header_level(12.0, 12.0, &tiers, false), None);

        // Body text excluded when tiers exist: 13pt (ratio 1.08) → None
        let tiers = vec![20.0];
        assert_eq!(detect_header_level(13.0, 12.0, &tiers, false), None);
    }

    #[test]
    fn test_to_markdown() {
        let text = "• First item\n• Second item\n\nRegular paragraph.";
        let md = to_markdown(text, MarkdownOptions::default());
        assert!(md.contains("- First item"));
        assert!(md.contains("- Second item"));
    }

    fn furniture_item(text: &str, x: f32, y: f32, page: u32) -> TextItem {
        let mut it = make_item(x, y, page);
        it.text = text.into();
        it
    }

    /// Items repeating verbatim at the same position on 3+ pages are running
    /// furniture; the same text on fewer pages, or at different positions, is
    /// not.
    #[test]
    fn running_furniture_requires_three_pages_at_same_position() {
        let mut items = Vec::new();
        for page in 1..=3 {
            // Body content so each page has a real vertical extent.
            items.push(furniture_item("body", 85.0, 700.0, page));
            items.push(furniture_item("TITULAR DEL", 85.0, 68.0, page));
        }
        // Same text but only two pages.
        for page in 1..=2 {
            items.push(furniture_item("SECRETARÍA", 200.0, 68.0, page));
        }
        // Same text on three pages but at drifting positions.
        for (page, x) in [(1, 300.0), (2, 320.0), (3, 340.0)] {
            items.push(furniture_item("MÉXICO", x, 68.0, page));
        }

        let running = running_furniture_keys(&items);
        assert!(running.contains(&furniture_key(&furniture_item(
            "TITULAR DEL",
            85.0,
            68.0,
            1
        ))));
        assert!(!running.contains(&furniture_key(&furniture_item(
            "SECRETARÍA",
            200.0,
            68.0,
            1
        ))));
        assert!(!running.contains(&furniture_key(&furniture_item("MÉXICO", 300.0, 68.0, 1))));
    }

    /// A table made of running-footer items is vetoed; a table whose body rows
    /// carry per-page content is kept even when its header row repeats.
    #[test]
    fn running_furniture_table_veto() {
        // The footer block, present identically on pages 1-3.
        let mut items = Vec::new();
        for page in 1..=3 {
            items.push(furniture_item("PROPOSICIÓN CON PUNTO", 85.0, 78.5, page));
            items.push(furniture_item("EL SENADO", 286.6, 78.5, page));
            items.push(furniture_item("TITULAR DEL", 85.0, 68.0, page));
            items.push(furniture_item("A TRAVÉS DE LA", 243.4, 68.0, page));
        }
        // A real table on page 1: repeated header row, per-page data rows.
        let header = [
            furniture_item("Year", 85.0, 500.0, 1),
            furniture_item("Total", 200.0, 500.0, 1),
        ];
        let data = [
            furniture_item("2023", 85.0, 488.0, 1),
            furniture_item("1,204", 200.0, 488.0, 1),
            furniture_item("2024", 85.0, 476.0, 1),
            furniture_item("1,377", 200.0, 476.0, 1),
        ];
        // Header repeats on every page (like a continued table's header).
        for page in 2..=3 {
            items.push(furniture_item("Year", 85.0, 500.0, page));
            items.push(furniture_item("Total", 200.0, 500.0, page));
        }
        items.extend(header.iter().cloned());
        items.extend(data.iter().cloned());

        let running = running_furniture_keys(&items);

        let table_of = |detection_items: &[TextItem]| crate::tables::Table {
            columns: vec![],
            rows: vec![],
            cells: vec![],
            item_indices: (0..detection_items.len()).collect(),
            kind: crate::tables::TableKind::Data,
        };

        // Footer-only candidate: every item is furniture -> vetoed.
        let footer_items: Vec<TextItem> = (1..=1)
            .flat_map(|page| {
                vec![
                    furniture_item("PROPOSICIÓN CON PUNTO", 85.0, 78.5, page),
                    furniture_item("EL SENADO", 286.6, 78.5, page),
                    furniture_item("TITULAR DEL", 85.0, 68.0, page),
                    furniture_item("A TRAVÉS DE LA", 243.4, 68.0, page),
                ]
            })
            .collect();
        assert!(is_running_furniture_table(
            &footer_items,
            &table_of(&footer_items),
            &running
        ));

        // Real table: header row repeats across pages, body rows do not ->
        // 2 furniture of 6 items (33%) stays under the 80% threshold.
        let real_items: Vec<TextItem> =
            header.iter().cloned().chain(data.iter().cloned()).collect();
        assert!(!is_running_furniture_table(
            &real_items,
            &table_of(&real_items),
            &running
        ));
    }

    /// A form template repeated per record carries identical labels at
    /// identical mid-page coordinates on every page — those are real table
    /// cells, not furniture. Only the page-edge bands qualify.
    #[test]
    fn mid_page_repetition_is_not_furniture() {
        let mut items = Vec::new();
        for page in 1..=4 {
            // Content spanning the page: y 60 (bottom) to 740 (top).
            items.push(furniture_item("body top", 85.0, 740.0, page));
            items.push(furniture_item("body bottom", 85.0, 60.0, page));
            // Form labels repeated dead centre on every page.
            items.push(furniture_item("Name of creditor", 85.0, 400.0, page));
            items.push(furniture_item("Amount of claim", 300.0, 400.0, page));
            // A genuine footer inside the bottom band.
            items.push(furniture_item("FORM 78 — page footer", 85.0, 70.0, page));
        }

        let running = running_furniture_keys(&items);
        assert!(
            !running.contains(&furniture_key(&furniture_item(
                "Name of creditor",
                85.0,
                400.0,
                1
            ))),
            "mid-page form labels must not be furniture"
        );
        assert!(running.contains(&furniture_key(&furniture_item(
            "FORM 78 — page footer",
            85.0,
            70.0,
            1
        ))));
    }

    #[test]
    fn running_furniture_empty_on_short_documents() {
        let mut items = Vec::new();
        for page in 1..=2 {
            items.push(furniture_item("body", 85.0, 700.0, page));
            items.push(furniture_item("FOOTER", 85.0, 68.0, page));
        }
        assert!(running_furniture_keys(&items).is_empty());
    }

    /// A page whose text has no vertical span (a single line) gives no
    /// evidence of where its edges are; its items never become furniture.
    #[test]
    fn zero_span_page_contributes_no_furniture() {
        let mut items = Vec::new();
        for page in 1..=4 {
            items.push(furniture_item("ROW LABEL", 85.0, 400.0, page));
            items.push(furniture_item("ROW VALUE", 300.0, 400.0, page));
        }
        assert!(running_furniture_keys(&items).is_empty());
    }

    fn make_item(x: f32, y: f32, page: u32) -> TextItem {
        TextItem {
            text: "A".into(),
            x,
            y,
            width: 5.0,
            height: 10.0,
            font: String::new(),
            font_tag: String::new(),
            legacy_symbol_rewrite: false,
            font_size: 10.0,
            page,
            is_bold: false,
            is_italic: false,
            font_weight: None,
            bold_source: None,
            fixed_pitch: None,
            is_underline: false,
            is_strikeout: false,
            rotation: 0.0,
            advance_known: true,
            item_type: crate::types::ItemType::Text,
            mcid: None,
            baseline_shift: 0.0,
        }
    }

    fn make_item_w(x: f32, y: f32, width: f32, page: u32) -> TextItem {
        let mut it = make_item(x, y, page);
        it.width = width;
        it
    }

    #[test]
    fn precomputed_folio_mask_preserves_numeric_table_cells() {
        let mut items = Vec::new();
        let mut rects = Vec::new();
        for row in 0..4 {
            for column in 0..2 {
                let mut item = make_item_w(
                    110.0 + column as f32 * 100.0,
                    30.0 + row as f32 * 20.0,
                    20.0,
                    1,
                );
                item.text = (row * 2 + column + 1).to_string();
                items.push(item);
                rects.push(PdfRect {
                    x: 100.0 + column as f32 * 100.0,
                    y: 20.0 + row as f32 * 20.0,
                    width: 100.0,
                    height: 20.0,
                    page: 1,
                });
            }
        }

        // Simulate document-level folio decisions that would remove every
        // short numeric item if applied before structural table detection.
        let removal_mask = vec![true; items.len()];
        let removed_pages = HashSet::from([1]);
        let markdown = to_markdown_from_items_with_rects_and_lines(
            items,
            MarkdownOptions::default(),
            &rects,
            &[],
            MarkdownDocumentContext {
                page_thresholds: &HashMap::new(),
                struct_roles: None,
                struct_tables: &[],
                page_count: 1,
                prefiltered_page_number_pages: Some(&removed_pages),
                prefiltered_page_number_mask: Some(&removal_mask),
                precomputed_chart_regions: None,
            },
        );

        assert!(markdown.contains("|1|2|"), "{markdown}");
        assert!(markdown.contains("|7|8|"), "{markdown}");
    }

    #[test]
    fn early_layout_excludes_chart_items_before_column_detection() {
        let mut items = Vec::new();
        for row in 0..20 {
            let y = 100.0 + row as f32 * 14.0;
            items.push(make_item_w(90.0, y, 180.0, 1));
            items.push(make_item_w(340.0, y, 180.0, 1));
        }
        for col in 0..12 {
            items.push(make_item_w(
                100.0 + col as f32 * 35.0,
                620.0 - col as f32 * 4.0,
                25.0,
                1,
            ));
        }

        let chart_regions = vec![(90.0, 540.0, 530.0, 700.0)];
        let layout_items = items_outside_chart_regions(&items, &chart_regions);

        assert_eq!(layout_items.len(), 40);
        assert_eq!(
            crate::extractor::detect_columns(&layout_items, 1, false).len(),
            2
        );
    }

    #[test]
    fn chart_padding_claims_labels_but_not_adjacent_prose() {
        let regions = vec![(100.0, 100.0, 500.0, 300.0)];

        let mut label = make_item_w(220.0, 90.0, 60.0, 1);
        label.text = "January 2021".into();
        assert!(item_is_in_chart_region(&label, &regions));

        let mut long_label = make_item_w(210.0, 90.0, 180.0, 1);
        long_label.text = "Share of respondents by employment sector".into();
        assert!(item_is_in_chart_region(&long_label, &regions));

        let mut caption = make_item_w(120.0, 310.0, 350.0, 1);
        caption.text = "Figure 3. Results across every survey phase and sector".into();
        assert!(item_is_in_chart_region(&caption, &regions));

        let mut wide_short_prose = make_item_w(120.0, 90.0, 350.0, 1);
        wide_short_prose.text = "Results improved across all sectors".into();
        assert!(!item_is_in_chart_region(&wide_short_prose, &regions));

        let mut prose = make_item_w(120.0, 90.0, 350.0, 1);
        prose.text = "This paragraph continues below the chart into the next prose column".into();
        assert!(!item_is_in_chart_region(&prose, &regions));

        let mut bullet = make_item_w(340.0, 90.0, 5.0, 1);
        bullet.text = "•".into();
        assert!(!item_is_in_chart_region(&bullet, &regions));
    }

    #[test]
    fn chart_page_detects_two_short_prose_columns() {
        let mut items = Vec::new();
        for row in 0..12 {
            let y = 100.0 + row as f32 * 14.0;
            let mut left = make_item_w(90.0, y, 180.0, 1);
            left.text = "A complete prose line in the left column".into();
            items.push(left);
            let mut right = make_item_w(340.0, y, 180.0, 1);
            right.text = "A complete prose line in the right column".into();
            items.push(right);
        }

        let split = chart_page_prose_column_split(&items).unwrap();
        assert!(split > 200.0 && split < 300.0);
    }

    #[test]
    fn chart_page_rejects_non_overlapping_prose_anchors() {
        let mut items = Vec::new();
        for row in 0..8 {
            let mut left = make_item_w(90.0, 100.0 + row as f32 * 14.0, 180.0, 1);
            left.text = "A complete prose line in the upper column".into();
            items.push(left);
            let mut right = make_item_w(340.0, 300.0 + row as f32 * 14.0, 180.0, 1);
            right.text = "A complete prose line in the lower column".into();
            items.push(right);
        }

        assert_eq!(chart_page_prose_column_split(&items), None);
    }

    #[test]
    fn chart_separator_must_span_both_prose_columns() {
        let split_x = 280.0;

        assert!(chart_spans_prose_split(
            (100.0, 300.0, 500.0, 500.0),
            split_x
        ));
        assert!(!chart_spans_prose_split(
            (90.0, 300.0, 260.0, 500.0),
            split_x
        ));
        assert!(!chart_spans_prose_split(
            (300.0, 300.0, 520.0, 500.0),
            split_x
        ));
    }

    #[test]
    fn chart_prose_order_survives_physical_gutter_detection() {
        let mut items = Vec::new();
        for row in 0..20 {
            let y = 760.0 - row as f32 * 13.0;
            let mut left = make_item_w(90.0, y, 100.0, 1);
            left.text = format!("Left prose line {row} has several words");
            items.push(left);
            let mut right = make_item_w(340.0, y, 100.0, 1);
            right.text = format!("Right prose line {row} has several words");
            items.push(right);
        }
        assert!(
            !split_side_by_side(&items).is_empty(),
            "fixture must exercise the physical-gutter path"
        );

        // Connected, variably sized bars form a full-width chart separator.
        let mut rects = vec![PdfRect {
            x: 80.0,
            y: 280.0,
            width: 440.0,
            height: 150.0,
            page: 1,
        }];
        for (x, height) in [(120.0, 55.0), (210.0, 90.0), (300.0, 70.0), (390.0, 115.0)] {
            rects.push(PdfRect {
                x,
                y: 280.0,
                width: 45.0,
                height,
                page: 1,
            });
        }
        // A second connected family keeps the bar cluster above the detector's
        // minimum while retaining data-driven height variation.
        for (x, y, height) in [
            (120.0, 332.0, 45.0),
            (210.0, 367.0, 55.0),
            (300.0, 347.0, 35.0),
            (390.0, 392.0, 30.0),
        ] {
            rects.push(PdfRect {
                x,
                y,
                width: 45.0,
                height,
                page: 1,
            });
        }

        // Detection/input order is deliberately right before left. Only the
        // chart-scoped logical order can put these blocks into prose-column
        // order after the physical bands have already scoped table detection.
        let mut right_image = make_item(340.0, 650.0, 1);
        right_image.text = "[Image: RightFigure]".into();
        right_image.item_type = crate::types::ItemType::Image;
        items.push(right_image);
        let mut left_image = make_item(90.0, 600.0, 1);
        left_image.text = "[Image: LeftFigure]".into();
        left_image.item_type = crate::types::ItemType::Image;
        items.push(left_image);

        let options = MarkdownOptions {
            include_images: true,
            ..MarkdownOptions::default()
        };
        let markdown = to_markdown_from_items_with_rects(items, options, &rects);
        let left_last = markdown.find("Left prose line 19").unwrap();
        let right_first = markdown.find("Right prose line 0").unwrap();
        assert!(
            left_last < right_first,
            "chart prose must read down the left column before the right even when table detection found physical bands:\n{markdown}"
        );
        let left_image = markdown.find("LeftFigure").unwrap();
        let right_image = markdown.find("RightFigure").unwrap();
        assert!(
            left_image < right_image,
            "chart blocks must retain column order when physical bands are present:\n{markdown}"
        );
    }

    #[test]
    fn weave_and_list_projections_are_rejected() {
        // Tiny 2-column weave: prose continues down each column across an
        // empty cell, so lookback (not just adjacent rows) must connect it.
        let tiny_weave = crate::tables::Table::new(
            vec![70.0, 300.0],
            vec![320.0, 300.0, 280.0],
            vec![
                vec![
                    "spawning aggregations, 45% were unknown, 33% were".into(),
                    "of exploited grouper aggregations globally, as noted by".into(),
                ],
                vec![
                    "".into(),
                    "fisher interviews, monitoring, or underwater surveys done".into(),
                ],
                vec![
                    "decreasing, and 5% were already gone from the region".into(),
                    "records collected over the previous two survey decades".into(),
                ],
            ],
            (0..5).collect(),
        );
        assert!(is_parallel_prose_table(&tiny_weave), "tiny grid weave");

        // Wide projection: continuations outnumber rows and flow in 3+
        // columns of long prose fragments.
        let wide_weave = crate::tables::Table::new(
            vec![60.0, 160.0, 260.0, 360.0, 460.0],
            vec![400.0, 380.0, 360.0, 340.0, 320.0, 300.0],
            vec![
                vec![
                    "check all of your emotions carefully before".into(),
                    "causes strong visceral reactions and it".into(),
                    "if a claim ever seems clearly designed to".into(),
                    "chapters we are still reviewing in the".into(),
                    "".into(),
                ],
                vec![
                    "sharing anything further with your own".into(),
                    "should always give you pause and reason".into(),
                    "provoke rather than genuinely inform the".into(),
                    "focusing our efforts on researching a".into(),
                    "".into(),
                ],
                vec![
                    "closest colleagues and all of their many".into(),
                    "before finally deciding to pass along the".into(),
                    "reader it deserves careful scrutiny and".into(),
                    "wicked problem, and these are certainly".into(),
                    "".into(),
                ],
                vec![
                    "networks of trusted contacts and friends".into(),
                    "message along to anyone else who asks it".into(),
                    "verification through multiple separate and".into(),
                    "not simple topics anyone can summarize".into(),
                    "".into(),
                ],
                vec![
                    "who might spread it much further still and".into(),
                    "in the entire organization or well beyond".into(),
                    "independent sources of record and archive".into(),
                    "quickly for the busy executive readers".into(),
                    "".into(),
                ],
                vec![
                    "without ever checking any part of it first".into(),
                    "the original recipients list and beyond it".into(),
                    "before acting on the contents of anything".into(),
                    "who needs only the short version of this".into(),
                    "".into(),
                ],
            ],
            (0..24).collect(),
        );
        assert!(
            is_parallel_prose_table(&wide_weave),
            "continuation-dominated"
        );

        // Reference-table protection: wrapped short citation fragments
        // continue in every column but never read as running text
        // (long_prose = 0) — must stay a table.
        let reference_grid = crate::tables::Table::new(
            vec![70.0, 220.0, 370.0],
            vec![400.0, 380.0, 360.0, 340.0, 320.0, 300.0],
            vec![
                vec![
                    "§1.338(h)(10)-1(f)".into(),
                    "§1.331-1(d), and".into(),
                    "§1.331-1T(d) and".into(),
                ],
                vec!["".into(), "§1.332-6".into(), "§1.332-6T".into()],
                vec!["§1.382-2T(h)(4)(vi)".into(), "section".into(), "11T".into()],
                vec!["§1.382-8(a)".into(), "and (c)(5) of this".into(), "".into()],
                vec![
                    "The last sentence of".into(),
                    "paragraph (a)(2)(ii)".into(),
                    "paragraph (a) of".into(),
                ],
                vec![
                    "§1.382-2T(h)(4)(vi)(B)".into(),
                    "section".into(),
                    "11T".into(),
                ],
            ],
            (0..16).collect(),
        );
        assert!(!is_parallel_prose_table(&reference_grid), "reference grid");

        // Citation block over-fragmented into a wide word grid.
        let word_grid = crate::tables::Table::new(
            vec![60.0, 130.0, 200.0, 270.0, 340.0, 410.0],
            vec![400.0, 380.0, 360.0, 340.0],
            vec![
                vec![
                    "Songfang".into(),
                    "Huang,".into(),
                    "and".into(),
                    "Fei Huang.".into(),
                    "2023.".into(),
                    "Rrhf:".into(),
                ],
                vec![
                    "Rank".into(),
                    "responses".into(),
                    "to align".into(),
                    "language".into(),
                    "models".into(),
                    "with".into(),
                ],
                vec![
                    "human".into(),
                    "feedback".into(),
                    "without".into(),
                    "tears.".into(),
                    "arXiv".into(),
                    "preprint".into(),
                ],
                vec![
                    "arXiv:2304".into(),
                    "".into(),
                    "".into(),
                    "".into(),
                    "".into(),
                    "".into(),
                ],
            ],
            (0..19).collect(),
        );
        assert!(is_parallel_prose_table(&word_grid), "word-fragment grid");

        // Numbered list projected onto marker|text columns.
        let marker_list = crate::tables::Table::new(
            vec![70.0, 100.0],
            vec![400.0, 380.0, 360.0, 340.0],
            vec![
                vec!["1.".into(), "Edward Bernays".into()],
                vec!["2.".into(), "Wikipedia. Public Relations".into()],
                vec!["3.".into(), "Pinterest. Retrieved June 10, 2021.".into()],
                vec!["4.".into(), "Museum of Public Relations".into()],
            ],
            (0..8).collect(),
        );
        assert!(is_parallel_prose_table(&marker_list), "ordinal marker list");

        // Numbered list items woven beside sidebar fragments.
        let prefixed_list = crate::tables::Table::new(
            vec![70.0, 340.0],
            vec![400.0, 380.0, 360.0, 340.0],
            vec![
                vec![
                    "1. Restructuring the administration of the program".into(),
                    "".into(),
                ],
                vec![
                    "2. Shifting priorities for resource allocation".into(),
                    "freedom to decide".into(),
                ],
                vec![
                    "3. Pursuing regulatory reform across agencies".into(),
                    "".into(),
                ],
                vec![
                    "4. Reinvesting savings from system reorganization".into(),
                    "control over resources".into(),
                ],
            ],
            (0..6).collect(),
        );
        assert!(
            is_parallel_prose_table(&prefixed_list),
            "ordinal-prefixed list"
        );

        // Ranked data tables use bare numbers (no list-marker punctuation)
        // and stay tables.
        let ranked = crate::tables::Table::new(
            vec![70.0, 340.0],
            vec![400.0, 380.0, 360.0, 340.0],
            vec![
                vec!["1".into(), "Golden Eagle Aviation".into()],
                vec!["2".into(), "Northern Star Freight".into()],
                vec!["3".into(), "Pacific Rim Cargo".into()],
                vec!["4".into(), "Atlas Air Services".into()],
            ],
            (0..8).collect(),
        );
        assert!(!is_parallel_prose_table(&ranked), "ranked table");
    }

    #[test]
    fn parallel_prose_table_is_rejected_but_real_table_is_preserved() {
        assert!(merged_retry_skips_body_font(true, false));
        assert!(!merged_retry_skips_body_font(true, true));
        assert!(!merged_retry_skips_body_font(false, true));

        assert!(looks_like_numbered_section_heading(
            "9.5. Adapting to the New Normal: Changing Business Models"
        ));
        assert!(!looks_like_numbered_section_heading(
            "2024 revenue by business segment"
        ));

        let prose = crate::tables::Table::new(
            vec![90.0, 340.0],
            vec![320.0, 300.0, 280.0, 260.0],
            vec![
                vec![
                    "This section investigates the impact of public health measures".into(),
                    "course of the research period and the impacts continued".into(),
                ],
                vec![
                    "measures on business operations during the national lockdown".into(),
                    "".into(),
                ],
                vec![
                    "asked about their expectations for business recovery".into(),
                    "felt by firms working under reduced operating conditions".into(),
                ],
                vec![
                    "respondents described their expectations for business recovery".into(),
                    "while many other businesses remained temporarily closed".into(),
                ],
            ],
            (0..7).collect(),
        );
        assert!(is_parallel_prose_table(&prose));

        let data = crate::tables::Table::new(
            vec![90.0, 340.0],
            vec![300.0, 280.0, 260.0],
            vec![
                vec!["Sector".into(), "Revenue".into()],
                vec!["Tourism".into(), "$1,240".into()],
                vec!["Agriculture".into(), "$980".into()],
            ],
            (0..6).collect(),
        );
        assert!(!is_parallel_prose_table(&data));

        // A compact header row atop parallel prose columns: cross-row prose
        // continuations outnumber the rows, so the header cannot save the
        // candidate — this is page prose with two short fragments on top.
        let headed_parallel_prose = crate::tables::Table::new(
            vec![90.0, 340.0],
            vec![340.0, 320.0, 300.0, 280.0, 260.0],
            vec![
                vec!["June 2023".into(), "Page 5".into()],
                vec![
                    "the committee reviewed the proposal and decided that the".into(),
                    "funding for the second phase would continue subject to the".into(),
                ],
                vec![
                    "implementation schedule should be extended by another".into(),
                    "quarterly reviews established during the first phase of the".into(),
                ],
                vec![
                    "six months to accommodate the revised procurement rules".into(),
                    "".into(),
                ],
                vec![
                    "adopted at the previous meeting of the governing board".into(),
                    "participating institutions across the partner regions".into(),
                ],
            ],
            (0..10).collect(),
        );
        assert!(is_parallel_prose_table(&headed_parallel_prose));

        let headed_text_table = crate::tables::Table::new(
            vec![90.0, 340.0],
            vec![320.0, 300.0, 280.0],
            vec![
                vec!["Program".into(), "Description".into()],
                vec![
                    "Business recovery and continuity planning support".into(),
                    "Provides tailored guidance to firms affected by disruptions".into(),
                ],
                vec![
                    "Regional market access and supplier development".into(),
                    "Connects eligible producers with new distribution partners".into(),
                ],
            ],
            (0..6).collect(),
        );
        assert!(!is_parallel_prose_table(&headed_text_table));

        let sparse_first_row_then_compact_body = crate::tables::Table::new(
            vec![90.0, 340.0],
            vec![340.0, 320.0, 300.0, 280.0],
            vec![
                vec![
                    "This introductory prose fragment occupies only the left column".into(),
                    "".into(),
                ],
                vec!["short".into(), "row".into()],
                vec![
                    "continuation text remains aligned with the left prose anchor".into(),
                    "parallel text continues down the right prose column".into(),
                ],
                vec![
                    "another wrapped fragment follows in the left column".into(),
                    "while its neighboring prose fragment continues on the right".into(),
                ],
            ],
            (0..8).collect(),
        );
        assert!(is_parallel_prose_table(&sparse_first_row_then_compact_body));

        let headerless_description_table = crate::tables::Table::new(
            vec![90.0, 340.0],
            vec![320.0, 300.0, 280.0],
            vec![
                vec![
                    "community preparedness and emergency response planning".into(),
                    "provides detailed support for local continuity programs".into(),
                ],
                vec![
                    "regional supplier and market development assistance".into(),
                    "connects eligible producers with new distribution partners".into(),
                ],
                vec![
                    "financial continuity and business recovery program".into(),
                    "offers tailored guidance to firms affected by disruptions".into(),
                ],
            ],
            (0..6).collect(),
        );
        assert!(!is_parallel_prose_table(&headerless_description_table));

        let sparse_rowspanning_description_table = crate::tables::Table::new(
            vec![90.0, 340.0],
            vec![360.0, 340.0, 320.0, 300.0, 280.0, 260.0],
            vec![
                vec![
                    "community preparedness and emergency response planning support".into(),
                    "provides detailed support for local continuity program delivery".into(),
                ],
                vec![
                    "through coordinated training and regional response exercises".into(),
                    "".into(),
                ],
                vec![
                    "".into(),
                    "with technical assistance for participating local organizations".into(),
                ],
                vec![
                    "regional supplier and market development assistance program".into(),
                    "connects eligible producers with new distribution opportunities".into(),
                ],
                vec![
                    "through procurement guidance and tailored readiness workshops".into(),
                    "".into(),
                ],
                vec![
                    "".into(),
                    "while expanding access to qualified commercial partners".into(),
                ],
            ],
            (0..8).collect(),
        );
        assert!(!is_parallel_prose_table(
            &sparse_rowspanning_description_table
        ));
    }

    /// 4-row × 2-col ruled grid from x=100..300 (rows every 20pt from y=600).
    fn ruled_cluster_rects() -> Vec<PdfRect> {
        let mut rects = Vec::new();
        for row in 0..4 {
            for col in 0..2 {
                rects.push(PdfRect {
                    x: 100.0 + col as f32 * 100.0,
                    y: 600.0 + row as f32 * 20.0,
                    width: 100.0,
                    height: 20.0,
                    page: 1,
                });
            }
        }
        rects
    }

    #[test]
    fn band_veto_cluster_ruled_across_boundary() {
        // Rects on both sides of the boundary and cell text on both sides,
        // row-aligned → the split cuts straight through a drawn table.
        let mut rects = ruled_cluster_rects();
        for r in &mut rects {
            r.width = 150.0; // right column now spans 250..400, past b=320
        }
        let mut items = Vec::new();
        for row in 0..4 {
            let y = 610.0 + row as f32 * 20.0;
            items.push(make_item_w(110.0, y, 80.0, 1)); // left cells
            items.push(make_item_w(330.0, y, 30.0, 1)); // right cells past b
        }
        assert!(rect_cluster_spans_band_boundary(
            &items,
            &rects,
            1,
            &[(90.0, 320.0), (320.0, 500.0)]
        ));
    }

    #[test]
    fn band_veto_ignores_spanning_figure() {
        // A figure's rects span the boundary at the top of the page, but the
        // far side is dominated by column prose below it → keep the split.
        let mut rects = ruled_cluster_rects(); // y 600..680
        for r in &mut rects {
            r.width = 150.0; // spans past b=320
        }
        let mut items = Vec::new();
        // A few figure labels inside the cluster, aligned rows.
        for row in 0..4 {
            let y = 610.0 + row as f32 * 20.0;
            items.push(make_item_w(110.0, y, 40.0, 1));
            items.push(make_item_w(330.0, y, 20.0, 1));
        }
        // Dense prose column far below the figure (outside cluster y-range).
        let mut y = 100.0;
        while y < 560.0 {
            items.push(make_item_w(330.0, y, 140.0, 1));
            y += 12.0;
        }
        assert!(!rect_cluster_spans_band_boundary(
            &items,
            &rects,
            1,
            &[(90.0, 320.0), (320.0, 500.0)]
        ));
    }

    #[test]
    fn band_veto_borderless_columns_continue_rows() {
        // Rects end at x=300 (just short of b=320); cell-like text at x=340
        // aligns with every grid row → the "gutter" is inside the table.
        let rects = ruled_cluster_rects();
        let mut items = Vec::new();
        for row in 0..4 {
            let y = 610.0 + row as f32 * 20.0;
            items.push(make_item_w(110.0, y, 80.0, 1)); // label cells
            items.push(make_item_w(340.0, y, 30.0, 1)); // borderless column
        }
        assert!(rect_cluster_spans_band_boundary(
            &items,
            &rects,
            1,
            &[(90.0, 320.0), (320.0, 500.0)]
        ));
    }

    #[test]
    fn band_veto_ignores_prose_column() {
        // Dense prose right of the boundary: wide lines, three per grid row,
        // mostly not row-aligned → keep the side-by-side split.
        let rects = ruled_cluster_rects();
        let mut items = Vec::new();
        for row in 0..4 {
            items.push(make_item_w(110.0, 610.0 + row as f32 * 20.0, 80.0, 1));
        }
        let mut y = 602.0;
        while y < 680.0 {
            items.push(make_item_w(340.0, y, 200.0, 1)); // full-width prose lines
            y += 7.0;
        }
        assert!(!rect_cluster_spans_band_boundary(
            &items,
            &rects,
            1,
            &[(90.0, 320.0), (320.0, 500.0)]
        ));
    }

    #[test]
    fn split_from_hint_regions_too_few_rects() {
        // Fewer than 60 rects → no split
        let items = vec![make_item(10.0, 100.0, 1)];
        let rects: Vec<PdfRect> = (0..30)
            .map(|i| PdfRect {
                x: 10.0 + (i % 7) as f32 * 15.0,
                y: 100.0 + (i / 7) as f32 * 15.0,
                width: 10.0,
                height: 10.0,
                page: 1,
            })
            .collect();
        assert!(split_from_hint_regions(&items, &rects, 1).is_empty());
    }

    #[test]
    fn split_from_hint_regions_no_pairs() {
        // Enough rects but all in one X zone → no left/right pairs → no split
        let items = vec![make_item(10.0, 100.0, 1)];
        // 80 rects all in left half
        let rects: Vec<PdfRect> = (0..80)
            .map(|i| PdfRect {
                x: 10.0 + (i % 10) as f32 * 15.0,
                y: 100.0 + (i / 10) as f32 * 15.0,
                width: 10.0,
                height: 10.0,
                page: 1,
            })
            .collect();
        assert!(split_from_hint_regions(&items, &rects, 1).is_empty());
    }

    #[test]
    fn no_split_label_plus_number_table() {
        // Balance sheet layout: text labels on left, numbers on right.
        // Should NOT split because it's one table, not side-by-side regions.
        let mut items = Vec::new();
        for row in 0..30 {
            // Label at x=50
            let mut label = make_item(50.0, 700.0 - row as f32 * 15.0, 1);
            label.text = format!("Row label {}", row);
            label.width = 100.0;
            items.push(label);
            // Number at x=400
            let mut num1 = make_item(400.0, 700.0 - row as f32 * 15.0, 1);
            num1.text = format!("{},000.0", 100 + row);
            num1.width = 50.0;
            items.push(num1);
            // Number at x=470
            let mut num2 = make_item(470.0, 700.0 - row as f32 * 15.0, 1);
            num2.text = format!("{},500.0", 200 + row);
            num2.width = 50.0;
            items.push(num2);
        }
        let split = split_side_by_side(&items);
        assert!(
            split.is_empty(),
            "label+number table should not be split side-by-side"
        );
    }
}