docling-pdf 0.50.0

//! Layout-driven assembly: map detected [`Region`]s + text cells to a
//! [`DoclingDocument`], mirroring docling's page-assembly + reading-order.
//!
//! Overlapping detections are resolved greedily by score, each text cell is
//! assigned to its best-containing region, regions are ordered in reading order
//! (two-column aware), and each becomes a typed node by its layout label.

use docling_core::{Node, PictureClass, PictureImage, Table};
#[cfg(feature = "ml")]
use image::RgbImage;

use crate::layout::Region;
use crate::pdfium_backend::{PdfPage, TextCell};

fn area(l: f32, t: f32, r: f32, b: f32) -> f32 {
    ((r - l).max(0.0)) * ((b - t).max(0.0))
}

/// Intersection area of two boxes.
fn inter(a: &Region, l: f32, t: f32, r: f32, b: f32) -> f32 {
    let il = a.l.max(l);
    let it = a.t.max(t);
    let ir = a.r.min(r);
    let ib = a.b.min(b);
    area(il, it, ir, ib)
}

/// Wrapper (structured-region) labels, ported from docling
/// `LayoutPostprocessor.WRAPPER_TYPES`: a region that *contains* other regions
/// and renders as a structured block (a table / table-of-contents index), not as
/// its own flat text.
fn is_wrapper(label: &str) -> bool {
    matches!(
        label,
        "table" | "document_index" | "form" | "key_value_region"
    )
}

/// Labels docling's table-structure (TableFormer) model runs on and that render
/// as a Markdown table: a plain `table` and a `document_index` (a table of
/// contents), which docling assembles as a `TableItem` too.
pub fn is_table_like(label: &str) -> bool {
    matches!(label, "table" | "document_index")
}

/// Greedily keep regions by descending score, dropping a region that is mostly
/// covered by an already-kept one (RT-DETR emits overlapping duplicates).
fn greedy(mut regions: Vec<Region>) -> Vec<Region> {
    regions.sort_by(|a, b| b.score.total_cmp(&a.score));
    let mut kept: Vec<Region> = Vec::new();
    for r in regions {
        let ra = area(r.l, r.t, r.r, r.b).max(1.0);
        let covered = kept.iter().any(|k| {
            let i = inter(&r, k.l, k.t, k.r, k.b);
            let ka = area(k.l, k.t, k.r, k.b).max(1.0);
            // drop if most of r is inside k, or they strongly mutually overlap
            i / ra > 0.7 || i / (ra + ka - i) > 0.5
        });
        if !covered {
            kept.push(r);
        }
    }
    kept
}

/// Resolve overlapping RT-DETR detections, ported from the bucket structure of
/// docling's `LayoutPostprocessor`: regular, picture and wrapper clusters live in
/// **separate** spatial indexes and are de-overlapped independently, so a
/// high-score picture never suppresses a lower-score table or table-of-contents
/// index (the redp5110 TOC that was otherwise replaced by a picture box). A
/// cross-type pass first drops a picture that nearly coincides with a table
/// (`_handle_cross_type_overlaps`), keeping the structured table.
pub fn resolve(regions: Vec<Region>) -> Vec<Region> {
    // Cross-type: a region proposed as BOTH a picture and a table survives twice
    // (the two buckets de-overlap independently). Keep the structured table and
    // drop the coincident picture — IoU (not containment), so a genuine small
    // figure fully inside a large table region is not removed.
    let tables: Vec<(f32, f32, f32, f32)> = regions
        .iter()
        .filter(|r| r.label == "table")
        .map(|r| (r.l, r.t, r.r, r.b))
        .collect();
    let mut regions = regions;
    regions.retain(|r| {
        if r.label != "picture" {
            return true;
        }
        let ra = area(r.l, r.t, r.r, r.b).max(1.0);
        !tables.iter().any(|&(l, t, rr, b)| {
            let i = inter(r, l, t, rr, b);
            let u = ra + area(l, t, rr, b) - i;
            u > 0.0 && i / u > 0.8
        })
    });
    // De-overlap each bucket on its own.
    let pictures = greedy(
        regions
            .iter()
            .filter(|r| r.label == "picture")
            .cloned()
            .collect(),
    );
    let wrappers = greedy(
        regions
            .iter()
            .filter(|r| is_wrapper(r.label))
            .cloned()
            .collect(),
    );
    let mut kept = greedy(
        regions
            .iter()
            .filter(|r| r.label != "picture" && !is_wrapper(r.label))
            .cloned()
            .collect(),
    );
    dedup_nested_code(&mut kept);
    kept.extend(pictures);
    kept.extend(wrappers);
    kept
}

/// Drop a regular region that is >80% contained in a surviving special region we
/// render **as a single unit** — a table/table-of-contents index — ported from
/// docling's "Remove regular clusters that are included in wrappers" step: the
/// special absorbs it as a child (a table cell), so it must not also be emitted
/// as its own paragraph/list-item. This stops the survey list-items from
/// appearing both inside the detected table and again as bullets
/// (`table_mislabeled_as_picture`).
///
/// `picture` regions stay in the swallow set even after #165: docling keeps a
/// picture's contained clusters as the `PictureItem`'s *children* in the
/// document JSON (`ReadingOrderModel._add_child_elements`), but its
/// `MarkdownPictureSerializer` prints only the caption and the image — the
/// children never reach the Markdown (verified against the corpus groundtruth:
/// `amt_handbook`'s in-figure callout labels are absent). Dropping the
/// fully-contained regulars here reproduces exactly that. What #165 *does*
/// change is upstream, in [`add_orphan_regions`]: pictures no longer claim
/// cells, so a line only partially under a figure box (straddling its border,
/// ≤80 % contained) now forms an orphan region that survives this drop — those
/// words were silently erased before, and docling emits them.
///
/// `form` / `key_value_region` wrappers are deliberately **excluded**: this
/// pipeline does not render them as a structured block (they are skipped), so
/// their textual content comes precisely from the contained regular regions —
/// dropping those would erase the page (e.g. `right_to_left_03`'s form-heavy
/// pages). Runs *after* [`drop_false_pictures`] so a phantom picture can't
/// swallow real text on its way out.
pub fn drop_contained_regulars(regions: &mut Vec<Region>) {
    let specials: Vec<(f32, f32, f32, f32)> = regions
        .iter()
        .filter(|r| r.label == "picture" || is_table_like(r.label))
        .map(|r| (r.l, r.t, r.r, r.b))
        .collect();
    if specials.is_empty() {
        return;
    }
    regions.retain(|r| {
        if r.label == "picture" || is_wrapper(r.label) {
            return true;
        }
        let ra = area(r.l, r.t, r.r, r.b).max(1.0);
        !specials
            .iter()
            .any(|&(l, t, rr, b)| inter(r, l, t, rr, b) / ra > 0.8)
    });
}

/// True for a bare, single-token source-code language label (`XML`, `C#`, `JSON`,
/// `bash`, …) — the little header the docs render above a code block. Matched
/// case-insensitively; anything with whitespace or longer than a token is out.
fn is_code_language(t: &str) -> bool {
    let t = t.trim();
    if t.is_empty() || t.chars().any(char::is_whitespace) || t.chars().count() > 12 {
        return false;
    }
    const LANGS: &[&str] = &[
        "xml",
        "html",
        "xhtml",
        "json",
        "jsonc",
        "yaml",
        "yml",
        "toml",
        "ini",
        "c#",
        "csharp",
        "f#",
        "fsharp",
        "vb",
        "c",
        "c++",
        "cpp",
        "java",
        "kotlin",
        "scala",
        "go",
        "golang",
        "rust",
        "swift",
        "javascript",
        "js",
        "typescript",
        "ts",
        "jsx",
        "tsx",
        "python",
        "py",
        "ruby",
        "rb",
        "php",
        "perl",
        "lua",
        "r",
        "dart",
        "bash",
        "sh",
        "shell",
        "powershell",
        "zsh",
        "batch",
        "cmd",
        "sql",
        "tsql",
        "plsql",
        "graphql",
        "dockerfile",
        "makefile",
        "css",
        "scss",
        "sass",
        "less",
        "markdown",
        "md",
        "tex",
        "latex",
        "diff",
        "proto",
        "razor",
        "cshtml",
        "xaml",
        "aspx",
        "http",
    ];
    let lower = t.to_ascii_lowercase();
    LANGS.contains(&lower.as_str())
}

/// Mark the region indices that are a code block's **language label** — a bare
/// `XML`/`C#`/… token sitting directly above a `code` region — so they are consumed
/// rather than emitted as their own stray paragraph/heading. The label may also be
/// captured inside a wider code box (rendered as the fence's first line); dropping
/// the standalone copy just removes the duplicate.
fn code_language_labels(regions: &[Region], cells: &[TextCell]) -> Vec<bool> {
    let mut drop = vec![false; regions.len()];
    for (i, r) in regions.iter().enumerate() {
        if matches!(r.label, "code" | "picture" | "table") {
            continue;
        }
        if !is_code_language(&region_text(r, cells)) {
            continue;
        }
        // The label sits just above the code (a blank line's gap) or is swallowed
        // into the top of a wider code box; either way it is that block's label.
        // The window is generous because the label's own font is small, so a
        // one-line gap is several times its height.
        let line_h = (r.b - r.t).abs().max(1.0);
        let window = (line_h * 4.0).max(28.0);
        let labels_code = regions.iter().enumerate().any(|(j, c)| {
            if j == i || c.label != "code" {
                return false;
            }
            let gap = c.t - r.b; // >0 when the code is below the label
            let h_overlap = (r.r.min(c.r) - r.l.max(c.l)).max(0.0);
            gap > -line_h * 3.0 && gap < window && h_overlap > 0.0
        });
        if labels_code {
            drop[i] = true;
        }
    }
    drop
}

/// Collapse `code` regions where one is nested inside another, keeping the larger.
///
/// RT-DETR sometimes emits a tight code box *and* a wider near-duplicate that also
/// captures the block's language label (`XML`, `C#`, …). When the tight box scores
/// higher it is kept first, and the wider container — not "mostly inside" the tight
/// box — survives [`resolve`]'s greedy pass, so the block is emitted twice. Keeping
/// the **larger** box (rather than dropping it) collapses the pair without leaking
/// the container's extra cells back out as orphan text, since the larger box still
/// covers every cell. Restricted to `code` so genuinely distinct nested regions of
/// other kinds are untouched.
fn dedup_nested_code(kept: &mut Vec<Region>) {
    let mut drop = vec![false; kept.len()];
    for i in 0..kept.len() {
        if kept[i].label != "code" {
            continue;
        }
        let ai = area(kept[i].l, kept[i].t, kept[i].r, kept[i].b).max(1.0);
        for j in 0..kept.len() {
            if i == j || drop[j] || kept[j].label != "code" {
                continue;
            }
            let aj = area(kept[j].l, kept[j].t, kept[j].r, kept[j].b).max(1.0);
            // Drop i when it is mostly inside a strictly larger code box j.
            let overlap = inter(&kept[i], kept[j].l, kept[j].t, kept[j].r, kept[j].b);
            if aj > ai && overlap / ai > 0.7 {
                drop[i] = true;
                break;
            }
        }
    }
    let mut keep = drop.iter();
    kept.retain(|_| !*keep.next().unwrap());
}

/// Append `text` regions for cells the layout left uncovered ("orphan cells"),
/// the way docling's `LayoutPostprocessor` does (`create_orphan_clusters`): any
/// non-empty cell that no kept region covers (>50% of the cell's area) becomes a
/// text region of its own, so text the detector missed (a stray `.`, a small
/// label) is still emitted instead of silently dropped. Adjacent orphan cells on a
/// line are merged so a missed paragraph doesn't shatter into one block per line.
pub fn add_orphan_regions(regions: &mut Vec<Region>, cells: &[TextCell]) {
    // docling assigns a cell to its best-overlapping cluster at
    // intersection-over-self > 0.2; only cells below that for *every* region are
    // orphans. (Our text extraction uses a stricter 0.5, but matching docling's
    // 0.2 here avoids emitting cells it already placed in a neighbouring region.)
    //
    // Only *regular* clusters claim cells: docling's `_find_unassigned_cells`
    // walks `regular_clusters` alone, so a cell under a `picture` or a wrapper
    // (`table`/`document_index`/`form`/`key_value_region`) that no regular
    // cluster covers still becomes an orphan text cluster (#165). The orphans
    // that end up *fully* inside the special are re-dropped by
    // [`drop_contained_regulars`] (docling's Markdown drops them the same way
    // — a picture's children never reach its `MarkdownPictureSerializer`
    // output, a table's text renders through the reconstructed grid). The
    // observable fix is the border-straddlers: a line only partially under a
    // figure box used to lose its cells to the picture's 0.2 claim and vanish
    // — now it forms an orphan region and is emitted, as docling does.
    let assigned = |c: &TextCell| {
        let ca = area(c.l, c.t, c.r, c.b).max(1.0);
        regions
            .iter()
            .filter(|r| r.label != "picture" && !is_wrapper(r.label))
            .any(|r| inter(r, c.l, c.t, c.r, c.b) / ca > 0.2)
    };
    // Collect orphan cells (non-empty, unassigned), in page order.
    let mut orphans: Vec<&TextCell> = cells
        .iter()
        .filter(|c| !c.text.trim().is_empty() && !assigned(c))
        .collect();
    if orphans.is_empty() {
        return;
    }
    orphans.sort_by(|a, b| a.t.total_cmp(&b.t).then(a.l.total_cmp(&b.l)));
    // Merge cells that sit on the same line and nearly touch into one region, so a
    // dropped multi-word line stays one block (docling's refinement merges these).
    let mut merged: Vec<Region> = Vec::new();
    for c in orphans {
        let h = (c.b - c.t).abs().max(1.0);
        if let Some(last) = merged.last_mut() {
            let same_line = (last.t - c.t).abs() < h * 0.5;
            let touching = c.l <= last.r + h && c.l >= last.l - h;
            if same_line && touching {
                last.l = last.l.min(c.l);
                last.r = last.r.max(c.r);
                last.t = last.t.min(c.t);
                last.b = last.b.max(c.b);
                continue;
            }
        }
        merged.push(Region {
            label: "text",
            score: 0.0,
            l: c.l,
            t: c.t,
            r: c.r,
            b: c.b,
        });
    }
    regions.extend(merged);
}

/// Demote a `picture` region that is really a **text panel** — a paragraph block
/// the layout model boxed as a figure because it is typeset on a colored
/// background (terms-and-conditions callouts, quote boxes) — into ordinary
/// `text` regions, one per paragraph, so its words are read instead of shipped
/// as pixels. docling loses this text the same way (cells assigned to a picture
/// cluster are never serialized); this is a deliberate improvement, not parity.
///
/// The gate is conservative so a genuine figure keeps its crop: the region must
/// contain at least three text lines whose median width spans most of the panel
/// (axis labels and chat bubbles are narrow and varied) and whose cells cover a
/// substantial fraction of its area (a photo or chart with sparse labels does
/// not). Paragraph boundaries are re-derived from the line pitch: a vertical gap
/// clearly larger than the panel's own leading starts a new `text` region, so
/// the panel doesn't collapse into one giant paragraph.
///
/// Works on any cell source — the digital text layer or OCR lines recognized
/// from the picture crop — so the native and browser paths, with or without
/// force-OCR, demote identically.
pub fn recover_text_panels(regions: &mut Vec<Region>, cells: &[TextCell]) {
    // A *captioned* picture is a genuine figure whatever it contains — the
    // corpus is full of document screenshots ("Figure 3: …" above a page
    // image) that are exactly as dense and wide as a text panel. Only an
    // uncaptioned picture is a demotion candidate.
    let captioned: Vec<bool> = regions
        .iter()
        .map(|r| {
            r.label == "picture"
                && regions.iter().any(|c| {
                    c.label == "caption" && c.r.min(r.r) - c.l.max(r.l) > 0.0 && {
                        let gap = if c.t >= r.b {
                            c.t - r.b
                        } else if r.t >= c.b {
                            r.t - c.b
                        } else {
                            f32::MAX // vertically overlapping: not a caption
                        };
                        gap <= 25.0
                    }
                })
        })
        .collect();
    let mut out: Vec<Region> = Vec::with_capacity(regions.len());
    // Synthesized paragraphs and the demoted panels' boxes are kept separate
    // from `out` until the end: the dedup filter below must not confuse a
    // paragraph we just built with a pre-existing region inside the panel.
    let mut demoted_paras: Vec<Region> = Vec::new();
    let mut demoted_boxes: Vec<(f32, f32, f32, f32)> = Vec::new();
    for (i, r) in regions.drain(..).enumerate() {
        if r.label != "picture" || captioned[i] {
            out.push(r);
            continue;
        }
        let inside: Vec<&TextCell> = cells
            .iter()
            .filter(|c| {
                !c.text.trim().is_empty() && {
                    let ca = area(c.l, c.t, c.r, c.b).max(1.0);
                    inter(&r, c.l, c.t, c.r, c.b) / ca > 0.5
                }
            })
            .collect();
        // Group the contained cells into lines by vertical overlap (the same
        // rule region_text orders by), tracking each line's union box.
        let mut lines: Vec<(f32, f32, f32, f32)> = Vec::new(); // (t, b, l, r)
        for c in &inside {
            let (ct, cb) = (c.t.min(c.b), c.t.max(c.b));
            match lines.iter_mut().find(|(lt, lb, _, _)| {
                let ov = cb.min(*lb) - ct.max(*lt);
                ov > 0.5 * (cb - ct).min(*lb - *lt).max(1.0)
            }) {
                Some((lt, lb, ll, lr)) => {
                    *lt = lt.min(ct);
                    *lb = lb.max(cb);
                    *ll = ll.min(c.l);
                    *lr = lr.max(c.r);
                }
                None => lines.push((ct, cb, c.l, c.r)),
            }
        }
        let panel_w = (r.r - r.l).max(1.0);
        let coverage = inside.iter().map(|c| area(c.l, c.t, c.r, c.b)).sum::<f32>()
            / area(r.l, r.t, r.r, r.b).max(1.0);
        let mut widths: Vec<f32> = lines.iter().map(|(_, _, l, rr)| rr - l).collect();
        widths.sort_by(f32::total_cmp);
        let text_panel =
            lines.len() >= 3 && coverage >= 0.2 && widths[widths.len() / 2] >= 0.45 * panel_w;
        if !text_panel {
            out.push(r);
            continue;
        }
        lines.sort_by(|a, b| a.0.total_cmp(&b.0));
        let mut heights: Vec<f32> = lines.iter().map(|(t, b, _, _)| b - t).collect();
        heights.sort_by(f32::total_cmp);
        let h = heights[heights.len() / 2].max(1.0);
        let mut gaps: Vec<f32> = lines
            .windows(2)
            .map(|w| (w[1].0 - w[0].1).max(0.0))
            .collect();
        gaps.sort_by(f32::total_cmp);
        let leading = if gaps.is_empty() {
            0.0
        } else {
            gaps[gaps.len() / 2]
        };
        let brk = (1.8 * leading).max(0.75 * h);
        let mut para: Option<(f32, f32, f32, f32)> = None; // (l, t, r, b) union
        for (t, b, l, rr) in &lines {
            match &mut para {
                Some((pl, _, pr, pb)) if *t - *pb <= brk => {
                    *pl = pl.min(*l);
                    *pr = pr.max(*rr);
                    *pb = pb.max(*b);
                }
                _ => {
                    if let Some((pl, pt, pr, pb)) = para.take() {
                        demoted_paras.push(Region {
                            label: "text",
                            score: r.score,
                            l: pl,
                            t: pt,
                            r: pr,
                            b: pb,
                        });
                    }
                    para = Some((*l, *t, *rr, *b));
                }
            }
        }
        if let Some((pl, pt, pr, pb)) = para {
            demoted_paras.push(Region {
                label: "text",
                score: r.score,
                l: pl,
                t: pt,
                r: pr,
                b: pb,
            });
        }
        demoted_boxes.push((r.l, r.t, r.r, r.b));
    }
    // The paragraphs are rebuilt from *all* of the panel's cells, so any
    // surviving text region inside a demoted panel (an orphan cluster or a
    // layout-detected fragment — pictures no longer swallow them, #165) would
    // say the same words twice. Consume those; wrappers and pictures stay.
    if !demoted_boxes.is_empty() {
        out.retain(|r| {
            r.label == "picture" || is_wrapper(r.label) || {
                let ra = area(r.l, r.t, r.r, r.b).max(1.0);
                !demoted_boxes
                    .iter()
                    .any(|&(l, t, rr, b)| inter(r, l, t, rr, b) / ra > 0.5)
            }
        });
    }
    out.extend(demoted_paras);
    *regions = out;
}

/// Drop a `picture` detection that is a small, empty, low-confidence margin box on
/// a **text page** — a false positive the RT-DETR layout sometimes emits (e.g.
/// `right_to_left_02`'s phantom right-column picture, score 0.40); docling does not
/// emit it. The gate is deliberately narrow so a genuine figure is never dropped:
/// (1) only on pages with a digital text layer — image/scanned/figure pages have
/// no `cells` yet at this point (OCR runs later), so their pictures, which *are*
/// the content, are kept; (2) only a box covering < 25 % of the page (a margin
/// artifact, not a dominant figure); (3) only when it contains no text and scores
/// below 0.5 (real empty figures in the corpus all score ≥ 0.86).
pub fn drop_false_pictures(
    regions: &mut Vec<Region>,
    cells: &[TextCell],
    page_w: f32,
    page_h: f32,
) {
    if cells.iter().all(|c| c.text.trim().is_empty()) {
        return; // no digital text layer (image/scanned page) — keep all pictures
    }
    // A text-document page carries several text-bearing non-picture regions (so a
    // spurious margin picture is clearly extra). A slide / figure page has at most
    // one — there the picture is the content, so never drop it.
    let content_regions = regions
        .iter()
        .filter(|r| r.label != "picture" && !region_text(r, cells).trim().is_empty())
        .count();
    if content_regions < 2 {
        return;
    }
    let page_area = (page_w * page_h).max(1.0);
    regions.retain(|r| {
        if r.label != "picture" || r.score >= 0.5 {
            return true;
        }
        if area(r.l, r.t, r.r, r.b) / page_area >= 0.25 {
            return true; // a dominant figure, not a margin artifact
        }
        // Keep it if any text cell falls mostly inside (a real captioned/labelled
        // figure); drop only the genuinely empty low-confidence boxes.
        cells.iter().any(|c| {
            let ca = area(c.l, c.t, c.r, c.b).max(1.0);
            !c.text.trim().is_empty() && inter(r, c.l, c.t, c.r, c.b) / ca > 0.5
        })
    });
}

/// A small digit-only region in the top/bottom margin: a page number. docling
/// emits `right_to_left_02`'s bottom `11` as the page's *first* text item (its
/// reading-order model floats the page number to the front), whereas our
/// position-based ordering would place a bottom region last.
fn is_page_number(region: &Region, cells: &[TextCell], page_h: f32) -> bool {
    let t = region_text(region, cells);
    let t = t.trim();
    !t.is_empty()
        && t.chars().all(|c| c.is_ascii_digit())
        && (region.b - region.t).abs() < 30.0
        && (region.t < page_h * 0.12 || region.b > page_h * 0.88)
}

/// Furniture / not-yet-emitted labels.
fn is_skipped(label: &str) -> bool {
    matches!(
        label,
        "page_header"
            | "page_footer"
            | "checkbox_selected"
            | "checkbox_unselected"
            | "form"
            | "key_value_region"
    )
}

/// Reading-order sort of a page's regions, via the ported rule-based
/// [`reading_order`](crate::reading_order) predictor (docling's
/// `ReadingOrderPredictor`): an up/down geometry graph, horizontal dilation and a
/// depth-first traversal, with `page_header`/`page_footer` ordered as their own
/// groups (first/last) as docling does.
fn order_regions<T: Clone>(
    items: &mut Vec<T>,
    page_w: f32,
    page_h: f32,
    reg: impl Fn(&T) -> &Region,
) {
    let boxes: Vec<(f32, f32, f32, f32)> = items
        .iter()
        .map(|it| {
            let r = reg(it);
            (r.l, r.t, r.r, r.b)
        })
        .collect();
    let is_header: Vec<bool> = items
        .iter()
        .map(|it| reg(it).label == "page_header")
        .collect();
    let is_footer: Vec<bool> = items
        .iter()
        .map(|it| reg(it).label == "page_footer")
        .collect();
    let order = crate::reading_order::order_page(&boxes, &is_header, &is_footer, page_w, page_h);
    *items = order.iter().map(|&i| items[i].clone()).collect();
}

/// Clean a region's assembled text: undo soft-hyphen line wraps, map curly
/// quotes and the ellipsis to ASCII (matching docling), and collapse runs of
/// whitespace. pdfium emits the line-wrap hyphen as U+0002 in this corpus
/// (U+00AD elsewhere), so `word\u{2} continuation` is one hyphenated word —
/// drop the hyphen + the joining space and merge (`com\u{2} pact` → `compact`,
/// `end-to\u{2} end` → `end-toend`), exactly as docling does.
///
/// Token spacing is otherwise left as the geometric join produced it. We do not
/// tighten punctuation spacing: docling preserves the PDF's own spaces (it keeps
/// `{ ahn }`, `Name 1 .`, `[ 9 ]`), and a geometric gap heuristic diverges from
/// it more than a plain single-space join does.
/// An ordered-list enumeration marker at the start of a list item: leading ASCII
/// digits followed by `.`, e.g. `1. Undo/Redo` → `(1, "Undo/Redo")`. Returns
/// `None` when the text doesn't start with `digits.`.
fn parse_ordered_marker(s: &str) -> Option<(u64, String)> {
    let digits: String = s.chars().take_while(|c| c.is_ascii_digit()).collect();
    if digits.is_empty() {
        return None;
    }
    let rest = s[digits.len()..].strip_prefix('.')?;
    let number = digits.parse().ok()?;
    Some((number, rest.trim_start().to_string()))
}

/// Escape markdown special characters the way docling-core's markdown serializer
/// does (`markdown.py` post_process): `_` → `\_`, then HTML-escape `&`, `<`, `>`
/// (quote=False, so quotes are left). Applied to prose (headings, list items,
/// paragraphs); code blocks, the formula placeholder, and table cells are left raw.
fn md_escape(text: &str) -> String {
    text.replace('_', "\\_")
        .replace('&', "&amp;")
        .replace('<', "&lt;")
        .replace('>', "&gt;")
}

fn clean_text(text: &str) -> String {
    // Typographic-quote normalization follows docling-parse's sanitizer table
    // (`pdf_sanitators/constants.h`): every curly quote — single *and double* —
    // becomes the ASCII apostrophe `'`, and `‚` a comma. A `"` in docling's
    // output only ever comes from a literal `quotedbl` glyph, never from `“ ”`
    // (2206's `'text in the wild"` pairs a curly open with a literal-quote
    // close). This replaces an earlier Hangul-only special case that patched
    // one symptom of mapping `“ ”` to `"`.
    let replaced = text
        .replace("\u{2} ", "")
        .replace("\u{ad} ", "")
        .replace(['\u{2}', '\u{ad}'], "") // any stray wrap hyphens not at a join
        .replace(
            [
                '\u{2018}', '\u{2019}', '\u{201b}', '\u{201c}', '\u{201d}', '\u{201e}', '\u{201f}',
            ],
            "'",
        ) // ‘ ’ ‛ “ ” „ ‟ → '
        .replace('\u{201a}', ",") // ‚ → ,
        .replace(
            [
                '\u{2010}', '\u{2011}', '\u{2012}', '\u{2013}', '\u{2014}', '\u{2015}', '\u{2212}',
            ],
            "-",
        ) // hyphen/dash family → -
        .replace('\u{2044}', "/") // ⁄ fraction slash → /
        .replace('\u{2022}', "\u{b7}") // • → · (docling never emits •; inline CCS-concept separators)
        .replace('\u{2026}', "..."); // … → ...
    let out = if crate::pdfium_backend::use_dp_lines() {
        // The docling-parse sanitizer already placed the correct spacing (e.g.
        // justified double spaces); preserve internal runs of spaces, only
        // normalizing line breaks/tabs and trimming the ends.
        replaced.replace(['\n', '\r', '\t'], " ").trim().to_string()
    } else {
        // Legacy: collapse all whitespace runs to single spaces.
        replaced.split_whitespace().collect::<Vec<_>>().join(" ")
    };
    fix_arabic_lam_alef(&out)
}

/// pdfium decomposes the Arabic lam-alef ligature (لا / لإ / لأ / لآ) into its
/// glyph constituents in *visual* order — `alef-variant, lam` — but docling keeps
/// logical order, `lam, alef-variant`. Swap a mid-word `alef-variant + lam` back
/// to `lam + alef-variant`. "Mid-word" (the previous char is an Arabic letter)
/// distinguishes the ligature from the definite article `ال` (word-initial
/// `alef + lam`), which must stay. No-op for non-Arabic text.
fn fix_arabic_lam_alef(s: &str) -> String {
    let is_arabic_letter = |c: char| ('\u{0620}'..='\u{064A}').contains(&c);
    let chars: Vec<char> = s.chars().collect();
    if !chars.iter().any(|&c| is_arabic_letter(c)) {
        return s.to_string(); // no-op for non-Arabic text
    }
    // Pass 1: swap mid-word `alef-variant + lam` → `lam + alef-variant`. Only the
    // hamza/madda alef variants (إ أ آ) are safe: the definite article is always
    // plain `ا + ل`, so plain `alef + lam` is ambiguous (a legitimate `فعالة` vs a
    // reversed `لا` ligature look identical) — leaving plain alef alone avoids
    // corrupting legitimate words.
    let mut a: Vec<char> = Vec::with_capacity(chars.len());
    let mut i = 0;
    while i < chars.len() {
        let c = chars[i];
        if matches!(c, '\u{0622}' | '\u{0623}' | '\u{0625}')
            && chars.get(i + 1) == Some(&'\u{0644}')
            && i > 0
            && is_arabic_letter(chars[i - 1])
            // A preceding lam means this alef-variant is *already* the logical
            // `lam + alef` ligature; the following lam is the next syllable's
            // letter, not a reversed ligature — swapping it corrupts `لآل` → `للآ`
            // (e.g. التعلم الآلي → الآلي, not اللآي).
            && chars[i - 1] != '\u{0644}'
        {
            a.push('\u{0644}');
            a.push(c);
            i += 2;
            continue;
        }
        a.push(c);
        i += 1;
    }
    // Pass 2: insert a space at Arabic↔Latin boundaries (bidi script switch) that
    // pdfium runs together — docling separates the embedded Latin run (`وPython`
    // → `و Python`).
    let mut out: Vec<char> = Vec::with_capacity(a.len());
    for (j, &c) in a.iter().enumerate() {
        if j > 0 {
            let p = a[j - 1];
            if (is_arabic_letter(p) && c.is_ascii_alphabetic())
                || (p.is_ascii_alphabetic() && is_arabic_letter(c))
            {
                out.push(' ');
            }
        }
        out.push(c);
    }
    out.into_iter().collect()
}

/// Resolve each page hyperlink to the visible text it covers, as `(anchor, uri)`
/// in reading order. The anchor is the cells whose centre falls in the link rect,
/// joined left-to-right and cleaned the same way prose is (so it matches the
/// serialized text), deduped against the immediately-preceding link so pdfium's
/// occasional duplicate annotation doesn't double-list. Empty anchors are dropped.
pub(crate) fn resolve_link_anchors(page: &PdfPage) -> Vec<(String, String)> {
    let mut out: Vec<(String, String)> = Vec::new();
    // Use per-word cells, not the line-merged `cells`: a link rect covers a few
    // words on a line, and a whole merged line cell would over-capture (its centre
    // lands in one link's rect, grabbing the entire line as that link's anchor).
    let words = if page.word_cells.is_empty() {
        &page.cells
    } else {
        &page.word_cells
    };
    for link in &page.links {
        // A cell participates when its centre row is inside the rect and it
        // overlaps the rect horizontally. A cell can be *wider* than the rect:
        // PDFs often draw a whole header line as one text run ("LinkedIn |
        // GitHub | Credly"), which docling-parse's word grouping keeps as one
        // cell even though each label carries its own link annotation —
        // centre-in-rect alone would hand the entire line to every link.
        // [`cell_text_in_rect`] clips such a cell to the tokens under the rect.
        let mut inside: Vec<(&TextCell, String)> = words
            .iter()
            .filter(|c| {
                let cy = (c.t + c.b) / 2.0;
                cy >= link.t && cy <= link.b && c.r.min(link.r) > c.l.max(link.l)
            })
            .filter_map(|c| {
                let text = cell_text_in_rect(c, link.l, link.r);
                (!text.is_empty()).then_some((c, text))
            })
            .collect();
        // Reading order: top band then left-to-right (link anchors are LTR).
        let band = inside
            .iter()
            .map(|(c, _)| (c.b - c.t).abs())
            .fold(0.0f32, f32::max)
            .max(1.0);
        inside.sort_by_key(|(c, _)| ((c.t / band).round() as i64, (c.l * 10.0) as i64));
        let anchor = clean_text(
            &inside
                .iter()
                .map(|(_, t)| t.trim())
                .filter(|t| !t.is_empty())
                .collect::<Vec<_>>()
                .join(" "),
        );
        if anchor.is_empty() {
            continue;
        }
        if out
            .last()
            .is_some_and(|(a, u)| a == &anchor && u == &link.uri)
        {
            continue;
        }
        out.push((anchor, link.uri.clone()));
    }
    out
}

/// The part of a cell's text that lies under a link rect's x-range. A cell
/// fully inside the rect (by centre) returns its whole text. A wider cell is
/// split into whitespace tokens whose x-spans are estimated proportionally to
/// their character positions (kerning makes this approximate, so selection
/// snaps to whole tokens, never characters); tokens whose estimated centre
/// falls inside the rect are kept. Returns "" when nothing falls inside.
fn cell_text_in_rect(c: &TextCell, l: f32, r: f32) -> String {
    let cx = (c.l + c.r) / 2.0;
    if cx >= l && cx <= r && c.l >= l - (c.r - c.l) * 0.25 && c.r <= r + (c.r - c.l) * 0.25 {
        return c.text.trim().to_string();
    }
    let chars: Vec<char> = c.text.chars().collect();
    let n = chars.len();
    if n == 0 || c.r <= c.l {
        return String::new();
    }
    let per = (c.r - c.l) / n as f32;
    let mut out: Vec<String> = Vec::new();
    let mut token = String::new();
    let mut start = 0usize;
    // A trailing sentinel space flushes the last token.
    for (i, &ch) in chars.iter().enumerate().chain(std::iter::once((n, &' '))) {
        if ch.is_whitespace() {
            if !token.is_empty() {
                let mid = c.l + (start as f32 + (i - start) as f32 / 2.0) * per;
                if mid >= l && mid <= r {
                    out.push(std::mem::take(&mut token));
                } else {
                    token.clear();
                }
            }
        } else {
            if token.is_empty() {
                start = i;
            }
            token.push(ch);
        }
    }
    out.join(" ")
}

/// Cells assigned to a region (best container), in reading order, joined.
fn region_text(region: &Region, cells: &[TextCell]) -> String {
    let mut inside: Vec<&TextCell> = cells
        .iter()
        .filter(|c| {
            let ca = area(c.l, c.t, c.r, c.b).max(1.0);
            inter(region, c.l, c.t, c.r, c.b) / ca > 0.5
        })
        .collect();
    // Quantize the top coordinate into ~line bands so cells on the same line
    // sort in reading order; this is a strict total order (a raw fuzzy comparator
    // is not transitive and makes Rust's sort panic). For a right-to-left
    // (Arabic-majority) region, cells on a line read right→left, so sort the band
    // by descending left edge.
    let band = inside
        .iter()
        .map(|c| (c.b - c.t).abs())
        .fold(0.0f32, f32::max)
        .max(1.0);
    let arabic = inside
        .iter()
        .flat_map(|c| c.text.chars())
        .filter(|&c| ('\u{0600}'..='\u{06FF}').contains(&c))
        .count();
    let latin = inside
        .iter()
        .flat_map(|c| c.text.chars())
        .filter(|c| c.is_ascii_alphabetic())
        .count();
    let rtl = arabic > latin;
    let dp = crate::pdfium_backend::use_dp_lines();
    if dp {
        // docling orders a cluster's cells by their docling-parse cell index
        // (`LayoutPostprocessor._sort_cells`: `sorted(cells, key=c.index)`) —
        // the sanitizer's output order, which our `cells` slice already is.
        // That is *stream* order, and a generator that paints a line's bold
        // runs after the surrounding regular text strands them out of place
        // ("C.6. Zur Wahrung …" read "C. Zur Wahrung … 6."). Restore reading
        // order geometrically — but group lines by vertical *overlap*, not a
        // quantized band: 2206's inline math `>` sits ~2 pt above its line's
        // band and a plain band-sort drifted it into the next line
        // (`( > 10 pages)` → `( 10 pages) … complex > tables`); overlap
        // grouping keeps it home. Sorts are stable, so cells sharing a line
        // and an x-position keep their index order.
        let mut lines: Vec<(f32, f32, Vec<&TextCell>)> = Vec::new();
        for c in inside.drain(..) {
            let (ct, cb) = (c.t.min(c.b), c.t.max(c.b));
            let line = lines.iter_mut().find(|(lt, lb, _)| {
                let ov = cb.min(*lb) - ct.max(*lt);
                ov > 0.5 * (cb - ct).min(*lb - *lt).max(1.0)
            });
            match line {
                Some((lt, lb, cs)) => {
                    *lt = lt.min(ct);
                    *lb = lb.max(cb);
                    cs.push(c);
                }
                None => lines.push((ct, cb, vec![c])),
            }
        }
        lines.sort_by(|a, b| a.0.total_cmp(&b.0));
        for (_, _, mut cs) in lines {
            cs.sort_by(|a, b| {
                if rtl {
                    b.l.total_cmp(&a.l)
                } else {
                    a.l.total_cmp(&b.l)
                }
            });
            inside.extend(cs);
        }
    } else {
        inside.sort_by_key(|c| {
            let x = (c.l * 10.0) as i64;
            ((c.t / band).round() as i64, if rtl { -x } else { x })
        });
    }
    // Join cells in reading order. With the docling-parse sanitizer the cells are
    // already correctly spaced words/lines, so adjacent cells join with a single
    // space (docling joins its line cells with a space) — matching e.g. a bold
    // label and its value, `LABEL` | `: value` → `LABEL : value`. The legacy
    // reconstruction instead joins same-band cells with a space only across a real
    // gap, because it can split a word into abutting segments (`الت`|`ي` → `التي`).
    let mut joined = String::new();
    let mut prev: Option<&&TextCell> = None;
    for c in &inside {
        let t = c.text.trim();
        // Skip whitespace-only cells (e.g. a justified line's trailing space glyph
        // at a wrap): the join already inserts a separator, so an empty cell would
        // double it (`all-metal  construction`).
        if t.is_empty() {
            continue;
        }
        if let Some(p) = prev {
            let same_band = ((p.t / band).round() as i64) == ((c.t / band).round() as i64);
            let h = (c.b - c.t).abs().max((p.b - p.t).abs()).max(1.0);
            let gap = if rtl { p.l - c.r } else { c.l - p.r };
            // Dehyphenate a wrapped word: a line ending in a hyphen/dash followed
            // by a continuation joins without the dash or a space (`platforms—` +
            // `reflects` → `platformsreflects`). The dash is still raw here
            // (clean_text normalizes em/en dashes later), so match them all.
            let ends_dash = matches!(
                joined.chars().last(),
                Some('-' | '\u{2010}' | '\u{2013}' | '\u{2014}')
            );
            let before = joined.chars().nth_back(1); // char before the dash
            let next = t.chars().next();
            let alpha_dehyph = before.is_some_and(|c| c.is_alphabetic())
                && next.is_some_and(|n| {
                    // Ordinary hyphenation (lowercase continuation), or a CamelCase
                    // compound name wrapped at the hyphen — a lowercase letter before
                    // the dash continuing with an uppercase one (`PubTab-Net` →
                    // `PubTabNet`, `Table-Former` → `TableFormer`). Excludes runs like
                    // `MS-COCO` (uppercase before the dash) and `PubTables-1M` (digit).
                    n.is_lowercase()
                        || (n.is_uppercase() && before.is_some_and(|b| b.is_lowercase()))
                });
            // A number range wrapped at its hyphen (`pp. 545-` + `561` → `545561`):
            // docling drops the line-wrap hyphen between two digit runs. A same-line
            // range (`1162-1167`) never reaches this continuation path, so it keeps
            // its hyphen.
            let num_dehyph = before.is_some_and(|c| c.is_ascii_digit())
                && next.is_some_and(|n| n.is_ascii_digit());
            let dehyph = dp && ends_dash && (alpha_dehyph || num_dehyph);
            if dehyph {
                joined.pop();
            } else if dp || !same_band || gap > h * 0.25 {
                joined.push(' ');
            }
        }
        joined.push_str(t);
        prev = Some(c);
    }
    clean_text(&joined)
}

/// Tighten the spaces pdfium leaves around tight punctuation in a code line
/// (`console .log` → `console.log`, `add (3 , 5)` → `add(3, 5)`), matching
/// docling-parse's source spacing.
fn tighten_code_punct(s: &str) -> String {
    s.replace(" .", ".")
        .replace(" ,", ",")
        .replace(" ;", ";")
        .replace(" )", ")")
        .replace(" (", "(")
}

/// Assemble a **code** region's text with its line structure preserved.
///
/// Unlike [`region_text`] — which joins every cell with a single space, the right
/// thing for prose reflow — a code block's line breaks and indentation are
/// significant. The `code_cells` are already one physical source line each
/// (grouped space-glyph-only, so monospace runs keep their spacing), so this:
///
/// 1. groups the cells into vertical line bands and orders them top→bottom,
///    left→right;
/// 2. joins the lines with `\n` (rather than spaces), keeping the carriage
///    returns; and
/// 3. reconstructs each line's leading indentation from its left offset, in units
///    of the block's estimated monospace character width, so nesting survives.
///
/// Typography is normalized per line via [`clean_text`] (smart quotes, dashes,
/// ellipsis), which never merges lines. Returns an empty string if the region has
/// no code cells (the caller falls back to the prose text).
fn code_region_text(region: &Region, cells: &[TextCell]) -> String {
    let mut inside: Vec<&TextCell> = cells
        .iter()
        .filter(|c| {
            let ca = area(c.l, c.t, c.r, c.b).max(1.0);
            inter(region, c.l, c.t, c.r, c.b) / ca > 0.5
        })
        .filter(|c| !c.text.trim().is_empty())
        .collect();
    if inside.is_empty() {
        return String::new();
    }

    // Quantize the top edge into ~line bands (like `region_text`), then order the
    // cells by band (top→bottom) and, within a band, by left edge.
    let band = inside
        .iter()
        .map(|c| (c.b - c.t).abs())
        .fold(0.0f32, f32::max)
        .max(1.0);
    let line_of = |c: &TextCell| (c.t / band).round() as i64;
    inside.sort_by_key(|c| (line_of(c), (c.l * 10.0) as i64));

    // Estimate one monospace character's width (total ink width / total glyphs) to
    // convert a line's left offset into a count of leading spaces. Measured over
    // all lines so a single short line can't skew it.
    let (mut total_w, mut total_chars) = (0.0f32, 0usize);
    for c in &inside {
        let n = c.text.trim().chars().count();
        if n > 0 {
            total_w += (c.r - c.l).max(0.0);
            total_chars += n;
        }
    }
    let char_w = if total_chars > 0 {
        (total_w / total_chars as f32).max(1.0)
    } else {
        1.0
    };
    // The block's own left margin is the zero-indent baseline.
    let base_l = inside.iter().map(|c| c.l).fold(f32::INFINITY, f32::min);

    let mut lines: Vec<String> = Vec::new();
    let mut cur: Option<i64> = None;
    for c in &inside {
        // Tighten pdfium's spaced punctuation per line (on the trimmed content, so
        // the reconstructed leading indentation is never nibbled).
        let text = tighten_code_punct(&clean_text(c.text.trim()));
        if Some(line_of(c)) == cur {
            // A second cell sharing this band (rare — e.g. split columns): keep it
            // on the same source line, separated by a space.
            if let Some(last) = lines.last_mut() {
                last.push(' ');
                last.push_str(&text);
            }
            continue;
        }
        let indent = ((c.l - base_l) / char_w).round().max(0.0) as usize;
        lines.push(format!("{}{}", " ".repeat(indent), text));
        cur = Some(line_of(c));
    }
    lines.join("\n")
}

/// Reconstruct a table's grid geometrically from the text cells inside its
/// region: cluster cells into rows (by vertical centre) and columns (by clustered
/// left edges), then place each cell. A model-free stand-in for TableFormer that
/// recovers grid-aligned tables from the precise PDF text layer (it does not
/// resolve row/column spans).
pub fn reconstruct_table(region: &Region, cells: &[TextCell]) -> Vec<Vec<String>> {
    let mut inside: Vec<&TextCell> = cells
        .iter()
        .filter(|c| {
            let ca = area(c.l, c.t, c.r, c.b).max(1.0);
            inter(region, c.l, c.t, c.r, c.b) / ca > 0.5
        })
        .collect();
    if inside.is_empty() {
        return Vec::new();
    }
    inside.sort_by(|a, b| a.t.total_cmp(&b.t));

    // Rows: consecutive cells whose vertical centre is within ~0.7 line height.
    let mut rows: Vec<(f32, Vec<&TextCell>)> = Vec::new();
    for c in &inside {
        let cyc = (c.t + c.b) / 2.0;
        let lh = (c.b - c.t).abs().max(1.0);
        if let Some((ryc, row)) = rows.last_mut() {
            if (cyc - *ryc).abs() < lh * 0.7 {
                row.push(c);
                continue;
            }
        }
        rows.push((cyc, vec![c]));
    }

    // Columns: cluster left edges (merge those within a tolerance).
    let tol = {
        let mut hs: Vec<f32> = inside.iter().map(|c| (c.b - c.t).abs()).collect();
        hs.sort_by(f32::total_cmp);
        hs[hs.len() / 2].max(4.0) * 1.5
    };
    let mut lefts: Vec<f32> = inside.iter().map(|c| c.l).collect();
    lefts.sort_by(f32::total_cmp);
    let mut col_starts: Vec<f32> = Vec::new();
    for l in lefts {
        if col_starts.last().is_none_or(|&last| l - last > tol) {
            col_starts.push(l);
        }
    }
    let ncols = col_starts.len().max(1);
    let col_of = |l: f32| -> usize {
        col_starts
            .iter()
            .rposition(|&s| l + tol * 0.5 >= s)
            .unwrap_or(0)
            .min(ncols - 1)
    };

    let mut grid = Vec::with_capacity(rows.len());
    for (_, mut row) in rows {
        row.sort_by(|a, b| a.l.total_cmp(&b.l));
        let mut cols = vec![String::new(); ncols];
        for c in row {
            let ci = col_of(c.l);
            // Strip the wrap-hyphen control char so it never lands in a cell.
            let t = c.text.trim().replace(['\u{2}', '\u{ad}'], "");
            if cols[ci].is_empty() {
                cols[ci] = t;
            } else {
                cols[ci].push(' ');
                cols[ci].push_str(&t);
            }
        }
        grid.push(cols);
    }
    grid
}

/// Does the geometric reconstruction of a table look trustworthy enough to use
/// as-is, instead of paying for TableFormer?
///
/// [`reconstruct_table`] derives columns by clustering cell **left edges**. On a
/// clean grid that is exact, but when a column's entries are not left-aligned
/// (or the OCR boxes wobble) the clustering splits one real column into several,
/// and the result is a wide, mostly-empty grid — the "spurious empty columns"
/// failure TableFormer exists to fix.
///
/// Two symptoms separate the two cases, and both are properties of the grid
/// alone (no model needed):
/// * **density** — a real table is mostly full; a split-up one is mostly holes;
/// * **thin columns** — a column carrying at most one entry across several rows
///   is almost always a split artefact rather than a real column.
///
/// Deliberately conservative: it answers `true` only for grids that are plainly
/// well-formed, so the expensive path stays the default whenever there is doubt.
/// A caller that skips TableFormer on `true` trades no quality for the time.
pub fn geometric_table_is_reliable(rows: &[Vec<String>]) -> bool {
    let ncols = rows.iter().map(Vec::len).max().unwrap_or(0);
    // Fewer than two columns is not a grid this heuristic can vouch for: it is
    // exactly the shape a collapsed table takes, and TableFormer may recover
    // real structure from it.
    if rows.len() < 2 || ncols < 2 {
        return false;
    }
    let filled = |c: &String| !c.trim().is_empty();
    let total = rows.len() * ncols;
    let full = rows.iter().flatten().filter(|c| filled(c)).count();
    if (full as f32) < MIN_TABLE_FILL * total as f32 {
        return false;
    }
    // A column used by at most one row, when there are rows enough to tell.
    if rows.len() >= 3 {
        for ci in 0..ncols {
            let used = rows
                .iter()
                .filter(|r| r.get(ci).is_some_and(filled))
                .count();
            if used <= 1 {
                return false;
            }
        }
    }
    true
}

/// Share of a geometric grid's cells that must carry text for it to be trusted
/// without TableFormer. Chosen well above the density a left-edge split
/// produces (those land nearer a third) and below what a genuine table with a
/// few blank cells reaches.
const MIN_TABLE_FILL: f32 = 0.6;

/// The union bbox of the text cells assigned to a region (same >50%-overlap
/// rule as [`region_text`]), or `None` when no cell lands in it. docling's
/// LayoutPostprocessor shrinks a regular cluster's bbox to its cells, and the
/// enrichment crops are taken from that cell-tight box — cropping the raw
/// detector box instead hands the VLM surrounding chrome (e.g. the `Listing N:`
/// caption under a code block) that changes its output.
pub fn region_cell_bbox(region: &Region, cells: &[TextCell]) -> Option<[f32; 4]> {
    let mut bbox: Option<[f32; 4]> = None;
    for c in cells {
        let ca = area(c.l, c.t, c.r, c.b).max(1.0);
        if inter(region, c.l, c.t, c.r, c.b) / ca <= 0.5 {
            continue;
        }
        bbox = Some(match bbox {
            None => [c.l, c.t, c.r, c.b],
            Some([l, t, r, b]) => [l.min(c.l), t.min(c.t), r.max(c.r), b.max(c.b)],
        });
    }
    bbox
}

/// One region's enrichment-model result, produced by the pipeline's opt-in
/// passes (issue #76) and applied during assembly.
#[derive(Debug, Clone)]
pub enum Enrichment {
    /// DocumentPictureClassifier predictions, descending confidence.
    PictureClasses(Vec<PictureClass>),
    /// CodeFormulaV2 output for a `code` region: the rewritten source text and
    /// the `<_language_>` prefix (when the model emitted one).
    Code {
        language: Option<String>,
        text: String,
    },
    /// CodeFormulaV2 output for a `formula` region: the decoded LaTeX.
    Formula { latex: String },
}

/// Crop a region (page points, already expanded by the caller if needed) from
/// the rendered page image and resize it to `target_scale` pixels per point —
/// the enrichment-model equivalent of docling's
/// `page.get_image(scale=…, cropbox=…)`, sourced from the existing
/// [`crate::pdfium_backend::RENDER_SCALE`] render instead of a fresh pdfium
/// pass (the page bitmap is already the exact docling render at scale 2).
#[cfg(feature = "ml")]
pub fn crop_region_scaled(page: &PdfPage, bbox: [f32; 4], target_scale: f32) -> Option<RgbImage> {
    let s = page.scale;
    let [l, t, r, b] = bbox;
    let (iw, ih) = (page.image.width(), page.image.height());
    let x = (l * s).max(0.0) as u32;
    let y = (t * s).max(0.0) as u32;
    if x >= iw || y >= ih {
        return None;
    }
    let w = (((r - l.max(0.0)) * s) as u32).min(iw - x);
    let h = (((b - t.max(0.0)) * s) as u32).min(ih - y);
    if w == 0 || h == 0 {
        return None;
    }
    let crop = image::imageops::crop_imm(&page.image, x, y, w, h).to_image();
    // docling renders the crop at `target_scale` directly; from the scale-2
    // page render that is a resize to the same pixel geometry
    // (`round(width_points * scale)`, PIL's BICUBIC ≙ CatmullRom).
    let tw = ((w as f32 / s) * target_scale).round().max(1.0) as u32;
    let th = ((h as f32 / s) * target_scale).round().max(1.0) as u32;
    if (tw, th) == (w, h) {
        return Some(crop);
    }
    Some(image::imageops::resize(
        &crop,
        tw,
        th,
        image::imageops::FilterType::CatmullRom,
    ))
}

/// Crop a layout region from the rendered page image and encode it as PNG (the
/// figure bytes docling stores on a `PictureItem`). Region coordinates are page
/// points; the image is rendered at `page.scale`.
#[cfg(feature = "ocr-prep")]
fn crop_region(page: &PdfPage, region: &Region) -> Option<PictureImage> {
    let s = page.scale;
    let (iw, ih) = (page.image.width(), page.image.height());
    let x = (region.l * s).max(0.0) as u32;
    let y = (region.t * s).max(0.0) as u32;
    if x >= iw || y >= ih {
        return None;
    }
    let w = (((region.r - region.l) * s) as u32).min(iw - x);
    let h = (((region.b - region.t) * s) as u32).min(ih - y);
    if w == 0 || h == 0 {
        return None;
    }
    let sub = image::imageops::crop_imm(&page.image, x, y, w, h).to_image();
    let mut buf = std::io::Cursor::new(Vec::new());
    sub.write_to(&mut buf, image::ImageFormat::Png).ok()?;
    Some(PictureImage {
        mimetype: "image/png".into(),
        width: w,
        height: h,
        data: buf.into_inner(),
    })
}

/// For each `picture` region, find the `caption` region closest below it (and
/// horizontally overlapping); docling pairs them and emits the caption first.
/// Each caption is claimed by at most one picture.
fn pair_captions(regions: &[Region]) -> Vec<Option<usize>> {
    let mut pairs = vec![None; regions.len()];
    let mut taken = vec![false; regions.len()];
    for (pi, p) in regions.iter().enumerate() {
        if p.label != "picture" {
            continue;
        }
        let mut best: Option<(usize, f32)> = None;
        for (ci, c) in regions.iter().enumerate() {
            if c.label != "caption" || taken[ci] {
                continue;
            }
            let line_h = (c.b - c.t).abs().max(1.0);
            let gap = c.t - p.b; // caption sits below the picture
            let h_overlap = (p.r.min(c.r) - p.l.max(c.l)).max(0.0);
            if gap > -line_h && gap < line_h * 3.0 && h_overlap > 0.0 {
                let dist = gap.abs();
                if best.is_none_or(|(_, bd)| dist < bd) {
                    best = Some((ci, dist));
                }
            }
        }
        if let Some((ci, _)) = best {
            pairs[pi] = Some(ci);
            taken[ci] = true;
        }
    }
    pairs
}

/// Pair each `code` region with the `caption` region just **above** it (a
/// `Listing N:` label). docling renders the code block first, then its caption,
/// so the caption is consumed from its own (earlier) reading-order slot and
/// re-emitted after the code.
fn pair_code_captions(regions: &[Region]) -> Vec<Option<usize>> {
    let mut pairs = vec![None; regions.len()];
    let mut taken = vec![false; regions.len()];
    for (pi, p) in regions.iter().enumerate() {
        if p.label != "code" {
            continue;
        }
        let mut best: Option<(usize, f32)> = None;
        for (ci, c) in regions.iter().enumerate() {
            if c.label != "caption" || taken[ci] {
                continue;
            }
            let line_h = (c.b - c.t).abs().max(1.0);
            let gap = p.t - c.b; // caption sits above the code
            let h_overlap = (p.r.min(c.r) - p.l.max(c.l)).max(0.0);
            if gap > -line_h && gap < line_h * 3.0 && h_overlap > 0.0 {
                let dist = gap.abs();
                if best.is_none_or(|(_, bd)| dist < bd) {
                    best = Some((ci, dist));
                }
            }
        }
        if let Some((ci, _)) = best {
            pairs[pi] = Some(ci);
            taken[ci] = true;
        }
    }
    pairs
}

/// Assemble one page from its (already overlap-resolved) layout regions and
/// text cells.
/// Normalize a layout region (page points, top-left origin) to DocLang's 0–511
/// location grid: `clamp(round(512 · coord / page_dim), 0, 511)`, per axis,
/// order `[x0, y0, x1, y1]`. Mirrors docling_core's
/// `_doclang_utils._create_location_tokens_for_bbox` (resolution 512) so the
/// emitted `<location>` tokens line up with the Python groundtruth. Our heron
/// cluster boxes match docling's to within ~1 grid unit; the residual (mainly
/// the aspect-ratio-stretch vs letterbox preprocessing difference) is absorbed
/// by the conformance harness's geometry tolerance.
fn norm_loc(region: &Region, page_w: f32, page_h: f32) -> [u16; 4] {
    let q = |v: f32, dim: f32| -> u16 {
        if dim <= 0.0 {
            return 0;
        }
        let g = (512.0 * (v as f64) / (dim as f64)).round() as i64;
        g.clamp(0, 511) as u16
    };
    [
        q(region.l, page_w),
        q(region.t, page_h),
        q(region.r, page_w),
        q(region.b, page_h),
    ]
}

/// Wrap a node in its layout provenance so the DocLang serializer emits the four
/// `<location>` tokens as the element's head (Markdown/JSON render `inner`
/// unchanged).
fn located(loc: [u16; 4], inner: Node) -> Node {
    Node::Located {
        location: loc,
        inner: Box::new(inner),
    }
}

pub fn assemble_page(
    page: &PdfPage,
    regions: Vec<Region>,
    table_rows: &[Option<Vec<Vec<String>>>],
    enrichments: &[Option<Enrichment>],
) -> (Vec<Node>, Vec<(String, String)>) {
    let mut nodes: Vec<Node> = Vec::new();
    // Recover this page's hyperlinks (rendered only in strict Markdown).
    let links = resolve_link_anchors(page);
    // Pair each region with its precomputed TableFormer grid and enrichment
    // (indexed by original order) and order by reading order together, so they
    // stay aligned.
    type RegionItem = (Region, Option<Vec<Vec<String>>>, Option<Enrichment>);
    let mut items: Vec<RegionItem> = regions
        .into_iter()
        .enumerate()
        .map(|(i, r)| {
            (
                r,
                table_rows.get(i).cloned().flatten(),
                enrichments.get(i).cloned().flatten(),
            )
        })
        .collect();
    order_regions(&mut items, page.width, page.height, |it| &it.0);
    // Float a margin page number to the front of reading order (docling parity:
    // right_to_left_02's bottom `11` is its first item). Stable, so everything
    // else keeps its order; no-op on pages without such a region.
    let page_h = page.height;
    items.sort_by_key(|(r, _, _)| !is_page_number(r, &page.cells, page_h));
    let table_rows: Vec<Option<Vec<Vec<String>>>> =
        items.iter().map(|(_, t, _)| t.clone()).collect();
    let enrichments: Vec<Option<Enrichment>> = items.iter().map(|(_, _, e)| e.clone()).collect();
    let regions: Vec<Region> = items.into_iter().map(|(r, _, _)| r).collect();
    // docling emits a figure's caption *before* the image marker. Pair each
    // picture with the caption region nearest below it and consume that caption,
    // so it isn't also emitted in its own (lower) reading-order position.
    let caption_for = pair_captions(&regions);
    let code_caption_for = pair_code_captions(&regions);
    let mut consumed = vec![false; regions.len()];
    for ci in caption_for.iter().flatten() {
        consumed[*ci] = true;
    }
    for ci in code_caption_for.iter().flatten() {
        consumed[*ci] = true;
    }
    // A code block's language label (`XML`, `C#`, …) is chrome, not content — the
    // detector emits it as its own region above the code; consume it.
    for (i, is_label) in code_language_labels(&regions, &page.cells)
        .into_iter()
        .enumerate()
    {
        if is_label {
            consumed[i] = true;
        }
    }

    // docling `ReadingOrderPredictor.predict_merges`: join a text fragment with a
    // following text fragment strictly to its right (an author column that wraps
    // into the next, a paragraph continuing in the next column) into one block —
    // the intra-page half of docling's reading-order merges (cross-page/vertical
    // continuations stay with [`merge_continuations`]). Already-consumed regions
    // (paired captions, code labels) are excluded.
    let region_texts: Vec<String> = regions
        .iter()
        .map(|r| region_text(r, &page.cells))
        .collect();
    let is_text: Vec<bool> = regions
        .iter()
        .enumerate()
        .map(|(i, r)| r.label == "text" && !consumed[i])
        .collect();
    let is_skip: Vec<bool> = regions
        .iter()
        .enumerate()
        .map(|(i, r)| {
            consumed[i]
                || matches!(
                    r.label,
                    "page_header" | "page_footer" | "table" | "picture" | "caption" | "footnote"
                )
        })
        .collect();
    let boxes: Vec<(f32, f32, f32, f32)> = regions.iter().map(|r| (r.l, r.t, r.r, r.b)).collect();
    let mut merge_suffix: Vec<String> = vec![String::new(); regions.len()];
    for (head, children) in
        crate::reading_order::predict_merges(&boxes, &region_texts, &is_text, &is_skip)
            .into_iter()
            .enumerate()
    {
        for c in children {
            let t = region_texts[c].trim();
            if !t.is_empty() {
                merge_suffix[head].push(' ');
                merge_suffix[head].push_str(t);
            }
            consumed[c] = true;
        }
    }

    for (i, region) in regions.iter().enumerate() {
        if consumed[i] {
            continue;
        }
        // Page headers/footers: docling emits them as furniture blocks
        // (`<page_header>`/`<page_footer>` with a layer + location + text) at
        // their reading-order position, not as body — emit them, don't skip.
        if matches!(region.label, "page_header" | "page_footer") {
            let text = region_text(region, &page.cells);
            if !text.is_empty() {
                nodes.push(Node::PageFurniture {
                    footer: region.label == "page_footer",
                    location: norm_loc(region, page.width, page_h),
                    text: md_escape(&text),
                });
            }
            continue;
        }
        if is_skipped(region.label) {
            continue;
        }
        // Layout provenance for this region, normalized to docling's 0–511 grid.
        let loc = norm_loc(region, page.width, page_h);
        if region.label == "picture" {
            // The figure pixels are cropped from the page render for image export.
            let caption = caption_for[i]
                .map(|ci| region_text(&regions[ci], &page.cells))
                .filter(|t| !t.is_empty());
            let classification = match &enrichments[i] {
                Some(Enrichment::PictureClasses(classes)) => Some(classes.clone()),
                _ => None,
            };
            // Without the page render (text-layer-only build) a picture keeps
            // its caption/classification but carries no cropped pixels.
            #[cfg(feature = "ocr-prep")]
            let image = crate::timing::timed("crop_region", || crop_region(page, region));
            #[cfg(not(feature = "ocr-prep"))]
            let image: Option<PictureImage> = None;
            nodes.push(located(
                loc,
                Node::Picture {
                    caption,
                    image,
                    classification,
                },
            ));
            continue;
        }
        let mut text = region_text(region, &page.cells);
        text.push_str(&merge_suffix[i]);
        if text.is_empty() {
            continue;
        }
        match region.label {
            // docling renders both the document title and section headers as
            // `##` (it never emits a top-level `#` for PDFs), so match that.
            "title" | "section_header" => nodes.push(located(
                loc,
                Node::Heading {
                    level: 2,
                    text: md_escape(&text),
                },
            )),
            // docling drops the rendered bullet glyph; the Markdown serializer
            // adds its own `- ` marker. An item whose text opens with an `N.`
            // enumeration marker is an ordered item (rendered `N. text`).
            "list_item" => {
                let stripped = text
                    .trim_start_matches(['•', '◦', '▪', '·', '*', '-'])
                    .trim_start()
                    .to_string();
                if let Some((number, rest)) = parse_ordered_marker(&stripped) {
                    nodes.push(Node::ListItem {
                        ordered: true,
                        number,
                        first_in_list: false,
                        text: md_escape(&rest),
                        level: 0,
                        marker: None,
                        location: Some(loc),
                        dclx: None,
                        href: None,
                        layer: None,
                    });
                } else {
                    nodes.push(Node::ListItem {
                        ordered: false,
                        number: 0,
                        first_in_list: false,
                        text: md_escape(&stripped),
                        level: 0,
                        // docling keeps the bullet as the DocLang list marker
                        // (`<ldiv><marker>·</marker></ldiv>`); Markdown ignores it.
                        marker: Some("·".into()),
                        location: Some(loc),
                        dclx: None,
                        href: None,
                        layer: None,
                    });
                }
            }
            // TableFormer structure (cells + spans, text matched from word cells)
            // when available; otherwise geometric grid reconstruction; finally a
            // single cell.
            "table" | "document_index" => {
                let rows = table_rows[i].clone().unwrap_or_else(|| {
                    let rows = reconstruct_table(region, &page.cells);
                    if rows.iter().any(|r| r.len() > 1) {
                        rows
                    } else {
                        vec![vec![text.clone()]]
                    }
                });
                nodes.push(located(
                    loc,
                    Node::Table(Table {
                        rows,
                        location: None,
                        structure: None,
                        cell_blocks: None,
                    }),
                ));
            }
            // With formula enrichment the CodeFormula model decodes the region
            // to LaTeX; otherwise docling emits a placeholder comment rather
            // than the (garbled) raw glyph text.
            "formula" => match &enrichments[i] {
                Some(Enrichment::Formula { latex }) => nodes.push(Node::Formula {
                    latex: latex.clone(),
                    orig: text.clone(),
                    location: Some(loc),
                }),
                _ => nodes.push(Node::Paragraph {
                    text: "<!-- formula-not-decoded -->".into(),
                }),
            },
            // Code blocks: use the space-glyph-only grouping (monospace keeps its
            // source spacing) and emit a fenced block, preserving the line breaks
            // and indentation of the source (unlike prose, which reflows). pdfium
            // still inserts spaces around tight punctuation (`console .log`,
            // `add (3 , 5)`); tighten them to match docling-parse's source spacing.
            "code" => {
                // `code_region_text` preserves line breaks/indentation and tightens
                // each line itself; the fallback prose `text` is tightened here.
                let code = code_region_text(region, &page.code_cells);
                let code = if code.is_empty() {
                    tighten_code_punct(&text)
                } else {
                    code
                };
                // With code enrichment the CodeFormula model rewrites the block
                // (and names its language); `orig` keeps the raw extraction in
                // docling's shape — its parser has no line-preserving code
                // path, so its `orig` is the same code with the lines joined
                // by single spaces (indentation collapsed).
                let node = match &enrichments[i] {
                    Some(Enrichment::Code {
                        language,
                        text: enriched,
                    }) => {
                        let flat = code
                            .lines()
                            .map(str::trim)
                            .filter(|l| !l.is_empty())
                            .collect::<Vec<_>>()
                            .join(" ");
                        Node::Code {
                            language: language.clone(),
                            text: enriched.clone(),
                            orig: Some(flat),
                        }
                    }
                    _ => Node::Code {
                        language: None,
                        text: code,
                        orig: None,
                    },
                };
                nodes.push(located(loc, node));
                // docling emits the `Listing N:` caption after the code block.
                if let Some(ci) = code_caption_for[i] {
                    let cap = region_text(&regions[ci], &page.cells);
                    if !cap.is_empty() {
                        nodes.push(Node::Paragraph { text: cap });
                    }
                }
            }
            // text, caption, footnote → paragraph
            _ => nodes.push(located(
                loc,
                Node::Paragraph {
                    text: md_escape(&text),
                },
            )),
        }
    }
    (nodes, links)
}

/// Merge paragraph fragments split across a column or page break. docling joins a
/// paragraph whose previous fragment ends mid-sentence (a letter, not sentence
/// punctuation) with a lowercase continuation: `…definition of` + `lists in…` →
/// `…definition of lists in…`. The fragments are consecutive paragraphs, or
/// separated only by figure(s) the text wraps around: a column whose body flows
/// past a figure resumes below it (`…The wing type that is` ⟶[figure]⟶ `the most
/// common…`), and docling emits the whole paragraph before the figure. A heading,
/// table, or list between them ends the paragraph (no merge).
/// A paragraph that is really a figure/table caption (`Fig. 1. …`, `Table 2 …`).
/// Used to skip an unpaired caption when stitching a paragraph that wraps around
/// a figure.
fn looks_like_caption(text: &str) -> bool {
    let head: String = text.trim_start().chars().take(14).collect();
    (head.starts_with("Fig") || head.starts_with("Table"))
        && head.contains(|c: char| c.is_ascii_digit())
}

/// A paragraph fragment is "open" — i.e. it might continue into the next
/// paragraph — when it ends mid-word (a letter) or with a wrap hyphen/dash.
/// docling joins `vocab-` + `ulary` → `vocab- ulary`.
fn paragraph_is_open(text: &str) -> bool {
    text.trim_end().chars().next_back().is_some_and(|c| {
        c.is_alphabetic() || matches!(c, '-' | '\u{2010}' | '\u{2013}' | '\u{2014}')
    })
}

/// The paragraph text inside a node, looking through a [`Node::Located`]
/// provenance wrapper (PDF body paragraphs are wrapped since they carry a
/// `<location>`). Returns `None` for non-paragraph nodes.
fn as_paragraph(n: &Node) -> Option<&str> {
    match n {
        Node::Paragraph { text } => Some(text),
        Node::Located { inner, .. } => match inner.as_ref() {
            Node::Paragraph { text } => Some(text),
            _ => None,
        },
        _ => None,
    }
}

/// Whether a node is a picture, looking through a [`Node::Located`] wrapper.
fn is_picture_node(n: &Node) -> bool {
    match n {
        Node::Picture { .. } => true,
        Node::Located { inner, .. } => matches!(inner.as_ref(), Node::Picture { .. }),
        _ => false,
    }
}

/// A node a forward paragraph merge looks straight past: a figure the text wraps
/// around, or a page header/footer that falls between the two fragments of a
/// paragraph continuing across a page break (docling merges the body text and
/// keeps the header/footer as a separate furniture layer).
fn is_merge_trailer(n: &Node) -> bool {
    is_picture_node(n)
        || matches!(n, Node::PageFurniture { .. })
        || as_paragraph(n).is_some_and(looks_like_caption)
}

/// Rebuild node `i` as a paragraph with `text`, preserving its `<location>`
/// wrapper (and thus provenance) if it had one.
fn reparagraph(node: &Node, text: String) -> Node {
    match node {
        Node::Located { location, .. } => located(*location, Node::Paragraph { text }),
        _ => Node::Paragraph { text },
    }
}

pub(crate) fn merge_continuations(nodes: &mut Vec<Node>) {
    let mut i = 0;
    while i + 1 < nodes.len() {
        let Some(a) = as_paragraph(&nodes[i]) else {
            i += 1;
            continue;
        };
        // A figure/table caption is a self-contained unit; body text resuming
        // after a figure is the continuation case, not the caption itself. Never
        // stitch *from* a caption — otherwise a caption that ends in a lone glyph
        // (`Fig. 5. … PubTabNet. μ`) would swallow a following stray figure label
        // (a standalone `μ`) into `… μ μ`.
        if looks_like_caption(a) {
            i += 1;
            continue;
        }
        if !paragraph_is_open(a) {
            i += 1;
            continue;
        }
        // The continuation is the next paragraph, looking past any figures the
        // text wraps around — and a figure/table caption that was emitted as its
        // own paragraph (an above-the-figure caption that didn't pair), since the
        // body text resumes after the whole figure+caption block.
        let mut j = i + 1;
        while nodes.get(j).is_some_and(is_merge_trailer) {
            j += 1;
        }
        let cont = nodes.get(j).and_then(as_paragraph).is_some_and(|b| {
            b.trim_start()
                .chars()
                .next()
                .is_some_and(char::is_lowercase)
        });
        if cont {
            let a = as_paragraph(&nodes[i]).unwrap().trim_end().to_string();
            let b = as_paragraph(&nodes[j]).unwrap().trim_start().to_string();
            // Keep node i's provenance wrapper; docling's merged paragraph keeps
            // the first fragment's geometry as its primary location.
            nodes[i] = reparagraph(&nodes[i], format!("{a} {b}"));
            nodes.remove(j);
            // Re-check i: the merged paragraph may continue further.
        } else {
            i += 1;
        }
    }
}

/// How many leading nodes of `nodes` are safe to flush now — i.e. cannot be
/// rewritten by a future [`merge_continuations`] once more pages are appended.
///
/// A forward merge can only start from an "open" paragraph (ends mid-word) and
/// only reaches across trailing pictures and figure/table captions. So we scan
/// from the end past those skippable trailers: if the first non-skippable node is
/// an open paragraph, it (and the trailers after it) must be held; anything else —
/// a closed paragraph, a heading, a table, a list — blocks any forward merge, so
/// the whole buffer is safe to flush.
fn hold_start(nodes: &[Node]) -> usize {
    for k in (0..nodes.len()).rev() {
        // Skippable trailers (figures, page furniture, captions): a forward merge
        // looks straight past them.
        if is_merge_trailer(&nodes[k]) {
            continue;
        }
        match as_paragraph(&nodes[k]) {
            // An open body paragraph might still pull a continuation off the next
            // page — hold from here to the end.
            Some(text) if paragraph_is_open(text) => return k,
            // A closed paragraph, heading, table, list, etc. ends the paragraph:
            // nothing after it can merge backwards across it. Flush everything.
            _ => return nodes.len(),
        }
    }
    // Only skippable trailers (or empty) and no open paragraph to anchor a merge.
    nodes.len()
}

/// Streaming counterpart of [`merge_continuations`]: feed per-page node batches in
/// document order and get back the prefix that is final (its cross-page merges are
/// resolved and no future page can change it), holding back only the small tail
/// that might still merge into the next page. Concatenating every flushed batch
/// (then [`finish`](Self::finish)) yields exactly the same nodes as running
/// [`merge_continuations`] once over the whole document.
pub(crate) struct StreamAssembler {
    pending: Vec<Node>,
}

impl StreamAssembler {
    pub(crate) fn new() -> Self {
        Self {
            pending: Vec::new(),
        }
    }

    /// Append one page's nodes, resolve merges within the buffer, and return the
    /// now-final prefix to emit (possibly empty).
    pub(crate) fn push(&mut self, mut nodes: Vec<Node>) -> Vec<Node> {
        self.pending.append(&mut nodes);
        merge_continuations(&mut self.pending);
        let cut = hold_start(&self.pending);
        let tail = self.pending.split_off(cut);
        std::mem::replace(&mut self.pending, tail)
    }

    /// Flush whatever is left after the last page (the held tail is final once no
    /// more pages can follow).
    pub(crate) fn finish(self) -> Vec<Node> {
        self.pending
    }
}

#[cfg(test)]
mod tests {
    use super::clean_text;
    use super::{code_region_text, merge_continuations, resolve_link_anchors, StreamAssembler};
    use crate::layout::Region;
    use crate::pdfium_backend::{LinkAnnot, PdfPage, TextCell};
    use docling_core::Node;

    /// #165: a picture no longer claims cells at 0.2 intersection-over-self.
    /// A line straddling the figure border (≤80 % contained) becomes an orphan
    /// region and survives the contained-regulars drop — before the fix its
    /// cells were silently erased. A line fully inside the picture is still
    /// re-dropped, matching docling's Markdown (a picture's children never
    /// reach its serializer's output).
    #[test]
    fn border_straddling_lines_survive_picture_interior_is_still_dropped() {
        let pic = Region {
            label: "picture",
            score: 0.9,
            l: 0.0,
            t: 0.0,
            r: 100.0,
            b: 100.0,
        };
        // ~35 % of this cell overlaps the picture (l=90..120 of 0..100): above
        // the old 0.2 claim (was swallowed), below full containment (survives).
        let straddler = TextCell {
            text: "axis label".into(),
            l: 90.0,
            t: 40.0,
            r: 120.0,
            b: 48.0,
        };
        let interior = TextCell {
            text: "in-figure callout".into(),
            l: 10.0,
            t: 10.0,
            r: 60.0,
            b: 18.0,
        };
        let mut regions = vec![pic];
        super::add_orphan_regions(&mut regions, &[straddler, interior]);
        assert_eq!(
            regions.iter().filter(|r| r.label == "text").count(),
            2,
            "both unclaimed lines become orphans"
        );
        super::drop_contained_regulars(&mut regions);
        let texts: Vec<(f32, f32)> = regions
            .iter()
            .filter(|r| r.label == "text")
            .map(|r| (r.l, r.r))
            .collect();
        assert_eq!(
            texts,
            [(90.0, 120.0)],
            "the straddler is emitted, the fully-contained callout is not"
        );
    }

    /// A colored terms-and-conditions panel detected as `picture` demotes into
    /// per-paragraph `text` regions (the blank line between C.7 and C.8 splits
    /// them); a chart whose only text is a few narrow axis labels keeps its
    /// crop untouched.
    #[test]
    fn text_panels_demote_to_paragraphs_but_charts_keep_their_crop() {
        let cell = |text: &str, l: f32, t: f32, r: f32, b: f32| TextCell {
            text: text.to_string(),
            l,
            t,
            r,
            b,
        };
        let panel = Region {
            label: "picture",
            score: 0.9,
            l: 0.0,
            t: 0.0,
            r: 100.0,
            b: 100.0,
        };
        // Three tight lines, a blank-line gap, two more: two paragraphs.
        let cells = vec![
            cell(
                "C.7. Wenn Sie diesen Vertrag widerrufen,",
                5.0,
                10.0,
                95.0,
                18.0,
            ),
            cell(
                "haben wir Ihnen alle Zahlungen, die wir",
                5.0,
                20.0,
                95.0,
                28.0,
            ),
            cell(
                "von Ihnen erhalten haben, zurückzuzahlen.",
                5.0,
                30.0,
                90.0,
                38.0,
            ),
            cell(
                "C.8. Wir können die Rückzahlung verweigern,",
                5.0,
                52.0,
                95.0,
                60.0,
            ),
            cell(
                "bis wir die Waren wieder zurückerhalten haben.",
                5.0,
                62.0,
                92.0,
                70.0,
            ),
        ];
        let mut regions = vec![panel.clone()];
        super::recover_text_panels(&mut regions, &cells);
        assert_eq!(
            regions.iter().map(|r| r.label).collect::<Vec<_>>(),
            ["text", "text"],
            "dense panel must demote into one text region per paragraph"
        );
        assert!(regions[0].b < regions[1].t, "paragraphs split at the gap");
        // Sparse narrow labels (a chart): picture survives.
        let labels = vec![
            cell("0", 5.0, 90.0, 8.0, 95.0),
            cell("50", 5.0, 50.0, 10.0, 55.0),
            cell("100", 5.0, 10.0, 12.0, 15.0),
            cell("t, s", 45.0, 96.0, 55.0, 100.0),
        ];
        let mut regions = vec![panel];
        super::recover_text_panels(&mut regions, &labels);
        assert_eq!(
            regions.iter().map(|r| r.label).collect::<Vec<_>>(),
            ["picture"]
        );
    }

    /// A generator that paints a line's bold runs *after* its regular text
    /// hands the sanitizer the cells out of visual order ("C." | "Zur Wahrung
    /// …" | "6." — the bold label number drawn last). docling's index order
    /// would strand the bold token at the line's end ("… Mitteilung 6."); the
    /// overlap-grouped line sort restores reading order, and an off-baseline
    /// glyph (2206's inline `>` sits ~2 pt above its line) still belongs to
    /// its own line rather than drifting into the next band.
    #[test]
    fn interleaved_font_runs_read_in_visual_order() {
        let cell = |text: &str, l: f32, t: f32, r: f32, b: f32| TextCell {
            text: text.to_string(),
            l,
            t,
            r,
            b,
        };
        let cells = vec![
            cell("C.", 10.0, 100.0, 18.0, 110.0),
            cell("Zur Wahrung der Widerrufsfrist", 30.0, 100.0, 150.0, 110.0),
            cell("über die Ausübung", 10.0, 112.0, 90.0, 122.0),
            cell("6.", 19.0, 100.0, 27.0, 110.0), // bold run, drawn last
        ];
        let region = Region {
            label: "text",
            score: 1.0,
            l: 0.0,
            t: 95.0,
            r: 200.0,
            b: 130.0,
        };
        assert_eq!(
            super::region_text(&region, &cells),
            "C. 6. Zur Wahrung der Widerrufsfrist über die Ausübung"
        );
        // Off-baseline glyph: raised but overlapping its line by more than half
        // its height — stays on that line, ordered by x.
        let raised = vec![
            cell("(", 10.0, 100.0, 14.0, 110.0),
            cell(">", 15.0, 97.0, 20.0, 104.0),
            cell("10 pages)", 21.0, 100.0, 60.0, 110.0),
        ];
        assert_eq!(super::region_text(&region, &raised), "( > 10 pages)");
    }

    /// The geometric-reliability gate, on the two shapes it has to tell apart.
    #[test]
    fn geometric_reliability_rejects_split_column_grids() {
        let g = |rows: &[&[&str]]| -> Vec<Vec<String>> {
            rows.iter()
                .map(|r| r.iter().map(|c| c.to_string()).collect())
                .collect()
        };
        // A genuine grid: dense, every column carrying entries. Nothing for
        // TableFormer to improve, so geometry is used as-is.
        assert!(super::geometric_table_is_reliable(&g(&[
            &["Datum", "Leistung", "Anzahl", "Kosten"],
            &["04.07", "Internet", "1", "40.30"],
            &["04.07", "Telefon", "2", "8.06"],
        ])));
        // The left-edge split artefact (the shape a scanned invoice produced):
        // one real label column plus values scattered across three sparse ones.
        assert!(!super::geometric_table_is_reliable(&g(&[
            &["www.magenta.at/faq", "", "", ""],
            &["Serviceteam", "", "", ""],
            &["Telefon", "0676/2000", "", ""],
            &["Kundennummer", "", "", "1.21699482"],
            &["Rechnungsnummer", "", "922769430725", ""],
            &["Rechnungsdatum", "", "", "04.07.2025"],
        ])));
        // A column only one row ever uses is a split artefact even when the
        // grid is otherwise dense.
        assert!(!super::geometric_table_is_reliable(&g(&[
            &["a", "b", ""],
            &["c", "d", ""],
            &["e", "f", "g"],
        ])));
        // Degenerate shapes are never vouched for — TableFormer may recover
        // structure a collapsed reconstruction lost.
        assert!(!super::geometric_table_is_reliable(&g(&[&[
            "only one column"
        ]])));
        assert!(!super::geometric_table_is_reliable(&[]));
    }

    /// A `picture` region is cropped out of the rendered page, whatever built
    /// that page. The browser pipeline (#157) has no pdfium but does hand over
    /// the rasterized bitmap through `from_cells_with_image`, so it must get
    /// the same figure bytes the native path does — that is what makes
    /// `images = "embedded"` inline real pixels instead of a placeholder.
    #[cfg(feature = "ocr-prep")]
    #[test]
    fn picture_regions_are_cropped_from_a_host_supplied_page_image() {
        let mut img = image::RgbImage::new(200, 200);
        // Paint the figure area so the crop is distinguishable from the page.
        for y in 100..160 {
            for x in 20..120 {
                img.put_pixel(x, y, image::Rgb([255, 0, 0]));
            }
        }
        // scale 2.0: the region is in page points, the bitmap in pixels.
        let page = PdfPage::from_cells_with_image(100.0, 100.0, 2.0, Vec::new(), img);
        let region = Region {
            label: "picture",
            score: 0.9,
            l: 10.0,
            t: 50.0,
            r: 60.0,
            b: 80.0,
        };
        let (nodes, _) = super::assemble_page(&page, vec![region], &[None], &[None]);
        // Layout-derived nodes carry provenance, so the picture arrives wrapped.
        let image = nodes
            .iter()
            .find_map(|n| match n {
                Node::Located { inner, .. } => match &**inner {
                    Node::Picture { image, .. } => image.as_ref(),
                    _ => None,
                },
                Node::Picture { image, .. } => image.as_ref(),
                _ => None,
            })
            .expect("a picture node with cropped pixels");
        assert_eq!(image.mimetype, "image/png");
        assert_eq!((image.width, image.height), (100, 60), "region × scale");
        assert!(!image.data.is_empty(), "PNG bytes were encoded");
    }

    #[test]
    fn link_anchors_split_a_shared_word_cell_between_adjacent_links() {
        // A common header layout: one text run holds several pipe-separated
        // labels, each carrying its own link annotation. Every link must get
        // its own label as the anchor (and the "|" separators must belong to
        // none), not the whole run.
        let annot = |l: f32, r: f32, uri: &str| LinkAnnot {
            l,
            t: 100.0,
            r,
            b: 114.0,
            uri: uri.into(),
        };
        let page = PdfPage {
            width: 600.0,
            height: 800.0,
            scale: 2.0,
            cells: Vec::new(),
            code_cells: Vec::new(),
            // "LinkedIn | GitHub | Credly" = 26 chars over x 100..360.
            word_cells: vec![cell(
                "LinkedIn | GitHub | Credly",
                100.0,
                100.0,
                360.0,
                114.0,
            )],
            image: image::RgbImage::new(1, 1),
            links: vec![
                annot(100.0, 180.0, "https://l"),
                annot(200.0, 260.0, "https://g"),
                annot(290.0, 360.0, "https://c"),
            ],
        };
        assert_eq!(
            resolve_link_anchors(&page),
            vec![
                ("LinkedIn".to_string(), "https://l".to_string()),
                ("GitHub".to_string(), "https://g".to_string()),
                ("Credly".to_string(), "https://c".to_string()),
            ]
        );
    }

    /// A one-line code cell at `[l, r] × [t, b]` (top-left coords).
    fn cell(text: &str, l: f32, t: f32, r: f32, b: f32) -> TextCell {
        TextCell {
            text: text.into(),
            l,
            t,
            r,
            b,
        }
    }

    fn region(label: &'static str, score: f32, l: f32, t: f32, r: f32, b: f32) -> Region {
        Region {
            label,
            score,
            l,
            t,
            r,
            b,
        }
    }

    #[test]
    fn resolve_collapses_nested_code_keeping_the_larger_box() {
        // A tight high-score `code` box and a taller lower-score near-duplicate that
        // contains it must collapse to one — the *larger* box, so every cell stays
        // covered and nothing leaks out as orphan text.
        let tight = region("code", 0.95, 78.0, 292.0, 300.0, 330.0);
        let wide = region("code", 0.66, 63.0, 260.0, 320.0, 346.0);
        let kept = super::resolve(vec![tight, wide]);
        assert_eq!(kept.len(), 1, "nested code boxes must collapse to one");
        assert!(
            kept[0].l == 63.0 && kept[0].b == 346.0,
            "the larger containing box is kept"
        );
    }

    #[test]
    fn resolve_keeps_distinct_and_differently_typed_regions() {
        // A text box fully inside a lower-score *table* must NOT be collapsed (the
        // code dedup is code-only), and two separate code blocks stay separate.
        let text = region("text", 0.95, 90.0, 210.0, 200.0, 230.0);
        let table = region("table", 0.60, 80.0, 200.0, 400.0, 500.0);
        assert_eq!(super::resolve(vec![text, table]).len(), 2);

        let code_a = region("code", 0.9, 78.0, 100.0, 300.0, 140.0);
        let code_b = region("code", 0.9, 78.0, 300.0, 300.0, 360.0); // far below, no overlap
        assert_eq!(super::resolve(vec![code_a, code_b]).len(), 2);
    }

    #[test]
    fn code_language_label_above_code_is_detected() {
        // A bare "XML" token directly above a code box is a language label; a real
        // heading above the same code is not; a language word with no code below is
        // left alone.
        let label = region("section_header", 0.9, 76.0, 540.0, 96.0, 549.0);
        let code = region("code", 0.7, 77.0, 552.0, 290.0, 640.0);
        let heading = region("section_header", 0.9, 76.0, 500.0, 260.0, 512.0);
        let cells = vec![
            cell("XML", 78.0, 541.0, 94.0, 548.0),       // inside `label`
            cell("Overview", 78.0, 501.0, 250.0, 511.0), // inside `heading`
        ];
        let drop = super::code_language_labels(&[label, code, heading], &cells);
        assert_eq!(drop, vec![true, false, false], "only the label is consumed");

        // Same label with no code region present → not consumed.
        let label2 = region("section_header", 0.9, 76.0, 540.0, 96.0, 549.0);
        let only = vec![cell("XML", 78.0, 541.0, 94.0, 548.0)];
        assert_eq!(super::code_language_labels(&[label2], &only), vec![false]);

        // A label swallowed into the top of a wider code box (negative gap) is still
        // recognized.
        let inside_lbl = region("text", 0.9, 76.0, 540.0, 96.0, 549.0);
        let wide_code = region("code", 0.7, 63.0, 531.0, 320.0, 654.0);
        let cells2 = vec![cell("XML", 78.0, 541.0, 94.0, 548.0)];
        assert_eq!(
            super::code_language_labels(&[inside_lbl, wide_code], &cells2),
            vec![true, false]
        );

        assert!(super::is_code_language("XML") && super::is_code_language("c#"));
        assert!(!super::is_code_language("Configure") && !super::is_code_language("XML schema"));
    }

    #[test]
    fn code_region_text_keeps_lines_and_indentation() {
        // Three source lines; each glyph is 6 units wide (width / chars = 6), so the
        // `int X;` line indented to x=22 is (22-10)/6 = 2 spaces in.
        let region = Region {
            label: "code",
            score: 1.0,
            l: 0.0,
            t: -5.0,
            r: 100.0,
            b: 40.0,
        };
        let cells = vec![
            cell("struct P {", 10.0, 0.0, 70.0, 10.0),
            cell("int X;", 22.0, 12.0, 58.0, 22.0),
            cell("}", 10.0, 24.0, 16.0, 34.0),
        ];
        assert_eq!(code_region_text(&region, &cells), "struct P {\n  int X;\n}");
    }

    #[test]
    fn code_region_text_tightens_punctuation_without_eating_indentation() {
        // A fluent `.Foo()` line at x=22 (2 chars in). Per-line tightening must not
        // consume the leading indent space by matching " ." across it.
        let region = Region {
            label: "code",
            score: 1.0,
            l: 0.0,
            t: -5.0,
            r: 100.0,
            b: 40.0,
        };
        let cells = vec![
            cell("builder", 10.0, 0.0, 52.0, 10.0),
            // pdfium spaced the call: ".Foo (x)" tightens to ".Foo(x)", still 2-indented.
            cell(".Foo (x)", 22.0, 12.0, 70.0, 22.0),
        ];
        assert_eq!(code_region_text(&region, &cells), "builder\n  .Foo(x)");
    }

    #[test]
    fn code_region_text_orders_out_of_order_cells_and_ignores_blank_lines() {
        let region = Region {
            label: "code",
            score: 1.0,
            l: 0.0,
            t: -5.0,
            r: 100.0,
            b: 60.0,
        };
        // Fed bottom-up and with a whitespace-only cell; output is top-down, no blank.
        let cells = vec![
            cell("b();", 10.0, 24.0, 34.0, 34.0),
            cell("   ", 10.0, 12.0, 20.0, 22.0),
            cell("a();", 10.0, 0.0, 34.0, 10.0),
        ];
        assert_eq!(code_region_text(&region, &cells), "a();\nb();");
        // No code cells → empty, so the caller falls back to the prose text.
        assert_eq!(code_region_text(&region, &[]), "");
    }

    fn para(text: &str) -> Node {
        Node::Paragraph { text: text.into() }
    }

    /// Run a node sequence through [`StreamAssembler`] with the given page splits
    /// and assert the flushed result equals one-shot [`merge_continuations`].
    fn assert_stream_eq(nodes: &[Node], splits: &[usize]) {
        let mut want = nodes.to_vec();
        merge_continuations(&mut want);

        let mut asm = StreamAssembler::new();
        let mut got = Vec::new();
        let mut start = 0;
        for &end in splits {
            got.extend(asm.push(nodes[start..end].to_vec()));
            start = end;
        }
        got.extend(asm.push(nodes[start..].to_vec()));
        got.extend(asm.finish());
        assert_eq!(got, want, "stream assembly diverged (splits={splits:?})");
    }

    #[test]
    fn stream_assembler_matches_merge_continuations() {
        // Open fragment + lowercase continuation split across a page boundary.
        let cross = [para("the definition of"), para("lists in scope")];
        assert_stream_eq(&cross, &[1]);
        assert_stream_eq(&cross, &[]);

        // Continuation that wraps around a figure (+ its caption) on the boundary.
        let wrap = [
            para("the wing type that is"),
            Node::Picture {
                caption: None,
                image: None,
                classification: None,
            },
            para("Fig. 1. a diagram"),
            para("the most common kind"),
        ];
        for splits in [&[][..], &[1][..], &[2][..], &[3][..], &[1, 3][..]] {
            assert_stream_eq(&wrap, splits);
        }

        // A heading between fragments blocks the merge (must still flush correctly).
        let blocked = [
            para("ends mid word and"),
            Node::Heading {
                level: 2,
                text: "New Section".into(),
            },
            para("more body here"),
        ];
        for splits in [&[][..], &[1][..], &[2][..]] {
            assert_stream_eq(&blocked, splits);
        }

        // A chain across three pages: each page is one open lowercase fragment.
        let chain = [
            para("alpha beta"),
            para("gamma delta"),
            para("epsilon zeta"),
        ];
        assert_stream_eq(&chain, &[1, 2]);
    }

    #[test]
    fn clean_text_dehyphenates_and_normalizes_typography() {
        // U+0002 line-wrap hyphen + the join space → merged word (like docling).
        assert_eq!(clean_text("com\u{2} pact"), "compact");
        assert_eq!(clean_text("end-to\u{2} end deep"), "end-toend deep");
        // A stray wrap hyphen (no following join) is dropped.
        assert_eq!(clean_text("word\u{2}"), "word");
        // Typographic punctuation → ASCII: every curly quote becomes `'`
        // (docling-parse's sanitizer table), a literal `"` stays.
        assert_eq!(
            clean_text("Graph\u{2019}s \u{201c}x\u{201d} \"y\""),
            "Graph's 'x' \"y\""
        );
        assert_eq!(clean_text("a\u{2026}"), "a...");
        // The dp default (the docling-parse sanitizer) preserves internal spacing
        // it placed deliberately; line breaks/tabs normalize to a space, ends trim.
        assert_eq!(clean_text("a   b\nc"), "a   b c");
    }

    #[test]
    fn lam_alef_only_swaps_a_genuinely_reversed_ligature() {
        // A mid-word `alef-variant + lam` is pdfium's reversed lam-alef ligature and
        // is swapped back to logical `lam + alef-variant` (`ب أ ل` → `ب ل أ`).
        assert_eq!(
            clean_text("\u{0628}\u{0623}\u{0644}"),
            "\u{0628}\u{0644}\u{0623}"
        );
        // But when the alef-variant is *already* preceded by a lam it is the logical
        // ligature `لآ`; the following lam is the next syllable's letter and must not
        // move. `التعلم الآلي` must stay `الآلي`, not become `اللآي`.
        assert_eq!(
            clean_text("\u{0627}\u{0644}\u{0622}\u{0644}\u{064a}"),
            "\u{0627}\u{0644}\u{0622}\u{0644}\u{064a}"
        );
    }
}