Skip to main content

docling_pdf/
assemble.rs

1//! Layout-driven assembly: map detected [`Region`]s + text cells to a
2//! [`DoclingDocument`], mirroring docling's page-assembly + reading-order.
3//!
4//! Overlapping detections are resolved greedily by score, each text cell is
5//! assigned to its best-containing region, regions are ordered in reading order
6//! (two-column aware), and each becomes a typed node by its layout label.
7
8use docling_core::{CaptionParent, ListItemDclx, Node, PictureClass, PictureImage, Table};
9#[cfg(any(feature = "ml", feature = "ocr-prep"))]
10use image::RgbImage;
11
12use crate::layout::Region;
13use crate::pdfium_backend::{PdfPage, TextCell};
14
15fn area(l: f32, t: f32, r: f32, b: f32) -> f32 {
16    ((r - l).max(0.0)) * ((b - t).max(0.0))
17}
18
19/// Intersection area of two boxes.
20fn inter(a: &Region, l: f32, t: f32, r: f32, b: f32) -> f32 {
21    let il = a.l.max(l);
22    let it = a.t.max(t);
23    let ir = a.r.min(r);
24    let ib = a.b.min(b);
25    area(il, it, ir, ib)
26}
27
28/// Wrapper (structured-region) labels, ported from docling
29/// `LayoutPostprocessor.WRAPPER_TYPES`: a region that *contains* other regions
30/// and renders as a structured block (a table / table-of-contents index), not as
31/// its own flat text.
32fn is_wrapper(label: &str) -> bool {
33    matches!(
34        label,
35        "table" | "document_index" | "form" | "key_value_region"
36    )
37}
38
39/// Labels docling's table-structure (TableFormer) model runs on and that render
40/// as a Markdown table: a plain `table` and a `document_index` (a table of
41/// contents), which docling assembles as a `TableItem` too.
42pub fn is_table_like(label: &str) -> bool {
43    matches!(label, "table" | "document_index")
44}
45
46/// Greedily keep regions by descending score, dropping a region that is mostly
47/// covered by an already-kept one (RT-DETR emits overlapping duplicates).
48fn greedy(mut regions: Vec<Region>) -> Vec<Region> {
49    regions.sort_by(|a, b| b.score.total_cmp(&a.score));
50    let mut kept: Vec<Region> = Vec::new();
51    for r in regions {
52        let ra = area(r.l, r.t, r.r, r.b).max(1.0);
53        let covered = kept.iter().any(|k| {
54            let i = inter(&r, k.l, k.t, k.r, k.b);
55            let ka = area(k.l, k.t, k.r, k.b).max(1.0);
56            // drop if most of r is inside k, or they strongly mutually overlap
57            i / ra > 0.7 || i / (ra + ka - i) > 0.5
58        });
59        if !covered {
60            kept.push(r);
61        }
62    }
63    kept
64}
65
66/// Resolve overlapping RT-DETR detections, ported from the bucket structure of
67/// docling's `LayoutPostprocessor`: regular, picture and wrapper clusters live in
68/// **separate** spatial indexes and are de-overlapped independently, so a
69/// high-score picture never suppresses a lower-score table or table-of-contents
70/// index (the redp5110 TOC that was otherwise replaced by a picture box). A
71/// cross-type pass first drops a picture that nearly coincides with a table
72/// (`_handle_cross_type_overlaps`), keeping the structured table.
73/// docling's `_remove_overlapping_clusters("picture")`: same-label picture
74/// detections whose boxes heavily overlap (IoU > 0.8, or either box > 80 %
75/// contained in the other) form one group, and a single survivor is kept per
76/// group. Survivor selection ports `_should_prefer_cluster` /
77/// `_select_best_cluster_from_group` with the picture params
78/// (`area_threshold` 2.0, `conf_threshold` 0.3): a candidate is rejected only
79/// when a rival is both comparable in size (candidate ≤ 2× its area) and
80/// clearly more confident (> 0.3); among the survivors the *larger* box wins
81/// unless it is > 0.3 less confident. Net effect on the corpus: a figure the
82/// detector proposes both whole and as its sub-panels (2206's four-thumbnail
83/// Figure 1) collapses to the whole-figure box, exactly like docling.
84pub(crate) fn dedup_pictures(regions: &mut Vec<Region>) {
85    remove_overlapping_specials(regions, |l| l == "picture", 2.0, 0.3);
86}
87
88/// docling's `_remove_overlapping_clusters` for one special bucket (the
89/// regions whose label satisfies `in_bucket`), with that bucket's
90/// `OVERLAP_PARAMS`: picture (2.0, 0.3) — see [`dedup_pictures`] — or wrapper
91/// (2.0, 0.2) for the table bucket in [`resolve`]. Regions outside the bucket
92/// are untouched; a bucket member overlapping nothing is always kept.
93fn remove_overlapping_specials(
94    regions: &mut Vec<Region>,
95    in_bucket: impl Fn(&str) -> bool,
96    area_threshold: f32,
97    conf_threshold: f32,
98) {
99    let idx: Vec<usize> = (0..regions.len())
100        .filter(|&i| in_bucket(regions[i].label))
101        .collect();
102    if idx.len() < 2 {
103        return;
104    }
105    // Union-find over the bucket.
106    let mut parent: Vec<usize> = (0..idx.len()).collect();
107    fn find(parent: &mut [usize], i: usize) -> usize {
108        let mut root = i;
109        while parent[root] != root {
110            root = parent[root];
111        }
112        let mut cur = i;
113        while parent[cur] != root {
114            let next = parent[cur];
115            parent[cur] = root;
116            cur = next;
117        }
118        root
119    }
120    let boxed = |r: &Region| (r.l, r.t, r.r, r.b);
121    for a in 0..idx.len() {
122        for b in (a + 1)..idx.len() {
123            let (ra, rb) = (&regions[idx[a]], &regions[idx[b]]);
124            let (al, at, ar, ab_) = boxed(ra);
125            let (bl, bt, br, bb) = boxed(rb);
126            let ix = (ar.min(br) - al.max(bl)).max(0.0);
127            let iy = (ab_.min(bb) - at.max(bt)).max(0.0);
128            let inter = ix * iy;
129            let aa = area(al, at, ar, ab_).max(f32::EPSILON);
130            let ba = area(bl, bt, br, bb).max(f32::EPSILON);
131            let iou = inter / (aa + ba - inter).max(f32::EPSILON);
132            if iou > 0.8 || inter / aa > 0.8 || inter / ba > 0.8 {
133                let (pa, pb) = (find(&mut parent, a), find(&mut parent, b));
134                if pa != pb {
135                    parent[pa] = pb;
136                }
137            }
138        }
139    }
140    // Per group, run docling's pairwise preference + larger-wins selection.
141    let mut groups: std::collections::HashMap<usize, Vec<usize>> = std::collections::HashMap::new();
142    for i in 0..idx.len() {
143        let root = find(&mut parent, i);
144        groups.entry(root).or_default().push(i);
145    }
146    let mut drop = vec![false; regions.len()];
147    for group in groups.values() {
148        if group.len() < 2 {
149            continue;
150        }
151        let area_of = |i: usize| {
152            let r = &regions[idx[i]];
153            area(r.l, r.t, r.r, r.b).max(f32::EPSILON)
154        };
155        let mut best: Option<usize> = None;
156        for &cand in group {
157            let passes = group.iter().all(|&other| {
158                if other == cand {
159                    return true;
160                }
161                let area_ratio = area_of(cand) / area_of(other);
162                let conf_diff = regions[idx[other]].score - regions[idx[cand]].score;
163                !(area_ratio <= area_threshold && conf_diff > conf_threshold)
164            });
165            if passes {
166                best = Some(match best {
167                    None => cand,
168                    Some(cur) => {
169                        if area_of(cand) > area_of(cur)
170                            && regions[idx[cur]].score - regions[idx[cand]].score <= conf_threshold
171                        {
172                            cand
173                        } else {
174                            cur
175                        }
176                    }
177                });
178            }
179        }
180        // Every candidate rejected can't happen with docling's rule (rejection
181        // needs a strictly better rival); guard with highest score anyway.
182        let keep = best.unwrap_or_else(|| {
183            *group
184                .iter()
185                .max_by(|&&a, &&b| regions[idx[a]].score.total_cmp(&regions[idx[b]].score))
186                .expect("non-empty group")
187        });
188        for &i in group {
189            if i != keep {
190                drop[idx[i]] = true;
191            }
192        }
193    }
194    let mut keep_iter = drop.into_iter();
195    regions.retain(|_| !keep_iter.next().expect("aligned"));
196}
197
198/// docling's `_remove_overlapping_clusters("regular")` for what [`greedy`]
199/// leaves standing, run on OCR'd pages before the region-scoped OCR (found
200/// with #471's synthetic scan: every line of the page came out twice).
201///
202/// `greedy` keeps regions by descending score and drops a candidate mostly
203/// inside an already-kept one — so a *lower*-score block that contains
204/// several higher-score line boxes (RT-DETR's favourite reading of a sparse
205/// scanned page: every line, plus the paragraph) survives alongside them.
206/// On a digital page that is harmless: [`fit_regions_to_cells`] hands each
207/// text cell to one owner and drops the regions left empty. On a scanned
208/// page the cells don't exist yet — they come from OCR of *each* region's
209/// crop — so the block and its lines were each recognized, and the block,
210/// which then owned both cell sets, read every line twice.
211///
212/// Upstream never has this problem because its OCR runs over the bitmap
213/// before layout postprocessing and each cell is assigned once; the
214/// postprocessor's regular pass then groups clusters that overlap (IoU > 0.8,
215/// or either > 80 % contained in the other) with a union-find, keeps one
216/// survivor per group via `_should_prefer_cluster` /
217/// `_select_best_cluster_from_group` (`area_threshold` 1.3, `conf_threshold`
218/// 0.05; a LIST_ITEM beats a same-sized TEXT, a CODE box beats what it
219/// contains) and merges the losers' cells into it, whose box is then fitted
220/// to those cells. The equivalent for region-scoped OCR: one survivor per
221/// group, keeping its label and score, with the group's **union** box so the
222/// single crop still covers every merged line. Pictures and wrappers are not
223/// regulars and are left alone ([`dedup_pictures`], [`resolve`]).
224pub(crate) fn merge_overlapping_regulars(regions: &mut Vec<Region>) {
225    let idx: Vec<usize> = (0..regions.len())
226        .filter(|&i| regions[i].label != "picture" && !is_wrapper(regions[i].label))
227        .collect();
228    if idx.len() < 2 {
229        return;
230    }
231    let mut parent: Vec<usize> = (0..idx.len()).collect();
232    fn find(parent: &mut [usize], i: usize) -> usize {
233        let mut root = i;
234        while parent[root] != root {
235            root = parent[root];
236        }
237        let mut cur = i;
238        while parent[cur] != root {
239            let next = parent[cur];
240            parent[cur] = root;
241            cur = next;
242        }
243        root
244    }
245    for a in 0..idx.len() {
246        for b in (a + 1)..idx.len() {
247            let (ra, rb) = (&regions[idx[a]], &regions[idx[b]]);
248            let i = inter(ra, rb.l, rb.t, rb.r, rb.b);
249            let aa = area(ra.l, ra.t, ra.r, ra.b).max(f32::EPSILON);
250            let ba = area(rb.l, rb.t, rb.r, rb.b).max(f32::EPSILON);
251            if i / (aa + ba - i).max(f32::EPSILON) > 0.8 || i / aa > 0.8 || i / ba > 0.8 {
252                let (pa, pb) = (find(&mut parent, a), find(&mut parent, b));
253                if pa != pb {
254                    parent[pa] = pb;
255                }
256            }
257        }
258    }
259    let mut groups: std::collections::BTreeMap<usize, Vec<usize>> =
260        std::collections::BTreeMap::new();
261    for i in 0..idx.len() {
262        let root = find(&mut parent, i);
263        groups.entry(root).or_default().push(i);
264    }
265    const AREA_THRESHOLD: f32 = 1.3;
266    const CONF_THRESHOLD: f32 = 0.05;
267    let area_of = |i: usize| {
268        let r = &regions[idx[i]];
269        area(r.l, r.t, r.r, r.b).max(f32::EPSILON)
270    };
271    // `_should_prefer_cluster(candidate, other)` with the regular params.
272    let prefer = |cand: usize, other: usize| -> bool {
273        let (c, o) = (&regions[idx[cand]], &regions[idx[other]]);
274        let area_ratio = area_of(cand) / area_of(other);
275        if c.label == "list_item" && o.label == "text" && (1.0 - area_ratio).abs() < 0.2 {
276            return true;
277        }
278        if c.label == "code" && inter(o, c.l, c.t, c.r, c.b) / area_of(other) > 0.8 {
279            return true;
280        }
281        !(area_ratio <= AREA_THRESHOLD && o.score - c.score > CONF_THRESHOLD)
282    };
283    let mut drop = vec![false; regions.len()];
284    let mut unions: Vec<(usize, (f32, f32, f32, f32))> = Vec::new();
285    for group in groups.values() {
286        if group.len() < 2 {
287            continue;
288        }
289        let mut best: Option<usize> = None;
290        for &cand in group {
291            if group
292                .iter()
293                .all(|&other| other == cand || prefer(cand, other))
294            {
295                best = Some(match best {
296                    None => cand,
297                    Some(cur)
298                        if area_of(cand) > area_of(cur)
299                            && regions[idx[cur]].score - regions[idx[cand]].score
300                                <= CONF_THRESHOLD =>
301                    {
302                        cand
303                    }
304                    Some(cur) => cur,
305                });
306            }
307        }
308        // docling falls back to the group's first cluster; the highest score
309        // is the deterministic equivalent for a set with no insertion order.
310        let keep = best.unwrap_or_else(|| {
311            *group
312                .iter()
313                .max_by(|&&a, &&b| regions[idx[a]].score.total_cmp(&regions[idx[b]].score))
314                .expect("non-empty group")
315        });
316        let mut u = (
317            f32::INFINITY,
318            f32::INFINITY,
319            f32::NEG_INFINITY,
320            f32::NEG_INFINITY,
321        );
322        for &i in group {
323            let r = &regions[idx[i]];
324            u = (u.0.min(r.l), u.1.min(r.t), u.2.max(r.r), u.3.max(r.b));
325            if i != keep {
326                drop[idx[i]] = true;
327            }
328        }
329        unions.push((idx[keep], u));
330    }
331    for (i, (l, t, r, b)) in unions {
332        let k = &mut regions[i];
333        (k.l, k.t, k.r, k.b) = (l, t, r, b);
334    }
335    let mut keep_iter = drop.into_iter();
336    regions.retain(|_| !keep_iter.next().expect("aligned"));
337}
338
339/// `intersection_over_union` of two regions.
340fn iou(a: &Region, b: &Region) -> f32 {
341    let i = inter(a, b.l, b.t, b.r, b.b);
342    let u = area(a.l, a.t, a.r, a.b) + area(b.l, b.t, b.r, b.b) - i;
343    if u > 0.0 {
344        i / u
345    } else {
346        0.0
347    }
348}
349
350/// docling's `_resolve_coincident_pairs` (#4059, 2.122): for every (loser,
351/// winner) pair at a near-identical box (IoU > 0.8) whose confidences are
352/// within 0.1 (`loser.score - winner.score < 0.1`), the loser label is dropped
353/// so the label with the richer downstream semantic survives. Nothing else —
354/// containment, area — is considered; a clearly more confident loser stays.
355fn coincident_losers(regions: &[Region], losers: &[usize], winners: &[usize]) -> Vec<usize> {
356    let mut out = Vec::new();
357    for &li in losers {
358        for &wi in winners {
359            if iou(&regions[li], &regions[wi]) > 0.8 && regions[li].score - regions[wi].score < 0.1
360            {
361                out.push(li);
362                break;
363            }
364        }
365    }
366    out
367}
368
369/// docling's `_handle_cross_type_overlaps` (2.122/2.123 shape): the layout
370/// model can emit one grounded region under several labels, and the picture /
371/// table / container buckets are de-overlapped independently, so such a region
372/// survives twice. Elect a winner for the near-identical pairs:
373///
374/// | pair                                  | loser     | winner              |
375/// |---------------------------------------|-----------|---------------------|
376/// | TABLE vs DOCUMENT_INDEX               | table     | document_index      |
377/// | PICTURE vs TABLE / DOCUMENT_INDEX     | picture   | the table-like      |
378/// | FORM / KEY_VALUE_REGION vs *surviving* TABLE / DOCUMENT_INDEX / PICTURE | container | structured element |
379///
380/// IoU (not containment) so a genuine small figure inside a large table region
381/// is not removed; the confidence tolerance keeps a clearly more confident
382/// loser (an earlier port dropped every coincident picture regardless).
383fn handle_cross_type_overlaps(regions: Vec<Region>) -> Vec<Region> {
384    let by = |pred: &dyn Fn(&str) -> bool| -> Vec<usize> {
385        (0..regions.len())
386            .filter(|&i| pred(regions[i].label))
387            .collect()
388    };
389    let tables = by(&|l| l == "table");
390    let doc_indices = by(&|l| l == "document_index");
391    let pictures = by(&|l| l == "picture");
392    let containers = by(&|l| matches!(l, "form" | "key_value_region"));
393    let mut drop = vec![false; regions.len()];
394    for i in coincident_losers(&regions, &tables, &doc_indices) {
395        drop[i] = true;
396    }
397    let table_like: Vec<usize> = tables.iter().chain(&doc_indices).copied().collect();
398    for i in coincident_losers(&regions, &pictures, &table_like) {
399        drop[i] = true;
400    }
401    let structured: Vec<usize> = table_like
402        .iter()
403        .chain(&pictures)
404        .copied()
405        .filter(|&i| !drop[i])
406        .collect();
407    for i in coincident_losers(&regions, &containers, &structured) {
408        drop[i] = true;
409    }
410    let mut drop = drop.into_iter();
411    let mut regions = regions;
412    regions.retain(|_| !drop.next().expect("aligned"));
413    regions
414}
415
416pub fn resolve(regions: Vec<Region>) -> Vec<Region> {
417    let regions = handle_cross_type_overlaps(regions);
418    // De-overlap each bucket on its own.
419    let pictures = greedy(
420        regions
421            .iter()
422            .filter(|r| r.label == "picture")
423            .cloned()
424            .collect(),
425    );
426    // Tables and containers are separate buckets since docling 2.123
427    // (`TABLE_TYPES` vs `CONTAINER_TYPES`, docling#4064): a form drawn around a
428    // table no longer competes with it for survival — the table nests inside
429    // the container instead (`order_with_containers`).
430    let mut tables = greedy(
431        regions
432            .iter()
433            .filter(|r| is_table_like(r.label))
434            .cloned()
435            .collect(),
436    );
437    // `greedy` only drops a table mostly inside a *more* confident one, so a
438    // low-score whole-page table proposed over the column tables it contains
439    // (a two-column glossary page: 0.53 over 0.71/0.67/0.66) survived next to them and every
440    // cell was emitted twice. docling's `_remove_overlapping_clusters(tables,
441    // "wrapper")` groups tables whose boxes overlap (IoU > 0.8, or either one
442    // > 80 % inside the other) and keeps one per group: run it on what
443    // `greedy` leaves, like `merge_overlapping_regulars` does for regulars.
444    remove_overlapping_specials(&mut tables, |_| true, 2.0, 0.2);
445    let containers = greedy(
446        regions
447            .iter()
448            .filter(|r| matches!(r.label, "form" | "key_value_region"))
449            .cloned()
450            .collect(),
451    );
452    let mut kept = greedy(
453        regions
454            .iter()
455            .filter(|r| r.label != "picture" && !is_wrapper(r.label))
456            .cloned()
457            .collect(),
458    );
459    dedup_nested_code(&mut kept);
460    kept.extend(pictures);
461    kept.extend(tables);
462    kept.extend(containers);
463    kept
464}
465
466/// Drop a regular region that is >80% contained in a surviving special region we
467/// render **as a single unit** — a table/table-of-contents index — ported from
468/// docling's "Remove regular clusters that are included in wrappers" step: the
469/// special absorbs it as a child (a table cell), so it must not also be emitted
470/// as its own paragraph/list-item. This stops the survey list-items from
471/// appearing both inside the detected table and again as bullets
472/// (`table_mislabeled_as_picture`).
473///
474/// `picture` regions are **not** in the swallow set: docling keeps a picture's
475/// contained clusters as the `PictureItem`'s *children* in the document JSON
476/// (`_set_cluster_children`, `ReadingOrderModel._add_child_elements`), while
477/// its `MarkdownPictureSerializer` prints only the caption and the image — the
478/// children never reach the Markdown (verified against the corpus groundtruth:
479/// `amt_handbook`'s in-figure callout labels are absent). So the regulars
480/// inside a picture stay in the region list, claim their cells and are fitted
481/// like any regular, and [`assemble_page`] lifts them out of the page's
482/// reading order into a [`Node::PictureChildren`] after their picture (see
483/// [`picture_parents`]). A line only partially under a figure box (straddling
484/// its border, ≤80 % contained) is no picture's child and is emitted, as
485/// docling emits it (#165).
486///
487/// `form` / `key_value_region` wrappers are deliberately **excluded**: this
488/// pipeline does not render them as a structured block (they are skipped), so
489/// their textual content comes precisely from the contained regular regions —
490/// A `page_footer` that is really the body of the last heading on the page
491/// becomes `text`.
492///
493/// The layout model labels the bottom margin by position as much as by
494/// content: a one-line paragraph that happens to sit where a running footer
495/// would — a CV's `Languages` line under its `## Languages` heading, the last
496/// entry of a section that runs to the page edge — comes back as
497/// `page_footer` (0.88 against 0.49 for `text` on the reporting file, with
498/// the fp32 model and either renderer alike), and Markdown drops furniture,
499/// so the document loses real content while the JSON keeps it under the
500/// wrong label. docling fails the same way. Deliberate deviation, gated so a
501/// real footer keeps its label: the nearest `section_header` above the footer
502/// must overlap it horizontally, end within 2.5 footer heights of its top,
503/// and be the **last body element** of its column — no other non-furniture
504/// region starts at or below the heading's bottom edge over the same span —
505/// and the footer must be a line of text, not a page number: at least 40 %
506/// of the page width. A running footer never follows a heading that has no
507/// body of its own, so the combination is the misread heading body.
508pub fn reclaim_heading_body_footers(regions: &mut [Region], page_w: f32) {
509    let n = regions.len();
510    let overlap_x = |a: &Region, b: &Region| a.r.min(b.r) - a.l.max(b.l) > 0.0;
511    for fi in 0..n {
512        let f = regions[fi].clone();
513        if f.label != "page_footer" || f.r - f.l < 0.4 * page_w {
514            continue;
515        }
516        let fh = (f.b - f.t).max(1.0);
517        // The nearest heading above the footer, over the footer's span.
518        let heading = (0..n)
519            .filter(|&j| {
520                let h = &regions[j];
521                j != fi && h.label == "section_header" && h.b <= f.t + 0.5 * fh && overlap_x(h, &f)
522            })
523            .min_by(|&a, &b| regions[b].b.total_cmp(&regions[a].b));
524        let Some(hi) = heading else {
525            continue;
526        };
527        let h = regions[hi].clone();
528        if f.t - h.b > 2.5 * fh {
529            continue;
530        }
531        // The heading must have no body of its own: nothing but the footer
532        // starts at or below its bottom edge over the heading's or footer's
533        // span (a heading whose paragraph follows is not this case, and a
534        // heading with the footer far below it was filtered above).
535        let has_body = (0..n).any(|j| {
536            let r = &regions[j];
537            j != fi
538                && j != hi
539                && !matches!(r.label, "page_footer" | "page_header")
540                && r.t >= h.b - 0.5 * fh
541                && (overlap_x(r, &h) || overlap_x(r, &f))
542        });
543        if has_body {
544            continue;
545        }
546        regions[fi].label = "text";
547    }
548}
549
550/// dropping those would erase the page (e.g. `right_to_left_03`'s form-heavy
551/// pages). Runs *after* [`drop_false_pictures`] so a phantom picture can't
552/// swallow real text on its way out.
553pub fn drop_contained_regulars(regions: &mut Vec<Region>) {
554    let specials: Vec<(f32, f32, f32, f32)> = regions
555        .iter()
556        .filter(|r| is_table_like(r.label))
557        .map(|r| (r.l, r.t, r.r, r.b))
558        .collect();
559    if specials.is_empty() {
560        return;
561    }
562    regions.retain(|r| {
563        if r.label == "picture" || is_wrapper(r.label) {
564            return true;
565        }
566        let ra = area(r.l, r.t, r.r, r.b).max(1.0);
567        !specials
568            .iter()
569            .any(|&(l, t, rr, b)| inter(r, l, t, rr, b) / ra > 0.8)
570    });
571}
572
573/// docling's `_set_cluster_children` for pictures: a regular region (one that
574/// claims cells) whose box is > 80 % inside a picture's box
575/// (`intersection_over_self`) is that picture's child — not a page element
576/// (it leaves the reading order and the layout score), but an item under the
577/// `PictureItem` in the JSON. Returns, per region, the index of its parent
578/// picture: the smallest containing one when pictures nest (upstream would
579/// attach it to each).
580pub fn picture_parents(regions: &[Region]) -> Vec<Option<usize>> {
581    regions
582        .iter()
583        .map(|r| {
584            if !claims_cells(r) {
585                return None;
586            }
587            let ra = area(r.l, r.t, r.r, r.b).max(1.0);
588            regions
589                .iter()
590                .enumerate()
591                .filter(|(_, p)| p.label == "picture" && inter(r, p.l, p.t, p.r, p.b) / ra > 0.8)
592                .min_by(|(_, a), (_, b)| {
593                    area(a.l, a.t, a.r, a.b).total_cmp(&area(b.l, b.t, b.r, b.b))
594                })
595                .map(|(i, _)| i)
596        })
597        .collect()
598}
599
600/// True for a bare, single-token source-code language label (`XML`, `C#`, `JSON`,
601/// `bash`, …) — the little header the docs render above a code block. Matched
602/// case-insensitively; anything with whitespace or longer than a token is out.
603fn is_code_language(t: &str) -> bool {
604    let t = t.trim();
605    if t.is_empty() || t.chars().any(char::is_whitespace) || t.chars().count() > 12 {
606        return false;
607    }
608    const LANGS: &[&str] = &[
609        "xml",
610        "html",
611        "xhtml",
612        "json",
613        "jsonc",
614        "yaml",
615        "yml",
616        "toml",
617        "ini",
618        "c#",
619        "csharp",
620        "f#",
621        "fsharp",
622        "vb",
623        "c",
624        "c++",
625        "cpp",
626        "java",
627        "kotlin",
628        "scala",
629        "go",
630        "golang",
631        "rust",
632        "swift",
633        "javascript",
634        "js",
635        "typescript",
636        "ts",
637        "jsx",
638        "tsx",
639        "python",
640        "py",
641        "ruby",
642        "rb",
643        "php",
644        "perl",
645        "lua",
646        "r",
647        "dart",
648        "bash",
649        "sh",
650        "shell",
651        "powershell",
652        "zsh",
653        "batch",
654        "cmd",
655        "sql",
656        "tsql",
657        "plsql",
658        "graphql",
659        "dockerfile",
660        "makefile",
661        "css",
662        "scss",
663        "sass",
664        "less",
665        "markdown",
666        "md",
667        "tex",
668        "latex",
669        "diff",
670        "proto",
671        "razor",
672        "cshtml",
673        "xaml",
674        "aspx",
675        "http",
676    ];
677    let lower = t.to_ascii_lowercase();
678    LANGS.contains(&lower.as_str())
679}
680
681/// Mark the region indices that are a code block's **language label** — a bare
682/// `XML`/`C#`/… token sitting directly above a `code` region — so they are consumed
683/// rather than emitted as their own stray paragraph/heading. The label may also be
684/// captured inside a wider code box (rendered as the fence's first line); dropping
685/// the standalone copy just removes the duplicate.
686fn code_language_labels(regions: &[Region], cells: &[TextCell]) -> Vec<bool> {
687    let mut drop = vec![false; regions.len()];
688    for (i, r) in regions.iter().enumerate() {
689        if matches!(r.label, "code" | "picture" | "table") {
690            continue;
691        }
692        if !is_code_language(&region_text(r, cells)) {
693            continue;
694        }
695        // The label sits just above the code (a blank line's gap) or is swallowed
696        // into the top of a wider code box; either way it is that block's label.
697        // The window is generous because the label's own font is small, so a
698        // one-line gap is several times its height.
699        let line_h = (r.b - r.t).abs().max(1.0);
700        let window = (line_h * 4.0).max(28.0);
701        let labels_code = regions.iter().enumerate().any(|(j, c)| {
702            if j == i || c.label != "code" {
703                return false;
704            }
705            let gap = c.t - r.b; // >0 when the code is below the label
706            let h_overlap = (r.r.min(c.r) - r.l.max(c.l)).max(0.0);
707            gap > -line_h * 3.0 && gap < window && h_overlap > 0.0
708        });
709        if labels_code {
710            drop[i] = true;
711        }
712    }
713    drop
714}
715
716/// Collapse `code` regions where one is nested inside another, keeping the larger.
717///
718/// RT-DETR sometimes emits a tight code box *and* a wider near-duplicate that also
719/// captures the block's language label (`XML`, `C#`, …). When the tight box scores
720/// higher it is kept first, and the wider container — not "mostly inside" the tight
721/// box — survives [`resolve`]'s greedy pass, so the block is emitted twice. Keeping
722/// the **larger** box (rather than dropping it) collapses the pair without leaking
723/// the container's extra cells back out as orphan text, since the larger box still
724/// covers every cell. Restricted to `code` so genuinely distinct nested regions of
725/// other kinds are untouched.
726fn dedup_nested_code(kept: &mut Vec<Region>) {
727    let mut drop = vec![false; kept.len()];
728    for i in 0..kept.len() {
729        if kept[i].label != "code" {
730            continue;
731        }
732        let ai = area(kept[i].l, kept[i].t, kept[i].r, kept[i].b).max(1.0);
733        for j in 0..kept.len() {
734            if i == j || drop[j] || kept[j].label != "code" {
735                continue;
736            }
737            let aj = area(kept[j].l, kept[j].t, kept[j].r, kept[j].b).max(1.0);
738            // Drop i when it is mostly inside a strictly larger code box j.
739            let overlap = inter(&kept[i], kept[j].l, kept[j].t, kept[j].r, kept[j].b);
740            if aj > ai && overlap / ai > 0.7 {
741                drop[i] = true;
742                break;
743            }
744        }
745    }
746    let mut keep = drop.iter();
747    kept.retain(|_| !*keep.next().unwrap());
748}
749
750/// Fraction of the page's non-empty text cells that some detected region
751/// claims (>0.2 intersection-over-self, docling's assignment rule). 1.0 for a
752/// page without text cells.
753///
754/// The int8-layout guard keys off this: a dense digital page whose detections
755/// cover almost none of its text is the signature of quantized confidences
756/// flipping under the 0.5 label thresholds on this CPU's kernels — not of a
757/// genuinely empty layout — and is worth re-running on the fp32 graph.
758pub fn layout_cell_coverage(regions: &[Region], cells: &[TextCell]) -> f32 {
759    let mut total = 0usize;
760    let mut covered = 0usize;
761    for c in cells {
762        if c.text.trim().is_empty() {
763            continue;
764        }
765        total += 1;
766        let ca = area(c.l, c.t, c.r, c.b).max(1.0);
767        if regions
768            .iter()
769            .any(|r| inter(r, c.l, c.t, c.r, c.b) / ca > 0.2)
770        {
771            covered += 1;
772        }
773    }
774    if total == 0 {
775        1.0
776    } else {
777        covered as f32 / total as f32
778    }
779}
780
781/// Whether a run reads as a note call: a number or reference mark
782/// (`1`, `12`, `*`, `†`, `¹`), or a short bracketed note (`(Anm. 1)`, `[3]`).
783/// Geometry alone is not enough: a figure's labels come in every size, and
784/// moving a smaller word behind the larger one beside it scrambled the
785/// figure text (`The Law will will never perfect be`, the OTSL rows run
786/// together) — the PDF snapshots of three figures.
787fn looks_like_note_call(text: &str) -> bool {
788    let t = text.trim();
789    let n = t.chars().count();
790    if n == 0 || n > 16 {
791        return false;
792    }
793    let mark = |c: char| c.is_ascii_digit() || "*†‡§¶#,⁰¹²³⁴⁵⁶⁷⁸⁹".contains(c);
794    // `1,2` is two calls; a lone comma is punctuation.
795    (t.chars().all(mark) && t.chars().any(|c| c != ','))
796        || matches!(
797            (t.chars().next(), t.chars().last()),
798            (Some('('), Some(')')) | (Some('['), Some(']'))
799        )
800}
801
802/// Does `note` — a small run raised above its line (a footnote call, `(Anm. 1)`)
803/// — sit at the right end of `host`'s line?
804fn is_note_of(note: &TextCell, host: &TextCell) -> bool {
805    if !looks_like_note_call(&note.text) {
806        return false;
807    }
808    let (nt, nb) = (note.t.min(note.b), note.t.max(note.b));
809    let (ht, hb) = (host.t.min(host.b), host.t.max(host.b));
810    let (nh, hh) = ((nb - nt).max(1.0), (hb - ht).max(1.0));
811    nh <= 0.8 * hh
812        && nb.min(hb) - nt.max(ht) >= 0.5 * nh
813        && note.l >= host.r - 0.5 * hh
814        && note.l <= host.r + hh
815}
816
817/// Reading order of the orphan cells `sorted` by (top, left), with a raised
818/// note moved behind its host line (#923). The note's top edge is above the
819/// host's, so the plain sort emitted `(Anm. 1)` ahead of the line it ends
820/// (`…besteht.` is the end of the line starting `und die Belegschaft`). The
821/// flags mark the cells that were moved, for the same-line merge below.
822fn place_raised_notes(sorted: Vec<&TextCell>) -> (Vec<&TextCell>, Vec<bool>) {
823    let n = sorted.len();
824    let mut out: Vec<&TextCell> = Vec::with_capacity(n);
825    let mut flags: Vec<bool> = Vec::with_capacity(n);
826    let mut held: Vec<&TextCell> = Vec::new();
827    for (i, &c) in sorted.iter().enumerate() {
828        // Release a held note the scan has passed without finding its host.
829        let mut k = 0;
830        while k < held.len() {
831            if c.t >= held[k].t.max(held[k].b) {
832                out.push(held.remove(k));
833                flags.push(false);
834            } else {
835                k += 1;
836            }
837        }
838        // A cell with a host line ahead of it (starting within its own band)
839        // waits for that host.
840        let waits = sorted[i + 1..]
841            .iter()
842            .take_while(|h| h.t < c.t.max(c.b))
843            .any(|h| is_note_of(c, h));
844        if waits {
845            held.push(c);
846            continue;
847        }
848        out.push(c);
849        flags.push(false);
850        let mut k = 0;
851        while k < held.len() {
852            if is_note_of(held[k], c) {
853                out.push(held.remove(k));
854                flags.push(true);
855            } else {
856                k += 1;
857            }
858        }
859    }
860    for c in held {
861        out.push(c);
862        flags.push(false);
863    }
864    (out, flags)
865}
866
867/// Append `text` regions for cells the layout left uncovered ("orphan cells"),
868/// the way docling's `LayoutPostprocessor` does (`create_orphan_clusters`): any
869/// non-empty cell that no kept region covers (>50% of the cell's area) becomes a
870/// text region of its own, so text the detector missed (a stray `.`, a small
871/// label) is still emitted instead of silently dropped. Adjacent orphan cells on a
872/// line are merged so a missed paragraph doesn't shatter into one block per line.
873pub fn add_orphan_regions(regions: &mut Vec<Region>, cells: &[TextCell]) {
874    // docling assigns each cell to its single best-overlapping cluster at
875    // intersection-over-self > 0.2 and serializes exactly the assigned cells —
876    // and since [`region_texts_exclusive`] now emits under that very rule, the
877    // claim test here matches it: any cell over 0.2 will actually render in
878    // its best region, everything else becomes an orphan. Completeness by
879    // construction, with no (0.2, 0.5] hole (the old > 0.5 serializer needed
880    // the claim test raised to > 0.5 to keep right_to_left_03's `20300` from
881    // vanishing; the exclusive port closes that structurally).
882    //
883    // Only *regular* clusters claim cells: docling's `_find_unassigned_cells`
884    // walks `regular_clusters` alone, so a cell under a `picture` or a wrapper
885    // (`table`/`document_index`/`form`/`key_value_region`) that no regular
886    // cluster covers still becomes an orphan text cluster (#165). The orphans
887    // that end up *fully* inside the special are re-dropped by
888    // [`drop_contained_regulars`] (docling's Markdown drops them the same way
889    // — a picture's children never reach its `MarkdownPictureSerializer`
890    // output, a table's text renders through the reconstructed grid). The
891    // observable fix is the border-straddlers: a line only partially under a
892    // figure box used to lose its cells to the picture's 0.2 claim and vanish
893    // — now it forms an orphan region and is emitted, as docling does.
894    let assigned = |c: &TextCell| {
895        let ca = area(c.l, c.t, c.r, c.b).max(1.0);
896        regions
897            .iter()
898            .filter(|r| r.label != "picture" && !is_wrapper(r.label))
899            .any(|r| inter(r, c.l, c.t, c.r, c.b) / ca > 0.2)
900    };
901    // Collect orphan cells (non-empty, unassigned), in page order.
902    let mut orphans: Vec<&TextCell> = cells
903        .iter()
904        .filter(|c| !c.text.trim().is_empty() && !assigned(c))
905        .collect();
906    if orphans.is_empty() {
907        return;
908    }
909    orphans.sort_by(|a, b| a.t.total_cmp(&b.t).then(a.l.total_cmp(&b.l)));
910    let (orphans, notes) = place_raised_notes(orphans);
911    // Merge cells that sit on the same line and nearly touch into one region, so a
912    // dropped multi-word line stays one block (docling's refinement merges these).
913    let mut merged: Vec<Region> = Vec::new();
914    for (k, c) in orphans.into_iter().enumerate() {
915        let mut h = (c.b - c.t).abs().max(1.0);
916        if let Some(last) = merged.last_mut() {
917            // A raised note is judged by its host line's height, not its own.
918            if notes[k] {
919                h = h.max((last.b - last.t).abs());
920            }
921            let same_line = (last.t - c.t).abs() < h * 0.5;
922            let touching = c.l <= last.r + h && c.l >= last.l - h;
923            // Both tolerances scale with the cell's own height, so a run set
924            // vertically (the arXiv stamp up the margin, #528 — a cell a few
925            // points wide and hundreds tall) would read as "on the line" of
926            // whatever precedes it and glue a whole column into one region.
927            // Lines of one row differ by a drop cap's few multiples at most.
928            let lh = (last.b - last.t).abs().max(1.0);
929            let comparable = h <= 4.0 * lh && lh <= 4.0 * h;
930            if same_line && touching && comparable {
931                last.l = last.l.min(c.l);
932                last.r = last.r.max(c.r);
933                last.t = last.t.min(c.t);
934                last.b = last.b.max(c.b);
935                continue;
936            }
937        }
938        merged.push(Region {
939            label: "text",
940            score: 0.0,
941            l: c.l,
942            t: c.t,
943            r: c.r,
944            b: c.b,
945        });
946    }
947    regions.extend(merged);
948}
949
950/// Demote a `picture` region that is really a **text panel** — a paragraph block
951/// the layout model boxed as a figure because it is typeset on a colored
952/// background (terms-and-conditions callouts, quote boxes) — into ordinary
953/// `text` regions, one per paragraph, so its words are read instead of shipped
954/// as pixels. docling loses this text the same way (cells assigned to a picture
955/// cluster are never serialized); this is a deliberate improvement, not parity.
956///
957/// The gate is conservative so a genuine figure keeps its crop: the region must
958/// contain at least three text lines whose median width spans most of the panel
959/// (axis labels and chat bubbles are narrow and varied) and whose cells cover a
960/// substantial fraction of its area (a photo or chart with sparse labels does
961/// not). Paragraph boundaries are re-derived from the line pitch: a vertical gap
962/// clearly larger than the panel's own leading starts a new `text` region, so
963/// the panel doesn't collapse into one giant paragraph.
964///
965/// Works on any cell source — the digital text layer or OCR lines recognized
966/// from the picture crop — so the native and browser paths, with or without
967/// force-OCR, demote identically.
968pub fn recover_text_panels(regions: &mut Vec<Region>, cells: &[TextCell]) {
969    // A *captioned* picture is a genuine figure whatever it contains — the
970    // corpus is full of document screenshots ("Figure 3: …" above a page
971    // image) that are exactly as dense and wide as a text panel. Only an
972    // uncaptioned picture is a demotion candidate.
973    let captioned: Vec<bool> = regions
974        .iter()
975        .map(|r| {
976            r.label == "picture"
977                && regions.iter().any(|c| {
978                    c.label == "caption" && c.r.min(r.r) - c.l.max(r.l) > 0.0 && {
979                        let gap = if c.t >= r.b {
980                            c.t - r.b
981                        } else if r.t >= c.b {
982                            r.t - c.b
983                        } else {
984                            f32::MAX // vertically overlapping: not a caption
985                        };
986                        gap <= 25.0
987                    }
988                })
989        })
990        .collect();
991    let mut out: Vec<Region> = Vec::with_capacity(regions.len());
992    // Synthesized paragraphs and the demoted panels' boxes are kept separate
993    // from `out` until the end: the dedup filter below must not confuse a
994    // paragraph we just built with a pre-existing region inside the panel.
995    let mut demoted_paras: Vec<Region> = Vec::new();
996    let mut demoted_boxes: Vec<(f32, f32, f32, f32)> = Vec::new();
997    for (i, r) in regions.drain(..).enumerate() {
998        if r.label != "picture" || captioned[i] {
999            out.push(r);
1000            continue;
1001        }
1002        let inside: Vec<&TextCell> = cells
1003            .iter()
1004            .filter(|c| {
1005                !c.text.trim().is_empty() && {
1006                    let ca = area(c.l, c.t, c.r, c.b).max(1.0);
1007                    inter(&r, c.l, c.t, c.r, c.b) / ca > 0.5
1008                }
1009            })
1010            .collect();
1011        // Group the contained cells into lines by vertical overlap (the same
1012        // rule region_text orders by), tracking each line's union box.
1013        let mut lines: Vec<(f32, f32, f32, f32)> = Vec::new(); // (t, b, l, r)
1014        for c in &inside {
1015            let (ct, cb) = (c.t.min(c.b), c.t.max(c.b));
1016            match lines.iter_mut().find(|(lt, lb, _, _)| {
1017                let ov = cb.min(*lb) - ct.max(*lt);
1018                ov > 0.5 * (cb - ct).min(*lb - *lt).max(1.0)
1019            }) {
1020                Some((lt, lb, ll, lr)) => {
1021                    *lt = lt.min(ct);
1022                    *lb = lb.max(cb);
1023                    *ll = ll.min(c.l);
1024                    *lr = lr.max(c.r);
1025                }
1026                None => lines.push((ct, cb, c.l, c.r)),
1027            }
1028        }
1029        if lines.len() < 3 {
1030            out.push(r);
1031            continue;
1032        }
1033        let panel_w = (r.r - r.l).max(1.0);
1034        let coverage = inside.iter().map(|c| area(c.l, c.t, c.r, c.b)).sum::<f32>()
1035            / area(r.l, r.t, r.r, r.b).max(1.0);
1036        let mut widths: Vec<f32> = lines.iter().map(|(_, _, l, rr)| rr - l).collect();
1037        widths.sort_by(f32::total_cmp);
1038        // A figure's text is ragged: a title line, small axis/tick labels, and
1039        // OCR boxes over the plot area come out at wildly different heights,
1040        // whereas a real text panel is set in one face with constant leading.
1041        // Require near-uniform line heights (median absolute deviation ≤ 35%
1042        // of the median) so an uncaptioned chart keeps its crop even when its
1043        // labels are dense enough to pass the coverage gate (#173) — garbled
1044        // OCR of its bars is not content.
1045        let mut heights: Vec<f32> = lines.iter().map(|(t, b, _, _)| b - t).collect();
1046        heights.sort_by(f32::total_cmp);
1047        let h_med = heights[heights.len() / 2].max(1.0);
1048        let mut devs: Vec<f32> = heights.iter().map(|h| (h - h_med).abs()).collect();
1049        devs.sort_by(f32::total_cmp);
1050        let uniform = devs[devs.len() / 2] <= 0.35 * h_med;
1051        let text_panel = coverage >= 0.2 && widths[widths.len() / 2] >= 0.45 * panel_w && uniform;
1052        if !text_panel {
1053            out.push(r);
1054            continue;
1055        }
1056        lines.sort_by(|a, b| a.0.total_cmp(&b.0));
1057        let mut heights: Vec<f32> = lines.iter().map(|(t, b, _, _)| b - t).collect();
1058        heights.sort_by(f32::total_cmp);
1059        let h = heights[heights.len() / 2].max(1.0);
1060        let mut gaps: Vec<f32> = lines
1061            .windows(2)
1062            .map(|w| (w[1].0 - w[0].1).max(0.0))
1063            .collect();
1064        gaps.sort_by(f32::total_cmp);
1065        let leading = if gaps.is_empty() {
1066            0.0
1067        } else {
1068            gaps[gaps.len() / 2]
1069        };
1070        let brk = (1.8 * leading).max(0.75 * h);
1071        let mut para: Option<(f32, f32, f32, f32)> = None; // (l, t, r, b) union
1072        for (t, b, l, rr) in &lines {
1073            match &mut para {
1074                Some((pl, _, pr, pb)) if *t - *pb <= brk => {
1075                    *pl = pl.min(*l);
1076                    *pr = pr.max(*rr);
1077                    *pb = pb.max(*b);
1078                }
1079                _ => {
1080                    if let Some((pl, pt, pr, pb)) = para.take() {
1081                        demoted_paras.push(Region {
1082                            label: "text",
1083                            score: r.score,
1084                            l: pl,
1085                            t: pt,
1086                            r: pr,
1087                            b: pb,
1088                        });
1089                    }
1090                    para = Some((*l, *t, *rr, *b));
1091                }
1092            }
1093        }
1094        if let Some((pl, pt, pr, pb)) = para {
1095            demoted_paras.push(Region {
1096                label: "text",
1097                score: r.score,
1098                l: pl,
1099                t: pt,
1100                r: pr,
1101                b: pb,
1102            });
1103        }
1104        demoted_boxes.push((r.l, r.t, r.r, r.b));
1105    }
1106    // The paragraphs are rebuilt from *all* of the panel's cells, so any
1107    // surviving text region inside a demoted panel (an orphan cluster or a
1108    // layout-detected fragment — pictures no longer swallow them, #165) would
1109    // say the same words twice. Consume those; wrappers and pictures stay.
1110    if !demoted_boxes.is_empty() {
1111        out.retain(|r| {
1112            r.label == "picture" || is_wrapper(r.label) || {
1113                let ra = area(r.l, r.t, r.r, r.b).max(1.0);
1114                !demoted_boxes
1115                    .iter()
1116                    .any(|&(l, t, rr, b)| inter(r, l, t, rr, b) / ra > 0.5)
1117            }
1118        });
1119    }
1120    // docling's "Remove regular clusters that are included in wrappers" (a
1121    // regular > 80 % inside a table is absorbed by it) already ran as
1122    // [`drop_contained_regulars`], but before this demotion created new
1123    // regulars. Apply it to them too: a panel that coincides with a table (a
1124    // dense data table detected as picture 0.80 and table 0.62 on one box;
1125    // `_handle_cross_type_overlaps` keeps both once the picture is ≥ 0.1 more
1126    // confident) rebuilds the table's words as a paragraph the grid already
1127    // renders. A panel inside another picture is left as it was.
1128    demoted_paras.retain(|p| {
1129        let pa = area(p.l, p.t, p.r, p.b).max(1.0);
1130        !out.iter()
1131            .any(|s| is_table_like(s.label) && inter(p, s.l, s.t, s.r, s.b) / pa > 0.8)
1132    });
1133    out.extend(demoted_paras);
1134    *regions = out;
1135}
1136
1137/// Drop a `picture` detection covering more than 90 % of the page — docling's
1138/// `LayoutPostprocessor._process_special_clusters` "Filter out full-page
1139/// pictures" (upstream since 2.15), applied to the thresholded detections
1140/// before overlap resolution. A box that big is the page itself, not a figure
1141/// on it: the layout model emits one for a whole-page diagram (a LaTeX figure
1142/// PDF cropped to its drawing), a plate, or a scan, and keeping it would swallow
1143/// every text cell on the page as picture children — the diagram's labels and
1144/// caption vanish behind a lone `<!-- image -->`, where docling reads them out as
1145/// text. `page_w`/`page_h` is the display-frame page box.
1146pub fn drop_full_page_pictures(regions: &mut Vec<Region>, page_w: f32, page_h: f32) {
1147    let page_area = (page_w * page_h).max(1.0);
1148    regions.retain(|r| r.label != "picture" || area(r.l, r.t, r.r, r.b) / page_area <= 0.90);
1149}
1150
1151/// Drop a `picture` detection that is a small, empty, low-confidence margin box on
1152/// a **text page** — a false positive the RT-DETR layout sometimes emits (e.g.
1153/// `right_to_left_02`'s phantom right-column picture, score 0.40); docling does not
1154/// emit it. The gate is deliberately narrow so a genuine figure is never dropped:
1155/// (1) only on pages with a digital text layer — image/scanned/figure pages have
1156/// no `cells` yet at this point (OCR runs later), so their pictures, which *are*
1157/// the content, are kept; (2) only a box covering < 25 % of the page (a margin
1158/// artifact, not a dominant figure); (3) only when it contains no text and scores
1159/// below 0.5 (real empty figures in the corpus all score ≥ 0.86).
1160pub fn drop_false_pictures(
1161    regions: &mut Vec<Region>,
1162    cells: &[TextCell],
1163    page_w: f32,
1164    page_h: f32,
1165) {
1166    if cells.iter().all(|c| c.text.trim().is_empty()) {
1167        return; // no digital text layer (image/scanned page) — keep all pictures
1168    }
1169    // A text-document page carries several text-bearing non-picture regions (so a
1170    // spurious margin picture is clearly extra). A slide / figure page has at most
1171    // one — there the picture is the content, so never drop it.
1172    let content_regions = regions
1173        .iter()
1174        .filter(|r| r.label != "picture" && !region_text(r, cells).trim().is_empty())
1175        .count();
1176    if content_regions < 2 {
1177        return;
1178    }
1179    let page_area = (page_w * page_h).max(1.0);
1180    regions.retain(|r| {
1181        if r.label != "picture" || r.score >= 0.5 {
1182            return true;
1183        }
1184        if area(r.l, r.t, r.r, r.b) / page_area >= 0.25 {
1185            return true; // a dominant figure, not a margin artifact
1186        }
1187        // Keep it if any text cell falls mostly inside (a real captioned/labelled
1188        // figure); drop only the genuinely empty low-confidence boxes.
1189        cells.iter().any(|c| {
1190            let ca = area(c.l, c.t, c.r, c.b).max(1.0);
1191            !c.text.trim().is_empty() && inter(r, c.l, c.t, c.r, c.b) / ca > 0.5
1192        })
1193    });
1194}
1195
1196/// A small digit-only region in the top/bottom margin: a page number. docling
1197/// emits `right_to_left_02`'s bottom `11` as the page's *first* text item (its
1198/// reading-order model floats the page number to the front), whereas our
1199/// position-based ordering would place a bottom region last.
1200fn is_page_number(region: &Region, cells: &[TextCell], page_h: f32) -> bool {
1201    let t = region_text(region, cells);
1202    let t = t.trim();
1203    !t.is_empty()
1204        && t.chars().all(|c| c.is_ascii_digit())
1205        && (region.b - region.t).abs() < 30.0
1206        && (region.t < page_h * 0.12 || region.b > page_h * 0.88)
1207}
1208
1209/// docling's `form` / `key_value_region` *containers* (2.123, docling#4064):
1210/// every region sitting > 0.8 inside one — text, list items, and since #4064
1211/// tables and pictures too — is that container's child. Children are
1212/// reading-ordered among themselves and emitted as one block where the
1213/// container falls in the page's top-level order (a `form_area` /
1214/// `key_value_area` group upstream), instead of interleaving with the text
1215/// around the form. A child inside several containers belongs to the smallest
1216/// (then most confident, then first); a container with children shrinks to
1217/// their union for the top-level ordering, like upstream's bbox adjustment.
1218///
1219/// The containers themselves are still not emitted (`is_skipped`), so the
1220/// Markdown is exactly upstream's — a group prints only its children.
1221///
1222/// `cids` are the items' positions in docling's assembly order
1223/// ([`cluster_cids`]) — the reading-order predictor's same-row rule (#424)
1224/// pairs consecutive ones, within the top level and within each container.
1225fn order_with_containers<T: Clone>(
1226    items: &mut Vec<T>,
1227    cids: &[usize],
1228    page_w: f32,
1229    page_h: f32,
1230    reg: impl Fn(&T) -> &Region,
1231) {
1232    let is_container = |r: &Region| matches!(r.label, "form" | "key_value_region");
1233    let containers: Vec<usize> = (0..items.len())
1234        .filter(|&i| is_container(reg(&items[i])))
1235        .collect();
1236    if containers.is_empty() {
1237        order_regions(items, cids, page_w, page_h, reg);
1238        return;
1239    }
1240    // Parent container per item (containers never nest in each other here —
1241    // upstream assigns regulars and tables/pictures only).
1242    let mut parent: Vec<Option<usize>> = vec![None; items.len()];
1243    for i in 0..items.len() {
1244        let r = reg(&items[i]);
1245        if is_container(r) {
1246            continue;
1247        }
1248        let ra = area(r.l, r.t, r.r, r.b).max(1.0);
1249        let mut best: Option<(usize, f32, f32)> = None; // (idx, area, -score)
1250        for &c in &containers {
1251            let cr = reg(&items[c]);
1252            if inter(r, cr.l, cr.t, cr.r, cr.b) / ra > 0.8 {
1253                let key = (area(cr.l, cr.t, cr.r, cr.b), -cr.score);
1254                if best.is_none_or(|(_, a, s)| key.0 < a || (key.0 == a && key.1 < s)) {
1255                    best = Some((c, key.0, key.1));
1256                }
1257            }
1258        }
1259        parent[i] = best.map(|(c, _, _)| c);
1260    }
1261    // Top-level pass: non-children plus the containers, the latter shrunk to
1262    // their children's union.
1263    let mut top: Vec<(usize, Region)> = Vec::new();
1264    for i in 0..items.len() {
1265        if parent[i].is_some() {
1266            continue;
1267        }
1268        let mut r = reg(&items[i]).clone();
1269        if is_container(&r) {
1270            let kids: Vec<&Region> = (0..items.len())
1271                .filter(|&k| parent[k] == Some(i))
1272                .map(|k| reg(&items[k]))
1273                .collect();
1274            if !kids.is_empty() {
1275                r.l = kids.iter().map(|k| k.l).fold(f32::INFINITY, f32::min);
1276                r.t = kids.iter().map(|k| k.t).fold(f32::INFINITY, f32::min);
1277                r.r = kids.iter().map(|k| k.r).fold(f32::NEG_INFINITY, f32::max);
1278                r.b = kids.iter().map(|k| k.b).fold(f32::NEG_INFINITY, f32::max);
1279            }
1280        }
1281        top.push((i, r));
1282    }
1283    let top_cids: Vec<usize> = top.iter().map(|(i, _)| cids[*i]).collect();
1284    order_regions(&mut top, &top_cids, page_w, page_h, |it| &it.1);
1285    let mut out: Vec<T> = Vec::with_capacity(items.len());
1286    for (i, _) in top {
1287        if is_container(reg(&items[i])) {
1288            let kid_idx: Vec<usize> = (0..items.len()).filter(|&k| parent[k] == Some(i)).collect();
1289            let mut kids: Vec<T> = kid_idx.iter().map(|&k| items[k].clone()).collect();
1290            let kid_cids: Vec<usize> = kid_idx.iter().map(|&k| cids[k]).collect();
1291            order_regions(&mut kids, &kid_cids, page_w, page_h, &reg);
1292            out.push(items[i].clone());
1293            out.extend(kids);
1294        } else {
1295            out.push(items[i].clone());
1296        }
1297    }
1298    *items = out;
1299}
1300
1301/// Furniture / not-yet-emitted labels.
1302fn is_skipped(label: &str) -> bool {
1303    matches!(
1304        label,
1305        "page_header" | "page_footer" | "form" | "key_value_region"
1306    )
1307}
1308
1309/// Reading-order sort of a page's regions, via the ported rule-based
1310/// [`reading_order`](crate::reading_order) predictor (docling's
1311/// `ReadingOrderPredictor`): an up/down geometry graph with same-row links
1312/// between `cids`-consecutive elements (#424), horizontal dilation and a
1313/// depth-first traversal, with `page_header`/`page_footer` ordered as their own
1314/// groups (first/last) as docling does.
1315fn order_regions<T: Clone>(
1316    items: &mut Vec<T>,
1317    cids: &[usize],
1318    page_w: f32,
1319    page_h: f32,
1320    reg: impl Fn(&T) -> &Region,
1321) {
1322    let boxes: Vec<(f32, f32, f32, f32)> = items
1323        .iter()
1324        .map(|it| {
1325            let r = reg(it);
1326            (r.l, r.t, r.r, r.b)
1327        })
1328        .collect();
1329    let is_header: Vec<bool> = items
1330        .iter()
1331        .map(|it| reg(it).label == "page_header")
1332        .collect();
1333    let is_footer: Vec<bool> = items
1334        .iter()
1335        .map(|it| reg(it).label == "page_footer")
1336        .collect();
1337    let order =
1338        crate::reading_order::order_page(&boxes, cids, &is_header, &is_footer, page_w, page_h);
1339    *items = order.iter().map(|&i| items[i].clone()).collect();
1340}
1341
1342/// docling's assembly order of a page's clusters (`LayoutPostprocessor`'s
1343/// final `_sort_clusters(mode="id")`, #424): each region's rank when sorted by
1344/// its first source cell, then by top edge, then left edge; a region with no
1345/// cells sorts after every one that has some. docling numbers its page
1346/// elements (`cid`) in this order, and the reading-order predictor's same-row
1347/// rule pairs elements with consecutive numbers, so the ranks are what
1348/// [`order_with_containers`] hands the predictor.
1349///
1350/// A regular region's first cell is the smallest index among the cells it
1351/// claims. A table, picture or container has no cells of its own upstream
1352/// either — its cells are its *children's*: the regular clusters > 0.8 inside
1353/// it, and upstream every cell no regular cluster claimed is an orphan cluster
1354/// of its own, so a table's interior text (which no regular cluster claims)
1355/// reaches the table through those orphans. Here that is the cells > 0.8
1356/// inside the region plus the claimed cells of the regular regions > 0.8
1357/// inside it. Without the interior cells every table would sort last, and two
1358/// side-by-side tables would then be consecutive and row-linked — reading the
1359/// right table's caption ahead of the left column's headings (2206 page 8).
1360pub fn cluster_cids(regions: &[Region], cells: &[TextCell]) -> Vec<usize> {
1361    let owned = assign_cells(regions, cells);
1362    let first_cell: Vec<usize> = regions
1363        .iter()
1364        .enumerate()
1365        .map(|(i, r)| {
1366            if claims_cells(r) {
1367                return owned[i].iter().copied().min().unwrap_or(usize::MAX);
1368            }
1369            let interior = cells
1370                .iter()
1371                .enumerate()
1372                .filter(|(_, c)| {
1373                    !c.text.trim().is_empty()
1374                        && inter(r, c.l, c.t, c.r, c.b) / area(c.l, c.t, c.r, c.b).max(1.0) > 0.8
1375                })
1376                .map(|(ci, _)| ci)
1377                .min();
1378            let children = regions
1379                .iter()
1380                .enumerate()
1381                .filter(|(j, child)| {
1382                    *j != i && claims_cells(child) && {
1383                        let ca = area(child.l, child.t, child.r, child.b).max(1.0);
1384                        inter(r, child.l, child.t, child.r, child.b) / ca > 0.8
1385                    }
1386                })
1387                .filter_map(|(j, _)| owned[j].iter().copied().min())
1388                .min();
1389            interior
1390                .into_iter()
1391                .chain(children)
1392                .min()
1393                .unwrap_or(usize::MAX)
1394        })
1395        .collect();
1396    let mut by_source: Vec<usize> = (0..regions.len()).collect();
1397    // Stable, like Python's `sorted`: full ties keep the layout order.
1398    by_source.sort_by(|&a, &b| {
1399        first_cell[a]
1400            .cmp(&first_cell[b])
1401            .then(regions[a].t.total_cmp(&regions[b].t))
1402            .then(regions[a].l.total_cmp(&regions[b].l))
1403    });
1404    let mut cids = vec![0; regions.len()];
1405    for (rank, &i) in by_source.iter().enumerate() {
1406        cids[i] = rank;
1407    }
1408    cids
1409}
1410
1411/// Clean a region's assembled text: undo soft-hyphen line wraps, map curly
1412/// quotes and the ellipsis to ASCII (matching docling), and collapse runs of
1413/// whitespace. pdfium emits the line-wrap hyphen as U+0002 in this corpus
1414/// (U+00AD elsewhere), so `word\u{2} continuation` is one hyphenated word —
1415/// drop the hyphen + the joining space and merge (`com\u{2} pact` → `compact`,
1416/// `end-to\u{2} end` → `end-toend`), exactly as docling does.
1417///
1418/// Token spacing is otherwise left as the geometric join produced it. We do not
1419/// tighten punctuation spacing: docling preserves the PDF's own spaces (it keeps
1420/// `{ ahn }`, `Name 1 .`, `[ 9 ]`), and a geometric gap heuristic diverges from
1421/// it more than a plain single-space join does.
1422/// An ordered-list enumeration marker at the start of a list item: leading ASCII
1423/// digits followed by `.`, e.g. `1. Undo/Redo` → `(1, "Undo/Redo")`. Returns
1424/// `None` when the text doesn't start with `digits.`.
1425/// docling's `ListItemMarkerProcessor` bullet patterns
1426/// (`docling/models/postprocessing/list_marker_processor.py`), one glyph each.
1427/// The last eight are docling#5043's Symbol/Wingdings bullets as the fonts'
1428/// private-use code points ([`PUA_FONT_BULLETS`]).
1429const LIST_BULLET_MARKERS: &str = "\u{2022}\u{2023}\u{25E6}\u{2043}\u{204C}\u{204D}\u{2219}\u{25AA}\u{25AB}\u{25CF}\u{25CB}-*+•·‣⁃►▶▸➤➢✓✔✗✘\u{F0B7}\u{F0A7}\u{F06E}\u{F076}\u{F0D8}\u{F0E0}\u{F0FC}\u{F0A8}";
1430
1431/// Symbol and Wingdings bullets as the fonts' private-use code points
1432/// (docling#5043) — what the text layer of a Word-exported PDF holds when its
1433/// ToUnicode CMap maps the symbol-font byte to `U+F000 + byte` (`U+F0B7` is
1434/// Symbol's `•`; `U+F0A7`/`U+F06E`/`U+F0A8` Wingdings' squares, `U+F076`
1435/// its `❖`, `U+F0D8`/`U+F0E0` its arrows, `U+F0FC` its tick). A bullet
1436/// class of their own, so the glued-bullet strip in [`list_item_node`]
1437/// drops them too.
1438const PUA_FONT_BULLETS: [char; 8] = [
1439    '\u{F0B7}', '\u{F0A7}', '\u{F06E}', '\u{F076}', '\u{F0D8}', '\u{F0E0}', '\u{F0FC}', '\u{F0A8}',
1440];
1441
1442/// docling's numbered-marker patterns as byte-length scanners over the start
1443/// of the text, in its first-wins order (the compound ones first, as they are
1444/// the more specific). Each returns the marker's candidate lengths, longest
1445/// (greedy) first — the alternatives Python's regex would backtrack through
1446/// before the `\s(.+)` tail. Hand-rolled rather than `regex` because this file
1447/// is part of the `pdf-text` (wasm) build, where the `regex` crate is not.
1448/// `\d` is Unicode-aware in Python, hence `char::is_numeric`; the letters are
1449/// ASCII classes in both.
1450const LIST_NUMBERED_MARKERS: &[fn(&str) -> Vec<usize>] = &[
1451    // `\d+(?:\.\d+)+\.?` — 1.1  1.2.3  1.1.
1452    |s| {
1453        let mut i = digits(s, 0);
1454        if i == 0 {
1455            return Vec::new();
1456        }
1457        let mut groups = 0;
1458        while s[i..].starts_with('.') && digits(s, i + 1) > i + 1 {
1459            i = digits(s, i + 1);
1460            groups += 1;
1461        }
1462        if groups == 0 {
1463            return Vec::new();
1464        }
1465        if s[i..].starts_with('.') {
1466            vec![i + 1, i]
1467        } else {
1468            vec![i]
1469        }
1470    },
1471    // `\d+\.?[a-zA-Z]\.` — 9a. 3.a.
1472    |s| digits_dot_letter(s, 0, '.').into_iter().collect(),
1473    // `\d+\.?[a-zA-Z]\)` — 9a) 3.a)
1474    |s| digits_dot_letter(s, 0, ')').into_iter().collect(),
1475    // `\(\d+\.?[a-zA-Z]\)` — (9a) (3.a)
1476    |s| {
1477        if !s.starts_with('(') {
1478            return Vec::new();
1479        }
1480        digits_dot_letter(s, 1, ')').into_iter().collect()
1481    },
1482    // `\d+\.` — 1. 2. 3.
1483    |s| digits_then(s, 0, '.').into_iter().collect(),
1484    // `\d+\)` — 1) 2) 3)
1485    |s| digits_then(s, 0, ')').into_iter().collect(),
1486    // `\(\d+\)` — (1) (2) (3)
1487    |s| {
1488        if !s.starts_with('(') {
1489            return Vec::new();
1490        }
1491        digits_then(s, 1, ')').into_iter().collect()
1492    },
1493    // `\[\d+\]` — [1] [2] [3]
1494    |s| {
1495        if !s.starts_with('[') {
1496            return Vec::new();
1497        }
1498        digits_then(s, 1, ']').into_iter().collect()
1499    },
1500    // `[ivxlcdm]+\.` — i. ii. iii.
1501    |s| class_run_then(s, "ivxlcdm", '.').into_iter().collect(),
1502    // `[IVXLCDM]+\.` — I. II. III.
1503    |s| class_run_then(s, "IVXLCDM", '.').into_iter().collect(),
1504    // `[a-z]\.` / `[A-Z]\.` / `[a-z]\)` / `[A-Z]\)`
1505    |s| {
1506        letter_then(s, char::is_ascii_lowercase, '.')
1507            .into_iter()
1508            .collect()
1509    },
1510    |s| {
1511        letter_then(s, char::is_ascii_uppercase, '.')
1512            .into_iter()
1513            .collect()
1514    },
1515    |s| {
1516        letter_then(s, char::is_ascii_lowercase, ')')
1517            .into_iter()
1518            .collect()
1519    },
1520    |s| {
1521        letter_then(s, char::is_ascii_uppercase, ')')
1522            .into_iter()
1523            .collect()
1524    },
1525];
1526
1527/// Byte offset just past the run of `\d` characters starting at `from`
1528/// (`from` itself when there is none).
1529fn digits(s: &str, from: usize) -> usize {
1530    s[from..]
1531        .char_indices()
1532        .find(|(_, c)| !c.is_numeric())
1533        .map_or(s.len(), |(i, _)| from + i)
1534}
1535
1536/// `\d+<close>` from `from`: the length through `close`, if it matches.
1537fn digits_then(s: &str, from: usize, close: char) -> Option<usize> {
1538    let end = digits(s, from);
1539    (end > from && s[end..].starts_with(close)).then(|| end + close.len_utf8())
1540}
1541
1542/// `\d+\.?[a-zA-Z]<close>` from `from`.
1543fn digits_dot_letter(s: &str, from: usize, close: char) -> Option<usize> {
1544    let mut i = digits(s, from);
1545    if i == from {
1546        return None;
1547    }
1548    if s[i..].starts_with('.') {
1549        i += 1;
1550    }
1551    let letter = s[i..].chars().next().filter(char::is_ascii_alphabetic)?;
1552    i += letter.len_utf8();
1553    s[i..].starts_with(close).then(|| i + close.len_utf8())
1554}
1555
1556/// `[<class>]+<close>` at the start.
1557fn class_run_then(s: &str, class: &str, close: char) -> Option<usize> {
1558    let end = s
1559        .char_indices()
1560        .find(|(_, c)| !class.contains(*c))
1561        .map_or(s.len(), |(i, _)| i);
1562    (end > 0 && s[end..].starts_with(close)).then(|| end + close.len_utf8())
1563}
1564
1565/// `[<letter class>]<close>` at the start.
1566fn letter_then(s: &str, class: fn(&char) -> bool, close: char) -> Option<usize> {
1567    let letter = s.chars().next().filter(class)?;
1568    let i = letter.len_utf8();
1569    s[i..].starts_with(close).then(|| i + close.len_utf8())
1570}
1571
1572/// docling's `ListItemMarkerProcessor.process_list_item`: the item's original
1573/// text is matched against `^(marker)\s(.+)` (DOTALL) for the bullet patterns,
1574/// then the numbered ones in order; a hit splits it into
1575/// `(marker, text, enumerated)`. `\s` is one Unicode whitespace character and
1576/// `.+` everything after it, which must be non-empty.
1577fn split_list_marker(text: &str) -> Option<(&str, &str, bool)> {
1578    let tail_after = |len: usize| -> Option<&str> {
1579        let ws = text[len..].chars().next()?;
1580        if !ws.is_whitespace() {
1581            return None;
1582        }
1583        let rest = &text[len + ws.len_utf8()..];
1584        (!rest.is_empty()).then_some(rest)
1585    };
1586    let first = text.chars().next()?;
1587    if LIST_BULLET_MARKERS.contains(first) {
1588        if let Some(rest) = tail_after(first.len_utf8()) {
1589            return Some((&text[..first.len_utf8()], rest, false));
1590        }
1591    }
1592    for matcher in LIST_NUMBERED_MARKERS {
1593        for len in matcher(text) {
1594            if let Some(rest) = tail_after(len) {
1595                return Some((&text[..len], rest, true));
1596            }
1597        }
1598    }
1599    None
1600}
1601
1602/// A PDF `list_item` region as docling emits it: `ListItemMarkerProcessor`
1603/// splits the marker off (see [`split_list_marker`]), and docling-core's
1604/// Markdown list serializer (default `orig_list_item_marker_mode = AUTO`,
1605/// `ensure_valid_list_item_marker`) prints it as `N. text` for an `N.` marker
1606/// (`case_already_valid`: the marker verbatim); as `- text` for a bullet glyph
1607/// (no letter or digit in the marker: only the `-` the serializer adds); and
1608/// as `- a) text` / `- 1.2 text` / `- [3] text` for any other marker
1609/// (`case_auto`: the serializer's `-`, then the original marker) — spelled
1610/// here as a bullet item whose text carries the marker, the way the DOCX and
1611/// DOC backends already spell theirs: the Markdown fields then carry no
1612/// marker (the serializer applies `case_auto` to a marker it is given, so it
1613/// would print it a second time, `- (1) (1) text`), and the DocLang
1614/// `<marker>` rides in the [`ListItemDclx`] overlay. An item without a recognizable marker is
1615/// a plain bullet. The symbol-font bullets docling-parse filters out of its
1616/// cells (`•◦▪·*` glued to the text) are stripped before the match, as before.
1617fn list_item_node(text: &str, loc: [u16; 4], first_in_list: bool) -> Node {
1618    // docling's match runs on the text as docling-parse hands it over; the
1619    // glued symbol-font bullets it never sees are stripped only when the raw
1620    // text matches no marker (`•Text` → `Text`, but `• Text` → marker `•`).
1621    // The private-use Symbol/Wingdings bullets (docling#5043) are stripped
1622    // the same way: glued to the text they match no marker either, and a
1623    // PUA code left in the item text prints as tofu.
1624    let stripped = text
1625        .trim_start_matches(|c| {
1626            matches!(c, '•' | '◦' | '▪' | '·' | '*') || PUA_FONT_BULLETS.contains(&c)
1627        })
1628        .trim_start();
1629    let split = split_list_marker(text).or_else(|| split_list_marker(stripped));
1630    let bullet = |text: String, marker: &str| Node::ListItem {
1631        ordered: false,
1632        number: 0,
1633        first_in_list,
1634        text: md_escape(&text),
1635        level: 0,
1636        // docling keeps the marker as the DocLang list marker
1637        // (`<ldiv><marker>·</marker></ldiv>`); Markdown prints its own `-`.
1638        marker: Some(marker.to_string()),
1639        location: Some(loc),
1640        dclx: None,
1641        href: None,
1642        layer: None,
1643    };
1644    // docling-core's `case_already_valid` is a *full* `\d+\.` match: `3.a.`
1645    // and `1.2` are `case_auto` markers (`- 3.a. text`), not item numbers.
1646    let is_number_dot = |m: &str| {
1647        m.strip_suffix('.')
1648            .is_some_and(|d| !d.is_empty() && d.chars().all(char::is_numeric))
1649    };
1650    match split {
1651        Some((marker, body, true)) if is_number_dot(marker) => {
1652            let number = parse_ordered_marker(marker).map_or(0, |(n, _)| n);
1653            Node::ListItem {
1654                ordered: true,
1655                number,
1656                first_in_list,
1657                text: md_escape(body),
1658                level: 0,
1659                marker: Some(marker.to_string()),
1660                location: Some(loc),
1661                dclx: None,
1662                href: None,
1663                layer: None,
1664            }
1665        }
1666        // `case_auto`: a marker holding a letter or digit rides in the text,
1667        // so not in the Markdown `marker` (docling-core's serializer would add
1668        // it again); DocLang keeps it through the overlay, whose text is the
1669        // node's own, so the DocLang output is what it was.
1670        Some((marker, body, true)) => {
1671            let text = md_escape(&format!("{marker} {body}"));
1672            Node::ListItem {
1673                ordered: false,
1674                number: 0,
1675                first_in_list,
1676                text: text.clone(),
1677                level: 0,
1678                marker: None,
1679                location: Some(loc),
1680                dclx: Some(ListItemDclx {
1681                    ordered: false,
1682                    marker: Some(marker.to_string()),
1683                    text,
1684                    runs: Vec::new(),
1685                }),
1686                href: None,
1687                layer: None,
1688            }
1689        }
1690        Some((marker, body, false)) => bullet(body.to_string(), marker),
1691        None => bullet(stripped.to_string(), "·"),
1692    }
1693}
1694
1695fn parse_ordered_marker(s: &str) -> Option<(u64, String)> {
1696    let digits: String = s.chars().take_while(|c| c.is_ascii_digit()).collect();
1697    if digits.is_empty() {
1698        return None;
1699    }
1700    let rest = s[digits.len()..].strip_prefix('.')?;
1701    let number = digits.parse().ok()?;
1702    Some((number, rest.trim_start().to_string()))
1703}
1704
1705/// Escape markdown special characters the way docling-core's markdown serializer
1706/// does (`markdown.py` post_process): `_` → `\_`, then HTML-escape `&`, `<`, `>`
1707/// (quote=False, so quotes are left). Applied to prose (headings, list items,
1708/// paragraphs); code blocks, the formula placeholder, and table cells are left raw.
1709pub(crate) fn md_escape(text: &str) -> String {
1710    text.replace('_', "\\_")
1711        .replace('&', "&amp;")
1712        .replace('<', "&lt;")
1713        .replace('>', "&gt;")
1714}
1715
1716/// [`clean_text_keep_wrap`] without the trailing soft hyphen: for text that is
1717/// not a paragraph fragment (headings, captions, cells, code).
1718fn clean_text(text: &str) -> String {
1719    let mut out = clean_text_keep_wrap(text);
1720    if out.ends_with('\u{ad}') {
1721        out.pop();
1722    }
1723    out
1724}
1725
1726/// [`clean_text`], except that a soft hyphen ending the text survives: it is a
1727/// word split across a line break, and the paragraph merge
1728/// ([`merge_continuations`]) needs it to fuse `Veröffent\u{ad}` + `lichung`
1729/// into one word (#919). Stripped here, the fragment ended mid-word with
1730/// nothing to say so, and the two halves were joined with a space.
1731fn clean_text_keep_wrap(text: &str) -> String {
1732    let wrap = text.trim_end().ends_with('\u{ad}');
1733    let unwrapped = text
1734        .replace("\u{2} ", "")
1735        .replace("\u{ad} ", "")
1736        // any stray wrap hyphens not at a join
1737        .replace(['\u{2}', '\u{ad}'], "");
1738    // `pdf_keep_typography` (#921): legal citations and quoted terms tell
1739    // `„…“` and `‚…‘` from ASCII quotes, and `–` ranges from hyphens; the
1740    // folding erases that, so the opt-out keeps the glyphs as drawn.
1741    let replaced = if KEEP_TYPOGRAPHY.with(std::cell::Cell::get) {
1742        unwrapped
1743    } else {
1744        fold_typography(&unwrapped)
1745    };
1746    // The docling-parse sanitizer already placed the correct spacing (e.g.
1747    // justified double spaces); preserve internal runs of spaces, only
1748    // normalizing line breaks/tabs and trimming the ends.
1749    let out = replaced.replace(['\n', '\r', '\t'], " ").trim().to_string();
1750    let out = fold_presentation_forms(&out);
1751    let mut out = fix_arabic_lam_alef(&out);
1752    if wrap && !out.is_empty() {
1753        out.push('\u{ad}');
1754    }
1755    out
1756}
1757
1758thread_local! {
1759    /// Whether the page being assembled keeps its typographic punctuation
1760    /// ([`PdfPage::keep_typography`]); set for the length of one
1761    /// [`assemble_page`] by [`TypographyScope`], so the text cleaners deep in
1762    /// it need no extra parameter.
1763    static KEEP_TYPOGRAPHY: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
1764}
1765
1766/// Sets [`KEEP_TYPOGRAPHY`] for one page's assembly and restores it after.
1767struct TypographyScope(bool);
1768
1769impl TypographyScope {
1770    fn set(keep: bool) -> Self {
1771        Self(KEEP_TYPOGRAPHY.with(|k| k.replace(keep)))
1772    }
1773}
1774
1775impl Drop for TypographyScope {
1776    fn drop(&mut self) {
1777        KEEP_TYPOGRAPHY.with(|k| k.set(self.0));
1778    }
1779}
1780
1781/// `DOCLING_RS_PDF_KEEP_TYPOGRAPHY` (#921): whether this process keeps
1782/// typographic punctuation as drawn unless a conversion says otherwise.
1783pub fn keep_typography_from_env() -> bool {
1784    docling_core::env::flag("DOCLING_RS_PDF_KEEP_TYPOGRAPHY")
1785}
1786
1787/// docling-parse's sanitizer table (`pdf_sanitators/constants.h`) applied to
1788/// assembled text: single curly quotes become the ASCII apostrophe `'`,
1789/// double ones the ASCII `"` (docling-parse#349, 7.x — before it folded both
1790/// to `'`, which docling's groundtruth carried until its docling-parse 7
1791/// refresh), `‚` a comma, the dash family a hyphen.
1792fn fold_typography(text: &str) -> String {
1793    text.replace(['\u{2018}', '\u{2019}', '\u{201b}'], "'") // ‘ ’ ‛ → '
1794        .replace(['\u{201c}', '\u{201d}', '\u{201e}', '\u{201f}'], "\"") // “ ” „ ‟ → "
1795        .replace('\u{201a}', ",") // ‚ → ,
1796        .replace(
1797            [
1798                '\u{2010}', '\u{2011}', '\u{2012}', '\u{2013}', '\u{2014}', '\u{2015}', '\u{2212}',
1799            ],
1800            "-",
1801        ) // hyphen/dash family → -
1802        .replace('\u{2044}', "/") // ⁄ fraction slash → /
1803        .replace('\u{2022}', "\u{b7}") // • → · (docling never emits •; inline CCS-concept separators)
1804        .replace('\u{2026}', "...") // … → ...
1805}
1806
1807/// The character folds of docling's `normalize_extracted_text` (docling#3929,
1808/// docling#5069) beyond [`fold_typography`]: Latin ligatures spelled out
1809/// (`fi` → `fi`, `ſt`/`st` → `st`, Dutch `IJ`/`ij` → `IJ`/`ij`), the spurious
1810/// private-use glyph U+F0A0 some fonts emit dropped, and runs of Arabic
1811/// Presentation Forms-A/B (U+FB50–FDFF, U+FE70–FEFF) NFKC-folded back to the
1812/// canonical letters, so shaped Arabic (`ﻣﺎﺮﺱ` → `مارس`) is searchable. The
1813/// NFKC is per run, never global, so `x²`, `H₂O` and full-width forms
1814/// survive. The text layer already spells out U+FB00–FB06
1815/// (`textparse::decompose_ligatures`); folding them here too covers the
1816/// other text sources. Not typography: `pdf_keep_typography` keeps these.
1817pub(crate) fn fold_presentation_forms(text: &str) -> String {
1818    use unicode_normalization::UnicodeNormalization;
1819    let is_arabic_form = |c: char| matches!(c, '\u{FB50}'..='\u{FDFF}' | '\u{FE70}'..='\u{FEFF}');
1820    let needs = |c: char| {
1821        matches!(
1822            c,
1823            '\u{FB00}'..='\u{FB06}' | '\u{132}' | '\u{133}' | '\u{F0A0}'
1824        ) || is_arabic_form(c)
1825    };
1826    if !text.chars().any(needs) {
1827        return text.to_string();
1828    }
1829    let mut out = String::with_capacity(text.len());
1830    let mut run = String::new();
1831    for c in text.chars() {
1832        if is_arabic_form(c) {
1833            run.push(c);
1834            continue;
1835        }
1836        if !run.is_empty() {
1837            out.extend(run.nfkc());
1838            run.clear();
1839        }
1840        match c {
1841            '\u{FB00}' => out.push_str("ff"),
1842            '\u{FB01}' => out.push_str("fi"),
1843            '\u{FB02}' => out.push_str("fl"),
1844            '\u{FB03}' => out.push_str("ffi"),
1845            '\u{FB04}' => out.push_str("ffl"),
1846            '\u{FB05}' | '\u{FB06}' => out.push_str("st"),
1847            '\u{132}' => out.push_str("IJ"),
1848            '\u{133}' => out.push_str("ij"),
1849            '\u{F0A0}' => {}
1850            _ => out.push(c),
1851        }
1852    }
1853    if !run.is_empty() {
1854        out.extend(run.nfkc());
1855    }
1856    out
1857}
1858
1859/// docling's `normalize_extracted_text` for a TableFormer cell (docling#5069):
1860/// cells get the same folds as body text — before, only `@ ` joined, so a
1861/// cell kept `it’s`, `“quoted”` and `• dot` where the paragraph beside it
1862/// read `it's`, `"quoted"`, `· dot`. The typography half honours
1863/// `pdf_keep_typography` like [`clean_text_keep_wrap`]; the Arabic lam-alef
1864/// reordering stays a body-text step (pdfium's visual-order runs).
1865fn normalize_table_text(text: &str) -> String {
1866    let text = if KEEP_TYPOGRAPHY.with(std::cell::Cell::get) {
1867        text.to_string()
1868    } else {
1869        fold_typography(text)
1870    };
1871    fold_presentation_forms(&text)
1872}
1873
1874/// [`normalize_table_text`] over a TableFormer grid: the dense `rows` text and
1875/// the first-class `cells` the JSON / DocLang exports read.
1876fn normalize_grid(mut grid: TableGrid) -> TableGrid {
1877    for row in &mut grid.rows {
1878        for cell in row.iter_mut() {
1879            *cell = normalize_table_text(cell);
1880        }
1881    }
1882    for cell in &mut grid.cells {
1883        cell.text = normalize_table_text(&cell.text);
1884    }
1885    grid
1886}
1887
1888/// pdfium decomposes the Arabic lam-alef ligature (لا / لإ / لأ / لآ) into its
1889/// glyph constituents in *visual* order — `alef-variant, lam` — but docling keeps
1890/// logical order, `lam, alef-variant`. Swap a mid-word `alef-variant + lam` back
1891/// to `lam + alef-variant`. "Mid-word" (the previous char is an Arabic letter)
1892/// distinguishes the ligature from the definite article `ال` (word-initial
1893/// `alef + lam`), which must stay. No-op for non-Arabic text.
1894fn fix_arabic_lam_alef(s: &str) -> String {
1895    let is_arabic_letter = |c: char| ('\u{0620}'..='\u{064A}').contains(&c);
1896    let chars: Vec<char> = s.chars().collect();
1897    if !chars.iter().any(|&c| is_arabic_letter(c)) {
1898        return s.to_string(); // no-op for non-Arabic text
1899    }
1900    // Pass 1: swap mid-word `alef-variant + lam` → `lam + alef-variant`. Only the
1901    // hamza/madda alef variants (إ أ آ) are safe: the definite article is always
1902    // plain `ا + ل`, so plain `alef + lam` is ambiguous (a legitimate `فعالة` vs a
1903    // reversed `لا` ligature look identical) — leaving plain alef alone avoids
1904    // corrupting legitimate words.
1905    let mut a: Vec<char> = Vec::with_capacity(chars.len());
1906    let mut i = 0;
1907    while i < chars.len() {
1908        let c = chars[i];
1909        if matches!(c, '\u{0622}' | '\u{0623}' | '\u{0625}')
1910            && chars.get(i + 1) == Some(&'\u{0644}')
1911            && i > 0
1912            && is_arabic_letter(chars[i - 1])
1913            // A preceding lam means this alef-variant is *already* the logical
1914            // `lam + alef` ligature; the following lam is the next syllable's
1915            // letter, not a reversed ligature — swapping it corrupts `لآل` → `للآ`
1916            // (e.g. التعلم الآلي → الآلي, not اللآي).
1917            && chars[i - 1] != '\u{0644}'
1918        {
1919            a.push('\u{0644}');
1920            a.push(c);
1921            i += 2;
1922            continue;
1923        }
1924        a.push(c);
1925        i += 1;
1926    }
1927    // Pass 2: insert a space at Arabic↔Latin boundaries (bidi script switch) that
1928    // pdfium runs together — docling separates the embedded Latin run (`وPython`
1929    // → `و Python`).
1930    let mut out: Vec<char> = Vec::with_capacity(a.len());
1931    for (j, &c) in a.iter().enumerate() {
1932        if j > 0 {
1933            let p = a[j - 1];
1934            if (is_arabic_letter(p) && c.is_ascii_alphabetic())
1935                || (p.is_ascii_alphabetic() && is_arabic_letter(c))
1936            {
1937                out.push(' ');
1938            }
1939        }
1940        out.push(c);
1941    }
1942    out.into_iter().collect()
1943}
1944
1945/// docling's `PageAssembleModel._match_hyperlink`: the URI whose link
1946/// annotations cover at least half of the region's box, or `None`. Coverage is
1947/// intersection-over-region-area, **accumulated per URI** — a URL that wraps
1948/// across lines carries several annotation rects that sum toward the same
1949/// target. Ties resolve to the first-seen URI (Python's `max` over dict
1950/// insertion order); the winner still needs `>= 0.5`
1951/// (`_HYPERLINK_COVERAGE_THRESHOLD`).
1952pub(crate) fn region_hyperlink(
1953    region: &Region,
1954    links: &[crate::pdfium_backend::LinkAnnot],
1955) -> Option<String> {
1956    if links.is_empty() {
1957        return None;
1958    }
1959    let area = (region.r - region.l).max(0.0) * (region.b - region.t).max(0.0);
1960    if area <= 0.0 {
1961        return None;
1962    }
1963    let mut coverage: Vec<(&str, f32)> = Vec::new();
1964    for link in links {
1965        let ix = (region.r.min(link.r) - region.l.max(link.l)).max(0.0);
1966        let iy = (region.b.min(link.b) - region.t.max(link.t)).max(0.0);
1967        let c = ix * iy / area;
1968        match coverage.iter_mut().find(|(uri, _)| *uri == link.uri) {
1969            Some((_, acc)) => *acc += c,
1970            None => coverage.push((&link.uri, c)),
1971        }
1972    }
1973    let mut best: Option<(&str, f32)> = None;
1974    for (uri, c) in coverage {
1975        // Strictly greater keeps the first-seen URI on ties, like Python's max.
1976        if best.is_none_or(|(_, bc)| c > bc) {
1977            best = Some((uri, c));
1978        }
1979    }
1980    let (uri, c) = best?;
1981    (c >= 0.5).then(|| normalize_uri(uri))
1982}
1983
1984/// The pydantic-`AnyUrl` normalization docling's hyperlink value passes
1985/// through on its way to the serializer: a URL with an authority but no path
1986/// gains a trailing `/` (`https://arxiv.org` → `https://arxiv.org/`). Other
1987/// AnyUrl canonicalizations (scheme/host lowercasing, percent-encoding) don't
1988/// occur in PDF link annotations in practice, so they are not reproduced.
1989fn normalize_uri(uri: &str) -> String {
1990    if let Some((_, rest)) = uri.split_once("://") {
1991        if !rest.is_empty() && !rest.contains(['/', '?', '#']) {
1992            return format!("{uri}/");
1993        }
1994    }
1995    uri.to_string()
1996}
1997
1998/// Resolve each page hyperlink to the visible text it covers, as `(anchor, uri)`
1999/// in reading order. The anchor is the cells whose centre falls in the link rect,
2000/// joined left-to-right and cleaned the same way prose is (so it matches the
2001/// serialized text), deduped against the immediately-preceding link so pdfium's
2002/// occasional duplicate annotation doesn't double-list. Empty anchors are dropped.
2003pub(crate) fn resolve_link_anchors(page: &PdfPage) -> Vec<(String, String)> {
2004    let mut out: Vec<(String, String)> = Vec::new();
2005    // Use per-word cells, not the line-merged `cells`: a link rect covers a few
2006    // words on a line, and a whole merged line cell would over-capture (its centre
2007    // lands in one link's rect, grabbing the entire line as that link's anchor).
2008    let words = if page.word_cells.is_empty() {
2009        &page.cells
2010    } else {
2011        &page.word_cells
2012    };
2013    for link in &page.links {
2014        // A cell participates when its centre row is inside the rect and it
2015        // overlaps the rect horizontally. A cell can be *wider* than the rect:
2016        // PDFs often draw a whole header line as one text run ("LinkedIn |
2017        // GitHub | Credly"), which docling-parse's word grouping keeps as one
2018        // cell even though each label carries its own link annotation —
2019        // centre-in-rect alone would hand the entire line to every link.
2020        // [`cell_text_in_rect`] clips such a cell to the tokens under the rect.
2021        let mut inside: Vec<(&TextCell, String)> = words
2022            .iter()
2023            .filter(|c| {
2024                let cy = (c.t + c.b) / 2.0;
2025                cy >= link.t && cy <= link.b && c.r.min(link.r) > c.l.max(link.l)
2026            })
2027            .filter_map(|c| {
2028                let text = cell_text_in_rect(c, link.l, link.r);
2029                (!text.is_empty()).then_some((c, text))
2030            })
2031            .collect();
2032        // Reading order: top band then left-to-right (link anchors are LTR).
2033        let band = inside
2034            .iter()
2035            .map(|(c, _)| (c.b - c.t).abs())
2036            .fold(0.0f32, f32::max)
2037            .max(1.0);
2038        inside.sort_by_key(|(c, _)| ((c.t / band).round() as i64, (c.l * 10.0) as i64));
2039        let anchor = clean_text(
2040            &inside
2041                .iter()
2042                .map(|(_, t)| t.trim())
2043                .filter(|t| !t.is_empty())
2044                .collect::<Vec<_>>()
2045                .join(" "),
2046        );
2047        if anchor.is_empty() {
2048            continue;
2049        }
2050        if out
2051            .last()
2052            .is_some_and(|(a, u)| a == &anchor && u == &link.uri)
2053        {
2054            continue;
2055        }
2056        out.push((anchor, link.uri.clone()));
2057    }
2058    out
2059}
2060
2061/// The part of a cell's text that lies under a link rect's x-range. A cell
2062/// fully inside the rect (by centre) returns its whole text. A wider cell is
2063/// split into whitespace tokens whose x-spans are estimated proportionally to
2064/// their character positions (kerning makes this approximate, so selection
2065/// snaps to whole tokens, never characters); tokens whose estimated centre
2066/// falls inside the rect are kept. Returns "" when nothing falls inside.
2067fn cell_text_in_rect(c: &TextCell, l: f32, r: f32) -> String {
2068    let cx = (c.l + c.r) / 2.0;
2069    if cx >= l && cx <= r && c.l >= l - (c.r - c.l) * 0.25 && c.r <= r + (c.r - c.l) * 0.25 {
2070        return c.text.trim().to_string();
2071    }
2072    let chars: Vec<char> = c.text.chars().collect();
2073    let n = chars.len();
2074    if n == 0 || c.r <= c.l {
2075        return String::new();
2076    }
2077    let per = (c.r - c.l) / n as f32;
2078    let mut out: Vec<String> = Vec::new();
2079    let mut token = String::new();
2080    let mut start = 0usize;
2081    // A trailing sentinel space flushes the last token.
2082    for (i, &ch) in chars.iter().enumerate().chain(std::iter::once((n, &' '))) {
2083        if ch.is_whitespace() {
2084            if !token.is_empty() {
2085                let mid = c.l + (start as f32 + (i - start) as f32 / 2.0) * per;
2086                if mid >= l && mid <= r {
2087                    out.push(std::mem::take(&mut token));
2088                } else {
2089                    token.clear();
2090                }
2091            }
2092        } else {
2093            if token.is_empty() {
2094                start = i;
2095            }
2096            token.push(ch);
2097        }
2098    }
2099    out.join(" ")
2100}
2101
2102/// Cells assigned to a region (best container), in reading order, joined.
2103fn region_text(region: &Region, cells: &[TextCell]) -> String {
2104    let inside: Vec<&TextCell> = cells
2105        .iter()
2106        .filter(|c| {
2107            let ca = area(c.l, c.t, c.r, c.b).max(1.0);
2108            inter(region, c.l, c.t, c.r, c.b) / ca > 0.5
2109        })
2110        .collect();
2111    cells_text(inside)
2112}
2113
2114/// docling's exclusive cell assignment (`_assign_cells_to_clusters`): every
2115/// non-empty cell goes to the single best-overlapping *regular* region at
2116/// intersection-over-self > 0.2, and each region serializes exactly its
2117/// assigned cells. A cell under two overlapping boxes is emitted once (by the
2118/// better-covering one), and a cell only partially under its region — e.g.
2119/// normal_4pages' big section numeral, ~30 % inside the heading box — still
2120/// joins it (`## 들어가며 1`) instead of leaking as an orphan. Pictures and
2121/// wrappers never claim (docling walks regular clusters only); ties go to the
2122/// first region, like docling's strict `>` best-overlap scan.
2123pub fn region_texts_exclusive(regions: &[Region], cells: &[TextCell]) -> Vec<String> {
2124    let mut texts = region_texts_keep_wrap(regions, cells);
2125    for t in &mut texts {
2126        if t.ends_with('\u{ad}') {
2127            t.pop();
2128        }
2129    }
2130    texts
2131}
2132
2133/// [`region_texts_exclusive`] with a region's trailing soft hyphen kept
2134/// (#919): the page assembler strips it and hands it to the paragraph it ends.
2135fn region_texts_keep_wrap(regions: &[Region], cells: &[TextCell]) -> Vec<String> {
2136    let owned = assign_cells(regions, cells);
2137    // Non-claimers (tables/wrappers/pictures) keep the inclusive > 0.5 text:
2138    // docling fills a special cluster's cells from its contained children, and
2139    // downstream table assembly gates on that text being non-empty.
2140    regions
2141        .iter()
2142        .zip(owned)
2143        .map(|(r, cs)| {
2144            if claims_cells(r) {
2145                cells_text_keep_wrap(cs.iter().map(|&i| &cells[i]).collect())
2146            } else {
2147                region_text(r, cells)
2148            }
2149        })
2150        .collect()
2151}
2152
2153/// A *regular* region in docling's sense — one that claims cells. Pictures and
2154/// the wrappers (`table`, `document_index`, `form`, `key_value_region`) fill
2155/// their cells from contained children instead.
2156fn claims_cells(r: &Region) -> bool {
2157    r.label != "picture" && !is_wrapper(r.label)
2158}
2159
2160/// docling's `_assign_cells_to_clusters`: each non-empty cell's index goes to
2161/// the single best-overlapping regular region at intersection-over-self > 0.2
2162/// (ties to the first region, like docling's strict `>` scan). One entry per
2163/// region, in region order.
2164fn assign_cells(regions: &[Region], cells: &[TextCell]) -> Vec<Vec<usize>> {
2165    let mut owned: Vec<Vec<usize>> = vec![Vec::new(); regions.len()];
2166    for (ci, c) in cells.iter().enumerate() {
2167        if c.text.trim().is_empty() {
2168            continue;
2169        }
2170        let ca = area(c.l, c.t, c.r, c.b).max(1.0);
2171        let mut best: Option<(usize, f32)> = None;
2172        for (i, r) in regions.iter().enumerate() {
2173            if !claims_cells(r) {
2174                continue;
2175            }
2176            let ov = inter(r, c.l, c.t, c.r, c.b) / ca;
2177            if ov > 0.2 && best.is_none_or(|(_, b)| ov > b) {
2178                best = Some((i, ov));
2179            }
2180        }
2181        if let Some((i, _)) = best {
2182            owned[i].push(ci);
2183        }
2184    }
2185    owned
2186}
2187
2188/// The ballot-box glyph a checkbox line can open with (#609): `Some(checked)`.
2189/// Empty boxes — `☐`, the white squares `□ ▢ ◻` and the shadowed `❏ ❐ ❑ ❒`
2190/// Word's checkbox bullets use — are unchecked; `☑ ☒ ⊠ ⌧ ▣ 🗹 🗷 🗵` are
2191/// checked. A bare tick or cross (`✓ ✗`) is a list bullet
2192/// ([`LIST_BULLET_MARKERS`]), not a box. Symbol-font boxes that reach the text
2193/// layer as Private Use Area codes or as their ASCII byte (Wingdings `o`,
2194/// `þ`) are not recognized: without the font the code is ambiguous (`U+F06F`
2195/// is Symbol's omicron too). The private-use codes docling#5043 made list
2196/// bullets ([`PUA_FONT_BULLETS`] — Wingdings' `U+F0A8` square and `U+F0FC`
2197/// tick among them) stay bullets here as well, as docling reads them.
2198pub(crate) fn checkbox_glyph(c: char) -> Option<bool> {
2199    match c {
2200        '☐' | '□' | '▢' | '◻' | '❏' | '❐' | '❑' | '❒' => Some(false),
2201        '☑' | '☒' | '⊠' | '⌧' | '▣' | '\u{1F5F9}' | '\u{1F5F7}' | '\u{1F5F5}' => {
2202            Some(true)
2203        }
2204        _ => None,
2205    }
2206}
2207
2208/// `text` without its leading ballot-box glyph and the space after it — a
2209/// checkbox item's label is the option text, the box is its state.
2210pub(crate) fn strip_checkbox_glyph(text: &str) -> &str {
2211    let t = text.trim_start();
2212    match t.chars().next() {
2213        Some(c) if checkbox_glyph(c).is_some() => t[c.len_utf8()..].trim_start(),
2214        _ => text,
2215    }
2216}
2217
2218/// Give every checklist line its own checkbox region (#609). The layout
2219/// model often reads a checklist as one `text` (or `list_item`) block — on
2220/// the reporter's ReportLab page all four options are one region, which then
2221/// printed as `First option Second option …`, and docling itself splits it
2222/// into two garbled paragraphs. What marks an item is on the page: a drawn
2223/// square just left of the line ([`crate::checkbox`]), or a ballot-box glyph
2224/// the line opens with ([`checkbox_glyph`]: `☐ Yes`, `☒ Done`). A region
2225/// whose lines carry either splits into a `checkbox_selected` /
2226/// `checkbox_unselected` region per marked line (its box spans the square
2227/// and the line; a following unmarked line — a wrapped label — stays with
2228/// it), so assembly emits [`Node::CheckboxItem`]s exactly as for the model's
2229/// own checkbox labels, the glyph stripped from the label. Lines above the
2230/// first mark stay one region of the original label. The first piece
2231/// replaces the region in place and the rest are appended, so per-region
2232/// slices indexed by the original order (table grids, enrichments — neither
2233/// applies to text) stay aligned.
2234///
2235/// A square counts for a line when its vertical centre lies on the line and
2236/// it sits just left of the line's first cell: no more than 1.5 sides (at
2237/// least 6 pt) of gap, at most 1 pt of overlap. Text set *inside* squares (a
2238/// comb field) never matches. A glyph counts when it opens the line and is
2239/// the line's only box glyph; a line with several (`☐ Yes ☐ No`) stays a text
2240/// piece of its own rather than become one item labelled `Yes ☐ No`. A
2241/// region the model already labelled a checkbox is not split, but its state
2242/// follows its first line's mark when it has one — the glyph or the square
2243/// is what the page says, the model's label a guess from the pixels (it read
2244/// a `☑ Bread` line as unselected). With neither on the page this is a no-op.
2245pub fn split_checkbox_lines(
2246    regions: &mut Vec<Region>,
2247    cells: &[TextCell],
2248    boxes: &[crate::checkbox::CheckBox],
2249) {
2250    let has_glyph = || {
2251        cells
2252            .iter()
2253            .any(|c| c.text.chars().any(|ch| checkbox_glyph(ch).is_some()))
2254    };
2255    if boxes.is_empty() && !has_glyph() {
2256        return;
2257    }
2258    let owned = assign_cells(regions, cells);
2259    let mut used = vec![false; boxes.len()];
2260    let mut appended: Vec<Region> = Vec::new();
2261    for (i, mine) in owned.iter().enumerate() {
2262        let region = regions[i].clone();
2263        let model_checkbox = matches!(region.label, "checkbox_selected" | "checkbox_unselected");
2264        if !(model_checkbox || matches!(region.label, "text" | "list_item")) || mine.is_empty() {
2265            continue;
2266        }
2267        // The region's lines, top down: cells sharing a vertical centre band,
2268        // with the row's text in left-to-right order (for the glyph check).
2269        let mut sorted: Vec<&TextCell> = mine.iter().map(|&c| &cells[c]).collect();
2270        sorted.sort_by(|a, b| a.t.total_cmp(&b.t).then(a.l.total_cmp(&b.l)));
2271        // (l, t, r, b) of each line, and its cells.
2272        type Row<'c> = ((f32, f32, f32, f32), Vec<&'c TextCell>);
2273        let mut rows: Vec<Row> = Vec::new();
2274        for c in sorted {
2275            let mid = (c.t + c.b) / 2.0;
2276            match rows.last_mut() {
2277                Some((row, members)) if mid >= row.1 && mid <= row.3 => {
2278                    *row = (
2279                        row.0.min(c.l),
2280                        row.1.min(c.t),
2281                        row.2.max(c.r),
2282                        row.3.max(c.b),
2283                    );
2284                    members.push(c);
2285                }
2286                _ => rows.push(((c.l, c.t, c.r, c.b), vec![c])),
2287            }
2288        }
2289        // What each row does to the grouping.
2290        enum Mark {
2291            /// Opens a checkbox item (`square`: the drawn one, if any).
2292            Item {
2293                square: Option<usize>,
2294                checked: bool,
2295            },
2296            /// Several box glyphs on one line (`☐ Yes ☐ No`): a text piece of
2297            /// its own, closing the open item.
2298            Plain,
2299            /// Unmarked: continues the open piece (a wrapped label).
2300            Continue,
2301        }
2302        let mut marks: Vec<Mark> = Vec::with_capacity(rows.len());
2303        for ((l, t, _, b), members) in &rows {
2304            let square = boxes
2305                .iter()
2306                .enumerate()
2307                .filter(|&(k, sq)| {
2308                    let side = sq.r - sq.l;
2309                    let mid = (sq.t + sq.b) / 2.0;
2310                    let gap = l - sq.r;
2311                    !used[k]
2312                        && mid >= t - 2.0
2313                        && mid <= b + 2.0
2314                        && gap >= -1.0
2315                        && gap <= (1.5 * side).max(6.0)
2316                })
2317                .min_by(|a, b| (l - a.1.r).total_cmp(&(l - b.1.r)))
2318                .map(|(k, _)| k);
2319            let mark = match square {
2320                Some(k) => {
2321                    used[k] = true;
2322                    Mark::Item {
2323                        square: Some(k),
2324                        checked: boxes[k].checked,
2325                    }
2326                }
2327                None => {
2328                    let mut members = members.clone();
2329                    members.sort_by(|a, b| a.l.total_cmp(&b.l));
2330                    let text: String = members.iter().map(|c| c.text.as_str()).collect();
2331                    let first = text.trim_start().chars().next().and_then(checkbox_glyph);
2332                    let glyphs = text
2333                        .chars()
2334                        .filter(|&c| checkbox_glyph(c).is_some())
2335                        .count();
2336                    match (first, glyphs) {
2337                        (Some(checked), 1) => Mark::Item {
2338                            square: None,
2339                            checked,
2340                        },
2341                        (_, 0) => Mark::Continue,
2342                        _ => Mark::Plain,
2343                    }
2344                }
2345            };
2346            marks.push(mark);
2347        }
2348        // A region the model already labelled a checkbox keeps its extent;
2349        // only its state follows the mark — a glyph or a drawn square is the
2350        // page's own record, where the model reads it off the pixels (on a
2351        // `☑ Bread` line Heron says unselected).
2352        if model_checkbox {
2353            if let Some(Mark::Item { checked, .. }) = marks.first() {
2354                regions[i].label = if *checked {
2355                    "checkbox_selected"
2356                } else {
2357                    "checkbox_unselected"
2358                };
2359            }
2360            continue;
2361        }
2362        if !marks.iter().any(|m| matches!(m, Mark::Item { .. })) {
2363            continue;
2364        }
2365        // Group rows: a marked row opens an item; an unmarked one continues
2366        // the open group (the leading group keeps the region's own label).
2367        let mut pieces: Vec<Region> = Vec::new();
2368        for ((row, _), mark) in rows.iter().zip(&marks) {
2369            match mark {
2370                Mark::Item { square, checked } => {
2371                    let (mut l, mut t, mut r, mut b) = *row;
2372                    if let Some(s) = square.map(|k| &boxes[k]) {
2373                        (l, t, r, b) = (l.min(s.l), t.min(s.t), r.max(s.r), b.max(s.b));
2374                    }
2375                    pieces.push(Region {
2376                        label: if *checked {
2377                            "checkbox_selected"
2378                        } else {
2379                            "checkbox_unselected"
2380                        },
2381                        score: region.score,
2382                        l,
2383                        t,
2384                        r,
2385                        b,
2386                    });
2387                }
2388                Mark::Plain => pieces.push(Region {
2389                    l: row.0,
2390                    t: row.1,
2391                    r: row.2,
2392                    b: row.3,
2393                    ..region.clone()
2394                }),
2395                Mark::Continue => match pieces.last_mut() {
2396                    Some(p) => {
2397                        p.l = p.l.min(row.0);
2398                        p.t = p.t.min(row.1);
2399                        p.r = p.r.max(row.2);
2400                        p.b = p.b.max(row.3);
2401                    }
2402                    None => pieces.push(Region {
2403                        l: row.0,
2404                        t: row.1,
2405                        r: row.2,
2406                        b: row.3,
2407                        ..region.clone()
2408                    }),
2409                },
2410            }
2411        }
2412        let mut pieces = pieces.into_iter();
2413        if let Some(first) = pieces.next() {
2414            regions[i] = first;
2415        }
2416        appended.extend(pieces);
2417    }
2418    regions.extend(appended);
2419}
2420
2421/// docling's regular-cluster refinement after cell assignment
2422/// (`LayoutPostprocessor._process_regular_clusters`, #419), run once the page's
2423/// cells are final and before reading order:
2424///
2425/// 1. every regular region's box becomes the union of the cells it claimed
2426///    (`_adjust_cluster_bboxes` — a regular cluster's bbox *is* its cells'
2427///    bbox; a table's is the union with the model box, and pictures keep
2428///    theirs, so neither is touched here);
2429/// 2. a regular region that claimed no cell is dropped (`keep_empty_clusters`
2430///    is off; a `formula` is kept, as upstream keeps it);
2431/// 3. an orphan text region (`score == 0.0`, from [`add_orphan_regions`]) that
2432///    now sits > 0.8 inside another regular region's fitted box is folded into
2433///    it (`_remove_overlapping_clusters` at containment 0.8, the larger box
2434///    winning the group) — up to three rounds, like upstream's loop.
2435///
2436/// Why it matters: the layout model's box can end partway through a line. That
2437/// line fails the 0.2 claim and becomes an orphan — recoverable — but the
2438/// *model* box still overlaps the orphan's line by a few points, so the
2439/// reading-order graph, which links only strictly-above pairs, gets no edge
2440/// between them and may emit the next paragraph first, stranding the line
2441/// after the paragraph it belongs in (1540 of 6050 text blocks on the #419
2442/// book began mid-sentence). Fitted to its cells, the box ends on a line
2443/// boundary and the orphan slots in between; an orphan the fitted box
2444/// swallows joins the paragraph outright. Cell assignment is untouched: a
2445/// region's fitted box contains every cell it claimed, so
2446/// [`region_texts_exclusive`] hands it the same cells afterwards.
2447///
2448/// A page with no cells yet (a scan before OCR) is left alone: dropping every
2449/// text region for want of cells would be wrong, and the OCR paths call this
2450/// again once the cells exist.
2451pub fn fit_regions_to_cells(regions: &mut Vec<Region>, cells: &[TextCell]) {
2452    if !cells.iter().any(|c| !c.text.trim().is_empty()) {
2453        return;
2454    }
2455    for _ in 0..3 {
2456        let owned = assign_cells(regions, cells);
2457        let mut fitted: Vec<Region> = Vec::with_capacity(regions.len());
2458        for (r, own) in regions.iter().zip(&owned) {
2459            if !claims_cells(r) {
2460                fitted.push(r.clone());
2461                continue;
2462            }
2463            if own.is_empty() {
2464                if r.label == "formula" {
2465                    fitted.push(r.clone());
2466                }
2467                continue;
2468            }
2469            let mut f = r.clone();
2470            f.l = own
2471                .iter()
2472                .map(|&i| cells[i].l)
2473                .fold(f32::INFINITY, f32::min);
2474            f.t = own
2475                .iter()
2476                .map(|&i| cells[i].t)
2477                .fold(f32::INFINITY, f32::min);
2478            f.r = own
2479                .iter()
2480                .map(|&i| cells[i].r)
2481                .fold(f32::NEG_INFINITY, f32::max);
2482            f.b = own
2483                .iter()
2484                .map(|&i| cells[i].b)
2485                .fold(f32::NEG_INFINITY, f32::max);
2486            fitted.push(f);
2487        }
2488        let mut changed = fitted.len() != regions.len();
2489        // Fold orphans into the regular region whose fitted box holds them.
2490        let mut drop = vec![false; fitted.len()];
2491        for i in 0..fitted.len() {
2492            let o = &fitted[i];
2493            if !(o.score == 0.0 && o.label == "text") {
2494                continue;
2495            }
2496            let oa = area(o.l, o.t, o.r, o.b).max(1.0);
2497            let mut best: Option<(usize, f32)> = None;
2498            for (j, r) in fitted.iter().enumerate() {
2499                if j == i || drop[j] || r.score == 0.0 || !claims_cells(r) {
2500                    continue;
2501                }
2502                let ov = inter(r, o.l, o.t, o.r, o.b) / oa;
2503                if ov > 0.8 && best.is_none_or(|(_, b)| ov > b) {
2504                    best = Some((j, ov));
2505                }
2506            }
2507            if let Some((j, _)) = best {
2508                let (l, t, r, b) = (o.l, o.t, o.r, o.b);
2509                let host = &mut fitted[j];
2510                host.l = host.l.min(l);
2511                host.t = host.t.min(t);
2512                host.r = host.r.max(r);
2513                host.b = host.b.max(b);
2514                drop[i] = true;
2515                changed = true;
2516            }
2517        }
2518        let mut drop = drop.into_iter();
2519        fitted.retain(|_| !drop.next().expect("aligned"));
2520        *regions = fitted;
2521        if !changed {
2522            break;
2523        }
2524    }
2525}
2526
2527/// Join a prefiltered cell list into the region's text (docling's
2528/// `sanitize_text` over the sanitizer's cell order).
2529fn cells_text(inside: Vec<&TextCell>) -> String {
2530    let mut t = cells_text_keep_wrap(inside);
2531    if t.ends_with('\u{ad}') {
2532        t.pop();
2533    }
2534    t
2535}
2536
2537/// [`cells_text`] keeping a trailing soft hyphen (see [`clean_text_keep_wrap`]).
2538fn cells_text_keep_wrap(inside: Vec<&TextCell>) -> String {
2539    // docling orders a cluster's cells by their docling-parse cell index
2540    // alone (`LayoutPostprocessor._sort_cells`: `sorted(cells, key=c.index)`)
2541    // — the sanitizer's output order, which our `cells` slice already is.
2542    // No geometric re-sort: normal_4pages' big section numerals paint
2543    // *after* their heading text, and docling's `## 들어가며 1` (numeral
2544    // last) only falls out of pure index order — a band sort dragged the
2545    // numeral to the front. The overlap-grouped line restore this replaced
2546    // measured strictly worse on the corpus (it fixed nothing the index
2547    // order broke, and broke the numerals).
2548    let joined = {
2549        // docling's `PageAssembleModel.sanitize_text`, ported verbatim over the
2550        // parse-index-ordered lines: append a separating space to a line —
2551        // unless it ends with `-`. A dash-ending line whose last word and the
2552        // next line's first word are both alphanumeric is a wrapped word: the
2553        // dash is dropped and the lines fuse (`platforms-` + `reflects` →
2554        // `platformsreflects`, `pp. 545-` + `561` → `545561`). Any other
2555        // dash-ending line — e.g. the *bare* `-` cell a superscript ORCID or an
2556        // inline `–` bullet splits off (its word list is empty, so the fuse
2557        // test fails) — keeps its dash and still takes no trailing space:
2558        // `[0000` `-` `0002` joins as docling's `[0000 -0002`, and the OTSL
2559        // list's `-` + `"C" cell -` + `a new table cell` collapses to
2560        // `-"C" cell a new table cell`. Our cells still carry the raw dash
2561        // family (docling-parse normalizes to `-` before this; clean_text does
2562        // it after), so the endswith test matches them all.
2563        let texts: Vec<&str> = inside
2564            .iter()
2565            .map(|c| c.text.trim())
2566            // Skip whitespace-only cells (a justified line's trailing space
2567            // glyph): an empty line would double the separator.
2568            .filter(|t| !t.is_empty())
2569            .collect();
2570        let last_word_alnum = |s: &str| {
2571            s.split(|c: char| !(c.is_alphanumeric() || c == '_'))
2572                .rfind(|w| !w.is_empty())
2573                .is_some_and(|w| w.chars().all(char::is_alphanumeric))
2574        };
2575        let first_word_alnum = |s: &str| {
2576            s.split(|c: char| !(c.is_alphanumeric() || c == '_'))
2577                .find(|w| !w.is_empty())
2578                .is_some_and(|w| w.chars().all(char::is_alphanumeric))
2579        };
2580        let mut out = String::new();
2581        for (i, t) in texts.iter().enumerate() {
2582            if i > 0 {
2583                let prev = texts[i - 1];
2584                let dashish = matches!(
2585                    prev.chars().last(),
2586                    Some(
2587                        '-' | '\u{2010}'
2588                            | '\u{2011}'
2589                            | '\u{2012}'
2590                            | '\u{2013}'
2591                            | '\u{2014}'
2592                            | '\u{2015}'
2593                            | '\u{2212}'
2594                    )
2595                );
2596                // docling#4052 (2.122): a dash only splits a word when it is
2597                // *attached* to one — the character before it is alphanumeric.
2598                // A dash that follows whitespace (a separator dash, a bullet
2599                // marker, a wrapped `-prefixed` token, the bare `-` cell an
2600                // ORCID splits off) is a literal character: it is kept and the
2601                // lines join with the ordinary space.
2602                let attached = prev.chars().rev().nth(1).is_some_and(char::is_alphanumeric);
2603                if dashish && attached && starts_with_conjunction(t) {
2604                    // `Warte-` / `und Behaltefristen`: an elliptical compound
2605                    // keeps its dash and the line break's space (#920).
2606                    out.push(' ');
2607                } else if dashish && attached {
2608                    if last_word_alnum(prev) && first_word_alnum(t) {
2609                        out.pop(); // wrapped word: fuse without the dash
2610                    }
2611                    // an attached dash never takes a separating space
2612                } else {
2613                    out.push(' ');
2614                }
2615            }
2616            out.push_str(t);
2617        }
2618        out
2619    };
2620    clean_text_keep_wrap(&joined)
2621}
2622
2623/// Tighten the spaces pdfium leaves around tight punctuation in a code line
2624/// (`console .log` → `console.log`, `add (3 , 5)` → `add(3, 5)`), matching
2625/// docling-parse's source spacing.
2626fn tighten_code_punct(s: &str) -> String {
2627    s.replace(" .", ".")
2628        .replace(" ,", ",")
2629        .replace(" ;", ";")
2630        .replace(" )", ")")
2631        .replace(" (", "(")
2632}
2633
2634/// Assemble a **code** region's text with its line structure preserved.
2635///
2636/// Unlike [`region_text`] — which joins every cell with a single space, the right
2637/// thing for prose reflow — a code block's line breaks and indentation are
2638/// significant. The `code_cells` are already one physical source line each
2639/// (grouped space-glyph-only, so monospace runs keep their spacing), so this:
2640///
2641/// 1. groups the cells into vertical line bands and orders them top→bottom,
2642///    left→right;
2643/// 2. joins the lines with `\n` (rather than spaces), keeping the carriage
2644///    returns; and
2645/// 3. reconstructs each line's leading indentation from its left offset, in units
2646///    of the block's estimated monospace character width, so nesting survives.
2647///
2648/// Typography is normalized per line via [`clean_text`] (smart quotes, dashes,
2649/// ellipsis), which never merges lines. Returns an empty string if the region has
2650/// no code cells (the caller falls back to the prose text).
2651fn code_region_text(region: &Region, cells: &[TextCell]) -> String {
2652    let mut inside: Vec<&TextCell> = cells
2653        .iter()
2654        .filter(|c| {
2655            let ca = area(c.l, c.t, c.r, c.b).max(1.0);
2656            inter(region, c.l, c.t, c.r, c.b) / ca > 0.5
2657        })
2658        .filter(|c| !c.text.trim().is_empty())
2659        .collect();
2660    if inside.is_empty() {
2661        return String::new();
2662    }
2663
2664    // Quantize the top edge into ~line bands (like `region_text`), then order the
2665    // cells by band (top→bottom) and, within a band, by left edge.
2666    let band = inside
2667        .iter()
2668        .map(|c| (c.b - c.t).abs())
2669        .fold(0.0f32, f32::max)
2670        .max(1.0);
2671    let line_of = |c: &TextCell| (c.t / band).round() as i64;
2672    inside.sort_by_key(|c| (line_of(c), (c.l * 10.0) as i64));
2673
2674    // Estimate one monospace character's width (total ink width / total glyphs) to
2675    // convert a line's left offset into a count of leading spaces. Measured over
2676    // all lines so a single short line can't skew it.
2677    let (mut total_w, mut total_chars) = (0.0f32, 0usize);
2678    for c in &inside {
2679        let n = c.text.trim().chars().count();
2680        if n > 0 {
2681            total_w += (c.r - c.l).max(0.0);
2682            total_chars += n;
2683        }
2684    }
2685    let char_w = if total_chars > 0 {
2686        (total_w / total_chars as f32).max(1.0)
2687    } else {
2688        1.0
2689    };
2690    // The block's own left margin is the zero-indent baseline.
2691    let base_l = inside.iter().map(|c| c.l).fold(f32::INFINITY, f32::min);
2692
2693    let mut lines: Vec<String> = Vec::new();
2694    let mut cur: Option<i64> = None;
2695    for c in &inside {
2696        // Tighten pdfium's spaced punctuation per line (on the trimmed content, so
2697        // the reconstructed leading indentation is never nibbled).
2698        let text = tighten_code_punct(&clean_text(c.text.trim()));
2699        if Some(line_of(c)) == cur {
2700            // A second cell sharing this band (rare — e.g. split columns): keep it
2701            // on the same source line, separated by a space.
2702            if let Some(last) = lines.last_mut() {
2703                last.push(' ');
2704                last.push_str(&text);
2705            }
2706            continue;
2707        }
2708        let indent = ((c.l - base_l) / char_w).round().max(0.0) as usize;
2709        lines.push(format!("{}{}", " ".repeat(indent), text));
2710        cur = Some(line_of(c));
2711    }
2712    lines.join("\n")
2713}
2714
2715/// Reconstruct a table's grid geometrically from the text cells inside its
2716/// region: cluster cells into rows (by vertical centre) and columns (by clustered
2717/// left edges), then place each cell. A model-free stand-in for TableFormer that
2718/// recovers grid-aligned tables from the precise PDF text layer (it does not
2719/// resolve row/column spans).
2720pub fn reconstruct_table(region: &Region, cells: &[TextCell]) -> Vec<Vec<String>> {
2721    let mut inside: Vec<&TextCell> = cells
2722        .iter()
2723        .filter(|c| {
2724            let ca = area(c.l, c.t, c.r, c.b).max(1.0);
2725            inter(region, c.l, c.t, c.r, c.b) / ca > 0.5
2726        })
2727        .collect();
2728    if inside.is_empty() {
2729        return Vec::new();
2730    }
2731    inside.sort_by(|a, b| a.t.total_cmp(&b.t));
2732
2733    // Rows: consecutive cells whose vertical centre is within ~0.7 line height.
2734    let mut rows: Vec<(f32, Vec<&TextCell>)> = Vec::new();
2735    for c in &inside {
2736        let cyc = (c.t + c.b) / 2.0;
2737        let lh = (c.b - c.t).abs().max(1.0);
2738        if let Some((ryc, row)) = rows.last_mut() {
2739            if (cyc - *ryc).abs() < lh * 0.7 {
2740                row.push(c);
2741                continue;
2742            }
2743        }
2744        rows.push((cyc, vec![c]));
2745    }
2746
2747    // Columns: cluster left edges (merge those within a tolerance).
2748    let tol = {
2749        let mut hs: Vec<f32> = inside.iter().map(|c| (c.b - c.t).abs()).collect();
2750        hs.sort_by(f32::total_cmp);
2751        hs[hs.len() / 2].max(4.0) * 1.5
2752    };
2753    let mut lefts: Vec<f32> = inside.iter().map(|c| c.l).collect();
2754    lefts.sort_by(f32::total_cmp);
2755    let mut col_starts: Vec<f32> = Vec::new();
2756    for l in lefts {
2757        if col_starts.last().is_none_or(|&last| l - last > tol) {
2758            col_starts.push(l);
2759        }
2760    }
2761    let ncols = col_starts.len().max(1);
2762    let col_of = |l: f32| -> usize {
2763        col_starts
2764            .iter()
2765            .rposition(|&s| l + tol * 0.5 >= s)
2766            .unwrap_or(0)
2767            .min(ncols - 1)
2768    };
2769
2770    let mut grid = Vec::with_capacity(rows.len());
2771    for (_, mut row) in rows {
2772        row.sort_by(|a, b| a.l.total_cmp(&b.l));
2773        let mut cols = vec![String::new(); ncols];
2774        for c in row {
2775            let ci = col_of(c.l);
2776            // Strip the wrap-hyphen control char so it never lands in a cell.
2777            let t = c.text.trim().replace(['\u{2}', '\u{ad}'], "");
2778            if cols[ci].is_empty() {
2779                cols[ci] = t;
2780            } else {
2781                cols[ci].push(' ');
2782                cols[ci].push_str(&t);
2783            }
2784        }
2785        grid.push(cols);
2786    }
2787    grid
2788}
2789
2790/// Does the geometric reconstruction of a table look trustworthy enough to use
2791/// as-is, instead of paying for TableFormer?
2792///
2793/// [`reconstruct_table`] derives columns by clustering cell **left edges**. On a
2794/// clean grid that is exact, but when a column's entries are not left-aligned
2795/// (or the OCR boxes wobble) the clustering splits one real column into several,
2796/// and the result is a wide, mostly-empty grid — the "spurious empty columns"
2797/// failure TableFormer exists to fix.
2798///
2799/// Two symptoms separate the two cases, and both are properties of the grid
2800/// alone (no model needed):
2801/// * **density** — a real table is mostly full; a split-up one is mostly holes;
2802/// * **thin columns** — a column carrying at most one entry across several rows
2803///   is almost always a split artefact rather than a real column.
2804///
2805/// Deliberately conservative: it answers `true` only for grids that are plainly
2806/// well-formed, so the expensive path stays the default whenever there is doubt.
2807/// A caller that skips TableFormer on `true` trades no quality for the time.
2808pub fn geometric_table_is_reliable(rows: &[Vec<String>]) -> bool {
2809    let ncols = rows.iter().map(Vec::len).max().unwrap_or(0);
2810    // Fewer than two columns is not a grid this heuristic can vouch for: it is
2811    // exactly the shape a collapsed table takes, and TableFormer may recover
2812    // real structure from it.
2813    if rows.len() < 2 || ncols < 2 {
2814        return false;
2815    }
2816    let filled = |c: &String| !c.trim().is_empty();
2817    let total = rows.len() * ncols;
2818    let full = rows.iter().flatten().filter(|c| filled(c)).count();
2819    if (full as f32) < MIN_TABLE_FILL * total as f32 {
2820        return false;
2821    }
2822    // A column used by at most one row, when there are rows enough to tell.
2823    if rows.len() >= 3 {
2824        for ci in 0..ncols {
2825            let used = rows
2826                .iter()
2827                .filter(|r| r.get(ci).is_some_and(filled))
2828                .count();
2829            if used <= 1 {
2830                return false;
2831            }
2832        }
2833    }
2834    true
2835}
2836
2837/// Share of a geometric grid's cells that must carry text for it to be trusted
2838/// without TableFormer. Chosen well above the density a left-edge split
2839/// produces (those land nearer a third) and below what a genuine table with a
2840/// few blank cells reaches.
2841const MIN_TABLE_FILL: f32 = 0.6;
2842
2843/// The union bbox of the text cells assigned to a region (same >50%-overlap
2844/// rule as [`region_text`]), or `None` when no cell lands in it. docling's
2845/// LayoutPostprocessor shrinks a regular cluster's bbox to its cells, and the
2846/// enrichment crops are taken from that cell-tight box — cropping the raw
2847/// detector box instead hands the VLM surrounding chrome (e.g. the `Listing N:`
2848/// caption under a code block) that changes its output.
2849pub fn region_cell_bbox(region: &Region, cells: &[TextCell]) -> Option<[f32; 4]> {
2850    let mut bbox: Option<[f32; 4]> = None;
2851    for c in cells {
2852        let ca = area(c.l, c.t, c.r, c.b).max(1.0);
2853        if inter(region, c.l, c.t, c.r, c.b) / ca <= 0.5 {
2854            continue;
2855        }
2856        bbox = Some(match bbox {
2857            None => [c.l, c.t, c.r, c.b],
2858            Some([l, t, r, b]) => [l.min(c.l), t.min(c.t), r.max(c.r), b.max(c.b)],
2859        });
2860    }
2861    bbox
2862}
2863
2864/// One region's enrichment-model result, produced by the pipeline's opt-in
2865/// passes (issue #76) and applied during assembly.
2866#[derive(Debug, Clone)]
2867pub enum Enrichment {
2868    /// DocumentPictureClassifier predictions, descending confidence.
2869    PictureClasses(Vec<PictureClass>),
2870    /// CodeFormulaV2 output for a `code` region: the rewritten source text and
2871    /// the `<_language_>` prefix (when the model emitted one).
2872    Code {
2873        language: Option<String>,
2874        text: String,
2875    },
2876    /// CodeFormulaV2 output for a `formula` region: the decoded LaTeX.
2877    Formula { latex: String },
2878}
2879
2880/// Crop a region (page points, already expanded by the caller if needed) from
2881/// the rendered page image and resize it to `target_scale` pixels per point —
2882/// the enrichment-model equivalent of docling's
2883/// `page.get_image(scale=…, cropbox=…)`, sourced from the existing
2884/// [`crate::pdfium_backend::RENDER_SCALE`] render instead of a fresh pdfium
2885/// pass (the page bitmap is already the exact docling render at scale 2).
2886#[cfg(feature = "ml")]
2887pub fn crop_region_scaled(page: &PdfPage, bbox: [f32; 4], target_scale: f32) -> Option<RgbImage> {
2888    debug_assert!(
2889        !page.image_pending(),
2890        "a crop reached an unrendered page bitmap (#897)"
2891    );
2892    let s = page.scale;
2893    let [l, t, r, b] = bbox;
2894    let (iw, ih) = (page.image.width(), page.image.height());
2895    let x = (l * s).max(0.0) as u32;
2896    let y = (t * s).max(0.0) as u32;
2897    if x >= iw || y >= ih {
2898        return None;
2899    }
2900    let w = (((r - l.max(0.0)) * s) as u32).min(iw - x);
2901    let h = (((b - t.max(0.0)) * s) as u32).min(ih - y);
2902    if w == 0 || h == 0 {
2903        return None;
2904    }
2905    let crop = image::imageops::crop_imm(&page.image, x, y, w, h).to_image();
2906    // docling renders the crop at `target_scale` directly; from the scale-2
2907    // page render that is a resize to the same pixel geometry
2908    // (`round(width_points * scale)`, PIL's BICUBIC ≙ CatmullRom).
2909    let tw = ((w as f32 / s) * target_scale).round().max(1.0) as u32;
2910    let th = ((h as f32 / s) * target_scale).round().max(1.0) as u32;
2911    if (tw, th) == (w, h) {
2912        return Some(crop);
2913    }
2914    Some(image::imageops::resize(
2915        &crop,
2916        tw,
2917        th,
2918        image::imageops::FilterType::CatmullRom,
2919    ))
2920}
2921
2922/// Resample `img` (rendered at `from` px/pt, covering `w_pt`×`h_pt` points)
2923/// to `to` px/pt — docling's `round(points * scale)` pixel geometry, PIL's
2924/// BICUBIC ≙ CatmullRom. Unchanged when the geometry already matches.
2925#[cfg(feature = "ocr-prep")]
2926fn rescale(img: RgbImage, w_pt: f32, h_pt: f32, to: f32) -> RgbImage {
2927    let tw = (w_pt * to).round().max(1.0) as u32;
2928    let th = (h_pt * to).round().max(1.0) as u32;
2929    if (tw, th) == img.dimensions() {
2930        return img;
2931    }
2932    image::imageops::resize(&img, tw, th, image::imageops::FilterType::CatmullRom)
2933}
2934
2935/// Encode `img` as a PNG [`PictureImage`] rendered at `scale` px/pt.
2936#[cfg(feature = "ocr-prep")]
2937fn png_image(img: &RgbImage, scale: f32) -> Option<PictureImage> {
2938    let mut buf = std::io::Cursor::new(Vec::new());
2939    img.write_to(&mut buf, image::ImageFormat::Png).ok()?;
2940    Some(PictureImage {
2941        mimetype: "image/png".into(),
2942        width: img.width(),
2943        height: img.height(),
2944        data: buf.into_inner(),
2945        dpi: PictureImage::dpi_for_scale(scale),
2946        display_size: None,
2947    })
2948}
2949
2950/// The whole page render as docling's `PageItem.image` (#520): at `scale`
2951/// px/pt (`None` = the render's own), `None` when the page has no bitmap.
2952#[cfg(feature = "ocr-prep")]
2953pub fn page_image(page: &PdfPage, scale: Option<f32>) -> Option<PictureImage> {
2954    #[cfg(feature = "ml")]
2955    debug_assert!(
2956        !page.image_pending(),
2957        "a crop reached an unrendered page bitmap (#897)"
2958    );
2959    if page.image.width() == 0 || page.image.height() == 0 || page.scale <= 0.0 {
2960        return None;
2961    }
2962    let scale = scale.unwrap_or(page.scale);
2963    let img = rescale(page.image.clone(), page.width, page.height, scale);
2964    png_image(&img, scale)
2965}
2966
2967/// Crop a layout region from the rendered page image and encode it as PNG (the
2968/// figure bytes docling stores on a `PictureItem`). Region coordinates are page
2969/// points; the image is rendered at `page.scale` and resampled to `scale`
2970/// px/pt when one is given (docling's `images_scale`, #520). The image's `dpi`
2971/// is 72·scale (#519).
2972#[cfg(feature = "ocr-prep")]
2973fn crop_region(page: &PdfPage, region: &Region, scale: Option<f32>) -> Option<PictureImage> {
2974    // `use_embedded_picture_images` (#1007): the one safe image the region
2975    // holds, at its native size and docling's `dpi` of 72 — not resampled to
2976    // `images_scale`, which only sizes render crops (docling likewise).
2977    #[cfg(feature = "render")]
2978    if let Some(img) = crate::embedded::match_region(
2979        &page.embedded_images,
2980        &page.cells,
2981        [region.l, region.t, region.r, region.b],
2982    ) {
2983        return png_image(img, 1.0);
2984    }
2985    #[cfg(feature = "ml")]
2986    debug_assert!(
2987        !page.image_pending(),
2988        "a crop reached an unrendered page bitmap (#897)"
2989    );
2990    let s = page.scale;
2991    let (iw, ih) = (page.image.width(), page.image.height());
2992    let x = (region.l * s).max(0.0) as u32;
2993    let y = (region.t * s).max(0.0) as u32;
2994    if x >= iw || y >= ih {
2995        return None;
2996    }
2997    let w = (((region.r - region.l) * s) as u32).min(iw - x);
2998    let h = (((region.b - region.t) * s) as u32).min(ih - y);
2999    if w == 0 || h == 0 {
3000        return None;
3001    }
3002    let sub = image::imageops::crop_imm(&page.image, x, y, w, h).to_image();
3003    match scale {
3004        Some(to) if (to - s).abs() > f32::EPSILON => {
3005            // The crop's own point extent (the pixel box, back in points), so
3006            // the resampled geometry is `round(points * scale)`.
3007            let img = rescale(sub, w as f32 / s, h as f32 / s, to);
3008            png_image(&img, to)
3009        }
3010        _ => png_image(&sub, s),
3011    }
3012}
3013
3014/// A `text` region that *says* it is a caption — `Table 5: …`, `Figure 2.
3015/// …`, `Fig. 3: …` — and sits right next to a graphic in reading order is one
3016/// (a docling.rs extension, `docs/PDF_CONFORMANCE.md`): the layout model
3017/// labels a long caption block `text` now and then (2206.01062's four-line
3018/// Table 5 caption: `text` 0.99 here, `caption` in docling's own run), and a
3019/// mislabelled caption stays a loose paragraph while its table goes
3020/// uncaptioned. The gate is the label's own pattern — the word, a number,
3021/// then `:` or `.` — so a paragraph that merely opens with `Table 5 shows`
3022/// is left alone, plus the geometry docling's matcher wants: the graphic is
3023/// the region's neighbour in reading order, overlaps it horizontally and
3024/// stands within 25 pt above or below. Relabelling before
3025/// [`pair_graphic_captions`] lets the regular matcher bind it.
3026fn promote_labelled_captions(regions: &mut [Region], cells: &[TextCell]) {
3027    let body: Vec<usize> = (0..regions.len())
3028        .filter(|&i| !is_skipped(regions[i].label))
3029        .collect();
3030    let mut promote = Vec::new();
3031    for (pos, &i) in body.iter().enumerate() {
3032        let r = &regions[i];
3033        if r.label != "text" || !is_caption_label(&region_text(r, cells)) {
3034            continue;
3035        }
3036        let beside = |g: &Region| {
3037            is_graphic(g.label) && g.r.min(r.r) - g.l.max(r.l) > 0.0 && {
3038                let gap = if r.t >= g.b {
3039                    r.t - g.b
3040                } else if g.t >= r.b {
3041                    g.t - r.b
3042                } else {
3043                    f32::MAX
3044                };
3045                gap <= 25.0
3046            }
3047        };
3048        let prev = pos.checked_sub(1).map(|p| &regions[body[p]]);
3049        let next = body.get(pos + 1).map(|&n| &regions[n]);
3050        if prev.is_some_and(beside) || next.is_some_and(beside) {
3051            promote.push(i);
3052        }
3053    }
3054    for i in promote {
3055        regions[i].label = "caption";
3056    }
3057}
3058
3059/// `Table 5:`, `Figure 2.1.`, `Fig. 3:`, `Listing 4:` — a caption label: the
3060/// word, whitespace, a number (dotted or lettered: `2.1`, `3a`), then `:` or
3061/// `.`, as the publishers' styles print it. Case-sensitive on purpose.
3062fn is_caption_label(text: &str) -> bool {
3063    let t = text.trim_start();
3064    let Some(rest) = ["Table", "Figure", "Fig.", "Listing"]
3065        .iter()
3066        .find_map(|w| t.strip_prefix(w))
3067    else {
3068        return false;
3069    };
3070    let rest = rest.trim_start();
3071    if !rest.starts_with(|c: char| c.is_ascii_digit()) {
3072        return false;
3073    }
3074    let num_end = rest
3075        .find(|c: char| !(c.is_ascii_alphanumeric() || c == '.'))
3076        .unwrap_or(rest.len());
3077    let num = rest[..num_end].trim_end_matches('.');
3078    let after = &rest[num.len()..];
3079    after.starts_with(':') || after.starts_with('.')
3080}
3081
3082/// docling's `GRAPHIC_LABELS` (`reading_order_rb.py`): the elements a
3083/// `caption` can be attached to. `document_index` rides along as table-like
3084/// (#265) so an index's title binds the way a table's does.
3085fn is_graphic(label: &str) -> bool {
3086    is_table_like(label) || matches!(label, "picture" | "code")
3087}
3088
3089/// docling's `_shortest_box_gap`: the shortest distance between two boxes,
3090/// 0 once they touch or overlap — along either axis whatever the union has
3091/// left over once both boxes are laid down, combined as the hypotenuse.
3092fn shortest_box_gap(a: &Region, b: &Region) -> f32 {
3093    let dx = ((a.r.max(b.r) - a.l.min(b.l)) - (a.r - a.l) - (b.r - b.l)).max(0.0);
3094    let dy = ((a.b.max(b.b) - a.t.min(b.t)) - (a.b - a.t) - (b.b - b.t)).max(0.0);
3095    dx.hypot(dy)
3096}
3097
3098/// Pair each graphic (`picture`, `table`/`document_index`, `code`) with its
3099/// `caption` region (#265, #813): a port of docling's
3100/// `ReadingOrderPredictor._find_to_captions` — `_rank_caption_candidates` +
3101/// `_match_caption` — over the page's regions in reading order. Returns, per
3102/// region, the index of the caption it owns.
3103///
3104/// A caption reaches the graphics in the **unbroken run** of graphics on
3105/// either side of it in reading order — above *and* below (#813: the
3106/// geometric predecessor only looked below a picture, so a `Figure N:` label
3107/// printed above its figure never bound), never across intervening text, and
3108/// never across the gutter of a two-column page however close the boxes are.
3109/// The candidates rank by the gap each leaves on the page (nearest first,
3110/// equal gaps to the preceding graphic), not by position in the run.
3111/// Graphics are then matched to captions by augmenting paths rather than
3112/// best-first: a caption with a second choice gives way to one that has none,
3113/// so a figure between two captions and a figure with one do not end up
3114/// sharing the former while the latter stays orphaned.
3115///
3116/// Furniture (page headers/footers, form chrome) is not part of docling's
3117/// body-element sequence here, so it neither bonds nor breaks a run. The
3118/// matched caption is consumed from its own reading-order slot and rides on
3119/// its graphic — docling's `captions` ref: Markdown prints it before a
3120/// picture or table and after a code block, JSON emits the `$ref`, DocLang
3121/// the `<caption>`.
3122fn pair_graphic_captions(regions: &[Region]) -> Vec<Option<usize>> {
3123    let body: Vec<usize> = (0..regions.len())
3124        .filter(|&i| !is_skipped(regions[i].label))
3125        .collect();
3126    // `_rank_caption_candidates`: caption index → its graphics, nearest first.
3127    let mut captions: Vec<usize> = Vec::new();
3128    let mut preferred: Vec<Vec<usize>> = vec![Vec::new(); regions.len()];
3129    for (pos, &ci) in body.iter().enumerate() {
3130        if regions[ci].label != "caption" {
3131            continue;
3132        }
3133        let graphic = |&&i: &&usize| is_graphic(regions[i].label);
3134        let preceding = body[..pos].iter().rev().take_while(graphic);
3135        let following = body[pos + 1..].iter().take_while(graphic);
3136        // Tagged 0 preceding / 1 following so equal gaps go to the graphic
3137        // above; a stable sort keeps run order among equals on one side.
3138        let mut ranked: Vec<(f32, u8, usize)> = preceding
3139            .map(|&g| (shortest_box_gap(&regions[ci], &regions[g]), 0, g))
3140            .chain(following.map(|&g| (shortest_box_gap(&regions[ci], &regions[g]), 1, g)))
3141            .collect();
3142        ranked.sort_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1)));
3143        preferred[ci] = ranked.into_iter().map(|(_, _, g)| g).collect();
3144        captions.push(ci);
3145    }
3146    // `_match_caption`, caption by caption in reading order: graphic → caption.
3147    let mut matched: Vec<Option<usize>> = vec![None; regions.len()];
3148    for &caption in &captions {
3149        if let Some(next) = augment_caption_match(caption, &preferred, &matched) {
3150            matched = next;
3151        }
3152    }
3153    // Side-by-side figures (a docling.rs extension, `docs/PDF_CONFORMANCE.md`):
3154    // a row of pictures reads as a run of graphics followed (or preceded) by
3155    // the run of their captions, and upstream's run rule serves only the
3156    // first caption — for the others the caption before them breaks the run,
3157    // and docling leaves them loose. A caption still loose whose only
3158    // separation from a graphic run is other captions takes the nearest
3159    // unclaimed graphic of that run it sits under or over (horizontal
3160    // overlap, within three line heights — the old geometric rule).
3161    for (pos, &ci) in body.iter().enumerate() {
3162        let is_caption = |i: &usize| regions[*i].label == "caption";
3163        if !is_caption(&ci) || matched.contains(&Some(ci)) {
3164            continue;
3165        }
3166        let c = &regions[ci];
3167        let line_h = (c.b - c.t).abs().max(1.0);
3168        let graphic = |i: &usize| is_graphic(regions[*i].label);
3169        let before = body[..pos]
3170            .iter()
3171            .rev()
3172            .skip_while(|i| is_caption(i))
3173            .take_while(|i| graphic(i));
3174        let after = body[pos + 1..]
3175            .iter()
3176            .skip_while(|i| is_caption(i))
3177            .take_while(|i| graphic(i));
3178        let best = before
3179            .chain(after)
3180            .filter(|&&g| matched[g].is_none())
3181            .filter_map(|&g| {
3182                let p = &regions[g];
3183                let h_overlap = p.r.min(c.r) - p.l.max(c.l);
3184                let gap = (c.t - p.b).max(p.t - c.b); // below or above the graphic
3185                (h_overlap > 0.0 && gap > -line_h && gap < line_h * 3.0).then_some((g, gap))
3186            })
3187            .min_by(|a, b| a.1.total_cmp(&b.1));
3188        if let Some((g, _)) = best {
3189            matched[g] = Some(ci);
3190        }
3191    }
3192    matched
3193}
3194
3195/// docling's `_match_caption`: a new matching in which `caption` owns one of
3196/// its graphics, displacing an earlier caption when that one can rehouse
3197/// itself; `None` when no graphic can be freed up. The displacement chain is
3198/// walked on an explicit stack (it can grow as long as the page): each frame
3199/// is a claimant and the graphics it has left to try, and the moves in
3200/// `chain` take effect only once it reaches a graphic nobody holds. `seen`
3201/// keeps a chain from revisiting a graphic.
3202fn augment_caption_match(
3203    caption: usize,
3204    preferred: &[Vec<usize>],
3205    matched: &[Option<usize>],
3206) -> Option<Vec<Option<usize>>> {
3207    let mut seen = vec![false; matched.len()];
3208    // (claimant, next candidate to try) frames; (graphic, claimant) moves.
3209    let mut stack: Vec<(usize, usize)> = vec![(caption, 0)];
3210    let mut chain: Vec<(usize, usize)> = Vec::new();
3211    while let Some(frame) = stack.last_mut() {
3212        let (claimant, cursor) = *frame;
3213        let candidates = &preferred[claimant];
3214        let Some(off) = candidates[cursor..].iter().position(|&g| !seen[g]) else {
3215            // Out of options: undo the move that got here and let the
3216            // caption below resume its own search.
3217            stack.pop();
3218            chain.pop();
3219            continue;
3220        };
3221        let graphic = candidates[cursor + off];
3222        frame.1 = cursor + off + 1;
3223        seen[graphic] = true;
3224        chain.push((graphic, claimant));
3225        match matched[graphic] {
3226            None => {
3227                let mut next = matched.to_vec();
3228                for &(g, c) in &chain {
3229                    next[g] = Some(c);
3230                }
3231                return Some(next);
3232            }
3233            // The graphic is taken; let its caption look for another one.
3234            Some(holder) => stack.push((holder, 0)),
3235        }
3236    }
3237    None
3238}
3239
3240/// Assemble one page from its (already overlap-resolved) layout regions and
3241/// text cells.
3242/// Normalize a layout region (page points, top-left origin) to DocLang's 0–511
3243/// location grid: `clamp(round(512 · coord / page_dim), 0, 511)`, per axis,
3244/// order `[x0, y0, x1, y1]`. Mirrors docling_core's
3245/// `_doclang_utils._create_location_tokens_for_bbox` (resolution 512) so the
3246/// emitted `<location>` tokens line up with the Python groundtruth. Our heron
3247/// cluster boxes match docling's to within ~1 grid unit; the residual (mainly
3248/// the aspect-ratio-stretch vs letterbox preprocessing difference) is absorbed
3249/// by the conformance harness's geometry tolerance.
3250fn norm_loc(region: &Region, page_w: f32, page_h: f32) -> [u16; 4] {
3251    let q = |v: f32, dim: f32| -> u16 {
3252        if dim <= 0.0 {
3253            return 0;
3254        }
3255        let g = (512.0 * (v as f64) / (dim as f64)).round() as i64;
3256        g.clamp(0, 511) as u16
3257    };
3258    [
3259        q(region.l, page_w),
3260        q(region.t, page_h),
3261        q(region.r, page_w),
3262        q(region.b, page_h),
3263    ]
3264}
3265
3266/// Wrap a node in its layout provenance so the DocLang serializer emits the four
3267/// `<location>` tokens as the element's head (Markdown/JSON render `inner`
3268/// unchanged).
3269fn located(loc: [u16; 4], inner: Node) -> Node {
3270    Node::Located {
3271        location: loc,
3272        inner: Box::new(inner),
3273    }
3274}
3275
3276/// Stamp the real 1-based page number onto a page's leading marker (see
3277/// [`assemble_page`], which emits it with `page_no: 0` because only the
3278/// document-level collector knows the true index — `--pages` windows shift it).
3279pub fn stamp_page_no(nodes: &mut [Node], page_no: usize) {
3280    if let Some(Node::PageInfo { page_no: p, .. }) = nodes.first_mut() {
3281        *p = page_no;
3282    }
3283}
3284
3285/// Text-layer path only (#984): put a page's running head back first. With no
3286/// layout model nothing is labelled `page_header`, and docling's rule-based
3287/// reading order treats a short line far to one side (the EU Official
3288/// Journal's right-aligned `02014R0537 — DE — … — 7`) as a separate head of
3289/// its own: the body column under the left-hand `▼B` marker is read through
3290/// first and the header lands at the end of its own page — in the text
3291/// export, after the page's body and ahead of the next page's. The line that
3292/// lies entirely above every other cell of the page is the first thing on it
3293/// whatever the graph says, so it is moved to the front of the page's nodes
3294/// (after the page marker). Pages whose top line is already first (the usual
3295/// case) and pages whose top line shares its rows with other text (columns)
3296/// are untouched.
3297pub fn float_running_head(nodes: &mut [Node], cells: &[TextCell]) {
3298    let mut lines = cells.iter().filter(|c| !c.text.trim().is_empty());
3299    let Some(first) = lines.next() else { return };
3300    let mut top = first;
3301    let mut rest_t = f32::INFINITY;
3302    for c in lines {
3303        if c.t < top.t {
3304            rest_t = rest_t.min(top.t);
3305            top = c;
3306        } else {
3307            rest_t = rest_t.min(c.t);
3308        }
3309    }
3310    if top.b > rest_t {
3311        return;
3312    }
3313    let key = |t: &str| -> String { t.chars().filter(|c| c.is_alphanumeric()).collect() };
3314    let want = key(&clean_text(&top.text));
3315    if want.is_empty() {
3316        return;
3317    }
3318    let body = usize::from(matches!(nodes.first(), Some(Node::PageInfo { .. })));
3319    let Some(at) = nodes
3320        .iter()
3321        .position(|n| as_paragraph(n).is_some_and(|t| key(t) == want))
3322    else {
3323        return;
3324    };
3325    // A margin page number already floated to the front (`is_page_number`,
3326    // docling parity) keeps the first slot.
3327    let numbered = nodes.get(body).and_then(as_paragraph).is_some_and(|t| {
3328        let t = t.trim();
3329        !t.is_empty() && t.chars().all(|c| c.is_ascii_digit())
3330    });
3331    if !numbered && at > body && !nodes[body..at].iter().all(is_merge_trailer) {
3332        nodes[body..=at].rotate_right(1);
3333    }
3334}
3335
3336/// A dense table grid plus its first-class cells (#240): `rows` is the text
3337/// grid every serializer renders (spans replicate their anchor's text);
3338/// `cells` are the docling-parity per-cell records (text, page-point bbox,
3339/// span rectangle, OTSL header roles). Produced by the TableFormer paths
3340/// (`tf_core`); lives in this always-compiled module so the pure-text (wasm
3341/// `pdf-text`) build sees the type.
3342#[derive(Clone, Debug)]
3343pub struct TableGrid {
3344    pub rows: Vec<Vec<String>>,
3345    pub cells: Vec<docling_core::TableCell>,
3346}
3347
3348/// docling's `_RICH_CELL_PICTURE_COVERAGE_THRESHOLD`.
3349const RICH_CELL_PICTURE_COVERAGE: f32 = 0.8;
3350
3351/// docling `ReadingOrderModel._match_table_pictures` (#3906, 2.118.1): every
3352/// picture ≥ 80 % inside a TableFormer-structured table on the page is matched
3353/// to the cell covering it, and returned per table as `cell index → pictures`.
3354/// A picture that pairs with a caption stays a standalone figure (upstream
3355/// would nest it and lose the caption; keeping the caption is the better
3356/// failure). Tables without first-class cells (geometric fallback) have no cell
3357/// boxes to match against and nest nothing.
3358fn match_table_pictures(
3359    regions: &[Region],
3360    table_rows: &[Option<TableGrid>],
3361    caption_for: &[Option<usize>],
3362) -> std::collections::HashMap<usize, Vec<(usize, Vec<usize>)>> {
3363    let mut out: std::collections::HashMap<usize, Vec<(usize, Vec<usize>)>> =
3364        std::collections::HashMap::new();
3365    for (p, pic) in regions.iter().enumerate() {
3366        if pic.label != "picture" || caption_for.get(p).is_some_and(Option::is_some) {
3367            continue;
3368        }
3369        let pa = area(pic.l, pic.t, pic.r, pic.b).max(1.0);
3370        let mut best: Option<(f32, usize, usize)> = None; // (coverage, table, cell)
3371        for (t, tbl) in regions.iter().enumerate() {
3372            if !is_table_like(tbl.label) {
3373                continue;
3374            }
3375            let Some(grid) = table_rows.get(t).and_then(Option::as_ref) else {
3376                continue;
3377            };
3378            if inter(pic, tbl.l, tbl.t, tbl.r, tbl.b) / pa < RICH_CELL_PICTURE_COVERAGE {
3379                continue;
3380            }
3381            if let Some((cov, cell)) = match_picture_to_cell(pic, &grid.cells) {
3382                if best.is_none_or(|(b, _, _)| cov > b) {
3383                    best = Some((cov, t, cell));
3384                }
3385            }
3386        }
3387        if let Some((_, t, cell)) = best {
3388            let entry = out.entry(t).or_default();
3389            match entry.iter_mut().find(|(c, _)| *c == cell) {
3390                Some((_, pics)) => pics.push(p),
3391                None => entry.push((cell, vec![p])),
3392            }
3393        }
3394    }
3395    out
3396}
3397
3398/// docling `_match_picture_to_table_cell`: among the cells covering ≥ 80 % of
3399/// the picture, prefer the one at the picture's inferred grid position (the
3400/// row / column whose median cell center is nearest the picture's center —
3401/// cell boxes can overlap across logical rows and columns), else the best
3402/// coverage. Returns `(coverage, cell index)`.
3403fn match_picture_to_cell(pic: &Region, cells: &[docling_core::TableCell]) -> Option<(f32, usize)> {
3404    let pa = area(pic.l, pic.t, pic.r, pic.b).max(1.0);
3405    let cover = |b: &[f32; 4]| inter(pic, b[0], b[1], b[2], b[3]) / pa;
3406    let eligible: Vec<(f32, usize)> = cells
3407        .iter()
3408        .enumerate()
3409        .filter_map(|(i, c)| {
3410            let b = c.bbox.as_ref()?;
3411            let cov = cover(b);
3412            (cov >= RICH_CELL_PICTURE_COVERAGE).then_some((cov, i))
3413        })
3414        .collect();
3415    if eligible.is_empty() {
3416        return None;
3417    }
3418    let mut row_centers: std::collections::BTreeMap<usize, Vec<f32>> = Default::default();
3419    let mut col_centers: std::collections::BTreeMap<usize, Vec<f32>> = Default::default();
3420    for c in cells {
3421        let Some(b) = c.bbox.as_ref() else { continue };
3422        for r in c.start_row..c.start_row + c.row_span {
3423            row_centers.entry(r).or_default().push((b[1] + b[3]) / 2.0);
3424        }
3425        for k in c.start_col..c.start_col + c.col_span {
3426            col_centers.entry(k).or_default().push((b[0] + b[2]) / 2.0);
3427        }
3428    }
3429    let median = |v: &mut Vec<f32>| -> f32 {
3430        v.sort_by(f32::total_cmp);
3431        let n = v.len();
3432        if n % 2 == 1 {
3433            v[n / 2]
3434        } else {
3435            (v[n / 2 - 1] + v[n / 2]) / 2.0
3436        }
3437    };
3438    let (px, py) = ((pic.l + pic.r) / 2.0, (pic.t + pic.b) / 2.0);
3439    let nearest = |centers: &mut std::collections::BTreeMap<usize, Vec<f32>>, target: f32| {
3440        centers
3441            .iter_mut()
3442            .map(|(&i, v)| (i, (median(v) - target).abs()))
3443            .min_by(|a, b| a.1.total_cmp(&b.1))
3444            .map(|(i, _)| i)
3445    };
3446    let row = nearest(&mut row_centers, py);
3447    let col = nearest(&mut col_centers, px);
3448    let logical: Vec<(f32, usize)> = eligible
3449        .iter()
3450        .copied()
3451        .filter(|&(_, i)| {
3452            let c = &cells[i];
3453            row.is_some_and(|r| c.start_row <= r && r < c.start_row + c.row_span)
3454                && col.is_some_and(|k| c.start_col <= k && k < c.start_col + c.col_span)
3455        })
3456        .collect();
3457    let pool = if logical.is_empty() {
3458        &eligible
3459    } else {
3460        &logical
3461    };
3462    // Python's `max` over `(coverage, cell_index, cell)` tuples: highest
3463    // coverage, ties to the higher index.
3464    pool.iter()
3465        .copied()
3466        .max_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1)))
3467}
3468
3469/// The DocLang structure overlay derived from first-class cells: span
3470/// continuations (`lcel`/`ucel`/`xcel`) and per-cell header roles, so the
3471/// PDF path's DCLX carries real spans instead of a flat grid.
3472fn structure_from_cells(
3473    cells: &[docling_core::TableCell],
3474    nrows: usize,
3475    ncols: usize,
3476) -> docling_core::TableStructure {
3477    let grid = || vec![vec![false; ncols]; nrows];
3478    let mut col_cont = grid();
3479    let mut row_cont = grid();
3480    let mut row_header = grid();
3481    let mut col_header = grid();
3482    for c in cells {
3483        for r in c.start_row..(c.start_row + c.row_span).min(nrows) {
3484            for k in c.start_col..(c.start_col + c.col_span).min(ncols) {
3485                col_cont[r][k] = k > c.start_col;
3486                row_cont[r][k] = r > c.start_row;
3487                row_header[r][k] = c.row_header;
3488                col_header[r][k] = c.column_header;
3489            }
3490        }
3491    }
3492    docling_core::TableStructure {
3493        header_row: Vec::new(),
3494        col_continuation: col_cont,
3495        row_continuation: row_cont,
3496        row_header,
3497        col_header,
3498    }
3499}
3500
3501pub fn assemble_page(
3502    page: &PdfPage,
3503    mut regions: Vec<Region>,
3504    table_rows: &[Option<TableGrid>],
3505    enrichments: &[Option<Enrichment>],
3506    // Picture-crop scale in px/pt (docling's `images_scale`, #520); `None`
3507    // keeps the page render's own scale.
3508    picture_scale: Option<f32>,
3509) -> (Vec<Node>, Vec<(String, String)>) {
3510    // Without pixels (the text-layer-only wasm build) no picture is cropped.
3511    #[cfg(not(feature = "ocr-prep"))]
3512    let _ = picture_scale;
3513    let _typography = TypographyScope::set(page.keep_typography);
3514    let mut nodes: Vec<Node> = Vec::new();
3515    // Every page opens with an invisible page marker carrying its size in
3516    // points — what the JSON export needs to build docling's `pages` map and
3517    // denormalize the 0–511 `<location>` grid into point bboxes (#171). The
3518    // page *number* is stamped by the document-level collector (which knows
3519    // the real 1-based index, `--pages` windows included); every serializer
3520    // except JSON skips the marker, so Markdown/DocLang stay byte-identical.
3521    nodes.push(Node::PageInfo {
3522        page_no: 0,
3523        width: page.width,
3524        height: page.height,
3525    });
3526    // Recover this page's hyperlinks (anchor-precise pairs for strict
3527    // Markdown; whole-item docling-parity links are baked below and their
3528    // pairs dropped from this list so strict output doesn't double-wrap).
3529    let mut links = resolve_link_anchors(page);
3530    // Lines with a drawn checkbox square in front become checkbox items (#609).
3531    split_checkbox_lines(&mut regions, &page.cells, &page.checkboxes);
3532    // Pair each region with its precomputed TableFormer grid and enrichment
3533    // (indexed by original order) and order by reading order together, so they
3534    // stay aligned.
3535    // A picture's children (docling's `_set_cluster_children`: the regulars
3536    // > 80 % inside it) are not page elements — they leave the reading order
3537    // here and ride with their picture, to be written under it in the JSON.
3538    let parents = picture_parents(&regions);
3539    let mut kids: Vec<Vec<Region>> = vec![Vec::new(); regions.len()];
3540    let mut top: Vec<(usize, Region)> = Vec::with_capacity(regions.len());
3541    for (i, (r, parent)) in regions.into_iter().zip(parents).enumerate() {
3542        match parent {
3543            Some(p) => kids[p].push(r),
3544            None => top.push((i, r)),
3545        }
3546    }
3547    // docling's assembly order of the regions — what its reading-order
3548    // predictor knows as `cid` (#424) — before they are shuffled.
3549    let top_regions: Vec<Region> = top.iter().map(|(_, r)| r.clone()).collect();
3550    let cids = cluster_cids(&top_regions, &page.cells);
3551    type RegionItem = (Region, Option<TableGrid>, Option<Enrichment>, Vec<Region>);
3552    let mut items: Vec<RegionItem> = top
3553        .into_iter()
3554        .map(|(i, r)| {
3555            (
3556                r,
3557                table_rows.get(i).cloned().flatten(),
3558                enrichments.get(i).cloned().flatten(),
3559                std::mem::take(&mut kids[i]),
3560            )
3561        })
3562        .collect();
3563    order_with_containers(&mut items, &cids, page.width, page.height, |it| &it.0);
3564    // Float a margin page number to the front of reading order (docling parity:
3565    // right_to_left_02's bottom `11` is its first item). Stable, so everything
3566    // else keeps its order; no-op on pages without such a region.
3567    let page_h = page.height;
3568    items.sort_by_key(|(r, _, _, _)| !is_page_number(r, &page.cells, page_h));
3569    let table_rows: Vec<Option<TableGrid>> = items.iter().map(|(_, t, _, _)| t.clone()).collect();
3570    let enrichments: Vec<Option<Enrichment>> = items.iter().map(|(_, _, e, _)| e.clone()).collect();
3571    let mut picture_children: Vec<Vec<Region>> = items
3572        .iter_mut()
3573        .map(|it| std::mem::take(&mut it.3))
3574        .collect();
3575    let mut regions: Vec<Region> = items.into_iter().map(|(r, _, _, _)| r).collect();
3576    promote_labelled_captions(&mut regions, &page.cells);
3577    // Children in docling's `_sort_clusters(mode="id")` order: first source
3578    // cell, then top, then left.
3579    for kids in picture_children.iter_mut().filter(|k| k.len() > 1) {
3580        let rank = cluster_cids(kids, &page.cells);
3581        let mut ranked: Vec<(usize, Region)> = rank.into_iter().zip(kids.drain(..)).collect();
3582        ranked.sort_by_key(|(k, _)| *k);
3583        kids.extend(ranked.into_iter().map(|(_, r)| r));
3584    }
3585    // docling's `_find_to_captions` (#265, #813): each graphic claims its
3586    // caption by reading-order adjacency and gap; the caption is consumed
3587    // from its own slot and re-emitted with the graphic (before a picture or
3588    // table, after a code block, as docling renders a `captions` ref).
3589    let graphic_caption_for = pair_graphic_captions(&regions);
3590    let only = |pred: fn(&str) -> bool| -> Vec<Option<usize>> {
3591        regions
3592            .iter()
3593            .zip(&graphic_caption_for)
3594            .map(|(r, c)| c.filter(|_| pred(r.label)))
3595            .collect()
3596    };
3597    let caption_for = only(|l| l == "picture");
3598    let code_caption_for = only(|l| l == "code");
3599    let table_caption_for = only(is_table_like);
3600    let mut consumed = vec![false; regions.len()];
3601    for ci in graphic_caption_for.iter().flatten() {
3602        consumed[*ci] = true;
3603    }
3604    // Pictures inside a table become rich-cell content (docling#3906, 2.118.1):
3605    // the picture is nested in the cell it covers and not emitted standalone.
3606    let rich_cell_pictures = match_table_pictures(&regions, &table_rows, &caption_for);
3607    for (_, pics) in rich_cell_pictures.values().flatten() {
3608        for &p in pics {
3609            consumed[p] = true;
3610        }
3611    }
3612    // A code block's language label (`XML`, `C#`, …) is chrome, not content — the
3613    // detector emits it as its own region above the code; consume it.
3614    for (i, is_label) in code_language_labels(&regions, &page.cells)
3615        .into_iter()
3616        .enumerate()
3617    {
3618        if is_label {
3619            consumed[i] = true;
3620        }
3621    }
3622
3623    // docling `ReadingOrderPredictor.predict_merges`: join a text fragment with a
3624    // following text fragment strictly to its right (an author column that wraps
3625    // into the next, a paragraph continuing in the next column) into one block —
3626    // the intra-page half of docling's reading-order merges (cross-page/vertical
3627    // continuations stay with [`merge_continuations`]). Already-consumed regions
3628    // (paired captions, code labels) are excluded.
3629    // Exclusive docling cell assignment: computed once for the ordered region
3630    // list and reused for every serialization below, so a cell can never render
3631    // in two regions. The picture children take part (docling assigns cells to
3632    // every regular cluster before it nests any); their texts are split off.
3633    let with_children: Vec<Region> = regions
3634        .iter()
3635        .chain(picture_children.iter().flatten())
3636        .cloned()
3637        .collect();
3638    let mut region_texts: Vec<String> = region_texts_keep_wrap(&with_children, &page.cells);
3639    // A trailing soft hyphen marks a word split across the line break; only a
3640    // plain paragraph hands it on to `merge_continuations` (#919).
3641    let wraps: Vec<bool> = region_texts
3642        .iter_mut()
3643        .map(|t| t.ends_with('\u{ad}') && t.pop().is_some())
3644        .collect();
3645    let mut kid_texts = region_texts.split_off(regions.len()).into_iter();
3646    let child_texts: Vec<Vec<String>> = picture_children
3647        .iter()
3648        .map(|k| kid_texts.by_ref().take(k.len()).collect())
3649        .collect();
3650    let is_text: Vec<bool> = regions
3651        .iter()
3652        .enumerate()
3653        .map(|(i, r)| r.label == "text" && !consumed[i])
3654        .collect();
3655    let is_skip: Vec<bool> = regions
3656        .iter()
3657        .enumerate()
3658        .map(|(i, r)| {
3659            consumed[i]
3660                || matches!(
3661                    r.label,
3662                    "page_header" | "page_footer" | "table" | "picture" | "caption" | "footnote"
3663                )
3664        })
3665        .collect();
3666    let boxes: Vec<(f32, f32, f32, f32)> = regions.iter().map(|r| (r.l, r.t, r.r, r.b)).collect();
3667    if docling_core::env::flag("DOCLING_RS_DEBUG_MERGES") {
3668        for (i, r) in regions.iter().enumerate() {
3669            eprintln!(
3670                "MRG {i:2} {} text={} skip={} [{:.0},{:.0},{:.0},{:.0}] {:?}",
3671                r.label,
3672                is_text[i],
3673                is_skip[i],
3674                r.l,
3675                r.t,
3676                r.r,
3677                r.b,
3678                region_texts[i].chars().take(40).collect::<String>()
3679            );
3680        }
3681    }
3682    let mut merge_suffix: Vec<String> = vec![String::new(); regions.len()];
3683    for (head, children) in
3684        crate::reading_order::predict_merges(&boxes, &region_texts, &is_text, &is_skip)
3685            .into_iter()
3686            .enumerate()
3687    {
3688        for c in children {
3689            let t = region_texts[c].trim();
3690            if !t.is_empty() {
3691                merge_suffix[head].push(' ');
3692                merge_suffix[head].push_str(t);
3693            }
3694            consumed[c] = true;
3695        }
3696    }
3697
3698    for (i, region) in regions.iter().enumerate() {
3699        if consumed[i] {
3700            continue;
3701        }
3702        // Page headers/footers: docling emits them as furniture blocks
3703        // (`<page_header>`/`<page_footer>` with a layer + location + text) at
3704        // their reading-order position, not as body — emit them, don't skip.
3705        if matches!(region.label, "page_header" | "page_footer") {
3706            let text = region_texts[i].clone();
3707            if !text.is_empty() {
3708                nodes.push(Node::PageFurniture {
3709                    footer: region.label == "page_footer",
3710                    location: norm_loc(region, page.width, page_h),
3711                    text: md_escape(&text),
3712                });
3713            }
3714            continue;
3715        }
3716        if is_skipped(region.label) {
3717            continue;
3718        }
3719        // Layout provenance for this region, normalized to docling's 0–511 grid.
3720        let loc = norm_loc(region, page.width, page_h);
3721        if region.label == "picture" {
3722            // The figure pixels are cropped from the page render for image export.
3723            // Captions are prose: markdown-escaped like a paragraph (the JSON
3724            // export unescapes back to the raw text, matching docling).
3725            let caption = caption_for[i]
3726                .map(|ci| md_escape(&region_texts[ci]))
3727                .filter(|t| !t.is_empty());
3728            // The caption's own region box (#609): docling gives every caption
3729            // the cluster it came from as its `prov`, not the picture's.
3730            let caption_location = caption_for[i]
3731                .filter(|_| caption.is_some())
3732                .map(|ci| norm_loc(&regions[ci], page.width, page_h));
3733            let classification = match &enrichments[i] {
3734                Some(Enrichment::PictureClasses(classes)) => Some(classes.clone()),
3735                _ => None,
3736            };
3737            // Without the page render (text-layer-only build) a picture keeps
3738            // its caption/classification but carries no cropped pixels.
3739            #[cfg(feature = "ocr-prep")]
3740            let image =
3741                crate::timing::timed("crop_region", || crop_region(page, region, picture_scale));
3742            #[cfg(not(feature = "ocr-prep"))]
3743            let image: Option<PictureImage> = None;
3744            nodes.push(located(
3745                loc,
3746                Node::Picture {
3747                    caption,
3748                    caption_href: None,
3749                    image,
3750                    classification,
3751                    description: None,
3752                    // docling's layout pipeline parents a figure's caption to
3753                    // the picture itself (#390) — the one backend that does.
3754                    caption_parent: CaptionParent::Item,
3755                    caption_location,
3756                },
3757            ));
3758            let children: Vec<Node> = picture_children[i]
3759                .iter()
3760                .zip(&child_texts[i])
3761                .filter_map(|(r, text)| {
3762                    picture_child_node(r, text, norm_loc(r, page.width, page_h))
3763                })
3764                .collect();
3765            if !children.is_empty() {
3766                nodes.push(Node::PictureChildren(children));
3767            }
3768            continue;
3769        }
3770        let mut text = region_texts[i].clone();
3771        text.push_str(&merge_suffix[i]);
3772        if text.is_empty() {
3773            continue;
3774        }
3775        match region.label {
3776            // docling assembles checkboxes as TEXT_ELEM items (the region's
3777            // cells are the option label, e.g. right_to_left_03's بلی/خير)
3778            // and its Markdown serializer renders them as task-list lines
3779            // (`- [x] …`) — mirrored by [`Node::CheckboxItem`].
3780            // Located like every other text item, so the JSON item carries
3781            // its page and box (#609) — a chunk of checkboxes has a page.
3782            // The label without a leading ballot-box glyph: the item's state
3783            // already says what `☐` / `☒` drew (#609).
3784            "checkbox_selected" | "checkbox_unselected" => nodes.push(located(
3785                loc,
3786                Node::CheckboxItem {
3787                    checked: region.label == "checkbox_selected",
3788                    text: md_escape(strip_checkbox_glyph(&text)),
3789                },
3790            )),
3791            // docling renders both the document title and section headers as
3792            // `##` (it never emits a top-level `#` for PDFs), so match that.
3793            "title" | "section_header" => nodes.push(located(
3794                loc,
3795                Node::Heading {
3796                    level: 2,
3797                    text: md_escape(&text),
3798                },
3799            )),
3800            // docling's `ListItemMarkerProcessor.process_list_item` runs on
3801            // every PDF list item: a leading bullet glyph or enumeration marker
3802            // followed by whitespace is split off into the item's `marker`, and
3803            // docling-core's Markdown then prints `- text` for a bullet, `N. text`
3804            // for an `N.` marker and `- a) text` for any other marker holding a
3805            // letter or digit (see [`list_item_node`]). The symbol-font bullets
3806            // docling-parse filters out of its cells are stripped first.
3807            "list_item" => nodes.push(list_item_node(&text, loc, false)),
3808            // TableFormer structure (cells + spans, text matched from word cells)
3809            // when available; otherwise geometric grid reconstruction; finally a
3810            // single cell.
3811            "table" | "document_index" => {
3812                // TableFormer grids carry first-class cells (#240: text +
3813                // page-point bbox + span rectangle + OTSL header roles) into
3814                // the public model, and the DocLang structure overlay derives
3815                // from them so DCLX emits real span/header tokens. The
3816                // geometric fallback has no per-cell records.
3817                let (mut rows, cells, structure) = match table_rows[i].clone().map(normalize_grid) {
3818                    Some(grid) => {
3819                        let nrows = grid.rows.len();
3820                        let ncols = grid.rows.first().map_or(0, Vec::len);
3821                        let structure = structure_from_cells(&grid.cells, nrows, ncols);
3822                        (grid.rows, Some(grid.cells), Some(structure))
3823                    }
3824                    None => {
3825                        let rows = reconstruct_table(region, &page.cells);
3826                        let rows = if rows.iter().any(|r| r.len() > 1) {
3827                            rows
3828                        } else {
3829                            vec![vec![text.clone()]]
3830                        };
3831                        (rows, None, None)
3832                    }
3833                };
3834                // The paired caption (#265) rides on the table — docling's
3835                // TableItem.captions ref; Markdown prints it above the grid,
3836                // the JSON export emits the $ref, DocLang the <caption>.
3837                let caption = table_caption_for[i]
3838                    .map(|ci| md_escape(&region_texts[ci]))
3839                    .filter(|t| !t.is_empty());
3840                // Its own region box becomes the caption item's `prov` (#609).
3841                let caption_location = table_caption_for[i]
3842                    .filter(|_| caption.is_some())
3843                    .map(|ci| norm_loc(&regions[ci], page.width, page_h));
3844                // Rich cells (docling#3906): the covering cell's blocks are its
3845                // text followed by the nested picture(s). docling's Markdown
3846                // renders a `RichTableCell` through the serializer — the
3847                // group's children joined by blank lines, newlines flattened
3848                // to spaces — so the flat `rows` text becomes
3849                // `text  <!-- image -->`; the first-class `cells` (the JSON
3850                // `table_cells` / `grid`) keep the plain text, as upstream.
3851                let mut cell_blocks: Option<Vec<Vec<Vec<Node>>>> = None;
3852                if let (Some(by_cell), Some(fc)) = (rich_cell_pictures.get(&i), cells.as_ref()) {
3853                    let nrows = rows.len();
3854                    let ncols = rows.iter().map(Vec::len).max().unwrap_or(0);
3855                    let mut blocks = vec![vec![Vec::<Node>::new(); ncols]; nrows];
3856                    for (cell_idx, pics) in by_cell {
3857                        let cell = &fc[*cell_idx];
3858                        let (r, c) = (cell.start_row, cell.start_col);
3859                        if r >= nrows || c >= ncols {
3860                            continue;
3861                        }
3862                        let mut parts: Vec<String> = Vec::new();
3863                        let mut cell_nodes: Vec<Node> = Vec::new();
3864                        if !cell.text.trim().is_empty() {
3865                            parts.push(cell.text.clone());
3866                            cell_nodes.push(Node::Paragraph {
3867                                text: cell.text.clone(),
3868                            });
3869                        }
3870                        for &p in pics {
3871                            parts.push("<!-- image -->".to_string());
3872                            let classification = match &enrichments[p] {
3873                                Some(Enrichment::PictureClasses(classes)) => Some(classes.clone()),
3874                                _ => None,
3875                            };
3876                            #[cfg(feature = "ocr-prep")]
3877                            let image = crop_region(page, &regions[p], picture_scale);
3878                            #[cfg(not(feature = "ocr-prep"))]
3879                            let image: Option<PictureImage> = None;
3880                            cell_nodes.push(located(
3881                                norm_loc(&regions[p], page.width, page_h),
3882                                Node::Picture {
3883                                    caption: None,
3884                                    caption_href: None,
3885                                    image,
3886                                    classification,
3887                                    description: None,
3888                                    caption_parent: Default::default(),
3889                                    caption_location: None,
3890                                },
3891                            ));
3892                        }
3893                        let rendered = parts.join("  ");
3894                        for row in rows.iter_mut().skip(r).take(cell.row_span) {
3895                            for slot in row.iter_mut().skip(c).take(cell.col_span) {
3896                                *slot = rendered.clone();
3897                            }
3898                        }
3899                        blocks[r][c] = cell_nodes;
3900                    }
3901                    cell_blocks = Some(blocks);
3902                }
3903                nodes.push(located(
3904                    loc,
3905                    Node::Table(Table {
3906                        footnotes: Vec::new(),
3907                        rows,
3908                        location: None,
3909                        structure,
3910                        cell_blocks,
3911                        cells,
3912                        caption,
3913                        // As for pictures: the caption is the table's child.
3914                        caption_parent: CaptionParent::Item,
3915                        caption_location,
3916                    }),
3917                ));
3918            }
3919            // With formula enrichment the CodeFormula model decodes the region
3920            // to LaTeX; otherwise docling emits a placeholder comment rather
3921            // than the (garbled) raw glyph text.
3922            "formula" => match &enrichments[i] {
3923                Some(Enrichment::Formula { latex }) => nodes.push(Node::Formula {
3924                    latex: latex.clone(),
3925                    orig: text.clone(),
3926                    location: Some(loc),
3927                }),
3928                _ => nodes.push(Node::Paragraph {
3929                    text: "<!-- formula-not-decoded -->".into(),
3930                }),
3931            },
3932            // Code blocks: use the space-glyph-only grouping (monospace keeps its
3933            // source spacing) and emit a fenced block, preserving the line breaks
3934            // and indentation of the source (unlike prose, which reflows). pdfium
3935            // still inserts spaces around tight punctuation (`console .log`,
3936            // `add (3 , 5)`); tighten them to match docling-parse's source spacing.
3937            "code" => {
3938                // `code_region_text` preserves line breaks/indentation and tightens
3939                // each line itself; the fallback prose `text` is tightened here.
3940                let code = code_region_text(region, &page.code_cells);
3941                let code = if code.is_empty() {
3942                    tighten_code_punct(&text)
3943                } else {
3944                    code
3945                };
3946                // With code enrichment the CodeFormula model rewrites the block
3947                // (and names its language); `orig` keeps the raw extraction in
3948                // docling's shape — its parser has no line-preserving code
3949                // path, so its `orig` is the same code with the lines joined
3950                // by single spaces (indentation collapsed).
3951                // docling's parser has no line-preserving code path — its code
3952                // items carry the lines joined by single spaces. That flat
3953                // form is what every byte-conformance surface serializes
3954                // (legacy Markdown, JSON, DocLang); the line-preserving
3955                // extraction rides in `pretty` for strict Markdown only.
3956                let flat = code
3957                    .lines()
3958                    .map(str::trim)
3959                    .filter(|l| !l.is_empty())
3960                    .collect::<Vec<_>>()
3961                    .join(" ");
3962                let node = match &enrichments[i] {
3963                    Some(Enrichment::Code {
3964                        language,
3965                        text: enriched,
3966                    }) => Node::Code {
3967                        language: language.clone(),
3968                        text: enriched.clone(),
3969                        orig: Some(flat),
3970                        pretty: None,
3971                    },
3972                    _ => Node::Code {
3973                        language: None,
3974                        text: flat,
3975                        orig: None,
3976                        pretty: Some(code),
3977                    },
3978                };
3979                nodes.push(located(loc, node));
3980                // docling emits the `Listing N:` caption after the code block.
3981                if let Some(ci) = code_caption_for[i] {
3982                    let cap = md_escape(&region_texts[ci]);
3983                    if !cap.is_empty() {
3984                        // With its own region box, like every caption (#609).
3985                        nodes.push(located(
3986                            norm_loc(&regions[ci], page.width, page_h),
3987                            Node::Paragraph { text: cap },
3988                        ));
3989                    }
3990                }
3991            }
3992            // text, caption, footnote → paragraph
3993            _ => {
3994                // docling parity (`PageAssembleModel._match_hyperlink`): when
3995                // link annotations cover ≥ half of the region's box, the
3996                // hyperlink attaches to the item and the legacy Markdown
3997                // serializer wraps its full text — 2206.01062's footnote URLs
3998                // render as `[1 https://…](https://…)`. Sparse in-paragraph
3999                // citation links stay below the 0.5 coverage threshold and
4000                // remain plain text, exactly like docling.
4001                //
4002                // Scope: **footnote regions only.** Upstream's page_assemble
4003                // matches every TEXT_ELEM label, but published docling
4004                // observably carries the hyperlink into the document only for
4005                // footnote items — in both committed groundtruth generations
4006                // (docling-JSON and Markdown, independent runs) the fully
4007                // covered plain-text DOI line of 2206.01062 page 1 has
4008                // `hyperlink: None` while the equally covered footnotes carry
4009                // theirs. The corpus is the conformance reference, so match
4010                // the observed behavior; widen the label set if a future
4011                // groundtruth refresh starts linking plain text too.
4012                let escaped = md_escape(&text);
4013                let hyperlink = (region.label == "footnote")
4014                    .then(|| region_hyperlink(region, &page.links))
4015                    .flatten();
4016                if let Some(uri) = &hyperlink {
4017                    // The strict-mode anchor pairs this item covers are
4018                    // superseded by the whole-item link.
4019                    links.retain(|(anchor, href)| {
4020                        !(href == uri && region_texts[i].contains(anchor.as_str()))
4021                    });
4022                }
4023                // A footnote keeps docling's label and carries its link as
4024                // the item's `hyperlink` (the JSON's `text` stays the raw
4025                // footnote, not Markdown); every other serializer renders it
4026                // as the `[text](uri)` paragraph it was before.
4027                let node = if region.label == "footnote" {
4028                    Node::LabeledText {
4029                        label: "footnote".into(),
4030                        text: escaped,
4031                        href: hyperlink,
4032                    }
4033                } else {
4034                    // The wrap hyphen rides on the open paragraph (#919).
4035                    let mut escaped = escaped;
4036                    if wraps[i] {
4037                        escaped.push('\u{ad}');
4038                    }
4039                    Node::Paragraph { text: escaped }
4040                };
4041                nodes.push(located(loc, node))
4042            }
4043        }
4044    }
4045    // A `/Rotate`-normalized scanned page (see `pdfium_backend`) was assembled
4046    // in upright space; rotate the finished geometry back so locations and the
4047    // page size are display-space, like docling and every viewer report them.
4048    if page.rotation != 0 {
4049        rotate_nodes_to_display(&mut nodes, page.rotation);
4050    }
4051    (nodes, links)
4052}
4053
4054/// One child of a picture as docling's `ReadingOrderModel._add_child_elements`
4055/// writes it under the `PictureItem`: a heading for a `section_header` /
4056/// `title` (upstream remaps title to section header), a list item for a
4057/// `list_item`, a furniture-layer text for a page header/footer, a caption,
4058/// otherwise a text item. `None` for a child that claimed no text.
4059fn picture_child_node(region: &Region, text: &str, loc: [u16; 4]) -> Option<Node> {
4060    if text.is_empty() {
4061        return None;
4062    }
4063    Some(match region.label {
4064        "title" | "section_header" => located(
4065            loc,
4066            Node::Heading {
4067                level: 2,
4068                text: md_escape(text),
4069            },
4070        ),
4071        // docling-core's `add_list_item` under a non-list parent opens a
4072        // list group per item, so every child item starts its own list;
4073        // `_add_child_elements` runs the marker processor on it too.
4074        "list_item" => list_item_node(text, loc, true),
4075        "page_header" | "page_footer" => Node::PageFurniture {
4076            footer: region.label == "page_footer",
4077            location: loc,
4078            text: md_escape(text),
4079        },
4080        "caption" => located(
4081            loc,
4082            Node::Caption {
4083                text: md_escape(text),
4084                href: None,
4085            },
4086        ),
4087        _ => located(
4088            loc,
4089            Node::Paragraph {
4090                text: md_escape(text),
4091            },
4092        ),
4093    })
4094}
4095
4096/// Rotate one 0–511 location bbox 90° clockwise on the grid (top-left origin):
4097/// `(x, y) → (511 - y, x)`.
4098fn rot_loc_cw(l: [u16; 4]) -> [u16; 4] {
4099    [511 - l[3], l[0], 511 - l[1], l[2]]
4100}
4101
4102/// Map upright-space geometry back to display space for a page whose `/Rotate`
4103/// was normalized away before inference: every `<location>` rotates `rot`°
4104/// clockwise on the 0–511 grid (the grid is per-axis normalized, so no page
4105/// dims are needed), and the `PageInfo` size returns to the display box. Node
4106/// text and order are untouched — reading order was decided upright, which is
4107/// the whole point.
4108fn rotate_nodes_to_display(nodes: &mut [Node], rot: u16) {
4109    let quarter_turns = (rot / 90) as usize;
4110    let rot_loc = |l: &mut [u16; 4]| {
4111        for _ in 0..quarter_turns {
4112            *l = rot_loc_cw(*l);
4113        }
4114    };
4115    fn walk(node: &mut Node, rot_loc: &impl Fn(&mut [u16; 4]), swap_dims: bool) {
4116        match node {
4117            Node::PageInfo { width, height, .. } => {
4118                if swap_dims {
4119                    std::mem::swap(width, height);
4120                }
4121            }
4122            Node::Located { location, inner } => {
4123                rot_loc(location);
4124                walk(inner, rot_loc, swap_dims);
4125            }
4126            Node::Furniture { inner, .. } => walk(inner, rot_loc, swap_dims),
4127            Node::Group { children, .. } | Node::PictureChildren(children) => {
4128                for c in children {
4129                    walk(c, rot_loc, swap_dims);
4130                }
4131            }
4132            Node::ListItem { location, .. }
4133            | Node::Formula { location, .. }
4134            | Node::Chart { location, .. } => {
4135                if let Some(l) = location {
4136                    rot_loc(l);
4137                }
4138            }
4139            Node::PageFurniture { location, .. } => rot_loc(location),
4140            Node::Table(t) => {
4141                if let Some(l) = &mut t.location {
4142                    rot_loc(l);
4143                }
4144            }
4145            _ => {}
4146        }
4147    }
4148    let swap_dims = quarter_turns % 2 == 1;
4149    for node in nodes {
4150        walk(node, &rot_loc, swap_dims);
4151    }
4152}
4153
4154/// Merge paragraph fragments split across a column or page break. docling joins a
4155/// paragraph whose previous fragment ends mid-sentence (a letter, not sentence
4156/// punctuation) with a lowercase continuation: `…definition of` + `lists in…` →
4157/// `…definition of lists in…`. The fragments are consecutive paragraphs, or
4158/// separated only by figure(s) the text wraps around: a column whose body flows
4159/// past a figure resumes below it (`…The wing type that is` ⟶[figure]⟶ `the most
4160/// common…`), and docling emits the whole paragraph before the figure. A heading,
4161/// table, or list between them ends the paragraph (no merge).
4162/// A paragraph that is really a figure/table caption (`Fig. 1. …`, `Table 2 …`).
4163/// Used to skip an unpaired caption when stitching a paragraph that wraps around
4164/// a figure.
4165fn looks_like_caption(text: &str) -> bool {
4166    let head: String = text.trim_start().chars().take(14).collect();
4167    (head.starts_with("Fig") || head.starts_with("Table"))
4168        && head.contains(|c: char| c.is_ascii_digit())
4169}
4170
4171/// A paragraph fragment is "open" — i.e. it might continue into the next
4172/// paragraph — when it ends mid-word (a letter) or with a wrap hyphen/dash.
4173/// docling joins `vocab-` + `ulary` → `vocab- ulary`.
4174fn paragraph_is_open(text: &str) -> bool {
4175    // docling's merge head test (`.+([a-z,\-\u00AD])\s*`): at least two chars,
4176    // ending in an ASCII lowercase letter, a comma, a hyphen, or a soft
4177    // hyphen. The comma matters: 2206's "…In phase four," resumes across the
4178    // page break. Uppercase/non-Latin endings do not merge, exactly as
4179    // upstream (the dash family is already `-` here — clean_text normalized).
4180    let t = text.trim_end();
4181    t.chars().count() >= 2
4182        && t.chars()
4183            .next_back()
4184            .is_some_and(|c| matches!(c, 'a'..='z' | ',' | '-' | '\u{ad}'))
4185}
4186
4187/// The paragraph text inside a node, looking through a [`Node::Located`]
4188/// provenance wrapper (PDF body paragraphs are wrapped since they carry a
4189/// `<location>`). Returns `None` for non-paragraph nodes.
4190fn as_paragraph(n: &Node) -> Option<&str> {
4191    match n {
4192        Node::Paragraph { text } => Some(text),
4193        Node::Located { inner, .. } => match inner.as_ref() {
4194            Node::Paragraph { text } => Some(text),
4195            _ => None,
4196        },
4197        _ => None,
4198    }
4199}
4200
4201/// Whether a node is a picture, looking through a [`Node::Located`] wrapper.
4202fn is_picture_node(n: &Node) -> bool {
4203    match n {
4204        Node::Picture { .. } => true,
4205        Node::Located { inner, .. } => matches!(inner.as_ref(), Node::Picture { .. }),
4206        _ => false,
4207    }
4208}
4209
4210/// A node a forward paragraph merge looks straight past: a figure or *table*
4211/// the text wraps around, or a page header/footer that falls between the two
4212/// fragments of a paragraph continuing across a page break (docling's merge
4213/// skip-labels: page_header, page_footer, table, picture, caption, footnote —
4214/// 2206's "…In phase four," resumes after a full caption+table+figure block).
4215fn is_merge_trailer(n: &Node) -> bool {
4216    is_picture_node(n)
4217        || matches!(
4218            n,
4219            Node::PageFurniture { .. }
4220                | Node::PageInfo { .. }
4221                | Node::Table(_)
4222                | Node::PictureChildren(_)
4223        )
4224        || matches!(n, Node::Located { inner, .. } if matches!(inner.as_ref(), Node::Table(_)))
4225        || is_footnote_node(n)
4226        || as_paragraph(n).is_some_and(looks_like_caption)
4227}
4228
4229/// Whether a node is a footnote item (a [`Node::LabeledText`] labelled
4230/// `footnote`), looking through a [`Node::Located`] wrapper — one of
4231/// docling's merge skip-labels: a footnote between the two halves of a
4232/// paragraph is looked past, never merged into.
4233fn is_footnote_node(n: &Node) -> bool {
4234    let n = match n {
4235        Node::Located { inner, .. } => inner.as_ref(),
4236        other => other,
4237    };
4238    matches!(n, Node::LabeledText { label, .. } if label == "footnote")
4239}
4240
4241/// Rebuild node `i` as a paragraph with `text`, preserving its `<location>`
4242/// wrapper (and thus provenance) if it had one.
4243fn reparagraph(node: &Node, text: String) -> Node {
4244    match node {
4245        Node::Located { location, .. } => located(*location, Node::Paragraph { text }),
4246        _ => Node::Paragraph { text },
4247    }
4248}
4249
4250/// Whether `text` opens with a coordinating conjunction as a whole word
4251/// (German, English, French, Dutch, Spanish): after a line-final `-` it means
4252/// an elliptical compound, never the second half of a wrapped word.
4253fn starts_with_conjunction(text: &str) -> bool {
4254    let word: String = text
4255        .chars()
4256        .take_while(|c| c.is_alphabetic())
4257        .collect::<String>()
4258        .to_lowercase();
4259    matches!(
4260        word.as_str(),
4261        "und" | "oder" | "bzw" | "sowie" | "and" | "or" | "et" | "ou" | "en" | "y"
4262    )
4263}
4264
4265/// Drop the soft hyphen an unmerged paragraph still ends with (see
4266/// [`clean_text_keep_wrap`]): a word split nothing continued is just a word.
4267fn strip_wrap_marks(nodes: &mut [Node]) {
4268    for n in nodes {
4269        let n = match n {
4270            Node::Located { inner, .. } => inner.as_mut(),
4271            other => other,
4272        };
4273        if let Node::Paragraph { text } = n {
4274            if text.ends_with('\u{ad}') {
4275                text.pop();
4276            }
4277        }
4278    }
4279}
4280
4281pub(crate) fn merge_continuations(nodes: &mut Vec<Node>) {
4282    merge_open_paragraphs(nodes);
4283    strip_wrap_marks(nodes);
4284}
4285
4286/// [`merge_continuations`] without the final wrap-mark strip: the streaming
4287/// buffer still holds open paragraphs a later page may continue.
4288fn merge_open_paragraphs(nodes: &mut Vec<Node>) {
4289    let mut i = 0;
4290    while i + 1 < nodes.len() {
4291        let Some(a) = as_paragraph(&nodes[i]) else {
4292            i += 1;
4293            continue;
4294        };
4295        // A figure/table caption is a self-contained unit; body text resuming
4296        // after a figure is the continuation case, not the caption itself. Never
4297        // stitch *from* a caption — otherwise a caption that ends in a lone glyph
4298        // (`Fig. 5. … PubTabNet. μ`) would swallow a following stray figure label
4299        // (a standalone `μ`) into `… μ μ`.
4300        if looks_like_caption(a) {
4301            i += 1;
4302            continue;
4303        }
4304        if !paragraph_is_open(a) {
4305            i += 1;
4306            continue;
4307        }
4308        // The continuation is the next paragraph, looking past any figures the
4309        // text wraps around — and a figure/table caption that was emitted as its
4310        // own paragraph (an above-the-figure caption that didn't pair), since the
4311        // body text resumes after the whole figure+caption block.
4312        let mut j = i + 1;
4313        while nodes.get(j).is_some_and(is_merge_trailer) {
4314            j += 1;
4315        }
4316        // docling's continuation regex allows either case, but its merge runs
4317        // over the pre-assembly element stream; at node level an uppercase
4318        // start is overwhelmingly a new sentence/heading fragment (allowing it
4319        // swallowed 2305's formula blocks and redp's chapter openers), so the
4320        // continuation stays lowercase-start here.
4321        let cont = nodes.get(j).and_then(as_paragraph).is_some_and(|b| {
4322            b.trim_start()
4323                .chars()
4324                .next()
4325                .is_some_and(char::is_lowercase)
4326        });
4327        if cont {
4328            let a = as_paragraph(&nodes[i]).unwrap().trim_end().to_string();
4329            let b = as_paragraph(&nodes[j]).unwrap().trim_start().to_string();
4330            // A soft hyphen -- or a hard hyphen followed by a lowercase
4331            // continuation (guaranteed lowercase by the `cont` gate above) --
4332            // is a word split across the break: strip it and join without a
4333            // space, docling#3888 ("vocab-" + "ulary" -> "vocabulary");
4334            // docling's older serializer kept the artifact ("vocab- ulary").
4335            // Everything else joins with the space, as before.
4336            // Exception (#920): a hard hyphen before a coordinating conjunction
4337            // is the elliptical compound (`Warte- und Behaltefristen`), not a
4338            // wrapped word -- no word is split as `Warte` + `und`. Kept with
4339            // its space; docling fuses these (`Warteund`), a deliberate
4340            // divergence (docs/MIGRATION.md).
4341            let elliptical = !a.ends_with('\u{ad}') && starts_with_conjunction(&b);
4342            let merged = match a.strip_suffix('\u{ad}').or_else(|| a.strip_suffix('-')) {
4343                Some(stem) if !elliptical => format!("{stem}{b}"),
4344                _ => format!("{a} {b}"),
4345            };
4346            // Keep node i's provenance wrapper; docling's merged paragraph keeps
4347            // the first fragment's geometry as its primary location.
4348            nodes[i] = reparagraph(&nodes[i], merged);
4349            nodes.remove(j);
4350            // Re-check i: the merged paragraph may continue further.
4351        } else {
4352            i += 1;
4353        }
4354    }
4355}
4356
4357/// How many leading nodes of `nodes` are safe to flush now — i.e. cannot be
4358/// rewritten by a future [`merge_continuations`] once more pages are appended.
4359///
4360/// A forward merge can only start from an "open" paragraph (ends mid-word) and
4361/// only reaches across trailing pictures and figure/table captions. So we scan
4362/// from the end past those skippable trailers: if the first non-skippable node is
4363/// an open paragraph, it (and the trailers after it) must be held; anything else —
4364/// a closed paragraph, a heading, a table, a list — blocks any forward merge, so
4365/// the whole buffer is safe to flush.
4366fn hold_start(nodes: &[Node]) -> usize {
4367    for k in (0..nodes.len()).rev() {
4368        // Skippable trailers (figures, page furniture, captions): a forward merge
4369        // looks straight past them.
4370        if is_merge_trailer(&nodes[k]) {
4371            continue;
4372        }
4373        match as_paragraph(&nodes[k]) {
4374            // An open body paragraph might still pull a continuation off the next
4375            // page — hold from here to the end.
4376            Some(text) if paragraph_is_open(text) => return k,
4377            // A closed paragraph, heading, table, list, etc. ends the paragraph:
4378            // nothing after it can merge backwards across it. Flush everything.
4379            _ => return nodes.len(),
4380        }
4381    }
4382    // Only skippable trailers (or empty) and no open paragraph to anchor a merge.
4383    nodes.len()
4384}
4385
4386/// Streaming counterpart of [`merge_continuations`]: feed per-page node batches in
4387/// document order and get back the prefix that is final (its cross-page merges are
4388/// resolved and no future page can change it), holding back only the small tail
4389/// that might still merge into the next page. Concatenating every flushed batch
4390/// (then [`finish`](Self::finish)) yields exactly the same nodes as running
4391/// [`merge_continuations`] once over the whole document.
4392pub(crate) struct StreamAssembler {
4393    pending: Vec<Node>,
4394}
4395
4396impl StreamAssembler {
4397    pub(crate) fn new() -> Self {
4398        Self {
4399            pending: Vec::new(),
4400        }
4401    }
4402
4403    /// Append one page's nodes, resolve merges within the buffer, and return the
4404    /// now-final prefix to emit (possibly empty).
4405    pub(crate) fn push(&mut self, mut nodes: Vec<Node>) -> Vec<Node> {
4406        self.pending.append(&mut nodes);
4407        merge_open_paragraphs(&mut self.pending);
4408        let cut = hold_start(&self.pending);
4409        let tail = self.pending.split_off(cut);
4410        let mut done = std::mem::replace(&mut self.pending, tail);
4411        strip_wrap_marks(&mut done);
4412        done
4413    }
4414
4415    /// Flush whatever is left after the last page (the held tail is final once no
4416    /// more pages can follow).
4417    pub(crate) fn finish(mut self) -> Vec<Node> {
4418        strip_wrap_marks(&mut self.pending);
4419        self.pending
4420    }
4421}
4422
4423#[cfg(test)]
4424mod tests {
4425    use super::{
4426        cells_text, clean_text, is_note_of, looks_like_note_call, merge_overlapping_regulars,
4427        reclaim_heading_body_footers,
4428    };
4429
4430    /// docling drops a picture covering > 90 % of the page (its labels then
4431    /// read out as text); a dominant-but-not-full figure and any other label
4432    /// stay whatever their size.
4433    #[test]
4434    fn full_page_pictures_are_dropped_like_docling() {
4435        use super::drop_full_page_pictures;
4436        use crate::layout::Region;
4437        let region = |label: &'static str, l, t, r, b| Region {
4438            label,
4439            score: 0.99,
4440            l,
4441            t,
4442            r,
4443            b,
4444        };
4445        let mut regions = vec![
4446            region("picture", 0.0, 0.5, 478.9, 241.8),
4447            region("picture", 10.0, 10.0, 400.0, 200.0),
4448            region("table", 0.0, 0.0, 480.0, 243.0),
4449            region("text", 5.0, 5.0, 100.0, 20.0),
4450        ];
4451        drop_full_page_pictures(&mut regions, 480.75, 243.75);
4452        let labels: Vec<_> = regions.iter().map(|r| (r.label, r.l)).collect();
4453        assert_eq!(
4454            labels,
4455            vec![("picture", 10.0), ("table", 0.0), ("text", 5.0)]
4456        );
4457    }
4458    use super::{
4459        code_region_text, float_running_head, merge_continuations, resolve_link_anchors,
4460        StreamAssembler,
4461    };
4462    use crate::layout::Region;
4463    use crate::pdfium_backend::{LinkAnnot, PdfPage, TextCell};
4464    use docling_core::Node;
4465
4466    /// The int8-layout guard's coverage metric: cells under detections count,
4467    /// cells outside don't, whitespace cells are ignored, and a cell-less page
4468    /// reads as fully covered (nothing to rescue).
4469    #[test]
4470    fn layout_cell_coverage_counts_claimed_text_cells() {
4471        let cell = |text: &str, l: f32, t: f32| TextCell {
4472            text: text.into(),
4473            l,
4474            t,
4475            r: l + 40.0,
4476            b: t + 10.0,
4477        };
4478        let region = Region {
4479            label: "text",
4480            score: 0.9,
4481            l: 0.0,
4482            t: 0.0,
4483            r: 100.0,
4484            b: 50.0,
4485        };
4486        let cells = vec![
4487            cell("inside", 10.0, 10.0),
4488            cell("also inside", 10.0, 30.0),
4489            cell("outside", 10.0, 200.0),
4490            cell("   ", 10.0, 210.0), // whitespace: not counted at all
4491        ];
4492        let cov = super::layout_cell_coverage(std::slice::from_ref(&region), &cells);
4493        assert!((cov - 2.0 / 3.0).abs() < 1e-6, "got {cov}");
4494        assert_eq!(super::layout_cell_coverage(&[], &[]), 1.0);
4495        assert_eq!(super::layout_cell_coverage(&[], &cells), 0.0);
4496    }
4497
4498    /// #165: a picture no longer claims cells at 0.2 intersection-over-self.
4499    /// A line straddling the figure border (≤80 % contained) becomes an orphan
4500    /// region and is emitted as page text — before the fix its cells were
4501    /// silently erased. A line fully inside the picture is the picture's child
4502    /// (docling's `_set_cluster_children`): it survives the containment drop,
4503    /// leaves the page's reading order, and is written only under the picture
4504    /// in the JSON — never in the Markdown, like docling's picture serializer.
4505    #[test]
4506    fn border_straddlers_are_page_text_picture_interior_is_a_picture_child() {
4507        let pic = Region {
4508            label: "picture",
4509            score: 0.9,
4510            l: 0.0,
4511            t: 0.0,
4512            r: 100.0,
4513            b: 100.0,
4514        };
4515        // ~35 % of this cell overlaps the picture (l=90..120 of 0..100): above
4516        // the old 0.2 claim (was swallowed), below full containment (survives).
4517        let straddler = TextCell {
4518            text: "axis label".into(),
4519            l: 90.0,
4520            t: 40.0,
4521            r: 120.0,
4522            b: 48.0,
4523        };
4524        let interior = TextCell {
4525            text: "in-figure callout".into(),
4526            l: 10.0,
4527            t: 10.0,
4528            r: 60.0,
4529            b: 18.0,
4530        };
4531        let cells = vec![straddler, interior];
4532        let mut regions = vec![pic];
4533        super::add_orphan_regions(&mut regions, &cells);
4534        super::drop_contained_regulars(&mut regions);
4535        assert_eq!(
4536            regions.iter().filter(|r| r.label == "text").count(),
4537            2,
4538            "both unclaimed lines become orphans, and a picture swallows neither"
4539        );
4540        let parents = super::picture_parents(&regions);
4541        let parent_of = |l: f32| {
4542            regions
4543                .iter()
4544                .zip(&parents)
4545                .find(|(r, _)| r.label == "text" && r.l == l)
4546                .and_then(|(_, p)| *p)
4547        };
4548        assert_eq!(
4549            parent_of(10.0),
4550            Some(0),
4551            "the callout is the picture's child"
4552        );
4553        assert_eq!(parent_of(90.0), None, "the straddler is a page element");
4554
4555        let page = PdfPage::from_cells(200.0, 200.0, 2.0, cells);
4556        let n = regions.len();
4557        let (nodes, _) = super::assemble_page(&page, regions, &vec![None; n], &vec![None; n], None);
4558        let children: Vec<&Node> = nodes
4559            .iter()
4560            .filter_map(|n| match n {
4561                Node::PictureChildren(c) => Some(c),
4562                _ => None,
4563            })
4564            .flatten()
4565            .collect();
4566        assert!(
4567            matches!(children.as_slice(), [Node::Located { inner, .. }]
4568                if matches!(inner.as_ref(), Node::Paragraph { text } if text == "in-figure callout")),
4569            "{children:?}"
4570        );
4571        let mut doc = docling_core::DoclingDocument::new("t");
4572        doc.nodes = nodes;
4573        let md = doc.export_to_markdown();
4574        assert!(md.contains("axis label"), "{md}");
4575        assert!(!md.contains("in-figure callout"), "{md}");
4576        let json = doc.export_to_json_value();
4577        let pic = &json["pictures"][0];
4578        let child = pic["children"][0]["$ref"].as_str().expect("a child ref");
4579        let idx: usize = child.rsplit('/').next().unwrap().parse().unwrap();
4580        assert_eq!(json["texts"][idx]["text"], "in-figure callout");
4581        assert_eq!(json["texts"][idx]["parent"]["$ref"], "#/pictures/0");
4582        assert_eq!(json["texts"][idx]["content_layer"], "body");
4583    }
4584
4585    /// docling#3906's concern, pinned on our side: a picture detected fully
4586    /// inside a table region must survive the containment drop (upstream now
4587    /// attaches it to the table's cell; we keep it as a body sibling — either
4588    /// way it must not vanish). The text region inside the same table is the
4589    /// control: regulars are the ones the drop swallows.
4590    #[test]
4591    fn picture_inside_a_table_region_survives_the_containment_drop() {
4592        let mut regions = vec![
4593            region("table", 0.9, 0.0, 0.0, 200.0, 200.0),
4594            region("picture", 0.9, 20.0, 20.0, 120.0, 120.0),
4595            region("text", 0.9, 20.0, 140.0, 180.0, 180.0),
4596        ];
4597        super::drop_contained_regulars(&mut regions);
4598        let labels: Vec<&str> = regions.iter().map(|r| r.label).collect();
4599        assert_eq!(
4600            labels,
4601            ["table", "picture"],
4602            "the in-table picture stays; the in-table regular is the special's child"
4603        );
4604    }
4605
4606    /// Caption pairing (#265, #813) is docling's `_find_to_captions`: a
4607    /// caption binds a graphic in the unbroken run next to it in the region
4608    /// sequence — above-caption and below-caption both work, and geometry
4609    /// only ranks (a same-page caption in the other column of a two-column
4610    /// layout is *not* adjacent, however close its box is). A caption
4611    /// separated from its graphic by a text paragraph stays unattached; one
4612    /// between two graphics goes to the nearer, a tie to the one above.
4613    #[test]
4614    fn captions_pair_by_reading_order_adjacency() {
4615        // caption → table (above-caption), then table → caption (below-caption),
4616        // then a caption fenced off by a paragraph, then one between two tables.
4617        let regions = vec![
4618            region("text", 0.9, 0.0, 0.0, 100.0, 10.0), // 0 body text
4619            region("caption", 0.9, 0.0, 12.0, 60.0, 20.0), // 1 above-caption
4620            region("table", 0.9, 20.0, 22.0, 90.0, 60.0), // 2 ← pairs with 1
4621            region("table", 0.9, 0.0, 70.0, 100.0, 110.0), // 3 ← pairs with 4
4622            region("caption", 0.9, 0.0, 112.0, 60.0, 120.0), // 4 below-caption
4623            region("text", 0.9, 0.0, 130.0, 100.0, 140.0), // 5 body text
4624            region("caption", 0.9, 0.0, 142.0, 60.0, 150.0), // 6 fenced by 5/7
4625            region("text", 0.9, 0.0, 152.0, 100.0, 162.0), // 7 body text
4626            region("table", 0.9, 0.0, 170.0, 100.0, 200.0), // 8 ← tie, above wins
4627            region("caption", 0.9, 0.0, 202.0, 60.0, 210.0), // 9 2 pt from both
4628            region("table", 0.9, 0.0, 212.0, 100.0, 240.0), // 10 unpaired
4629            region("page_footer", 0.9, 0.0, 250.0, 100.0, 260.0), // 11 furniture
4630        ];
4631        let pairs = super::pair_graphic_captions(&regions);
4632        assert_eq!(pairs[2], Some(1), "caption directly above its table pairs");
4633        assert_eq!(pairs[3], Some(4), "caption directly below its table pairs");
4634        assert!(
4635            !pairs.contains(&Some(6)),
4636            "a text paragraph between caption and table breaks the bond"
4637        );
4638        assert_eq!(pairs[8], Some(9), "equal gaps go to the preceding graphic");
4639        assert_eq!(pairs[10], None);
4640        assert!(
4641            pairs.iter().filter(|p| p.is_some()).count() == 3,
4642            "{pairs:?}"
4643        );
4644    }
4645
4646    /// #813: a `Figure N:` label printed *above* its figure (DP-Bench
4647    /// 01030000000036: caption [40,255,287,266] over picture [118,277,423,469])
4648    /// binds the picture; the old geometric matcher only looked below. A
4649    /// caption below a code block binds too (docling's run is label-blind).
4650    #[test]
4651    fn a_caption_above_its_picture_binds() {
4652        let regions = vec![
4653            region("section_header", 0.9, 40.0, 200.0, 300.0, 215.0),
4654            region("text", 0.9, 40.0, 220.0, 500.0, 250.0),
4655            region("caption", 0.93, 40.0, 255.0, 287.0, 266.0), // 2
4656            region("picture", 0.98, 118.0, 277.0, 423.0, 469.0), // 3 ← 2
4657            region("text", 0.9, 40.0, 480.0, 500.0, 520.0),
4658            region("code", 0.9, 40.0, 530.0, 500.0, 600.0), // 5 ← 6
4659            region("caption", 0.9, 40.0, 604.0, 300.0, 614.0), // 6
4660        ];
4661        let pairs = super::pair_graphic_captions(&regions);
4662        assert_eq!(pairs[3], Some(2));
4663        assert_eq!(pairs[5], Some(6));
4664    }
4665
4666    /// Within a run the *nearest* graphic wins, not the adjacent one in the
4667    /// sequence; and the matching augments: a caption whose first choice is
4668    /// held by a caption that can rehouse itself displaces it, so no figure
4669    /// ends up orphaned while another holds a caption it did not need.
4670    #[test]
4671    fn captions_rank_by_gap_and_displace_rehousable_claims() {
4672        // P1 and P2 are an unbroken run; C1 under it sits right below P2
4673        // (gap 2) and far from P1 (gap 62), so it takes P2, not the run's
4674        // first graphic.
4675        let regions = vec![
4676            region("picture", 0.9, 0.0, 0.0, 100.0, 50.0),   // 0 P1
4677            region("picture", 0.9, 0.0, 60.0, 100.0, 110.0), // 1 P2
4678            region("caption", 0.9, 0.0, 112.0, 100.0, 120.0), // 2 C1: P2 (2) < P1 (62)
4679        ];
4680        let pairs = super::pair_graphic_captions(&regions);
4681        assert_eq!(pairs, vec![None, Some(2), None]);
4682
4683        // C1 above the run prefers P1 (gap 2) over P2 (62); C2 below the run
4684        // reaches P2 (2) then P1 (62). Each gets its nearest.
4685        let regions = vec![
4686            region("caption", 0.9, 0.0, 0.0, 100.0, 8.0),     // 0 C1
4687            region("picture", 0.9, 0.0, 10.0, 100.0, 50.0),   // 1 P1
4688            region("picture", 0.9, 0.0, 60.0, 100.0, 110.0),  // 2 P2
4689            region("caption", 0.9, 0.0, 112.0, 100.0, 120.0), // 3 C2
4690        ];
4691        let pairs = super::pair_graphic_captions(&regions);
4692        assert_eq!(pairs, vec![None, Some(0), Some(3), None]);
4693
4694        // Augmenting path: C1 (between P1 and P2, nearer P2) takes P2 first;
4695        // C2 below reaches only P2 and displaces C1, which rehouses on P1.
4696        let regions = vec![
4697            region("picture", 0.9, 0.0, 0.0, 100.0, 50.0),   // 0 P1
4698            region("caption", 0.9, 0.0, 70.0, 100.0, 78.0),  // 1 C1: P2 (2) < P1 (20)
4699            region("picture", 0.9, 0.0, 80.0, 100.0, 120.0), // 2 P2
4700            region("caption", 0.9, 0.0, 122.0, 100.0, 130.0), // 3 C2: P2 only
4701        ];
4702        let pairs = super::pair_graphic_captions(&regions);
4703        assert_eq!(pairs, vec![Some(1), None, Some(3), None]);
4704
4705        // No graphic can be freed: the first claim stands, the second stays loose.
4706        let regions = vec![
4707            region("caption", 0.9, 0.0, 0.0, 100.0, 8.0),   // 0 C1
4708            region("picture", 0.9, 0.0, 10.0, 100.0, 50.0), // 1 P1
4709            region("caption", 0.9, 0.0, 52.0, 100.0, 60.0), // 2 C2
4710        ];
4711        let pairs = super::pair_graphic_captions(&regions);
4712        assert_eq!(pairs, vec![None, Some(0), None]);
4713    }
4714
4715    #[test]
4716    fn caption_labels_are_recognized() {
4717        for t in [
4718            "Table 5: Prediction Performance",
4719            "Figure 2.1: Surveyed MSMEs",
4720            "Fig. 3: Overview",
4721            "Table 1. Pour volume by month (m3)",
4722            "  Figure 12a. Left panel",
4723            "Listing 4: main.rs",
4724        ] {
4725            assert!(super::is_caption_label(t), "{t}");
4726        }
4727        for t in [
4728            "Table 5 shows the results",
4729            "Tables 5: and 6:",
4730            "Figure",
4731            "Figure: no number",
4732            "table 5: lowercase",
4733            "Table : no number",
4734        ] {
4735            assert!(!super::is_caption_label(t), "{t}");
4736        }
4737    }
4738
4739    /// 2206.01062's Table 5 caption: a four-line `text` block the model did
4740    /// not call `caption`, 22 pt above its table, becomes one; the same block
4741    /// with a paragraph between it and the table, or without the label's
4742    /// punctuation, stays text.
4743    #[test]
4744    fn a_labelled_text_block_beside_a_graphic_becomes_a_caption() {
4745        let cells = vec![
4746            cell(
4747                "Table 5: Prediction Performance of a network",
4748                54.0,
4749                87.0,
4750                296.0,
4751                100.0,
4752            ),
4753            cell("across the datasets.", 54.0, 100.0, 200.0, 113.0),
4754            cell("In this section we discuss", 54.0, 400.0, 296.0, 412.0),
4755        ];
4756        let mut regions = vec![
4757            region("text", 0.99, 54.0, 87.0, 296.0, 150.0),
4758            region("table", 0.99, 73.0, 172.0, 275.0, 340.0),
4759            region("text", 0.99, 54.0, 400.0, 296.0, 500.0),
4760        ];
4761        super::promote_labelled_captions(&mut regions, &cells);
4762        assert_eq!(regions[0].label, "caption");
4763        assert_eq!(regions[2].label, "text");
4764
4765        // A paragraph between the label and the table breaks the adjacency.
4766        let mut regions = vec![
4767            region("text", 0.99, 54.0, 87.0, 296.0, 150.0),
4768            region("text", 0.99, 54.0, 152.0, 296.0, 170.0),
4769            region("table", 0.99, 73.0, 172.0, 275.0, 340.0),
4770        ];
4771        super::promote_labelled_captions(&mut regions, &cells);
4772        assert_eq!(regions[0].label, "text");
4773
4774        // `Table 5 shows …` is prose.
4775        let cells = vec![cell("Table 5 shows the results", 54.0, 87.0, 296.0, 100.0)];
4776        let mut regions = vec![
4777            region("text", 0.99, 54.0, 87.0, 296.0, 150.0),
4778            region("table", 0.99, 73.0, 172.0, 275.0, 340.0),
4779        ];
4780        super::promote_labelled_captions(&mut regions, &cells);
4781        assert_eq!(regions[0].label, "text");
4782    }
4783
4784    /// Side-by-side figures (`tests/snapshots/latex/…/swa.pdf`): three
4785    /// pictures in a row, their three captions in a row under them. docling's
4786    /// run rule binds only the first caption (the one before each other
4787    /// caption breaks its run); the extension gives the other two the
4788    /// unclaimed picture each sits under. A caption whose run is broken by
4789    /// *text* (2206.01062's Figure 4 behind a hash line) stays loose, as
4790    /// docling leaves it.
4791    #[test]
4792    fn side_by_side_figure_captions_bind_their_own_picture() {
4793        let regions = vec![
4794            region("picture", 0.8, 9.0, 2.0, 367.0, 347.0),   // 0
4795            region("picture", 0.7, 423.0, 3.0, 718.0, 346.0), // 1
4796            region("picture", 0.7, 756.0, 13.0, 1040.0, 341.0), // 2
4797            region("caption", 0.6, 128.0, 371.0, 315.0, 392.0), // 3 ← under 0
4798            region("caption", 0.6, 425.0, 371.0, 713.0, 396.0), // 4 ← under 1
4799            region("caption", 0.6, 785.0, 371.0, 1069.0, 397.0), // 5 ← under 2
4800        ];
4801        let pairs = super::pair_graphic_captions(&regions);
4802        assert_eq!(pairs[..3], [Some(3), Some(4), Some(5)]);
4803
4804        let regions = vec![
4805            region("picture", 0.9, 316.0, 82.0, 564.0, 459.0),
4806            region("text", 0.9, 400.0, 458.0, 476.0, 461.0), // a hash line
4807            region("caption", 0.9, 318.0, 473.0, 559.0, 504.0),
4808        ];
4809        assert_eq!(
4810            super::pair_graphic_captions(&regions),
4811            vec![None, None, None]
4812        );
4813    }
4814
4815    /// A colored terms-and-conditions panel detected as `picture` demotes into
4816    /// per-paragraph `text` regions (the blank line between C.7 and C.8 splits
4817    /// them); a chart whose only text is a few narrow axis labels keeps its
4818    /// crop untouched.
4819    #[test]
4820    fn text_panels_demote_to_paragraphs_but_charts_keep_their_crop() {
4821        let cell = |text: &str, l: f32, t: f32, r: f32, b: f32| TextCell {
4822            text: text.to_string(),
4823            l,
4824            t,
4825            r,
4826            b,
4827        };
4828        let panel = Region {
4829            label: "picture",
4830            score: 0.9,
4831            l: 0.0,
4832            t: 0.0,
4833            r: 100.0,
4834            b: 100.0,
4835        };
4836        // Three tight lines, a blank-line gap, two more: two paragraphs.
4837        let cells = vec![
4838            cell(
4839                "C.7. Wenn Sie diesen Vertrag widerrufen,",
4840                5.0,
4841                10.0,
4842                95.0,
4843                18.0,
4844            ),
4845            cell(
4846                "haben wir Ihnen alle Zahlungen, die wir",
4847                5.0,
4848                20.0,
4849                95.0,
4850                28.0,
4851            ),
4852            cell(
4853                "von Ihnen erhalten haben, zurückzuzahlen.",
4854                5.0,
4855                30.0,
4856                90.0,
4857                38.0,
4858            ),
4859            cell(
4860                "C.8. Wir können die Rückzahlung verweigern,",
4861                5.0,
4862                52.0,
4863                95.0,
4864                60.0,
4865            ),
4866            cell(
4867                "bis wir die Waren wieder zurückerhalten haben.",
4868                5.0,
4869                62.0,
4870                92.0,
4871                70.0,
4872            ),
4873        ];
4874        let mut regions = vec![panel.clone()];
4875        super::recover_text_panels(&mut regions, &cells);
4876        assert_eq!(
4877            regions.iter().map(|r| r.label).collect::<Vec<_>>(),
4878            ["text", "text"],
4879            "dense panel must demote into one text region per paragraph"
4880        );
4881        assert!(regions[0].b < regions[1].t, "paragraphs split at the gap");
4882        // Sparse narrow labels (a chart): picture survives.
4883        let labels = vec![
4884            cell("0", 5.0, 90.0, 8.0, 95.0),
4885            cell("50", 5.0, 50.0, 10.0, 55.0),
4886            cell("100", 5.0, 10.0, 12.0, 15.0),
4887            cell("t, s", 45.0, 96.0, 55.0, 100.0),
4888        ];
4889        let mut regions = vec![panel];
4890        super::recover_text_panels(&mut regions, &labels);
4891        assert_eq!(
4892            regions.iter().map(|r| r.label).collect::<Vec<_>>(),
4893            ["picture"]
4894        );
4895    }
4896
4897    /// An uncaptioned chart on a scanned page whose title, axis labels, and
4898    /// OCR boxes over the plot area are dense and wide enough to pass the
4899    /// coverage/width gates still keeps its crop: its line heights are ragged
4900    /// (title face vs tick labels vs bar-area OCR), failing the uniform-leading
4901    /// gate — a real text panel is set with constant leading (#173).
4902    #[test]
4903    fn dense_titled_chart_keeps_its_crop() {
4904        let cell = |text: &str, l: f32, t: f32, r: f32, b: f32| TextCell {
4905            text: text.to_string(),
4906            l,
4907            t,
4908            r,
4909            b,
4910        };
4911        let chart = Region {
4912            label: "picture",
4913            score: 0.9,
4914            l: 0.0,
4915            t: 0.0,
4916            r: 100.0,
4917            b: 100.0,
4918        };
4919        // Five wide lines at wildly different heights: a 12-pt title, 20-pt OCR
4920        // boxes over the bars, 4–5-pt tick/axis labels. Coverage and median
4921        // width both clear the panel thresholds.
4922        let cells = vec![
4923            cell("Underground Water Storage", 10.0, 5.0, 90.0, 17.0),
4924            cell("aquifer recharge zone", 15.0, 30.0, 75.0, 50.0),
4925            cell("confined | unconfined | perched", 12.0, 55.0, 80.0, 59.0),
4926            cell("saturated thickness", 8.0, 70.0, 60.0, 90.0),
4927            cell("distance from well, km", 20.0, 92.0, 85.0, 97.0),
4928        ];
4929        let mut regions = vec![chart];
4930        super::recover_text_panels(&mut regions, &cells);
4931        assert_eq!(
4932            regions.iter().map(|r| r.label).collect::<Vec<_>>(),
4933            ["picture"],
4934            "ragged line heights mark a figure, not a text panel"
4935        );
4936    }
4937
4938    /// docling serializes a cluster's cells in docling-parse index order
4939    /// (`_sort_cells`) and joins them with `PageAssembleModel.sanitize_text`:
4940    /// a space after every line except one ending in `-`, which either fuses a
4941    /// wrapped word (alnum on both sides — dash dropped) or glues verbatim (a
4942    /// bare `-` cell: `[0000` `-` `0002` → `[0000 -0002`, the 2305 ORCID line;
4943    /// `-` + `"C" cell -` + `a new table cell` → `-"C" cell a new table cell`,
4944    /// its OTSL list). Verified against the corpus: pure index order beats any
4945    /// geometric re-sort (normal_4pages' heading numerals paint after their
4946    /// text and belong last: `## 들어가며 1`).
4947    #[test]
4948    fn cells_join_in_index_order_with_sanitize_text_rules() {
4949        let cell = |text: &str, l: f32, t: f32, r: f32, b: f32| TextCell {
4950            text: text.to_string(),
4951            l,
4952            t,
4953            r,
4954            b,
4955        };
4956        let region = Region {
4957            label: "text",
4958            score: 1.0,
4959            l: 0.0,
4960            t: 95.0,
4961            r: 200.0,
4962            b: 130.0,
4963        };
4964        // ORCID superscript: a bare dash cell is a *detached* dash — kept, and
4965        // since docling#4052 (2.122) it joins with the ordinary space on both
4966        // sides (`[0000 -0002 -6960]` before that fix).
4967        let orcid = vec![
4968            cell("[0000", 10.0, 100.0, 30.0, 110.0),
4969            cell("−", 30.0, 100.0, 34.0, 110.0),
4970            cell("0002", 34.0, 100.0, 50.0, 110.0),
4971            cell("−", 50.0, 100.0, 54.0, 110.0),
4972            cell("6960]", 54.0, 100.0, 70.0, 110.0),
4973        ];
4974        assert_eq!(super::region_text(&region, &orcid), "[0000 - 0002 - 6960]");
4975        // Wrapped word: dash dropped, lines fused (both boundary words alnum).
4976        let wrapped = vec![
4977            cell("platforms-", 10.0, 100.0, 60.0, 110.0),
4978            cell("reflects the design", 10.0, 112.0, 90.0, 122.0),
4979        ];
4980        assert_eq!(
4981            super::region_text(&region, &wrapped),
4982            "platformsreflects the design"
4983        );
4984        // Dash-ending lines that are *detached* dashes (a bare bullet cell, a
4985        // `cell -` separator): the dash stays and the lines join with a space
4986        // — docling#4052; before it they glued (`-"C" cell a new table cell`,
4987        // 2305's OTSL list bullets).
4988        let otsl = vec![
4989            cell("–", 10.0, 100.0, 14.0, 110.0),
4990            cell("\"C\" cell -", 16.0, 100.0, 60.0, 110.0),
4991            cell("a new table cell", 10.0, 112.0, 80.0, 122.0),
4992        ];
4993        assert_eq!(
4994            super::region_text(&region, &otsl),
4995            "- \"C\" cell - a new table cell"
4996        );
4997        // Index order is authoritative — no geometric re-sort.
4998        let numeral = vec![
4999            cell("들어가며", 30.0, 100.0, 80.0, 110.0),
5000            cell("1", 10.0, 98.0, 25.0, 112.0), // big numeral painted last
5001        ];
5002        assert_eq!(super::region_text(&region, &numeral), "들어가며 1");
5003    }
5004
5005    /// The geometric-reliability gate, on the two shapes it has to tell apart.
5006    #[test]
5007    fn geometric_reliability_rejects_split_column_grids() {
5008        let g = |rows: &[&[&str]]| -> Vec<Vec<String>> {
5009            rows.iter()
5010                .map(|r| r.iter().map(|c| c.to_string()).collect())
5011                .collect()
5012        };
5013        // A genuine grid: dense, every column carrying entries. Nothing for
5014        // TableFormer to improve, so geometry is used as-is.
5015        assert!(super::geometric_table_is_reliable(&g(&[
5016            &["Datum", "Leistung", "Anzahl", "Kosten"],
5017            &["04.07", "Internet", "1", "40.30"],
5018            &["04.07", "Telefon", "2", "8.06"],
5019        ])));
5020        // The left-edge split artefact (the shape a scanned invoice produced):
5021        // one real label column plus values scattered across three sparse ones.
5022        assert!(!super::geometric_table_is_reliable(&g(&[
5023            &["www.magenta.at/faq", "", "", ""],
5024            &["Serviceteam", "", "", ""],
5025            &["Telefon", "0676/2000", "", ""],
5026            &["Kundennummer", "", "", "1.21699482"],
5027            &["Rechnungsnummer", "", "922769430725", ""],
5028            &["Rechnungsdatum", "", "", "04.07.2025"],
5029        ])));
5030        // A column only one row ever uses is a split artefact even when the
5031        // grid is otherwise dense.
5032        assert!(!super::geometric_table_is_reliable(&g(&[
5033            &["a", "b", ""],
5034            &["c", "d", ""],
5035            &["e", "f", "g"],
5036        ])));
5037        // Degenerate shapes are never vouched for — TableFormer may recover
5038        // structure a collapsed reconstruction lost.
5039        assert!(!super::geometric_table_is_reliable(&g(&[&[
5040            "only one column"
5041        ]])));
5042        assert!(!super::geometric_table_is_reliable(&[]));
5043    }
5044
5045    /// A `picture` region is cropped out of the rendered page, whatever built
5046    /// that page. The browser pipeline (#157) has no pdfium but does hand over
5047    /// the rasterized bitmap through `from_cells_with_image`, so it must get
5048    /// the same figure bytes the native path does — that is what makes
5049    /// `images = "embedded"` inline real pixels instead of a placeholder.
5050    #[cfg(feature = "ocr-prep")]
5051    #[test]
5052    fn picture_regions_are_cropped_from_a_host_supplied_page_image() {
5053        let mut img = image::RgbImage::new(200, 200);
5054        // Paint the figure area so the crop is distinguishable from the page.
5055        for y in 100..160 {
5056            for x in 20..120 {
5057                img.put_pixel(x, y, image::Rgb([255, 0, 0]));
5058            }
5059        }
5060        // scale 2.0: the region is in page points, the bitmap in pixels.
5061        let page = PdfPage::from_cells_with_image(100.0, 100.0, 2.0, Vec::new(), img);
5062        let region = Region {
5063            label: "picture",
5064            score: 0.9,
5065            l: 10.0,
5066            t: 50.0,
5067            r: 60.0,
5068            b: 80.0,
5069        };
5070        let (nodes, _) = super::assemble_page(&page, vec![region], &[None], &[None], None);
5071        // Layout-derived nodes carry provenance, so the picture arrives wrapped.
5072        let image = nodes
5073            .iter()
5074            .find_map(|n| match n {
5075                Node::Located { inner, .. } => match &**inner {
5076                    Node::Picture { image, .. } => image.as_ref(),
5077                    _ => None,
5078                },
5079                Node::Picture { image, .. } => image.as_ref(),
5080                _ => None,
5081            })
5082            .expect("a picture node with cropped pixels");
5083        assert_eq!(image.mimetype, "image/png");
5084        assert_eq!((image.width, image.height), (100, 60), "region × scale");
5085        assert!(!image.data.is_empty(), "PNG bytes were encoded");
5086    }
5087
5088    #[test]
5089    fn link_anchors_split_a_shared_word_cell_between_adjacent_links() {
5090        // A common header layout: one text run holds several pipe-separated
5091        // labels, each carrying its own link annotation. Every link must get
5092        // its own label as the anchor (and the "|" separators must belong to
5093        // none), not the whole run.
5094        let annot = |l: f32, r: f32, uri: &str| LinkAnnot {
5095            l,
5096            t: 100.0,
5097            r,
5098            b: 114.0,
5099            uri: uri.into(),
5100        };
5101        let page = PdfPage {
5102            width: 600.0,
5103            height: 800.0,
5104            scale: 2.0,
5105            cells: Vec::new(),
5106            code_cells: Vec::new(),
5107            checkboxes: Vec::new(),
5108            bitmaps: Vec::new(),
5109            // "LinkedIn | GitHub | Credly" = 26 chars over x 100..360.
5110            word_cells: vec![cell(
5111                "LinkedIn | GitHub | Credly",
5112                100.0,
5113                100.0,
5114                360.0,
5115                114.0,
5116            )],
5117            image: image::RgbImage::new(1, 1),
5118            image_layout: None,
5119            links: vec![
5120                annot(100.0, 180.0, "https://l"),
5121                annot(200.0, 260.0, "https://g"),
5122                annot(290.0, 360.0, "https://c"),
5123            ],
5124            rotation: 0,
5125            keep_typography: false,
5126            #[cfg(feature = "render")]
5127            embedded_images: Vec::new(),
5128            #[cfg(feature = "ml")]
5129            pending_image: None,
5130        };
5131        assert_eq!(
5132            resolve_link_anchors(&page),
5133            vec![
5134                ("LinkedIn".to_string(), "https://l".to_string()),
5135                ("GitHub".to_string(), "https://g".to_string()),
5136                ("Credly".to_string(), "https://c".to_string()),
5137            ]
5138        );
5139    }
5140
5141    /// A one-line code cell at `[l, r] × [t, b]` (top-left coords).
5142    fn cell(text: &str, l: f32, t: f32, r: f32, b: f32) -> TextCell {
5143        TextCell {
5144            text: text.into(),
5145            l,
5146            t,
5147            r,
5148            b,
5149        }
5150    }
5151
5152    /// OCR-path grouping (docling's `_remove_overlapping_clusters("regular")`):
5153    /// the low-score paragraph box RT-DETR draws over its own high-score line
5154    /// boxes collapses to one region — the group's union, with the survivor's
5155    /// label and score — so region-scoped OCR reads each line once. Regions
5156    /// that merely sit near each other, and specials, are untouched.
5157    #[test]
5158    fn merge_overlapping_regulars_collapses_a_block_over_its_lines() {
5159        let mut regions = vec![
5160            region("text", 0.84, 60.0, 186.0, 270.0, 198.0),
5161            region("text", 0.80, 60.0, 160.0, 294.0, 172.0),
5162            region("text", 0.79, 59.0, 107.0, 272.0, 119.0),
5163            // The paragraph box, lower score, containing all three lines.
5164            region("text", 0.52, 59.0, 107.0, 295.0, 200.0),
5165            // Elsewhere on the page: stays as is.
5166            region("section_header", 0.77, 60.0, 71.0, 253.0, 86.0),
5167            // A picture the block overlaps is not a regular — never grouped.
5168            region("picture", 0.9, 50.0, 100.0, 300.0, 210.0),
5169        ];
5170        merge_overlapping_regulars(&mut regions);
5171        assert_eq!(regions.len(), 3, "{regions:?}");
5172        let block = regions
5173            .iter()
5174            .find(|r| r.label == "text")
5175            .expect("one text");
5176        // docling keeps the largest passing candidate unless a rival is both
5177        // comparable in size and > 0.05 more confident; the 16× larger block
5178        // passes, and a smaller line never replaces a larger current best.
5179        // Either way the survivor spans the whole group.
5180        assert_eq!(
5181            (block.l, block.t, block.r, block.b),
5182            (59.0, 107.0, 295.0, 200.0)
5183        );
5184        assert!(regions.iter().any(|r| r.label == "section_header"));
5185        assert!(regions.iter().any(|r| r.label == "picture"));
5186    }
5187
5188    /// The pairwise rules, each in the arrangement where it decides the
5189    /// outcome: docling seeds the survivor with the group's first passing
5190    /// cluster and a later one replaces it only when larger *and* within
5191    /// 0.05 confidence, so a rule that merely lets a cluster pass matters
5192    /// exactly when that cluster comes first — a same-sized list item ahead
5193    /// of a far more confident text box, a code box ahead of the text it
5194    /// contains. Without the rule either would be rejected outright (similar
5195    /// size, rival > 0.05 more confident) and the text box would win.
5196    #[test]
5197    fn merge_overlapping_regulars_follows_the_preference_rules() {
5198        let mut regions = vec![
5199            region("list_item", 0.6, 0.0, 0.0, 102.0, 20.0),
5200            region("text", 0.9, 0.0, 0.0, 100.0, 20.0),
5201        ];
5202        merge_overlapping_regulars(&mut regions);
5203        assert_eq!(regions.len(), 1);
5204        assert_eq!(regions[0].label, "list_item");
5205
5206        let mut regions = vec![
5207            region("code", 0.6, 0.0, 0.0, 100.0, 100.0),
5208            region("text", 0.9, 2.0, 2.0, 98.0, 98.0),
5209        ];
5210        merge_overlapping_regulars(&mut regions);
5211        assert_eq!(regions.len(), 1);
5212        assert_eq!(regions[0].label, "code");
5213
5214        // No rule applies: a near-identical rival that is > 0.05 more
5215        // confident rejects the candidate whatever the order.
5216        for order in [[0.9, 0.6], [0.6, 0.9]] {
5217            let mut regions = vec![
5218                region("text", order[0], 0.0, 0.0, 100.0, 20.0),
5219                region("text", order[1], 0.0, 0.0, 105.0, 21.0),
5220            ];
5221            merge_overlapping_regulars(&mut regions);
5222            assert_eq!(regions.len(), 1);
5223            assert_eq!(regions[0].score, 0.9, "the confident twin wins");
5224            assert_eq!(
5225                (regions[0].r, regions[0].b),
5226                (105.0, 21.0),
5227                "on the union box"
5228            );
5229        }
5230
5231        // Side by side (no containment, IoU 0): nothing to merge.
5232        let mut regions = vec![
5233            region("text", 0.9, 0.0, 0.0, 100.0, 20.0),
5234            region("text", 0.9, 0.0, 22.0, 100.0, 42.0),
5235        ];
5236        merge_overlapping_regulars(&mut regions);
5237        assert_eq!(regions.len(), 2);
5238    }
5239
5240    #[test]
5241    fn footer_under_a_body_less_heading_is_its_text() {
5242        // The reporting CV's last page: `## Languages` at t=773.7..783.4 and
5243        // its one-line body at 798.5..808.7, labelled page_footer (A4, 595 pt).
5244        let mut regions = vec![
5245            region("list_item", 0.9, 66.0, 754.0, 353.0, 765.0),
5246            region("section_header", 0.94, 43.0, 773.7, 89.5, 783.4),
5247            region("page_footer", 0.88, 43.0, 798.5, 527.8, 808.7),
5248        ];
5249        reclaim_heading_body_footers(&mut regions, 595.28);
5250        assert_eq!(regions[2].label, "text");
5251        assert_eq!(regions[1].label, "section_header");
5252
5253        // A heading with its own paragraph and a running footer below: kept.
5254        let mut regions = vec![
5255            region("section_header", 0.9, 43.0, 700.0, 120.0, 710.0),
5256            region("text", 0.9, 43.0, 714.0, 520.0, 780.0),
5257            region("page_footer", 0.9, 43.0, 798.0, 520.0, 808.0),
5258        ];
5259        reclaim_heading_body_footers(&mut regions, 595.28);
5260        assert_eq!(regions[2].label, "page_footer");
5261
5262        // A page number under a trailing heading is too narrow to be a body.
5263        let mut regions = vec![
5264            region("section_header", 0.9, 43.0, 773.0, 120.0, 783.0),
5265            region("page_footer", 0.9, 280.0, 798.0, 300.0, 808.0),
5266        ];
5267        reclaim_heading_body_footers(&mut regions, 595.28);
5268        assert_eq!(regions[1].label, "page_footer");
5269
5270        // Too far below the heading (a real footer after a heading that ends
5271        // the page): kept.
5272        let mut regions = vec![
5273            region("section_header", 0.9, 43.0, 700.0, 120.0, 710.0),
5274            region("page_footer", 0.9, 43.0, 798.0, 520.0, 808.0),
5275        ];
5276        reclaim_heading_body_footers(&mut regions, 595.28);
5277        assert_eq!(regions[1].label, "page_footer");
5278    }
5279
5280    fn region(label: &'static str, score: f32, l: f32, t: f32, r: f32, b: f32) -> Region {
5281        Region {
5282            label,
5283            score,
5284            l,
5285            t,
5286            r,
5287            b,
5288        }
5289    }
5290
5291    #[test]
5292    fn resolve_collapses_nested_code_keeping_the_larger_box() {
5293        // A tight high-score `code` box and a taller lower-score near-duplicate that
5294        // contains it must collapse to one — the *larger* box, so every cell stays
5295        // covered and nothing leaks out as orphan text.
5296        let tight = region("code", 0.95, 78.0, 292.0, 300.0, 330.0);
5297        let wide = region("code", 0.66, 63.0, 260.0, 320.0, 346.0);
5298        let kept = super::resolve(vec![tight, wide]);
5299        assert_eq!(kept.len(), 1, "nested code boxes must collapse to one");
5300        assert!(
5301            kept[0].l == 63.0 && kept[0].b == 346.0,
5302            "the larger containing box is kept"
5303        );
5304    }
5305
5306    #[test]
5307    fn resolve_keeps_distinct_and_differently_typed_regions() {
5308        // A text box fully inside a lower-score *table* must NOT be collapsed (the
5309        // code dedup is code-only), and two separate code blocks stay separate.
5310        let text = region("text", 0.95, 90.0, 210.0, 200.0, 230.0);
5311        let table = region("table", 0.60, 80.0, 200.0, 400.0, 500.0);
5312        assert_eq!(super::resolve(vec![text, table]).len(), 2);
5313
5314        let code_a = region("code", 0.9, 78.0, 100.0, 300.0, 140.0);
5315        let code_b = region("code", 0.9, 78.0, 300.0, 300.0, 360.0); // far below, no overlap
5316        assert_eq!(super::resolve(vec![code_a, code_b]).len(), 2);
5317    }
5318
5319    /// A two-column glossary page came out as three column
5320    /// tables *and* one low-score whole-page table over them. docling's wrapper
5321    /// `_remove_overlapping_clusters` keeps one table per overlapping group
5322    /// (here the whole-page one: > 2× every rival's area and ≤ 0.2 less
5323    /// confident than the running best); `greedy` alone kept all four and
5324    /// emitted every cell twice.
5325    #[test]
5326    fn resolve_keeps_one_table_per_nested_group() {
5327        let kept = super::resolve(vec![
5328            region("table", 0.71, 26.0, 203.0, 183.0, 558.0),
5329            region("table", 0.67, 26.0, 55.0, 183.0, 196.0),
5330            region("table", 0.66, 196.0, 56.0, 354.0, 561.0),
5331            region("table", 0.53, 25.0, 53.0, 354.0, 561.0),
5332        ]);
5333        assert_eq!(kept.len(), 1, "one survivor per overlapping group");
5334        assert_eq!((kept[0].l, kept[0].b), (25.0, 561.0));
5335        // Side-by-side tables that don't overlap stay separate.
5336        let kept = super::resolve(vec![
5337            region("table", 0.9, 26.0, 55.0, 183.0, 558.0),
5338            region("table", 0.9, 196.0, 56.0, 354.0, 561.0),
5339        ]);
5340        assert_eq!(kept.len(), 2);
5341    }
5342
5343    /// A dense data table detected as both picture (0.80) and table (0.62) on one box.
5344    /// The picture is ≥ 0.1 more confident, so `_handle_cross_type_overlaps`
5345    /// keeps both, and the dense table text passed the text-panel gates: the
5346    /// demoted paragraph repeated every cell the table grid renders. A
5347    /// paragraph > 80 % inside a surviving table is the table's child and is
5348    /// not emitted; a panel with no table under it still demotes.
5349    #[test]
5350    fn text_panel_over_a_table_does_not_repeat_its_cells() {
5351        let lines = |t0: f32| -> Vec<TextCell> {
5352            (0..4)
5353                .map(|i| {
5354                    let t = t0 + 10.0 * i as f32;
5355                    cell("1 2 3 4 5 6 7 8 9 10 11 12", 5.0, t, 95.0, t + 8.0)
5356                })
5357                .collect()
5358        };
5359        let mut cells = lines(0.0);
5360        cells.extend(lines(200.0));
5361        let mut regions = vec![
5362            region("picture", 0.80, 0.0, 0.0, 100.0, 45.0),
5363            region("table", 0.62, 0.0, 0.0, 100.0, 45.0),
5364            region("picture", 0.80, 0.0, 200.0, 100.0, 245.0),
5365        ];
5366        super::recover_text_panels(&mut regions, &cells);
5367        assert_eq!(
5368            regions.iter().map(|r| r.label).collect::<Vec<_>>(),
5369            ["table", "text"]
5370        );
5371        assert_eq!(regions[1].t, 200.0, "the table-free panel still demotes");
5372    }
5373
5374    /// docling's `ListItemMarkerProcessor` + docling-core's Markdown list rules
5375    /// (see `list_item_node`): a bullet glyph or `-` is split off and the
5376    /// serializer's own `-` printed (2305's OTSL list reads `- "C" cell`, not
5377    /// `- - "C" cell`); `N.` is an ordered item; any other marker with a letter
5378    /// or digit stays in the text behind the bullet; no marker → plain bullet.
5379    #[test]
5380    fn list_item_markers_split_like_docling() {
5381        // The marker as DocLang reads it: the overlay's, else the node's.
5382        let text_of = |n: &Node| match n {
5383            Node::ListItem {
5384                ordered,
5385                number,
5386                text,
5387                marker,
5388                dclx,
5389                ..
5390            } => (
5391                *ordered,
5392                *number,
5393                text.clone(),
5394                dclx.as_ref().map_or(marker.clone(), |d| d.marker.clone()),
5395            ),
5396            other => panic!("{other:?}"),
5397        };
5398        let loc = [0, 0, 100, 10];
5399        assert_eq!(
5400            text_of(&super::list_item_node(
5401                "- \"C\" cell - a new table cell",
5402                loc,
5403                false
5404            )),
5405            (
5406                false,
5407                0,
5408                "\"C\" cell - a new table cell".into(),
5409                Some("-".into())
5410            )
5411        );
5412        assert_eq!(
5413            text_of(&super::list_item_node("• Bullet text", loc, false)),
5414            (false, 0, "Bullet text".into(), Some("•".into()))
5415        );
5416        assert_eq!(
5417            text_of(&super::list_item_node("3. Third step", loc, false)),
5418            (true, 3, "Third step".into(), Some("3.".into()))
5419        );
5420        assert_eq!(
5421            text_of(&super::list_item_node("a) Option", loc, false)),
5422            (false, 0, "a) Option".into(), Some("a)".into()))
5423        );
5424        assert_eq!(
5425            text_of(&super::list_item_node("1.2 Nested outline", loc, false)),
5426            (false, 0, "1.2 Nested outline".into(), Some("1.2".into()))
5427        );
5428        // 2203's `3.a. If all IOU scores…`: a compound marker is not an item
5429        // number — docling prints `- 3.a. If all…`.
5430        assert_eq!(
5431            text_of(&super::list_item_node("3.a. If all IOU scores", loc, false)),
5432            (
5433                false,
5434                0,
5435                "3.a. If all IOU scores".into(),
5436                Some("3.a.".into())
5437            )
5438        );
5439        // A glued symbol-font bullet is stripped, a spaced one is the marker.
5440        assert_eq!(
5441            text_of(&super::list_item_node("•Glued", loc, false)),
5442            (false, 0, "Glued".into(), Some("·".into()))
5443        );
5444        // docling#5043's `test_private_use_font_bullets_are_markers`: the
5445        // Symbol/Wingdings bullets as private-use code points (a Word-exported
5446        // PDF's text layer) are bullet markers, never part of the item text.
5447        for bullet in ['\u{F0B7}', '\u{F0A7}', '\u{F0D8}'] {
5448            let input = format!("{bullet} Transcribo: converts audio to text");
5449            assert_eq!(
5450                text_of(&super::list_item_node(&input, loc, false)),
5451                (
5452                    false,
5453                    0,
5454                    "Transcribo: converts audio to text".into(),
5455                    Some(bullet.to_string())
5456                )
5457            );
5458            assert_eq!(super::split_list_marker(&input).map(|m| m.2), Some(false));
5459            assert_eq!(super::checkbox_glyph(bullet), None);
5460        }
5461        for bullet in super::PUA_FONT_BULLETS {
5462            assert!(super::LIST_BULLET_MARKERS.contains(bullet), "{bullet:?}");
5463            // Glued to the text it is stripped like `•Glued`.
5464            assert_eq!(
5465                text_of(&super::list_item_node(
5466                    &format!("{bullet}Glued"),
5467                    loc,
5468                    false
5469                )),
5470                (false, 0, "Glued".into(), Some("·".into()))
5471            );
5472        }
5473        // No whitespace after the glyph → not a marker (docling's `\s` is required).
5474        assert_eq!(
5475            text_of(&super::list_item_node("-5 degrees", loc, false)),
5476            (false, 0, "-5 degrees".into(), Some("·".into()))
5477        );
5478        // The remaining numbered shapes, first-wins like docling's list.
5479        for (input, marker, body) in [
5480            ("1.2.3. Deep", "1.2.3.", "Deep"),
5481            ("9a) Nine-a", "9a)", "Nine-a"),
5482            ("(3.a) Paren", "(3.a)", "Paren"),
5483            ("12) Twelve", "12)", "Twelve"),
5484            ("(4) Four", "(4)", "Four"),
5485            ("[7] Seven", "[7]", "Seven"),
5486            ("iv. Roman", "iv.", "Roman"),
5487            ("IX. Roman", "IX.", "Roman"),
5488            ("b. Letter", "b.", "Letter"),
5489            ("B) Letter", "B)", "Letter"),
5490        ] {
5491            assert_eq!(
5492                super::split_list_marker(input),
5493                Some((marker, body, true)),
5494                "{input}"
5495            );
5496        }
5497        // A `1.2.` whose optional dot would eat the separator backtracks like
5498        // Python's regex; a marker with nothing after the whitespace is none.
5499        assert_eq!(
5500            super::split_list_marker("1.2.\tx"),
5501            Some(("1.2.", "x", true))
5502        );
5503        assert_eq!(super::split_list_marker("1. "), None);
5504        assert_eq!(super::split_list_marker("• "), None);
5505        assert_eq!(
5506            text_of(&super::list_item_node("Plain item", loc, false)),
5507            (false, 0, "Plain item".into(), Some("·".into()))
5508        );
5509    }
5510
5511    /// A `case_auto` marker is printed once: it rides in the item's text, so
5512    /// the Markdown serializer — which applies docling-core's AUTO rule to a
5513    /// marker it is handed — must not be handed it too (`- (1) (1) text`, the
5514    /// four drifted PDF snapshots of the docling 2.142 serializer change).
5515    /// The bullet glyph and `N.` shapes keep their Markdown marker.
5516    #[test]
5517    fn case_auto_marker_prints_once_in_markdown() {
5518        let loc = [0, 0, 100, 10];
5519        let nodes: Vec<Node> = [
5520            "(1) Human annotation",
5521            "[21] Peng Zhang",
5522            "a) Option",
5523            "• Bullet",
5524            "3. Third",
5525        ]
5526        .iter()
5527        .enumerate()
5528        .map(|(i, t)| super::list_item_node(t, loc, i == 0))
5529        .collect();
5530        for n in &nodes[..3] {
5531            let Node::ListItem { marker, dclx, .. } = n else {
5532                unreachable!()
5533            };
5534            assert_eq!(marker, &None, "{n:?}");
5535            assert!(dclx.as_ref().is_some_and(|d| d.marker.is_some()), "{n:?}");
5536        }
5537        let mut doc = docling_core::DoclingDocument::new("t");
5538        doc.nodes = nodes;
5539        let md = doc.export_to_markdown();
5540        for line in [
5541            "- (1) Human annotation",
5542            "- [21] Peng Zhang",
5543            "- a) Option",
5544            "- Bullet",
5545            "3. Third",
5546        ] {
5547            assert!(md.lines().any(|l| l == line), "missing {line:?} in:\n{md}");
5548        }
5549        assert!(!md.contains("(1) (1)") && !md.contains("[21] [21]"), "{md}");
5550    }
5551
5552    #[test]
5553    fn code_language_label_above_code_is_detected() {
5554        // A bare "XML" token directly above a code box is a language label; a real
5555        // heading above the same code is not; a language word with no code below is
5556        // left alone.
5557        let label = region("section_header", 0.9, 76.0, 540.0, 96.0, 549.0);
5558        let code = region("code", 0.7, 77.0, 552.0, 290.0, 640.0);
5559        let heading = region("section_header", 0.9, 76.0, 500.0, 260.0, 512.0);
5560        let cells = vec![
5561            cell("XML", 78.0, 541.0, 94.0, 548.0),       // inside `label`
5562            cell("Overview", 78.0, 501.0, 250.0, 511.0), // inside `heading`
5563        ];
5564        let drop = super::code_language_labels(&[label, code, heading], &cells);
5565        assert_eq!(drop, vec![true, false, false], "only the label is consumed");
5566
5567        // Same label with no code region present → not consumed.
5568        let label2 = region("section_header", 0.9, 76.0, 540.0, 96.0, 549.0);
5569        let only = vec![cell("XML", 78.0, 541.0, 94.0, 548.0)];
5570        assert_eq!(super::code_language_labels(&[label2], &only), vec![false]);
5571
5572        // A label swallowed into the top of a wider code box (negative gap) is still
5573        // recognized.
5574        let inside_lbl = region("text", 0.9, 76.0, 540.0, 96.0, 549.0);
5575        let wide_code = region("code", 0.7, 63.0, 531.0, 320.0, 654.0);
5576        let cells2 = vec![cell("XML", 78.0, 541.0, 94.0, 548.0)];
5577        assert_eq!(
5578            super::code_language_labels(&[inside_lbl, wide_code], &cells2),
5579            vec![true, false]
5580        );
5581
5582        assert!(super::is_code_language("XML") && super::is_code_language("c#"));
5583        assert!(!super::is_code_language("Configure") && !super::is_code_language("XML schema"));
5584    }
5585
5586    #[test]
5587    fn code_region_text_keeps_lines_and_indentation() {
5588        // Three source lines; each glyph is 6 units wide (width / chars = 6), so the
5589        // `int X;` line indented to x=22 is (22-10)/6 = 2 spaces in.
5590        let region = Region {
5591            label: "code",
5592            score: 1.0,
5593            l: 0.0,
5594            t: -5.0,
5595            r: 100.0,
5596            b: 40.0,
5597        };
5598        let cells = vec![
5599            cell("struct P {", 10.0, 0.0, 70.0, 10.0),
5600            cell("int X;", 22.0, 12.0, 58.0, 22.0),
5601            cell("}", 10.0, 24.0, 16.0, 34.0),
5602        ];
5603        assert_eq!(code_region_text(&region, &cells), "struct P {\n  int X;\n}");
5604    }
5605
5606    #[test]
5607    fn code_region_text_tightens_punctuation_without_eating_indentation() {
5608        // A fluent `.Foo()` line at x=22 (2 chars in). Per-line tightening must not
5609        // consume the leading indent space by matching " ." across it.
5610        let region = Region {
5611            label: "code",
5612            score: 1.0,
5613            l: 0.0,
5614            t: -5.0,
5615            r: 100.0,
5616            b: 40.0,
5617        };
5618        let cells = vec![
5619            cell("builder", 10.0, 0.0, 52.0, 10.0),
5620            // pdfium spaced the call: ".Foo (x)" tightens to ".Foo(x)", still 2-indented.
5621            cell(".Foo (x)", 22.0, 12.0, 70.0, 22.0),
5622        ];
5623        assert_eq!(code_region_text(&region, &cells), "builder\n  .Foo(x)");
5624    }
5625
5626    #[test]
5627    fn code_region_text_orders_out_of_order_cells_and_ignores_blank_lines() {
5628        let region = Region {
5629            label: "code",
5630            score: 1.0,
5631            l: 0.0,
5632            t: -5.0,
5633            r: 100.0,
5634            b: 60.0,
5635        };
5636        // Fed bottom-up and with a whitespace-only cell; output is top-down, no blank.
5637        let cells = vec![
5638            cell("b();", 10.0, 24.0, 34.0, 34.0),
5639            cell("   ", 10.0, 12.0, 20.0, 22.0),
5640            cell("a();", 10.0, 0.0, 34.0, 10.0),
5641        ];
5642        assert_eq!(code_region_text(&region, &cells), "a();\nb();");
5643        // No code cells → empty, so the caller falls back to the prose text.
5644        assert_eq!(code_region_text(&region, &[]), "");
5645    }
5646
5647    fn para(text: &str) -> Node {
5648        Node::Paragraph { text: text.into() }
5649    }
5650
5651    /// Run a node sequence through [`StreamAssembler`] with the given page splits
5652    /// and assert the flushed result equals one-shot [`merge_continuations`].
5653    fn assert_stream_eq(nodes: &[Node], splits: &[usize]) {
5654        let mut want = nodes.to_vec();
5655        merge_continuations(&mut want);
5656
5657        let mut asm = StreamAssembler::new();
5658        let mut got = Vec::new();
5659        let mut start = 0;
5660        for &end in splits {
5661            got.extend(asm.push(nodes[start..end].to_vec()));
5662            start = end;
5663        }
5664        got.extend(asm.push(nodes[start..].to_vec()));
5665        got.extend(asm.finish());
5666        assert_eq!(got, want, "stream assembly diverged (splits={splits:?})");
5667    }
5668
5669    #[test]
5670    fn running_head_floats_to_page_front() {
5671        let cells = vec![
5672            cell("HEAD — 7", 350.0, 40.0, 500.0, 50.0),
5673            cell("marker", 70.0, 70.0, 90.0, 80.0),
5674            cell("body", 100.0, 90.0, 300.0, 100.0),
5675        ];
5676        let mut nodes = vec![
5677            Node::PageInfo {
5678                page_no: 1,
5679                width: 600.0,
5680                height: 800.0,
5681            },
5682            para("marker"),
5683            para("body"),
5684            para("HEAD — 7"),
5685        ];
5686        float_running_head(&mut nodes, &cells);
5687        assert_eq!(nodes[1], para("HEAD — 7"));
5688        assert_eq!(nodes[3], para("body"));
5689    }
5690
5691    #[test]
5692    fn stream_assembler_matches_merge_continuations() {
5693        // Open fragment + lowercase continuation split across a page boundary.
5694        let cross = [para("the definition of"), para("lists in scope")];
5695        assert_stream_eq(&cross, &[1]);
5696        assert_stream_eq(&cross, &[]);
5697
5698        // Continuation that wraps around a figure (+ its caption) on the boundary.
5699        let wrap = [
5700            para("the wing type that is"),
5701            Node::Picture {
5702                caption: None,
5703                caption_href: None,
5704                image: None,
5705                classification: None,
5706                description: None,
5707                caption_parent: Default::default(),
5708                caption_location: None,
5709            },
5710            para("Fig. 1. a diagram"),
5711            para("the most common kind"),
5712        ];
5713        for splits in [&[][..], &[1][..], &[2][..], &[3][..], &[1, 3][..]] {
5714            assert_stream_eq(&wrap, splits);
5715        }
5716
5717        // A heading between fragments blocks the merge (must still flush correctly).
5718        let blocked = [
5719            para("ends mid word and"),
5720            Node::Heading {
5721                level: 2,
5722                text: "New Section".into(),
5723            },
5724            para("more body here"),
5725        ];
5726        for splits in [&[][..], &[1][..], &[2][..]] {
5727            assert_stream_eq(&blocked, splits);
5728        }
5729
5730        // A chain across three pages: each page is one open lowercase fragment.
5731        let chain = [
5732            para("alpha beta"),
5733            para("gamma delta"),
5734            para("epsilon zeta"),
5735        ];
5736        assert_stream_eq(&chain, &[1, 2]);
5737    }
5738
5739    #[test]
5740    fn clean_text_dehyphenates_and_normalizes_typography() {
5741        // U+0002 line-wrap hyphen + the join space → merged word (like docling).
5742        assert_eq!(clean_text("com\u{2} pact"), "compact");
5743        assert_eq!(clean_text("end-to\u{2} end deep"), "end-toend deep");
5744        // A stray wrap hyphen (no following join) is dropped.
5745        assert_eq!(clean_text("word\u{2}"), "word");
5746        // Typographic punctuation → ASCII (docling-parse's sanitizer table):
5747        // single curly quotes become `'`, double ones `"` (docling-parse#349).
5748        assert_eq!(
5749            clean_text("Graph\u{2019}s \u{201c}x\u{201d} \u{201e}y\u{201c}"),
5750            "Graph's \"x\" \"y\""
5751        );
5752        assert_eq!(clean_text("a\u{2026}"), "a...");
5753        // The docling-parse sanitizer's internal spacing is preserved as
5754        // placed; line breaks/tabs normalize to a space, ends trim.
5755        assert_eq!(clean_text("a   b\nc"), "a   b c");
5756    }
5757
5758    /// docling#4064: a form's children are emitted together where the form
5759    /// sits in the top-level order, not interleaved with surrounding text.
5760    #[test]
5761    fn form_children_stay_together_in_reading_order() {
5762        let reg = |label: &'static str, l: f32, t: f32, r: f32, b: f32| Region {
5763            label,
5764            score: 0.9,
5765            l,
5766            t,
5767            r,
5768            b,
5769        };
5770        // Page: intro text, then a form spanning the left column with two
5771        // fields and a table inside, while a right-column paragraph sits
5772        // level with the form's first field (it would otherwise be read
5773        // between the form's children).
5774        let mut items = vec![
5775            reg("text", 50.0, 50.0, 550.0, 70.0),    // 0 intro
5776            reg("form", 50.0, 100.0, 300.0, 400.0),  // 1 container
5777            reg("text", 60.0, 110.0, 290.0, 130.0),  // 2 field A (child)
5778            reg("text", 320.0, 110.0, 550.0, 130.0), // 3 right column paragraph
5779            reg("table", 60.0, 150.0, 290.0, 300.0), // 4 table (child)
5780            reg("text", 60.0, 320.0, 290.0, 340.0),  // 5 field B (child)
5781            reg("text", 50.0, 450.0, 550.0, 470.0),  // 6 outro
5782        ];
5783        let cids = super::cluster_cids(&items, &[]);
5784        super::order_with_containers(&mut items, &cids, 600.0, 800.0, |r| r);
5785        let order: Vec<(&str, f32)> = items.iter().map(|r| (r.label, r.t)).collect();
5786        // The form block (container, then its children top-down) is one unit.
5787        let form_pos = order.iter().position(|(l, _)| *l == "form").unwrap();
5788        assert_eq!(
5789            &order[form_pos..form_pos + 4],
5790            &[
5791                ("form", 100.0),
5792                ("text", 110.0),
5793                ("table", 150.0),
5794                ("text", 320.0)
5795            ]
5796        );
5797        assert_eq!(order[0], ("text", 50.0));
5798        assert_eq!(order[order.len() - 1], ("text", 450.0));
5799        // Without a container the plain order interleaves by geometry.
5800        let mut flat: Vec<Region> = items
5801            .iter()
5802            .filter(|r| r.label != "form")
5803            .cloned()
5804            .collect();
5805        let cids = super::cluster_cids(&flat, &[]);
5806        super::order_regions(&mut flat, &cids, 600.0, 800.0, |r| r);
5807        assert_ne!(
5808            flat.iter().map(|r| r.t).collect::<Vec<_>>(),
5809            order
5810                .iter()
5811                .filter(|(l, _)| *l != "form")
5812                .map(|(_, t)| *t)
5813                .collect::<Vec<_>>()
5814        );
5815    }
5816
5817    /// docling#3906: a picture inside a table lands in the covering cell,
5818    /// chosen by the picture's inferred grid position when cell boxes overlap.
5819    #[test]
5820    fn picture_matches_the_cell_at_its_grid_position() {
5821        let cell = |r: usize, c: usize, bbox: [f32; 4]| docling_core::TableCell {
5822            text: format!("r{r}c{c}"),
5823            bbox: Some(bbox),
5824            start_row: r,
5825            start_col: c,
5826            row_span: 1,
5827            col_span: 1,
5828            column_header: false,
5829            row_header: false,
5830            row_section: false,
5831        };
5832        // 2×2 grid; the (1,0) cell box is generous and also covers the picture.
5833        let cells = vec![
5834            cell(0, 0, [0.0, 0.0, 100.0, 50.0]),
5835            cell(0, 1, [100.0, 0.0, 200.0, 50.0]),
5836            cell(1, 0, [0.0, 50.0, 100.0, 100.0]),
5837            cell(1, 1, [100.0, 50.0, 200.0, 100.0]),
5838        ];
5839        let pic = Region {
5840            label: "picture",
5841            score: 0.9,
5842            l: 110.0,
5843            t: 60.0,
5844            r: 190.0,
5845            b: 95.0,
5846        };
5847        assert_eq!(super::match_picture_to_cell(&pic, &cells), Some((1.0, 3)));
5848        // A picture only half inside any cell is not nested.
5849        let straddling = Region {
5850            label: "picture",
5851            score: 0.9,
5852            l: 60.0,
5853            t: 60.0,
5854            r: 160.0,
5855            b: 95.0,
5856        };
5857        assert_eq!(super::match_picture_to_cell(&straddling, &cells), None);
5858    }
5859
5860    /// docling#4052 (2.122): a line-final dash fuses the wrapped word only
5861    /// when attached to it; a detached dash is a literal and the lines join
5862    /// with a space.
5863    #[test]
5864    fn line_final_hyphen_fuses_only_when_attached_to_a_word() {
5865        let line = |text: &str, t: f32| TextCell {
5866            text: text.to_string(),
5867            l: 0.0,
5868            t,
5869            r: 100.0,
5870            b: t + 10.0,
5871        };
5872        // `algo-` / `rithms`: attached hyphen, alnum on both sides → fused.
5873        assert_eq!(
5874            cells_text(vec![&line("algo-", 0.0), &line("rithms", 12.0)]),
5875            "algorithms"
5876        );
5877        // `pp. 545-` / `561`: attached, digits count as alnum → `545561` (upstream).
5878        assert_eq!(
5879            cells_text(vec![&line("pp. 545-", 0.0), &line("561", 12.0)]),
5880            "pp. 545561"
5881        );
5882        // A dash after whitespace — a separator or a lone `-` cell — is kept and
5883        // the lines take the ordinary joining space.
5884        assert_eq!(
5885            cells_text(vec![&line("range -", 0.0), &line("wide", 12.0)]),
5886            "range - wide"
5887        );
5888        assert_eq!(
5889            cells_text(vec![&line("-", 0.0), &line("item", 12.0)]),
5890            "- item"
5891        );
5892        // Attached but the next line opens with no word (`x-` / `...`): dash
5893        // kept and, as before, no separating space.
5894        assert_eq!(
5895            cells_text(vec![&line("x-", 0.0), &line("...", 12.0)]),
5896            "x-..."
5897        );
5898    }
5899
5900    #[test]
5901    fn lam_alef_only_swaps_a_genuinely_reversed_ligature() {
5902        // A mid-word `alef-variant + lam` is pdfium's reversed lam-alef ligature and
5903        // is swapped back to logical `lam + alef-variant` (`ب أ ل` → `ب ل أ`).
5904        assert_eq!(
5905            clean_text("\u{0628}\u{0623}\u{0644}"),
5906            "\u{0628}\u{0644}\u{0623}"
5907        );
5908        // But when the alef-variant is *already* preceded by a lam it is the logical
5909        // ligature `لآ`; the following lam is the next syllable's letter and must not
5910        // move. `التعلم الآلي` must stay `الآلي`, not become `اللآي`.
5911        assert_eq!(
5912            clean_text("\u{0627}\u{0644}\u{0622}\u{0644}\u{064a}"),
5913            "\u{0627}\u{0644}\u{0622}\u{0644}\u{064a}"
5914        );
5915    }
5916
5917    /// The #419 page, in points: three layout boxes over one paragraph, two of
5918    /// them ending partway through a line. The sliced lines miss the 0.2 claim
5919    /// and become orphans; the third model box starts above the second orphan,
5920    /// so unfitted the reading order emits that box first and strands the line.
5921    fn sliced_paragraph() -> (Vec<Region>, Vec<TextCell>) {
5922        let line = |text: &str, t: f32, r: f32| cell(text, 60.0, t, r, t + 11.0);
5923        let cells = vec![
5924            line("The mission of this series is to improve", 135.0, 458.0),
5925            line("The books in this series are technical,", 147.0, 458.0),
5926            line("substantial. The authors are", 159.0, 458.0),
5927            line("highly experienced craftsmen and", 171.5, 458.0), // sliced: 1.5/11 under box A
5928            line("actually works in practice, as opposed", 185.0, 458.0),
5929            line("about what the author has done, not", 197.0, 458.0),
5930            line("about programming, there will be lots", 210.5, 458.0), // sliced: 1.5/11 under box B
5931            line("will be lots of case studies from real", 223.0, 206.0), // C's line
5932        ];
5933        let regions = vec![
5934            region("text", 0.9, 60.0, 132.0, 458.0, 173.0), // A: three lines + a sliver of the 4th
5935            region("text", 0.9, 60.0, 184.0, 458.0, 212.0), // B: two lines + a sliver of the 7th
5936            region("text", 0.9, 60.0, 216.0, 206.0, 227.0), // C: last line, box opening 5.5pt too early
5937        ];
5938        (regions, cells)
5939    }
5940
5941    fn ordered_texts(regions: &[Region], cells: &[TextCell]) -> Vec<String> {
5942        let mut items: Vec<Region> = regions.to_vec();
5943        let cids = super::cluster_cids(&items, cells);
5944        super::order_regions(&mut items, &cids, 500.0, 700.0, |r| r);
5945        super::region_texts_exclusive(&items, cells)
5946            .into_iter()
5947            .map(|t| t.chars().take(9).collect())
5948            .collect()
5949    }
5950
5951    /// #419: fitted to its cells, a model box that cut a line in half no longer
5952    /// overlaps the orphan that line became, so the orphan orders where it
5953    /// reads; unfitted, the same page strands the line after the paragraph.
5954    #[test]
5955    fn fitting_boxes_to_cells_puts_a_sliced_line_back_in_order() {
5956        let (mut regions, cells) = sliced_paragraph();
5957        super::add_orphan_regions(&mut regions, &cells);
5958        assert_eq!(regions.len(), 5, "two orphan lines");
5959        // The defect, for the record: C (top 216) is not strictly below the
5960        // orphan at 210.5–221.5, so the graph orders C first.
5961        assert_eq!(
5962            ordered_texts(&regions, &cells).last().map(String::as_str),
5963            Some("about pro")
5964        );
5965
5966        super::fit_regions_to_cells(&mut regions, &cells);
5967        assert_eq!(regions.len(), 5);
5968        // A ends on its last claimed line, C starts on its only one.
5969        assert_eq!((regions[0].t, regions[0].b), (135.0, 170.0));
5970        assert_eq!((regions[2].t, regions[2].b), (223.0, 234.0));
5971        assert_eq!(
5972            ordered_texts(&regions, &cells),
5973            [
5974                "The missi",
5975                "highly ex",
5976                "actually ",
5977                "about pro",
5978                "will be l"
5979            ]
5980        );
5981    }
5982
5983    /// An orphan the fitted paragraph box surrounds (a short middle line the
5984    /// narrow model box missed while claiming the lines around it) is folded
5985    /// into the paragraph; an empty regular box goes away, a formula stays, a
5986    /// picture is never refitted, and a page with no cells is left untouched.
5987    #[test]
5988    fn fitting_folds_surrounded_orphans_and_drops_empty_regulars() {
5989        let wide = |text: &str, t: f32| cell(text, 60.0, t, 400.0, t + 11.0);
5990        let cells = vec![
5991            wide("first line of the paragraph", 100.0),
5992            cell("stray", 250.0, 112.0, 400.0, 123.0), // clear of the narrow box
5993            wide("third line of the paragraph", 124.0),
5994        ];
5995        let mut regions = vec![
5996            // Narrow box: claims the wide lines at 0.41, misses the short one.
5997            region("text", 0.9, 60.0, 98.0, 200.0, 136.0),
5998            region("section_header", 0.8, 60.0, 300.0, 200.0, 320.0), // no cells
5999            region("formula", 0.8, 60.0, 340.0, 200.0, 360.0),        // no cells, kept
6000            region("picture", 0.8, 0.0, 400.0, 500.0, 600.0),
6001        ];
6002        super::add_orphan_regions(&mut regions, &cells);
6003        assert_eq!(regions.len(), 5, "the short line became an orphan");
6004        super::fit_regions_to_cells(&mut regions, &cells);
6005        let labels: Vec<&str> = regions.iter().map(|r| r.label).collect();
6006        assert_eq!(labels, ["text", "formula", "picture"]);
6007        let para = &regions[0];
6008        assert_eq!(
6009            (para.l, para.t, para.r, para.b),
6010            (60.0, 100.0, 400.0, 135.0)
6011        );
6012        assert_eq!(
6013            super::region_texts_exclusive(&regions, &cells)[0],
6014            "first line of the paragraph stray third line of the paragraph"
6015        );
6016        assert_eq!(
6017            (regions[2].t, regions[2].b),
6018            (400.0, 600.0),
6019            "picture untouched"
6020        );
6021
6022        let mut untouched = vec![region("text", 0.9, 0.0, 0.0, 10.0, 10.0)];
6023        super::fit_regions_to_cells(&mut untouched, &[]);
6024        assert_eq!(untouched.len(), 1, "no cells yet: nothing dropped");
6025    }
6026
6027    /// #609: one `text` region over a checklist whose lines each have a drawn
6028    /// square in front splits into a checkbox region per boxed line — the
6029    /// wrapped second line of an option stays with it, the intro line above
6030    /// the first square keeps the region's label, and a ticked square gives
6031    /// `checkbox_selected`. Without squares nothing changes.
6032    #[test]
6033    fn checklist_lines_with_squares_split_into_checkbox_regions() {
6034        use crate::checkbox::CheckBox;
6035        let cells = vec![
6036            cell("Pick any:", 100.0, 90.0, 160.0, 100.0),
6037            cell("First option", 120.0, 110.0, 180.0, 120.0),
6038            cell("Second option that", 120.0, 129.0, 200.0, 139.0),
6039            cell("wraps onto a line", 120.0, 141.0, 196.0, 151.0),
6040            cell("Third option", 120.0, 160.0, 182.0, 170.0),
6041        ];
6042        let sq = |t: f32, checked| CheckBox {
6043            l: 100.0,
6044            t,
6045            r: 112.0,
6046            b: t + 12.0,
6047            checked,
6048        };
6049        let boxes = [sq(109.0, false), sq(128.0, true), sq(159.0, false)];
6050        let block = || vec![region("text", 0.6, 100.0, 90.0, 200.0, 170.0)];
6051
6052        let mut regions = block();
6053        super::split_checkbox_lines(&mut regions, &cells, &boxes);
6054        let texts = super::region_texts_exclusive(&regions, &cells);
6055        let got: Vec<(&str, &str)> = regions
6056            .iter()
6057            .zip(&texts)
6058            .map(|(r, t)| (r.label, t.as_str()))
6059            .collect();
6060        assert_eq!(
6061            got,
6062            [
6063                ("text", "Pick any:"),
6064                ("checkbox_unselected", "First option"),
6065                ("checkbox_selected", "Second option that wraps onto a line"),
6066                ("checkbox_unselected", "Third option"),
6067            ]
6068        );
6069        // A checkbox region spans its square and its line(s).
6070        assert_eq!((regions[2].l, regions[2].t), (100.0, 128.0));
6071        assert_eq!((regions[2].r, regions[2].b), (200.0, 151.0));
6072
6073        let shape = |rs: &[Region]| -> Vec<(&'static str, [f32; 4])> {
6074            rs.iter().map(|r| (r.label, [r.l, r.t, r.r, r.b])).collect()
6075        };
6076        let mut untouched = block();
6077        super::split_checkbox_lines(&mut untouched, &cells, &[]);
6078        assert_eq!(shape(&untouched), shape(&block()));
6079        // A square far left of the text (a margin mark), or one the text
6080        // sits inside (a comb field), does not make a checkbox.
6081        for far in [
6082            CheckBox {
6083                l: 40.0,
6084                r: 52.0,
6085                ..sq(109.0, false)
6086            },
6087            CheckBox {
6088                l: 115.0,
6089                r: 127.0,
6090                ..sq(109.0, false)
6091            },
6092        ] {
6093            let mut regions = block();
6094            super::split_checkbox_lines(&mut regions, &cells, &[far]);
6095            assert_eq!(shape(&regions), shape(&block()));
6096        }
6097    }
6098
6099    /// #609: a ballot-box glyph opening a line marks a checkbox item like a
6100    /// drawn square — `☐` unchecked, `☒` checked, the glyph stripped from the
6101    /// label at emission — while a line holding two boxes (`☐ Yes ☐ No`)
6102    /// stays text of its own.
6103    #[test]
6104    fn checklist_lines_opening_with_a_ballot_box_split_into_checkbox_regions() {
6105        let cells = vec![
6106            cell("Options:", 100.0, 90.0, 150.0, 100.0),
6107            cell("\u{2610} Tea", 100.0, 110.0, 140.0, 120.0),
6108            cell("\u{2612} Coffee", 100.0, 130.0, 150.0, 140.0),
6109            cell("\u{2610} Yes \u{2610} No", 100.0, 150.0, 170.0, 160.0),
6110        ];
6111        let mut regions = vec![region("text", 0.6, 100.0, 90.0, 170.0, 160.0)];
6112        super::split_checkbox_lines(&mut regions, &cells, &[]);
6113        let texts = super::region_texts_exclusive(&regions, &cells);
6114        let got: Vec<(&str, &str)> = regions
6115            .iter()
6116            .zip(&texts)
6117            .map(|(r, t)| match r.label {
6118                "text" => (r.label, t.as_str()),
6119                _ => (r.label, super::strip_checkbox_glyph(t)),
6120            })
6121            .collect();
6122        assert_eq!(
6123            got,
6124            [
6125                ("text", "Options:"),
6126                ("checkbox_unselected", "Tea"),
6127                ("checkbox_selected", "Coffee"),
6128                // Two boxes on one line: its own text, the item closed.
6129                ("text", "\u{2610} Yes \u{2610} No"),
6130            ]
6131        );
6132        assert_eq!(super::strip_checkbox_glyph("  \u{2611}  Done"), "Done");
6133        assert_eq!(super::strip_checkbox_glyph("No box"), "No box");
6134    }
6135
6136    #[test]
6137    fn only_note_calls_move_behind_their_line() {
6138        for t in ["(Anm. 1)", "[3]", "12", "*", "†", "¹²", "1,2"] {
6139            assert!(looks_like_note_call(t), "{t}");
6140        }
6141        // Figure labels a size smaller than their neighbours stay put.
6142        for t in [
6143            "be",
6144            "perfect",
6145            "C",
6146            "apply",
6147            "RoPE",
6148            "",
6149            ",",
6150            "(this is a long aside)",
6151        ] {
6152            assert!(!looks_like_note_call(t), "{t}");
6153        }
6154        let cell = |text: &str, l: f32, t: f32, r: f32, b: f32| TextCell {
6155            text: text.into(),
6156            l,
6157            t,
6158            r,
6159            b,
6160        };
6161        let host = cell("und die Belegschaft besteht.", 100.0, 100.0, 300.0, 112.0);
6162        assert!(is_note_of(
6163            &cell("(Anm. 1)", 302.0, 98.0, 330.0, 106.0),
6164            &host
6165        ));
6166        assert!(!is_note_of(
6167            &cell("perfect", 302.0, 98.0, 330.0, 106.0),
6168            &host
6169        ));
6170    }
6171
6172    /// docling#3929 `TestSanitizeTextArabicPresentationForms` (#1001): shaped
6173    /// Arabic folds to the canonical letters, per run — other scripts' NFKC
6174    /// compatibility forms are left alone.
6175    #[test]
6176    fn arabic_presentation_forms_fold_like_docling() {
6177        assert_eq!(clean_text("ﻣﺎﺮﺱ"), "مارس");
6178        assert_eq!(clean_text("ﻻ"), "لا");
6179        assert_eq!(clean_text("ﭘ"), "پ");
6180        assert!(clean_text("ﺍﻟﺤﺮﺱ").contains("الحرس"));
6181        assert_eq!(clean_text("مرحبا"), "مرحبا");
6182        assert_eq!(clean_text("x²"), "x²");
6183        assert_eq!(clean_text("H₂O"), "H₂O");
6184        let fullwidth: String = [0xFF46, 0xFF55, 0xFF4C, 0xFF4C]
6185            .iter()
6186            .map(|&c| char::from_u32(c).unwrap())
6187            .collect();
6188        assert_eq!(clean_text(&fullwidth), fullwidth);
6189        assert_eq!(clean_text("2024 ﻣﺎﺮﺱ x²"), "2024 مارس x²");
6190    }
6191
6192    /// docling#5069's ligature map (#1001): `ſt` is `ſt` → `st`, Dutch `IJ`/`ij`
6193    /// spell out, the spurious U+F0A0 glyph goes and a real word boundary
6194    /// after it stays.
6195    #[test]
6196    fn ligatures_and_the_private_use_glyph_fold_like_docling() {
6197        assert_eq!(clean_text("\u{FB05}ar \u{FB06}op"), "star stop");
6198        assert_eq!(clean_text("\u{132} is \u{133}s"), "IJ is ijs");
6199        assert_eq!(clean_text("hello\u{F0A0} world"), "hello world");
6200        assert_eq!(clean_text("con\u{FB01}gure"), "configure");
6201    }
6202
6203    /// docling#5069: TableFormer cells get body text's folds (they kept
6204    /// `it’s`, `“quoted”`, `• dot` and shaped Arabic), unless
6205    /// `pdf_keep_typography` keeps the typography.
6206    #[test]
6207    fn table_cells_are_normalized_like_body_text() {
6208        use super::{normalize_grid, TableGrid, TypographyScope};
6209        let grid = || TableGrid {
6210            rows: vec![vec![
6211                "it\u{2019}s".into(),
6212                "\u{201c}quoted\u{201d}".into(),
6213                "\u{2022} dot".into(),
6214                "ﻣﺎﺮﺱ".into(),
6215            ]],
6216            cells: vec![docling_core::TableCell {
6217                text: "\u{201c}a\u{FB01}\u{201d}".into(),
6218                bbox: None,
6219                start_row: 0,
6220                start_col: 0,
6221                row_span: 1,
6222                col_span: 1,
6223                column_header: false,
6224                row_header: false,
6225                row_section: false,
6226            }],
6227        };
6228        let g = normalize_grid(grid());
6229        assert_eq!(g.rows[0], ["it's", "\"quoted\"", "\u{b7} dot", "مارس"]);
6230        assert_eq!(g.cells[0].text, "\"afi\"");
6231        let _keep = TypographyScope::set(true);
6232        let g = normalize_grid(grid());
6233        assert_eq!(
6234            g.rows[0],
6235            [
6236                "it\u{2019}s",
6237                "\u{201c}quoted\u{201d}",
6238                "\u{2022} dot",
6239                "مارس"
6240            ]
6241        );
6242    }
6243}