xberg_native_pdf/document/span_postprocess.rs
1//! Post-extraction span rotation, composition, and banding.
2//!
3//! Split out of the parent's single 18,544-line `impl PdfDocument`, which made
4//! `document.rs` 1.2 MiB and tripped the 500 KiB file-safety limit. A child module's
5//! `impl` is the same inherent impl and sees the parent's private items unchanged. ~keep
6
7use super::*;
8
9impl PdfDocument {
10 /// Map a span rectangle (already translated so the page origin is at
11 /// `(0, 0)`) through a clockwise page `/Rotate` of `rot` degrees, returning
12 /// the axis-aligned bounding box in the displayed coordinate frame.
13 ///
14 /// `page_w` / `page_h` are the unrotated page dimensions; for 90° / 270° the
15 /// displayed page is `page_h × page_w`. Per ISO 32000-1:2008 §7.7.3.3 the
16 /// rotation is clockwise and §8.3.3 gives the point transform. `rot` must be
17 /// a normalised multiple of 90 (`0/90/180/270`); any other value returns the
18 /// rectangle unchanged. `rot == 0` is the identity and `rot == 180` is
19 /// numerically identical to the legacy mirror, preserving byte-for-byte
20 /// output for unrotated and 180° pages.
21 pub(crate) fn rotate_span_bbox(
22 bbox: crate::geometry::Rect,
23 rot: i32,
24 page_w: f32,
25 page_h: f32,
26 ) -> crate::geometry::Rect {
27 // Map a point (y-up) by the clockwise display rotation. ~keep
28 let map = |x: f32, y: f32| -> (f32, f32) {
29 match rot {
30 90 => (y, page_w - x),
31 180 => (page_w - x, page_h - y),
32 270 => (page_h - y, x),
33 _ => (x, y),
34 }
35 };
36 let (ax, ay) = map(bbox.x, bbox.y);
37 let (bx, by) = map(bbox.x + bbox.width, bbox.y + bbox.height);
38 crate::geometry::Rect::new(ax.min(bx), ay.min(by), (ax - bx).abs(), (ay - by).abs())
39 }
40
41 /// Map a single span's bbox into the displayed frame for a `/Rotate`d page
42 /// (translate to origin → [`rotate_span_bbox`] → translate back).
43 ///
44 /// This is the only place `TextSpan::page_rotation_applied` is ever written,
45 /// and it is reached only through `postprocess_spans` — i.e. from
46 /// [`Self::extract_spans`] / `extract_spans_filtered`, never from
47 /// [`Self::extract_spans_filtered_with_reading_order`] (see that function's
48 /// doc comment). Anything reading `page_rotation_applied` off a span must
49 /// know which of the two extraction paths produced it.
50 fn map_span_into_rotated_frame(s: &mut crate::layout::TextSpan, rot: i32, llx: f32, lly: f32, w: f32, h: f32) {
51 let rel = crate::geometry::Rect::new(s.bbox.x - llx, s.bbox.y - lly, s.bbox.width, s.bbox.height);
52 let m = Self::rotate_span_bbox(rel, rot, w, h);
53 s.bbox.x = llx + m.x;
54 s.bbox.y = lly + m.y;
55 s.bbox.width = m.width;
56 s.bbox.height = m.height;
57 // `rotation_degrees` stays raw (downstream passes select on it), so
58 // record the applied rotation for `TextSpan::page_bbox` — otherwise it
59 // would re-rotate the already-mapped rect. ~keep
60 s.page_rotation_applied = rot;
61 }
62
63 /// Order rotated runs that were segregated out of the horizontal reading
64 /// flow. Spans drawn with a rotated text matrix (`rotation_degrees != 0`)
65 /// break the axis-aligned assumptions of the row-band / XY-cut sort, so they
66 /// are pulled out, ordered here, and appended as their own blocks. Runs are
67 /// grouped by rotation (first-seen group order preserved); within a group
68 /// each span's origin is rotated back into an upright frame and the standard
69 /// row-aware comparator (top→bottom, left→right) is applied there.
70 pub(crate) fn order_rotated_blocks(spans: Vec<crate::layout::TextSpan>) -> Vec<crate::layout::TextSpan> {
71 let mut groups: Vec<(f32, Vec<crate::layout::TextSpan>)> = Vec::new();
72 for s in spans {
73 let key = s.rotation_degrees;
74 match groups.iter_mut().find(|(k, _)| (*k - key).abs() < 0.5) {
75 Some(g) => g.1.push(s),
76 None => groups.push((key, vec![s])),
77 }
78 }
79 let mut out = Vec::new();
80 for (deg, mut group) in groups {
81 let (sin, cos) = (-deg).to_radians().sin_cos();
82 // Upright frame: rotate each origin by -deg, then read top→bottom,
83 // left→right exactly as horizontal text. ~keep
84 group.sort_by(|a, b| {
85 let ax = a.bbox.x * cos - a.bbox.y * sin;
86 let ay = a.bbox.x * sin + a.bbox.y * cos;
87 let bx = b.bbox.x * cos - b.bbox.y * sin;
88 let by = b.bbox.x * sin + b.bbox.y * cos;
89 crate::utils::row_aware_span_cmp(ay, ax, by, bx)
90 });
91 out.extend(group);
92 }
93 out
94 }
95
96 /// Re-attach an oversized lone leading capital (a drop-cap / table-title
97 /// initial that the producer set in a larger font, so it became its own
98 /// span) to the body run immediately to its right on the same line —
99 /// otherwise reading-order strands it (`TABLE` → `T` … `ABLE`).
100 ///
101 /// Conservative gates so prose drop-caps / standalone capitals aren't glued
102 /// to the wrong word: the candidate must be a single uppercase ASCII letter
103 /// at ≥1.5× the body run's font size, its right edge within ~0.3 em of the
104 /// body's left edge, vertically overlapping it, and the body must start with
105 /// a letter. Runs in raw span order before reading-order sorting.
106 pub(super) fn merge_drop_cap_initials(spans: &mut Vec<crate::layout::TextSpan>) {
107 let n = spans.len();
108 if n < 2 {
109 return;
110 }
111 // A genuine drop cap is oversized relative to the page's *normal* body
112 // text, not merely relative to its right-hand neighbor. Inline math such
113 // as "A_st" pairs a normal-size capital with a shrunken subscript; gating
114 // on the neighbor alone would treat that capital as oversized and glue
115 // "A" + "st" into "Ast". Anchor the size gate to the median size of
116 // multi-character spans (real words) so a body-size capital cannot
117 // qualify. ~keep
118 let mut body_sizes: Vec<f32> = spans
119 .iter()
120 .filter(|s| s.font_size > 0.0 && s.text.chars().nth(1).is_some())
121 .map(|s| s.font_size)
122 .collect();
123 if body_sizes.is_empty() {
124 return;
125 }
126 body_sizes.sort_by(|a, b| crate::utils::safe_float_cmp(*a, *b));
127 let body_size = body_sizes[body_sizes.len() / 2];
128
129 // Span indices sorted by left edge, and the widest font on the page, so
130 // each initial only probes spans whose left edge falls in its narrow
131 // candidate x-window (was a full O(n) rescan per initial). A continuation
132 // satisfies `gap in [-fs*0.5, fs*0.12]`, i.e. its left edge is within
133 // [init_right - max_fs*0.5, init_right + max_fs*0.12]; using the page max
134 // font widens the window conservatively, and the exact per-candidate gap
135 // test below reproduces the original filter — so this is byte-identical. ~keep
136 let order: Vec<usize> = {
137 let mut o: Vec<usize> = (0..n).collect();
138 o.sort_by(|&a, &b| crate::utils::safe_float_cmp(spans[a].bbox.x, spans[b].bbox.x));
139 o
140 };
141 let max_fs = spans.iter().map(|s| s.font_size).fold(0.0_f32, f32::max);
142
143 let mut target: Vec<Option<usize>> = vec![None; n];
144 for i in 0..n {
145 let init = &spans[i];
146 if init.text.chars().count() != 1 || init.font_size <= 0.0 {
147 continue;
148 }
149 if !init.text.chars().next().is_some_and(|c| c.is_ascii_uppercase()) {
150 continue;
151 }
152 if init.font_size < body_size * 1.5 {
153 continue;
154 }
155 let init_right = init.bbox.x + init.bbox.width;
156 // Candidates: spans whose left edge is in the conservative window.
157 // Collect their indices and visit in ASCENDING ORIGINAL ORDER so the
158 // strict-`<` min keeps the same first-wins tie-break as the old scan. ~keep
159 let lo_x = init_right - max_fs * 0.5;
160 let hi_x = init_right + max_fs * 0.12;
161 let lo = order.partition_point(|&k| spans[k].bbox.x < lo_x);
162 let hi = order.partition_point(|&k| spans[k].bbox.x <= hi_x);
163 let mut cands: Vec<usize> = order[lo..hi].to_vec();
164 cands.sort_unstable();
165 let mut best: Option<usize> = None;
166 let mut best_gap = f32::MAX;
167 for &j in &cands {
168 let body = &spans[j];
169 if j == i || body.font_size <= 0.0 {
170 continue;
171 }
172 if !body.text.chars().next().is_some_and(|c| c.is_alphabetic()) {
173 continue;
174 }
175 // Continuation shares the initial's baseline (same text line). A
176 // tall oversized initial also vertically overlaps the line *above*
177 // it, so a raw bbox-overlap test would let it reach up and steal a
178 // word from the previous line (alice_old: the 16.8pt "A" of "A very
179 // heavy weight" overlapping "Or if" → "OrAif"). Baseline proximity
180 // (≈ bbox bottom) keeps the merge on the initial's own line. ~keep
181 if (init.bbox.y - body.bbox.y).abs() > body.font_size * 0.5 {
182 continue;
183 }
184 // Body immediately to the right, essentially touching. A genuine
185 // oversized initial is the first glyph of one word ("T" of
186 // "TABLE", "P" of "PENALTY"), so its continuation begins within a
187 // hair of the initial's advance — never across a word space. A
188 // word-space gap (~0.25 em) would wrongly glue a standalone "A"
189 // or "I" onto the next word ("A Perspective" → "APerspective"),
190 // so the upper bound stays well below it. ~keep
191 let gap = body.bbox.x - init_right;
192 if gap < -body.font_size * 0.5 || gap > body.font_size * 0.12 {
193 continue;
194 }
195 if gap.abs() < best_gap {
196 best_gap = gap.abs();
197 best = Some(j);
198 }
199 }
200 target[i] = best;
201 }
202
203 let mut taken = vec![false; n];
204 let mut remove = vec![false; n];
205 for i in 0..n {
206 let Some(j) = target[i] else { continue };
207 if taken[j] || remove[j] || remove[i] {
208 continue;
209 }
210 taken[j] = true;
211 remove[i] = true;
212 let init_text = spans[i].text.clone();
213 let init_left = spans[i].bbox.x;
214 let body = &mut spans[j];
215 body.text = format!("{init_text}{}", body.text);
216 let right = body.bbox.x + body.bbox.width;
217 body.bbox.x = init_left.min(body.bbox.x);
218 body.bbox.width = right - body.bbox.x;
219 }
220 let mut k = 0;
221 spans.retain(|_| {
222 let keep = !remove[k];
223 k += 1;
224 keep
225 });
226 }
227
228 /// True for Computer-Modern (`CM*`) or symbol font names, after stripping a
229 /// `ABCDEF+` subset tag. Used to scope the `¬`→`.` decimal recovery.
230 pub(super) fn is_cm_or_symbol_font(font_name: &str) -> bool {
231 let base = font_name.split('+').next_back().unwrap_or(font_name);
232 let lower = base.to_ascii_lowercase();
233 lower.starts_with("cm") || lower.contains("symbol")
234 }
235
236 /// Replace a `¬` (U+00AC) that a math subset drew from its `logicalnot`
237 /// slot as a decimal point. Two shapes are recovered:
238 ///
239 /// - `digit ¬ digit` → `digit.digit` (e.g. `1¬00` → `1.00`)
240 /// - `digit ¬ <space> digit` → `digit.digit` (e.g. `1¬ 00` → `1.00`)
241 ///
242 /// The second form covers subsets that emit a single space between the
243 /// decimal glyph and the fractional digits; the lone separating space is
244 /// dropped so the number reads as one token. The leading digit must abut
245 /// `¬` directly in both shapes, so a genuinely spaced negation (`5 ¬ 3`,
246 /// `A ¬ B`) is left untouched. Every other `¬` is preserved.
247 pub(super) fn fix_digit_logicalnot_decimal(text: &str) -> String {
248 let chars: Vec<char> = text.chars().collect();
249 let mut out = String::with_capacity(text.len());
250 let mut i = 0;
251 while i < chars.len() {
252 let c = chars[i];
253 if c == '\u{00AC}' && i > 0 && chars[i - 1].is_ascii_digit() {
254 if chars.get(i + 1).is_some_and(|n| n.is_ascii_digit()) {
255 out.push('.');
256 i += 1;
257 continue;
258 }
259 if chars.get(i + 1) == Some(&' ') && chars.get(i + 2).is_some_and(|n| n.is_ascii_digit()) {
260 out.push('.');
261 i += 2;
262 continue;
263 }
264 }
265 out.push(c);
266 i += 1;
267 }
268 out
269 }
270
271 /// Drop spans whose bbox lies ENTIRELY outside the page's MediaBox.
272 ///
273 /// PDFs that reuse one big Form XObject across pages (ExpertPdf and similar
274 /// tools - see issue B1 / nougat_005.pdf) rely on the content stream's `W n`
275 /// clip rectangle to hide the off-page portion. The text extractor does not
276 /// honour `W n` yet, so without this filter a page emits every page's worth of
277 /// spans at distinct but out-of-bounds Y coordinates. Spans that even
278 /// PARTIALLY overlap the MediaBox are kept, so legitimate bleed / trim-mark
279 /// content is never dropped.
280 ///
281 /// `get_page_media_box` returns `(llx, lly, urx, ury)` - absolute corner
282 /// coordinates per ISO 32000-1 s7.7.3.3, NOT `(x, y, width, height)`.
283 pub(super) fn drop_offpage_spans(&self, page_index: usize, spans: &mut Vec<crate::layout::TextSpan>) {
284 if let Ok((llx, lly, urx, ury)) = self.get_page_media_box(page_index) {
285 const EDGE_TOLERANCE_PT: f32 = 2.0;
286 // Normalise corners: some producers write the MediaBox with swapped
287 // corners (e.g. `[0 792 612 0]`, ury < lly). Taking min/max makes the
288 // bounds correct either way - without this a swapped box inverts the
289 // test below and drops the whole page's legitimate text. ~keep
290 let left = llx.min(urx) - EDGE_TOLERANCE_PT;
291 let right = llx.max(urx) + EDGE_TOLERANCE_PT;
292 let bottom = lly.min(ury) - EDGE_TOLERANCE_PT;
293 let top = lly.max(ury) + EDGE_TOLERANCE_PT;
294 spans.retain(|span| {
295 let sx1 = span.bbox.x;
296 let sx2 = span.bbox.x + span.bbox.width;
297 let sy1 = span.bbox.y;
298 let sy2 = span.bbox.y + span.bbox.height;
299 sx2 > left && sx1 < right && sy2 > bottom && sy1 < top
300 });
301 }
302 }
303
304 pub(super) fn postprocess_spans(
305 &self,
306 page_index: usize,
307 raw_spans: Vec<crate::layout::TextSpan>,
308 ) -> Result<Vec<crate::layout::TextSpan>> {
309 let mut spans = raw_spans;
310
311 self.drop_offpage_spans(page_index, &mut spans);
312
313 // Recover decimal points mis-decoded as `¬` (U+00AC) in Computer-Modern
314 // math subsets, where the `/Differences` names the decimal slot
315 // `logicalnot`. Only a `¬` sitting *directly between two digits* (no
316 // space) is rewritten — real logic/set `¬` is always spaced, so this
317 // cannot corrupt it. ~keep
318 for span in &mut spans {
319 if Self::is_cm_or_symbol_font(&span.font_name) && span.text.contains('\u{00AC}') {
320 span.text = Self::fix_digit_logicalnot_decimal(&span.text);
321 }
322 }
323
324 // Re-attach oversized lone leading capitals to their word before the
325 // reading-order sort can strand them (drop-cap / table-title initials). ~keep
326 Self::merge_drop_cap_initials(&mut spans);
327
328 // Apply page /Rotate to span geometry BEFORE reading-order sorting.
329 //
330 // A page with a /Rotate entry must be read in its DISPLAYED orientation
331 // or the row-aware sort emits text in the wrong order (pdf.js issue14415
332 // is a 180° English page that otherwise comes out word- and line-reversed).
333 //
334 // The transform is applied selectively, because a rotated page carries two
335 // very different kinds of run, distinguished by each span's own
336 // `rotation_degrees` (the content-stream text-matrix rotation):
337 //
338 // * **Horizontal content (`rotation_degrees == 0`) on a 90°/270° page** —
339 // e.g. a landscape table stored rotated (`/Rotate 90`, MediaBox already
340 // landscape). This text is horizontal in raw user space, so it reads and
341 // groups correctly THERE. Rotating its bbox by ±90° only rotates the
342 // RECTANGLE, but `TextSpan::to_chars` still lays glyphs horizontally with
343 // raw advance widths and cannot express a now-vertical run, so every raw
344 // row collapses onto one displayed band and perpendicular columns fuse
345 // into one 1000+ char token. These are LEFT RAW — matching
346 // `extract_chars`, which also returns raw coordinates.
347 //
348 // * **Rotated content (`rotation_degrees == ±90`) on a 90°/270° page** —
349 // e.g. a chart axis, a sideways table, or a whole landscape page authored
350 // by drawing every glyph sideways in a portrait MediaBox with `/Rotate 90`
351 // to present it upright. Here the page /Rotate must be applied so it
352 // COMBINES with the content rotation (which `order_rotated_blocks` undoes
353 // for ordering) into the correct upright displayed frame; leaving it raw
354 // reads the page sideways. These ARE mapped.
355 //
356 // 180° maps everything (text stays horizontal; both axes just mirror —
357 // numerically identical to the legacy mirror).
358 //
359 // Captured so the same transform is applied to annotation spans appended
360 // later (their /Rect is in unrotated page space too). `None` for rot == 0
361 // or unknown media box — those pages keep raw geometry. ~keep
362 let page_rotation: Option<(i32, f32, f32, f32, f32)> = match self.get_page_media_box(page_index) {
363 Ok((llx, lly, urx, ury)) => {
364 let rot = self.get_page_rotation(page_index).unwrap_or(0).rem_euclid(360);
365 matches!(rot, 90 | 180 | 270).then_some((rot, llx, lly, urx - llx, ury - lly))
366 }
367 Err(_) => None,
368 };
369 if let Some((rot, llx, lly, w, h)) = page_rotation {
370 for s in spans.iter_mut() {
371 // 90°/270°: only map runs whose own content is rotated; horizontal
372 // content stays in raw user space (see rationale above). ~keep
373 if rot != 180 && s.rotation_degrees == 0.0 {
374 continue;
375 }
376 Self::map_span_into_rotated_frame(s, rot, llx, lly, w, h);
377 }
378 }
379
380 // Tategaki (vertical writing) intercept. Pages whose majority of
381 // spans were emitted under WMode 1 (font /Encoding ends in -V or
382 // the CMap declares /WMode 1) need right-to-left, top-to-bottom
383 // ordering. Row-aware / XY-cut sorts assume horizontal flow and
384 // scramble vertical text; per-span wmode lets us route just those
385 // pages through a tategaki comparator while leaving every existing
386 // horizontal corpus untouched. ~keep
387 let vertical_count = spans.iter().filter(|s| s.wmode == 1).count();
388 if !spans.is_empty() && vertical_count * 2 >= spans.len() {
389 // See `crate::utils::sort_vertical_tategaki` for the
390 // column-clustering algorithm and the total-order rationale. ~keep
391 spans = crate::utils::sort_vertical_tategaki(spans, |s| &s.bbox);
392 } else if let Some(ordered) = Self::sidebar_body_reading_order(&spans) {
393 // RW-1: narrow-sidebar + wide-body first pages (full-width title band
394 // over a metadata sidebar + body). Handled before the XY-cut so the
395 // title is not sliced along the body gutter (§14.8.3). ~keep
396 spans = ordered;
397 } else if Self::is_multi_column_page(&spans) {
398 use crate::pipeline::reading_order::{
399 ReadingOrderContext as ROContext, ReadingOrderStrategy, XYCutStrategy,
400 };
401 let strategy = XYCutStrategy::new();
402 // `is_multi_column_page` has already established there IS a
403 // gutter but returns only a bool, so re-measure its X here for
404 // the heading-run pre-pass (GH#1757). ~keep
405 let column_gutter = Self::detect_column_gutter(&spans);
406 let mut context = ROContext::new().with_page(page_index as u32);
407 if let Some(gutter_x) = column_gutter {
408 context = context.with_column_gutter(gutter_x);
409 }
410 // Clone needed: apply() takes ownership, and the Err branch
411 // falls back to sorting the original vec in place. ~keep
412 match strategy.apply(spans.clone(), &context) {
413 Ok(ordered) => {
414 spans = ordered.into_iter().map(|o| o.span).collect();
415 }
416 Err(_) => {
417 tracing::warn!(target: LOG_TARGET,
418 page_index,
419 error_code = "reading_order_error",
420 "XY-cut reading order failed; falling back to row-aware sort"
421 );
422 spans.sort_by(|a, b| crate::utils::row_aware_span_cmp(a.bbox.y, a.bbox.x, b.bbox.y, b.bbox.x));
423 Self::reorder_rowspan_labels(&mut spans);
424 }
425 }
426 } else {
427 spans.sort_by(|a, b| crate::utils::row_aware_span_cmp(a.bbox.y, a.bbox.x, b.bbox.y, b.bbox.x));
428 Self::reorder_rowspan_labels(&mut spans);
429 }
430
431 reorder_ltr_key_value_rows(&mut spans);
432
433 // Per-span rotation firewall. Runs drawn with a rotated text matrix
434 // (the vertical `arXiv:…` margin stamp, figure/axis labels, rotated
435 // table headers, transit-poster route names) break the axis-aligned
436 // row-band / XY-cut assumptions, so interleaving them with the
437 // horizontal flow scrambles reading order. The reordering above ran on
438 // the FULL span set (so its column/XY-cut decisions are unchanged — the
439 // horizontal body keeps its exact baseline order); now stably lift the
440 // rotated runs out (preserving horizontal order) and re-append them as
441 // their own blocks, each ordered in an upright frame. No-op (and ~keep
442 // byte-identical) when the page has no rotated spans.
443 if spans.iter().any(|s| s.rotation_degrees != 0.0) {
444 let rotated: Vec<crate::layout::TextSpan> =
445 spans.iter().filter(|s| s.rotation_degrees != 0.0).cloned().collect();
446 spans.retain(|s| s.rotation_degrees == 0.0);
447 spans.extend(Self::order_rotated_blocks(rotated));
448 }
449
450 let erase = self.erase_regions.lock_or_recover().get(&page_index).cloned();
451 if let Some(regions) = erase {
452 spans.retain(|span| !regions.iter().any(|r| r.intersects(&span.bbox)));
453 }
454
455 // Append text from non-Widget annotations (/Subtype /Text, FreeText,
456 // Stamp, Highlight, etc.) that carry a /Contents entry. These are not
457 // part of the page content stream so they are not picked up by the
458 // regular extractor. On a /Rotate'd page their /Rect-derived bboxes are
459 // in unrotated page space, so map the appended spans into the same
460 // displayed frame as the content spans (no-op for unrotated pages).
461 // Annotation text is horizontal (rotation_degrees == 0), so on a 90°/270°
462 // page it stays raw, matching the horizontal content spans above. ~keep
463 let pre_annotation_len = spans.len();
464 spans.extend(self.annotation_content_spans(page_index));
465 if let Some((rot, llx, lly, w, h)) = page_rotation {
466 for s in spans[pre_annotation_len..].iter_mut() {
467 if rot != 180 && s.rotation_degrees == 0.0 {
468 continue;
469 }
470 Self::map_span_into_rotated_frame(s, rot, llx, lly, w, h);
471 }
472 }
473
474 // Mark running headers/footers (untagged-PDF heuristic). Spans whose
475 // normalized text recurs on >=50% of pages and sits near the top or
476 // bottom of the page are flagged as artifacts so downstream filters
477 // drop them. ~keep
478 self.mark_running_artifact_spans(page_index, &mut spans)?;
479
480 // Normalize Unicode typographic spaces (U+2000–U+200B, U+202F, U+205F)
481 // to ASCII space. Some PDF producers encode word separators as hair-space
482 // or thin-space variants in ToUnicode CMaps (e.g. justified text layouts);
483 // normalising here gives consistent word boundaries to every downstream
484 // consumer (extract_text, word-F1 scoring, etc.). ~keep
485 for span in &mut spans {
486 if span
487 .text
488 .chars()
489 .any(|c| matches!(c, '\u{2000}'..='\u{200B}' | '\u{202F}' | '\u{205F}'))
490 {
491 span.text =
492 crate::converters::text_post_processor::TextPostProcessor::normalize_unicode_spaces(&span.text)
493 .into_owned();
494 }
495 }
496
497 // Apply char_widths boundary splits directly to span.text so that every
498 // downstream consumer (to_markdown, to_html, extract_text) sees the same
499 // word boundaries. extract_text applies the same logic through push_span_text;
500 // after this normalization push_span_text sees a space at the boundary
501 // becomes a no-op, so there is no double-application risk. ~keep
502 for span in &mut spans {
503 if let Some(split) = Self::char_widths_boundary_split(span) {
504 let mut t = String::with_capacity(span.text.len() + 1);
505 t.push_str(&span.text[..split]);
506 t.push(' ');
507 t.push_str(&span.text[split..]);
508 span.text = t;
509 }
510 }
511
512 // Detect superscript / subscript runs and substitute ASCII
513 // digits with their Unicode super/sub-script equivalents
514 // (only when the run is sandwiched between alphabetic body
515 // spans on both sides — chemistry/math context like "S²X"
516 // or "H₂O"). The same substitution would otherwise fire on
517 // author-affiliation markers ("name¹,²") which the bench
518 // ground truth keeps in ASCII; gating on token-internal
519 // context keeps the desired cases without regressing the
520 // affiliation-block pages. ~keep
521 Self::apply_super_sub_script_substitutions(&mut spans);
522
523 // Fold spacing-diacritic spans (´, `, ^, ~, ¨, …) into the
524 // base letter of the following span when the diacritic is
525 // centred over the base glyph. PDFs that pre-shape accented
526 // Latin (LaTeX `\'E` → two glyphs, `acute` then `E`) emit
527 // the marks as separate `Tj` ops at the base glyph's X
528 // coordinate. Without this pass extract_text returns the
529 // raw two-glyph order "´Ecole" instead of "École". ~keep
530 Self::apply_combining_mark_composition(&mut spans);
531
532 // Stamp accurate per-glyph x-origins onto the finalized spans so that
533 // `to_chars()` (and thus extract_words / extract_spans /
534 // extract_text_lines, which all decompose spans through it) reports
535 // spec-aligned positions instead of drifting prefix-sums. Runs last, on
536 // the fully post-processed spans, so alignment sees the same text the ~keep
537 // consumers do.
538 self.stamp_char_x_offsets(page_index, &mut spans);
539
540 Ok(spans)
541 }
542
543 /// Copy the spec-aligned per-glyph baseline x-origins from the char-level
544 /// extractor onto each span's [`char_x_offsets`](crate::layout::TextSpan::char_x_offsets).
545 ///
546 /// # Why
547 ///
548 /// [`TextSpan::to_chars`](crate::layout::TextSpan::to_chars) otherwise
549 /// reconstructs each glyph's x by prefix-summing the span's nominal
550 /// `char_widths` from `bbox.x`. Those nominal widths omit the ISO
551 /// 32000-1:2008 §9.4.3 TJ-array adjustment (the number in a TJ array is
552 /// "expressed in thousandths of a unit of text space … subtracted from the
553 /// current … coordinate") and the full §9.4.4 text-space displacement
554 /// (`t_x = ((w0 − Tj/1000) · Tfs + Tc + Tw) · Th`). Prefix-summing the
555 /// nominal widths therefore drifts cumulatively along a line. The
556 /// char-level extractor that [`extract_chars`](Self::extract_chars) uses
557 /// implements §9.4.4 / §9.4.3 in full (it matches Poppler `pdftotext
558 /// -bbox`), so its `origin_x` values are the authoritative positions this
559 /// function stamps back onto the spans.
560 ///
561 /// # Alignment (robust, per span)
562 ///
563 /// A naive global greedy walk on char values mis-jumps on repeated
564 /// letters / spaces. Instead, for each span we take only the accurate chars
565 /// on the SAME baseline (`|origin_y − span.bbox.y| ≤ 0.5·font_size`), sort
566 /// them by x, and match the span's glyph sequence as a CONTIGUOUS run,
567 /// choosing the run whose first glyph's `origin_x` is nearest `span.bbox.x`.
568 ///
569 /// # Fallback (never guess)
570 ///
571 /// If a span cannot be fully, unambiguously aligned — no contiguous run of
572 /// exactly the span's glyphs exists on its line (count mismatch from a
573 /// post-processing text edit, ligature expansion, a synthetic space glyph
574 /// not present in the char stream, …) — its `char_x_offsets` is left empty
575 /// so `to_chars` uses the legacy prefix-sum path. A cleared span is a
576 /// no-op, never a regression. `char_widths` is never touched (a downstream
577 /// word-boundary heuristic keys off its length).
578 ///
579 /// 180° pages are skipped entirely: the spans are mirrored into the displayed
580 /// frame while the accurate chars remain in unrotated page space, so a
581 /// horizontal-x stamp would not correspond. On 90°/270° pages, horizontal
582 /// content spans stay in raw user space and ARE stamped, but rotated-content
583 /// spans (`rotation_degrees != 0`) have been mapped into the displayed frame
584 /// (see `postprocess_spans`) and their glyphs run along a rotated axis, so a
585 /// horizontal-x stamp would misalign — those individual spans are skipped.
586 fn stamp_char_x_offsets(&self, page_index: usize, spans: &mut [crate::layout::TextSpan]) {
587 // Horizontal-x offsets only make sense in an unrotated frame; the 180°
588 // mirror is the one rotation that leaves ALL spans in the displayed frame. ~keep
589 if self.get_page_rotation(page_index).unwrap_or(0).rem_euclid(360) == 180 {
590 return;
591 }
592
593 let accurate = match self.cached_page_chars(page_index) {
594 Ok(chars) if !chars.is_empty() => chars,
595 _ => return,
596 };
597
598 // Baseline index: char positions ordered by `origin_y`, so each span's
599 // baseline slice is a binary-searched range rather than a linear scan
600 // over every glyph on the page (that scan made this pass
601 // O(spans x chars) — the dominant per-page cost on long documents). ~keep
602 let mut by_y: Vec<u32> = (0..accurate.len() as u32).collect();
603 by_y.sort_by(|&a, &b| {
604 crate::utils::safe_float_cmp(accurate[a as usize].origin_y, accurate[b as usize].origin_y)
605 });
606 let ys: Vec<f32> = by_y.iter().map(|&i| accurate[i as usize].origin_y).collect();
607
608 for span in spans.iter_mut() {
609 // Rotated-content spans are in the displayed frame (mapped on 90/270)
610 // and their glyphs run vertically; a horizontal-x stamp from the raw
611 // chars would not correspond, so leave them to the prefix-sum path. ~keep
612 if span.rotation_degrees != 0.0 {
613 continue;
614 }
615 // Start clean: any offsets carried over via struct-update from a
616 // source span must not be trusted for this (possibly edited) text. ~keep
617 span.char_x_offsets.clear();
618
619 let glyphs: Vec<char> = span.text.chars().collect();
620 let n = glyphs.len();
621 if n == 0 {
622 continue;
623 }
624
625 let baseline_tol = 0.6 * span.font_size.max(1.0);
626 // Chars sharing this span's baseline, left-to-right. The y-sorted
627 // index brackets a candidate range; the exact `abs()` predicate then
628 // selects from it, so the result matches a full linear scan even
629 // where the bracket arithmetic rounds differently. The widened
630 // bracket keeps that range a superset. Ordering by
631 // (origin_x, source index) reproduces the previous stable
632 // filter-then-sort: ties on origin_x keep their `accurate` order. ~keep
633 let bracket = baseline_tol + baseline_tol.abs() * 1e-6 + f32::EPSILON;
634 let lo = ys.partition_point(|&y| y < span.bbox.y - bracket);
635 let hi = ys.partition_point(|&y| y <= span.bbox.y + bracket);
636 let mut idx: Vec<u32> = by_y[lo..hi]
637 .iter()
638 .copied()
639 .filter(|&i| (accurate[i as usize].origin_y - span.bbox.y).abs() <= baseline_tol)
640 .collect();
641 if idx.is_empty() {
642 continue;
643 }
644 idx.sort_by(|&a, &b| {
645 crate::utils::safe_float_cmp(accurate[a as usize].origin_x, accurate[b as usize].origin_x)
646 .then(a.cmp(&b))
647 });
648 let line: Vec<&crate::layout::TextChar> = idx.iter().map(|&i| &accurate[i as usize]).collect();
649
650 // Greedy per-glyph alignment. Anchor the scan cursor at the accurate
651 // char nearest this span's left edge, then walk the span's glyphs,
652 // matching each to the next equal accurate char within a small
653 // forward window. Unlike an all-or-nothing contiguous match, a single
654 // unmatched glyph (an inserted word-boundary space, a ligature split,
655 // a combining mark) no longer discards the whole span — such glyphs
656 // are interpolated below so `char_x_offsets` still fills every index. ~keep
657 let start = line.iter().position(|c| c.origin_x >= span.bbox.x - 0.5).unwrap_or(0);
658 let mut assigned: Vec<Option<f32>> = vec![None; n];
659 let mut li = start;
660 for (k, &g) in glyphs.iter().enumerate() {
661 let mut j = li;
662 let mut steps = 0;
663 while j < line.len() && steps < 6 {
664 if line[j].char == g {
665 assigned[k] = Some(line[j].origin_x);
666 li = j + 1;
667 break;
668 }
669 j += 1;
670 steps += 1;
671 }
672 }
673
674 // Need enough real anchors to trust the run; otherwise fall back. ~keep
675 let anchors = assigned.iter().filter(|a| a.is_some()).count();
676 if anchors * 5 < n * 3 {
677 // < 60% matched ~keep
678 continue;
679 }
680
681 // Fill the gaps: each unmatched glyph takes the nearest preceding
682 // anchor plus the prefix sum of the (locally accurate) char_widths
683 // between them; if there is no preceding anchor, walk back from the
684 // nearest following one. Over the short spans between anchors the
685 // cumulative drift these interpolations reintroduce is sub-point. ~keep
686 let cw = &span.char_widths;
687 let width_at = |i: usize| -> f32 {
688 if cw.len() == n {
689 cw[i]
690 } else {
691 span.bbox.width / n as f32
692 }
693 };
694 let mut offs = vec![0.0f32; n];
695 let mut last: Option<(usize, f32)> = None;
696 for k in 0..n {
697 if let Some(x) = assigned[k] {
698 offs[k] = x;
699 last = Some((k, x));
700 } else if let Some((lk, lx)) = last {
701 let acc: f32 = (lk..k).map(width_at).sum();
702 offs[k] = lx + acc;
703 }
704 }
705 if assigned[0].is_none()
706 && let Some(fk) = assigned.iter().position(|a| a.is_some())
707 {
708 let fx = assigned[fk].unwrap();
709 for k in 0..fk {
710 let acc: f32 = (k..fk).map(width_at).sum();
711 offs[k] = fx - acc;
712 }
713 }
714 span.char_x_offsets = offs;
715 }
716 }
717
718 /// Fold a one-char spacing-diacritic span into the following
719 /// span's first character when they overlap in X (the typical
720 /// LaTeX `\'E` → `(´)(E)` shape). Substitutes the relevant
721 /// combining mark from U+0300..U+0327 and lets
722 /// `unicode_normalization::nfc` precompose where it can
723 /// ("E\u{0301}" → "É"). The diacritic span is left empty so
724 /// downstream rendering skips it.
725 fn apply_combining_mark_composition(spans: &mut Vec<crate::layout::TextSpan>) {
726 use unicode_normalization::UnicodeNormalization;
727
728 fn combining_for(spacing: char) -> Option<char> {
729 Some(match spacing {
730 '\u{00B4}' => '\u{0301}',
731 '\u{0060}' => '\u{0300}',
732 '\u{005E}' => '\u{0302}',
733 '\u{02C6}' => '\u{0302}',
734 '\u{007E}' => '\u{0303}',
735 '\u{02DC}' => '\u{0303}',
736 '\u{00A8}' => '\u{0308}',
737 '\u{00AF}' => '\u{0304}',
738 '\u{02C9}' => '\u{0304}',
739 '\u{00B8}' => '\u{0327}',
740 '\u{02DA}' => '\u{030A}',
741 _ => return None,
742 })
743 }
744
745 // First pass: spans that already got merged at the extractor
746 // (when the LaTeX `(´)(Ecole)` pair both sit at the same
747 // text-matrix origin the upstream merge_adjacent_spans pulls
748 // them into a single "´Ecole" span). Fold the leading
749 // diacritic + base letter into the precomposed form. ~keep
750 for span in spans.iter_mut() {
751 let mut iter = span.text.chars();
752 let Some(d) = iter.next() else { continue };
753 let Some(base) = iter.next() else { continue };
754 let Some(combining) = combining_for(d) else {
755 continue;
756 };
757 if !base.is_alphabetic() {
758 continue;
759 }
760 let rest_start = d.len_utf8() + base.len_utf8();
761 let mut composed = String::with_capacity(span.text.len() + 2);
762 composed.push(base);
763 composed.push(combining);
764 composed.push_str(&span.text[rest_start..]);
765 span.text = composed.nfc().collect();
766 }
767
768 // Walk spans pairwise. The diacritic is on its own one-
769 // character span; the next span carries the base letter. ~keep
770 let mut i = 0;
771 while i + 1 < spans.len() {
772 let mark_char = {
773 let s = &spans[i];
774 let mut iter = s.text.chars();
775 let first = iter.next();
776 let rest = iter.next();
777 if rest.is_some() {
778 None
779 } else {
780 first.and_then(combining_for)
781 }
782 };
783 let Some(combining) = mark_char else {
784 i += 1;
785 continue;
786 };
787 // Geometric: same line, diacritic anchored over the base
788 // letter's left edge (within ±1 pt). ~keep
789 let (same_line, overlaps_x) = {
790 let p = &spans[i];
791 let n = &spans[i + 1];
792 let same = (p.bbox.y - n.bbox.y).abs() < p.font_size.max(n.font_size) * 0.6;
793 let dx = (p.bbox.x - n.bbox.x).abs();
794 (same, dx <= 1.5)
795 };
796 if !(same_line && overlaps_x) {
797 i += 1;
798 continue;
799 }
800 let Some(base) = spans[i + 1].text.chars().next() else {
801 i += 1;
802 continue;
803 };
804 if !base.is_alphabetic() {
805 i += 1;
806 continue;
807 }
808 let mut composed = String::with_capacity(spans[i + 1].text.len() + 2);
809 composed.push(base);
810 composed.push(combining);
811 let rest_start = base.len_utf8();
812 composed.push_str(&spans[i + 1].text[rest_start..]);
813 spans[i + 1].text = composed.nfc().collect();
814 // Empty out the diacritic span; downstream consumers
815 // skip zero-text spans. ~keep
816 spans[i].text.clear();
817 i += 2;
818 }
819
820 spans.retain(|s| !s.text.is_empty());
821 }
822
823 /// Substitute ASCII digits and a few punctuation characters in
824 /// super/sub-script spans with their Unicode counterparts
825 /// (U+2070..U+2079 / U+00B2/B3/B9 for superscripts,
826 /// U+2080..U+2089 for subscripts). A span is treated as
827 /// super- or sub-script when its font is meaningfully smaller
828 /// than the previous span on the same line and its baseline is
829 /// raised or lowered. Only spans whose text consists entirely
830 /// of substitutable characters are rewritten — mixed-content
831 /// or single-letter superscript callouts (e.g. footnote "a")
832 /// fall through unchanged so the existing citation-handling
833 /// path stays in control.
834 fn apply_super_sub_script_substitutions(spans: &mut [crate::layout::TextSpan]) {
835 fn super_for_char(c: char) -> Option<char> {
836 Some(match c {
837 '0' => '\u{2070}',
838 '1' => '\u{00B9}',
839 '2' => '\u{00B2}',
840 '3' => '\u{00B3}',
841 '4' => '\u{2074}',
842 '5' => '\u{2075}',
843 '6' => '\u{2076}',
844 '7' => '\u{2077}',
845 '8' => '\u{2078}',
846 '9' => '\u{2079}',
847 '+' => '\u{207A}',
848 '-' => '\u{207B}',
849 '=' => '\u{207C}',
850 '(' => '\u{207D}',
851 ')' => '\u{207E}',
852 _ => return None,
853 })
854 }
855 fn sub_for_char(c: char) -> Option<char> {
856 Some(match c {
857 '0' => '\u{2080}',
858 '1' => '\u{2081}',
859 '2' => '\u{2082}',
860 '3' => '\u{2083}',
861 '4' => '\u{2084}',
862 '5' => '\u{2085}',
863 '6' => '\u{2086}',
864 '7' => '\u{2087}',
865 '8' => '\u{2088}',
866 '9' => '\u{2089}',
867 '+' => '\u{208A}',
868 '-' => '\u{208B}',
869 '=' => '\u{208C}',
870 '(' => '\u{208D}',
871 ')' => '\u{208E}',
872 _ => return None,
873 })
874 }
875 // Two-pass: first compute the body-font baseline for each
876 // line band (largest font_size on that line), then walk
877 // spans and substitute any whose font is meaningfully
878 // smaller AND whose baseline is raised or lowered relative
879 // to the body baseline. ~keep
880 let n = spans.len();
881 if n < 2 {
882 return;
883 }
884 const LINE_BAND_PT: f32 = 4.0;
885 // band_anchor[i] = (body_font_size, body_y) of the line
886 // band that span `i` belongs to. Sorting span indices by Y
887 // once + sliding a two-pointer window over the sorted view
888 // reduces the per-span band-anchor scan from O(n) to amortised
889 // O(window_size), bringing the whole pass from O(n²) down to
890 // O(n log n) on thesis-style pages with thousands of spans. ~keep
891 let mut sorted_by_y: Vec<usize> = (0..n).collect();
892 sorted_by_y.sort_by(|&a, &b| crate::utils::safe_float_cmp(spans[a].bbox.y, spans[b].bbox.y));
893 let band_anchor = Self::compute_band_anchors(spans, &sorted_by_y, LINE_BAND_PT);
894 // Spatial index: bucket spans by Y-band so `span_is_token_internal`
895 // queries only nearby spans instead of all of them (its same-line
896 // neighbour scan was O(n) per candidate → O(n²) on dense pages). ~keep
897 let y_index = Self::build_y_band_index(spans, LINE_BAND_PT);
898 for i in 0..n {
899 let (anchor_fs, anchor_y) = band_anchor[i];
900 let curr_fs = spans[i].font_size;
901 if anchor_fs <= 0.0 || curr_fs >= anchor_fs * 0.85 {
902 continue;
903 }
904 let y_delta = spans[i].bbox.y - anchor_y;
905 let raised = y_delta > anchor_fs * 0.15;
906 let lowered = y_delta < -anchor_fs * 0.15;
907 if !raised && !lowered {
908 continue;
909 }
910 let map: fn(char) -> Option<char> = if raised { super_for_char } else { sub_for_char };
911 if spans[i].text.is_empty() || !spans[i].text.chars().all(|c| map(c).is_some()) {
912 continue;
913 }
914 // Leave a signed numeric exponent (scientific unit notation such as
915 // `s−1`, `m−2`) as ASCII. ToUnicode already decoded the intended
916 // characters, and the plaintext convention every reference extractor
917 // follows keeps these un-superscripted; rewriting `−1` to `₋₁` / `⁻¹`
918 // is both wrong against that convention and — because the geometric
919 // classifier fires inconsistently on borderline baselines — a source
920 // of non-determinism across identical occurrences. ~keep
921 if Self::run_is_signed_number(&spans[i].text) {
922 continue;
923 }
924 // Limit the substitution to clearly token-internal
925 // super/sub-scripts: the run must have a base-sized
926 // neighbour on BOTH sides whose first/last char is
927 // alphabetic and roughly adjacent in X. Author-
928 // affiliation markers like "name¹,²" sit at the END
929 // of a line with no following body letter; the bench
930 // GT renders those as plain ASCII digits, so substi-
931 // tuting them would regress. Restricting to sandwiched
932 // runs keeps the chemistry / exponent cases that the
933 // GT does want as Unicode (S², H₂O, k₁) and skips the
934 // trailing footnote callouts. ~keep
935 if !Self::span_is_token_internal(spans, i, &y_index, LINE_BAND_PT) {
936 continue;
937 }
938 let substituted: String = spans[i].text.chars().map(|c| map(c).unwrap()).collect();
939 spans[i].text = substituted;
940 }
941 }
942
943 /// A run is a signed numeric exponent — e.g. `-1`, `−2`, `‑3` — when it
944 /// opens with a minus/hyphen sign and contains at least one digit. Such runs
945 /// are scientific unit exponents (`s−1`, `m−2`) that the plaintext extraction
946 /// convention keeps as ASCII, so [`apply_super_sub_script_substitutions`]
947 /// must not rewrite them into Unicode sub/superscript glyphs.
948 ///
949 /// [`apply_super_sub_script_substitutions`]: Self::apply_super_sub_script_substitutions
950 pub(super) fn run_is_signed_number(text: &str) -> bool {
951 let is_minus = |c: char| matches!(c, '\u{002D}' | '\u{2212}' | '\u{2010}' | '\u{2011}');
952 matches!(text.chars().next(), Some(c) if is_minus(c)) && text.chars().any(|c| c.is_ascii_digit())
953 }
954
955 /// For every span, the `(max_font_size, anchor_y)` over the spans within
956 /// `±band` of its Y, in O(n) via a sliding-window maximum (monotonic deque)
957 /// over the Y-sorted order. Replaces a per-span window walk that was O(n²)
958 /// when many spans share a Y band (wide table rows).
959 ///
960 /// Tie-break on equal max font size: the lowest-Y span (deque keeps the
961 /// earliest sorted position). A substitution only fires when the span's own
962 /// font is strictly smaller than the anchor, so the tie-break merely picks
963 /// which equal-sized body span supplies `anchor_y`, all within `band`.
964 fn compute_band_anchors(spans: &[crate::layout::TextSpan], sorted_by_y: &[usize], band: f32) -> Vec<(f32, f32)> {
965 let n = sorted_by_y.len();
966 let mut band_anchor = vec![(0.0f32, 0.0f32); n];
967 let y = |p: usize| spans[sorted_by_y[p]].bbox.y;
968 let fs = |p: usize| spans[sorted_by_y[p]].font_size;
969 // Deque of sorted positions, font size non-increasing front→back;
970 // positions are pushed in increasing order so the deque is also
971 // position-increasing front→back (front = smallest position = max fs). ~keep
972 let mut deque: std::collections::VecDeque<usize> = std::collections::VecDeque::new();
973 let mut lo = 0usize;
974 let mut hi = 0usize;
975 for pos in 0..n {
976 let cy = y(pos);
977 while hi < n && y(hi) <= cy + band {
978 while let Some(&back) = deque.back() {
979 if fs(back) < fs(hi) {
980 deque.pop_back();
981 } else {
982 break;
983 }
984 }
985 deque.push_back(hi);
986 hi += 1;
987 }
988 while lo < n && y(lo) < cy - band {
989 if deque.front() == Some(&lo) {
990 deque.pop_front();
991 }
992 lo += 1;
993 }
994 let best = *deque.front().expect("window always contains pos");
995 band_anchor[sorted_by_y[pos]] = (fs(best), y(best));
996 }
997 band_anchor
998 }
999
1000 /// Return true when span `i` has a base-sized alphabetic
1001 /// neighbour both before and after it on the same line band,
1002 /// within ~1 em horizontally. That captures the "X²Y" /
1003 /// "H₂O" / "k₁ + …" pattern but excludes footnote markers
1004 /// that hang off the end of a word with no following body
1005 /// character.
1006 /// Bucket span indices by Y-band (`round(y / band)`) so same-line lookups
1007 /// scan only nearby bands instead of every span. Querying a band `k`'s
1008 /// `[k-2, k+2]` neighbours is a guaranteed superset of all spans within
1009 /// `band` points of any Y in band `k`, so an exact `|Δy|` filter on the
1010 /// result is byte-identical to a full scan.
1011 pub(super) fn build_y_band_index(spans: &[crate::layout::TextSpan], band: f32) -> HashMap<i32, Vec<usize>> {
1012 let mut idx: HashMap<i32, Vec<usize>> = HashMap::new();
1013 for (j, s) in spans.iter().enumerate() {
1014 idx.entry((s.bbox.y / band).round() as i32).or_default().push(j);
1015 }
1016 idx
1017 }
1018
1019 /// Indices in the Y-bands within ±2 of `y`'s band (superset of `|Δy| ≤ band`).
1020 pub(super) fn y_band_candidates<'a>(
1021 y_index: &'a HashMap<i32, Vec<usize>>,
1022 y: f32,
1023 band: f32,
1024 ) -> impl Iterator<Item = usize> + 'a {
1025 let k = (y / band).round() as i32;
1026 (k - 2..=k + 2).flat_map(move |b| y_index.get(&b).into_iter().flatten().copied())
1027 }
1028
1029 fn span_is_token_internal(
1030 spans: &[crate::layout::TextSpan],
1031 i: usize,
1032 y_index: &HashMap<i32, Vec<usize>>,
1033 band: f32,
1034 ) -> bool {
1035 let curr = &spans[i];
1036 let curr_y = curr.bbox.y;
1037 let curr_x = curr.bbox.x;
1038 let curr_right = curr.bbox.x + curr.bbox.width;
1039 let body_fs = Self::y_band_candidates(y_index, curr_y, band)
1040 .filter(|&j| (spans[j].bbox.y - curr_y).abs() <= 4.0)
1041 .map(|j| spans[j].font_size)
1042 .fold(0f32, f32::max)
1043 .max(1.0);
1044 let neighbour_fs_min = body_fs * 0.85;
1045 let max_em = body_fs;
1046 let mut has_left = false;
1047 let mut has_right = false;
1048 for j in Self::y_band_candidates(y_index, curr_y, band) {
1049 if j == i {
1050 continue;
1051 }
1052 let s = &spans[j];
1053 if (s.bbox.y - curr_y).abs() > 4.0 {
1054 continue;
1055 }
1056 if s.font_size < neighbour_fs_min {
1057 continue;
1058 }
1059 // Anchor must start or end with an alphabetic character
1060 // — a digit or punctuation neighbour does not signal a
1061 // token-internal context. ~keep
1062 let s_right = s.bbox.x + s.bbox.width;
1063 // Allow small overlap (super/sub glyphs nest slightly
1064 // under the body letter's bounding box). ~keep
1065 let dx_left = curr_x - s_right;
1066 if s_right < curr_right
1067 && dx_left <= max_em
1068 && dx_left >= -max_em * 0.5
1069 && s.text.chars().next_back().is_some_and(|c| c.is_alphabetic())
1070 {
1071 has_left = true;
1072 }
1073 let dx_right = s.bbox.x - curr_right;
1074 if s.bbox.x > curr_x
1075 && dx_right <= max_em
1076 && dx_right >= -max_em * 0.5
1077 && s.text.chars().next().is_some_and(|c| c.is_alphabetic())
1078 {
1079 has_right = true;
1080 }
1081 }
1082 has_left && has_right
1083 }
1084
1085 /// Return per-page font statistics for use in heading detection and layout analysis.
1086 ///
1087 /// [`crate::layout::PageFontStats`] contains:
1088 /// - `dominant_em`: the mode font size weighted by character count — the body text "1 em"
1089 /// - `dominant_line_height`: median baseline-to-baseline distance
1090 /// - `dominant_char_width`: average character advance width
1091 /// - `body_font_name`: name of the most-used font
1092 ///
1093 /// The primary use-case is heading detection in downstream tools: compare
1094 /// `span.font_size / stats.dominant_em` against a threshold (e.g. 1.4×
1095 /// for H2, 1.8× for H1) to classify large-font spans as headings without
1096 /// depending on any hardcoded point sizes.
1097 ///
1098 /// ```ignore
1099 /// let stats = doc.page_font_stats(0)?;
1100 /// let spans = doc.extract_spans(0)?;
1101 /// for span in &spans {
1102 /// let ratio = span.font_size / stats.dominant_em;
1103 /// if ratio >= 1.8 { println!("H1: {}", span.text); }
1104 /// else if ratio >= 1.4 { println!("H2: {}", span.text); }
1105 /// }
1106 /// ```
1107 pub fn page_font_stats(&self, page_index: usize) -> Result<crate::layout::PageFontStats> {
1108 let spans = self.extract_spans(page_index)?;
1109 Ok(crate::layout::PageFontStats::from_spans(&spans))
1110 }
1111}