Skip to main content

docling_pdf/
textparse.rs

1//! Pure-Rust PDF text extraction (replacing pdfium's glyph layer).
2//!
3//! pdfium reports *rendered* glyph boxes, which diverge from docling's
4//! `docling-parse` C++ parser at exactly the points that drive conformance:
5//! generated spaces get a zero-width box, combining diacritics get a real-width
6//! box, and ligature/fraction glyphs land at different x. This module instead
7//! reconstructs each glyph's box from the **font's own advance widths** and the
8//! PDF text/graphics matrices — the same information docling-parse uses — so a
9//! space is as wide as the font says and a combining mark has zero advance.
10//!
11//! The output is the same [`Glyph`] stream pdfium produces (native PDF
12//! coordinates, y-up), fed straight into the existing docling-parse line
13//! sanitizer ([`crate::dp_lines`]). Only the digital text layer is handled here;
14//! pages without one still fall back to OCR upstream.
15
16use std::collections::{HashMap, HashSet};
17use std::sync::Arc;
18
19use lopdf::{Dictionary, Document, Object};
20
21use crate::pdfium_backend::Glyph;
22use crate::tounicode::parse_tounicode;
23
24/// Per-document caches for the content-stream interpreter. Fonts are indirect
25/// objects shared by many pages, but were fully re-parsed — ToUnicode CMap
26/// decompression + tokenization, embedded Type1 program scan, width tables —
27/// for **every page and every Form XObject invocation**; decoded form content
28/// streams were likewise re-inflated on every `Do`. Cached per document,
29/// keyed by the referenced object id (fonts also by resource name — the
30/// `/Font` entry a page binds the object to). Inline (non-reference) dicts
31/// are rare and stay uncached.
32#[derive(Default)]
33struct DocCaches {
34    fonts: HashMap<(lopdf::ObjectId, Vec<u8>), Arc<Font>>,
35    forms: HashMap<lopdf::ObjectId, Arc<lopdf::content::Content>>,
36    /// Forms decoded once and not (yet) cached (#994). A form enters
37    /// [`Self::forms`] on its *second* sighting: a shared header/footer form
38    /// is decoded twice and then hits the cache on every later page, while
39    /// a generator's one-form-per-page body (each page's chart in its own
40    /// form, used once) is dropped with its page instead of every decoded
41    /// `Content` — many times its stream's size — living until the document
42    /// ends. Caching on first use made peak RSS grow ~45 MB per page on such
43    /// files (7 GiB at 60 pages of the issue's repro, flat ~0.6 GiB now).
44    /// Only ids are kept here, so the set itself stays tiny.
45    forms_seen: HashSet<lopdf::ObjectId>,
46}
47
48/// A 2×3 affine matrix `[a b c d e f]`: maps `(x,y)` → `(a·x+c·y+e, b·x+d·y+f)`.
49#[derive(Clone, Copy)]
50struct Mat {
51    a: f64,
52    b: f64,
53    c: f64,
54    d: f64,
55    e: f64,
56    f: f64,
57}
58
59impl Mat {
60    const ID: Mat = Mat {
61        a: 1.0,
62        b: 0.0,
63        c: 0.0,
64        d: 1.0,
65        e: 0.0,
66        f: 0.0,
67    };
68
69    /// `self ∘ m`: the matrix that applies `self` first, then `m`.
70    fn then(self, m: Mat) -> Mat {
71        Mat {
72            a: self.a * m.a + self.b * m.c,
73            b: self.a * m.b + self.b * m.d,
74            c: self.c * m.a + self.d * m.c,
75            d: self.c * m.b + self.d * m.d,
76            e: self.e * m.a + self.f * m.c + m.e,
77            f: self.e * m.b + self.f * m.d + m.f,
78        }
79    }
80
81    fn apply(self, x: f64, y: f64) -> (f64, f64) {
82        (
83            self.a * x + self.c * y + self.e,
84            self.b * x + self.d * y + self.f,
85        )
86    }
87}
88
89/// A parsed font: how to turn raw string bytes into (unicode, advance) pairs.
90struct Font {
91    /// 2-byte codes (Type0 / Identity-H) vs 1-byte (simple fonts).
92    two_byte: bool,
93    /// code → Unicode string (from ToUnicode; may be multi-char, e.g. ligatures).
94    to_unicode: HashMap<u32, String>,
95    /// For codes ToUnicode lacks (ISO 32000-1, 9.10.2).
96    cid_fallback: CidFallback,
97    /// code → glyph advance, in 1000-unit glyph space.
98    widths: HashMap<u32, f64>,
99    default_width: f64,
100    /// 1-byte fallback decoding when ToUnicode lacks a code (WinAnsi-ish).
101    simple_encoding: Option<HashMap<u8, char>>,
102    /// code → raw `/Differences` glyph name, for placeholder names (`g115`,
103    /// `SM590000`) that have no Unicode mapping; they override the base
104    /// encoding (see [`unknown_name_text`]).
105    fallback_names: HashMap<u8, String>,
106    /// The unmapped codes whose embedded glyph paints nothing — semantic
107    /// spaces (docling-parse 7.24's blank glyph programs: the redp5110
108    /// list's `/g3`). Every other unmapped code reads U+FFFD.
109    blank_codes: HashSet<u32>,
110    /// code → char from the embedded Type1 font program's own `/Encoding` vector,
111    /// used only as a last resort for glyphs the base encoding leaves unmapped
112    /// (standard TeX math fonts: `λ`, `≤`, …).
113    program_encoding: HashMap<u8, char>,
114    /// A symbolic simple font without a declared base encoding: the text of
115    /// each code its embedded program's builtin encoding resolves
116    /// (docling-parse 7.25, #1113 — 2203.01017's `Dingbats` check marks are
117    /// `a19` / `a23`, `✓` / `✗`, not the `3` / `7` the Latin tables read).
118    /// It outranks every encoding table but a `/Differences` entry
119    /// ([`Font::diff_codes`]); empty for every other font and in a build
120    /// without the renderer.
121    builtin: HashMap<u8, String>,
122    /// The codes `/Differences` names.
123    diff_codes: HashSet<u8>,
124    ascent: f64,
125    descent: f64,
126    /// code → its embedded glyph's ink extent `(y_min, y_max)` in 1000-unit
127    /// em, for a font whose descriptor metrics docling-parse rejects
128    /// ([`font_ascent_descent`]'s third value): docling-parse 7.24 boxes such
129    /// a font's glyphs by their outline's control box, so a glyph drawn
130    /// under the baseline — 2305.03393's `_` in `PMC5406406_003_01.png` —
131    /// is a cell of its own, not part of the word. Empty for every other
132    /// font, a glyph with no ink, and a build without the renderer.
133    ink_y: HashMap<u32, (f64, f64)>,
134    hash: u64,
135    /// Whether the font's docling-parse name marks a math face
136    /// ([`is_math_font_name`]) — for the word contraction's local
137    /// attachments (#921).
138    math: bool,
139    /// Weight and slant read from the `/BaseFont` name (the heading
140    /// hierarchy's style signal, #302).
141    style: crate::font_style::FontStyle,
142}
143
144impl Font {
145    /// The text of `code` and its advance; the flag marks a blank unmapped
146    /// glyph, whose box has no height.
147    ///
148    /// A code no source maps to Unicode is visibly painted but carries no
149    /// text: docling-parse 7.24 emits U+FFFD for it, and docling turns a NUL
150    /// into U+FFFD too (docling#5070), so the loss stays visible and counted
151    /// by the text-quality check instead of silently dropping the glyph
152    /// (`mL⁻¹` reading `mL1`). A glyph whose embedded program paints nothing
153    /// is a space — a word boundary — instead.
154    fn decode_code(&self, code: u32) -> (Option<String>, f64, bool) {
155        let w = self
156            .widths
157            .get(&code)
158            .copied()
159            .unwrap_or(self.default_width);
160        if let Some(s) = self.to_unicode.get(&code) {
161            return (Some(decompose_ligatures(s)), w, false);
162        }
163        let unmapped = |font: &Font| {
164            let blank = font.blank_codes.contains(&code);
165            (
166                Some(if blank { " " } else { "\u{FFFD}" }.to_string()),
167                w,
168                blank,
169            )
170        };
171        if !self.two_byte {
172            // A placeholder `/Differences` name (no Unicode) overrides the
173            // base encoding. docling-parse 7.24 reads a bare glyph index
174            // (`g115`, `gid00043`) as unmapped and any other unknown name as
175            // its text (`SM590000`, the redp5110 bullets).
176            if let Some(name) = self.fallback_names.get(&(code as u8)) {
177                return match unknown_name_text(name) {
178                    Some(text) => (Some(text), w, false),
179                    None => unmapped(self),
180                };
181            }
182            // docling-parse 7.25's `get_correct_character`: after
183            // `/Differences` and `/ToUnicode`, a symbolic font's builtin
184            // encoding — "the producer's own statement of what each code
185            // draws" — before any base-font or Latin table.
186            if !self.diff_codes.contains(&(code as u8)) {
187                if let Some(text) = self.builtin.get(&(code as u8)) {
188                    return (Some(decompose_ligatures(text)), w, false);
189                }
190            }
191            if let Some(enc) = &self.simple_encoding {
192                if let Some(&ch) = enc.get(&(code as u8)) {
193                    return (Some(decompose_ligatures(&ch.to_string())), w, false);
194                }
195            }
196            // Last resort: the embedded Type1 font program's own `/Encoding`
197            // vector (`dup N /glyphname put`). Standard TeX math fonts (CMMI, CMSY,
198            // …) ship no PDF `/Encoding` and no ToUnicode, so a glyph like `λ`
199            // (CMMI code 21 → `/lambda`) or `≤` (CMSY code 20 → `/lessequal`) has
200            // no other mapping and would otherwise be silently dropped. docling
201            // recovers these from the same font program. This only fills codes the
202            // base encoding left unmapped, so it never changes an existing decode.
203            if let Some(&ch) = self.program_encoding.get(&(code as u8)) {
204                return (Some(decompose_ligatures(&ch.to_string())), w, false);
205            }
206        } else {
207            match self.cid_fallback {
208                CidFallback::None => {}
209                CidFallback::UnicodeCodes { .. } => {
210                    if let Some(ch) = unicode_cmap_char(code).filter(|c| !c.is_control()) {
211                        return (Some(ch.to_string()), w, false);
212                    }
213                }
214                CidFallback::Collection(table) => {
215                    if let Some(s) = collection_text(table, code) {
216                        return (Some(decompose_ligatures(&s)), w, false);
217                    }
218                }
219            }
220        }
221        unmapped(self)
222    }
223}
224
225/// Spell out Latin presentation-form ligatures (`fi`→`fi`, `ffi`→`ffi`, …) the way
226/// docling does, so `configuration`/`difficult` don't keep the ligature glyph.
227/// The chars share the ligature's box, so the line sanitizer recomposes them.
228fn decompose_ligatures(s: &str) -> String {
229    if !s.chars().any(|c| ('\u{FB00}'..='\u{FB06}').contains(&c)) {
230        return s.to_string();
231    }
232    s.chars()
233        .map(|c| {
234            match c {
235                '\u{FB00}' => "ff",
236                '\u{FB01}' => "fi",
237                '\u{FB02}' => "fl",
238                '\u{FB03}' => "ffi",
239                '\u{FB04}' => "ffl",
240                // U+FB05 is LATIN SMALL LIGATURE LONG S T (`ſt`): `st`, as
241                // docling's ligature map and Unicode's decomposition (#1001).
242                '\u{FB05}' => "st",
243                '\u{FB06}' => "st",
244                _ => return c.to_string(),
245            }
246            .to_string()
247        })
248        .collect()
249}
250
251fn hash_name(name: &[u8]) -> u64 {
252    use std::hash::{Hash, Hasher};
253    let mut h = std::collections::hash_map::DefaultHasher::new();
254    name.hash(&mut h);
255    h.finish()
256}
257
258/// Resolve a possibly-indirect object to a dictionary.
259fn as_dict<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Dictionary> {
260    match obj {
261        Object::Dictionary(d) => Some(d),
262        Object::Reference(id) => doc.get_object(*id).ok().and_then(|o| o.as_dict().ok()),
263        _ => None,
264    }
265}
266
267fn deref<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Object> {
268    match obj {
269        Object::Reference(id) => doc.get_object(*id).ok(),
270        other => Some(other),
271    }
272}
273
274/// The font identity the line sanitizer's `enforce_same_font` compares —
275/// docling-parse's `page_font::init_font_name`: `/FontDescriptor/FontName`
276/// (the descendant CIDFont's for a Type0), else `/Name`, else `/BaseFont`
277/// (its `base_font`), else the literal `"null"`. The resource key (`/F1`,
278/// `/F2`) is deliberately **not** part of it: a producer that references one
279/// face twice — the same Helvetica under two keys, one re-encoded so a code
280/// maps to the `fi` glyph — draws one word across the two resources, and
281/// docling-parse reads both as the same font. Keyed by resource name (as
282/// this port was until #728) the `fi` ligature, which bridges fonts, glued
283/// itself to the preceding run while the font barrier then cut it off from
284/// its own word (`fi nancial`); docling's `financial` needs the two
285/// resources to compare equal.
286/// docling-parse's `has_math_positioning_semantics`, the font-name half: a
287/// TeX math or symbol face (`cmmi`, `cmsy`, `cmex`, `cmr`, `cmbx`, `msam`,
288/// `msbm`) or any name saying `math`, `symbol` or `mtextra`.
289fn is_math_font_name(name: &[u8]) -> bool {
290    let lower = name.to_ascii_lowercase();
291    [
292        "cmmi", "cmsy", "cmex", "cmr", "cmbx", "msam", "msbm", "math", "symbol", "mtextra",
293    ]
294    .iter()
295    .any(|t| lower.windows(t.len()).any(|w| w == t.as_bytes()))
296}
297
298fn sanitizer_font_name(doc: &Document, fdict: &Dictionary, two_byte: bool) -> Vec<u8> {
299    let descriptor_font_name = |owner: &Dictionary| {
300        owner
301            .get(b"FontDescriptor")
302            .ok()
303            .and_then(|o| as_dict(doc, o))
304            .and_then(|fd| fd.get(b"FontName").ok())
305            .and_then(|o| deref(doc, o))
306            .and_then(|o| o.as_name().ok().map(<[u8]>::to_vec))
307    };
308    if let Some(n) = descriptor_font_name(fdict) {
309        return n;
310    }
311    if two_byte {
312        let descendant = fdict
313            .get(b"DescendantFonts")
314            .ok()
315            .and_then(|o| deref(doc, o))
316            .and_then(|o| match o {
317                Object::Array(a) => a.first(),
318                _ => None,
319            })
320            .and_then(|o| as_dict(doc, o));
321        if let Some(n) = descendant.and_then(descriptor_font_name) {
322            return n;
323        }
324    }
325    if let Some(n) = fdict
326        .get(b"Name")
327        .ok()
328        .and_then(|o| deref(doc, o))
329        .and_then(|o| o.as_name().ok())
330    {
331        return n.to_vec();
332    }
333    // docling-parse's `base_font` keeps the subset prefix (`ABCDEF+Helvetica`):
334    // two subsets of one face are two fonts there, and so here.
335    if let Some(n) = fdict.get(b"BaseFont").ok().and_then(|o| o.as_name().ok()) {
336        return n.to_vec();
337    }
338    b"null".to_vec()
339}
340
341/// Parse one font dictionary into a [`Font`].
342fn parse_font(doc: &Document, name: &[u8], fdict: &Dictionary) -> Font {
343    let subtype: &[u8] = fdict
344        .get(b"Subtype")
345        .ok()
346        .and_then(|o| o.as_name().ok())
347        .unwrap_or(&[]);
348    let two_byte = subtype == b"Type0".as_slice();
349
350    let mut to_unicode = fdict
351        .get(b"ToUnicode")
352        .ok()
353        .and_then(|o| deref(doc, o))
354        .and_then(|o| o.as_stream().ok())
355        .and_then(|s| s.decompressed_content().ok())
356        .map(|data| parse_tounicode(&data))
357        .unwrap_or_default();
358    if !two_byte {
359        if let Some(diffs) = differences_array(doc, fdict) {
360            reconcile_stale_tounicode(&mut to_unicode, &diffs);
361        }
362    }
363    let sanitizer_name = sanitizer_font_name(doc, fdict, two_byte);
364    // Symbol / ZapfDingbats mapped into the private-use "symbol charset"
365    // (`U+F000` + byte): docling-parse 7.25 reads the glyph's text through
366    // the face's own encoding instead (docling-parse#431, #1113 — redp5110's
367    // `U+F0AE` is `→`, a Word bullet `U+F0B7` is `•`).
368    let raw_base = fdict.get(b"BaseFont").ok().and_then(|o| o.as_name().ok());
369    let names: Vec<&[u8]> = std::iter::once(sanitizer_name.as_slice())
370        .chain(raw_base)
371        .collect();
372    if let Some(face) = crate::symbol_fonts::symbol_face(&names) {
373        for value in to_unicode.values_mut() {
374            if let Some(ch) = crate::symbol_fonts::fold_private_use(face, value) {
375                *value = ch.to_string();
376            }
377        }
378    }
379
380    let cid_fallback = if two_byte {
381        cid_fallback(doc, fdict)
382    } else {
383        CidFallback::None
384    };
385
386    let (mut widths, mut default_width) = if two_byte {
387        cid_widths(doc, fdict)
388    } else {
389        simple_widths(doc, fdict)
390    };
391
392    let simple_encoding = if two_byte {
393        None
394    } else {
395        Some(simple_encoding_table(doc, fdict))
396    };
397
398    // A standard-14 font referenced without an embedded program usually ships
399    // no `/Widths` and no `/FontDescriptor` either (ReportLab's default, #187).
400    // Every advance then resolved to 0, the cells collapsed to zero width, and
401    // the page's whole text layer was silently dropped — while pdfium, with
402    // its built-in metrics, reads the same file fine. Fill the widths from the
403    // built-in Adobe Core 14 AFM tables via the font's own code→char decode
404    // (base encoding + `/Differences`), so an explicit `/Widths` always wins.
405    if !two_byte && widths.is_empty() && default_width == 0.0 {
406        if let Some(std14) = base_font_name(fdict).and_then(|n| crate::std14::widths_for(&n)) {
407            if let Some(enc) = &simple_encoding {
408                for (&code, &ch) in enc {
409                    if let Some(w) = std14.width(ch) {
410                        widths.insert(u32::from(code), w);
411                    }
412                }
413            }
414            // Codes the table misses still advance a typical width instead of
415            // stacking at x=0 (the failure mode this whole branch fixes).
416            default_width = 500.0;
417        } else if let Some(face) = crate::symbol_fonts::symbol_face(&[base_font_name(fdict)
418            .as_deref()
419            .unwrap_or_default()])
420        {
421            // The two symbol faces are no std14 text faces: their AFM
422            // advances by code, as docling-parse's base font reads them.
423            for code in 0..=255u8 {
424                if let Some(w) = crate::symbol_fonts::face_width(face, code) {
425                    widths.insert(u32::from(code), w);
426                }
427            }
428            default_width = 500.0;
429        }
430    }
431    let fallback_names = if two_byte {
432        HashMap::new()
433    } else {
434        differences_gid_names(doc, fdict)
435    };
436    let program_encoding = if two_byte {
437        HashMap::new()
438    } else {
439        type1_program_encoding(doc, fdict)
440    };
441
442    let (ascent, descent, ink_boxes) = font_ascent_descent(doc, fdict, two_byte);
443    let mut font = Font {
444        two_byte,
445        to_unicode,
446        cid_fallback,
447        widths,
448        default_width,
449        simple_encoding,
450        fallback_names,
451        blank_codes: HashSet::new(),
452        program_encoding,
453        builtin: HashMap::new(),
454        diff_codes: differences_codes(doc, fdict),
455        ascent,
456        descent,
457        ink_y: HashMap::new(),
458        hash: hash_name(&sanitizer_name),
459        math: is_math_font_name(&sanitizer_name),
460        style: crate::font_style::parse_font_style(&String::from_utf8_lossy(
461            &base_font_name(fdict).unwrap_or_else(|| name.to_vec()),
462        )),
463    };
464    font.blank_codes = blank_codes(doc, fdict, &font);
465    if !two_byte && subtype != b"Type3".as_slice() && uses_builtin_encoding(doc, fdict) {
466        font.builtin = builtin_encoding(doc, fdict, &font);
467    }
468    if ink_boxes {
469        font.ink_y = ink_extents(doc, fdict, &font);
470    }
471    font
472}
473
474/// docling-parse 7.25's `uses_builtin_encoding` for a simple, non-Type 3
475/// font: the descriptor flags it symbolic (bit 3) and the PDF declares no
476/// base encoding — no `/Encoding` name, no `/BaseEncoding` in an `/Encoding`
477/// dictionary (a bare `/Differences` overrides only the codes it lists,
478/// 9.6.6.1).
479fn uses_builtin_encoding(doc: &Document, fdict: &Dictionary) -> bool {
480    let symbolic = fdict
481        .get(b"FontDescriptor")
482        .ok()
483        .and_then(|o| as_dict(doc, o))
484        .and_then(|fd| fd.get(b"Flags").ok())
485        .and_then(|o| deref(doc, o))
486        .and_then(num)
487        .is_some_and(|f| (f as i64) & 4 != 0);
488    let declared_base = match fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o)) {
489        Some(Object::Name(_)) => true,
490        Some(Object::Dictionary(d)) => d.get(b"BaseEncoding").is_ok(),
491        _ => false,
492    };
493    symbolic && !declared_base
494}
495
496/// The codes a simple font's `/Differences` names.
497fn differences_codes(doc: &Document, fdict: &Dictionary) -> HashSet<u8> {
498    let mut codes = HashSet::new();
499    let mut code = 0i64;
500    for el in differences_array(doc, fdict).unwrap_or_default() {
501        match el {
502            Object::Integer(i) => code = i,
503            Object::Real(r) => code = r as i64,
504            Object::Name(_) => {
505                if (0..=255).contains(&code) {
506                    codes.insert(code as u8);
507                }
508                code += 1;
509            }
510            _ => {}
511        }
512    }
513    codes
514}
515
516/// [`Font::builtin`]: read from the embedded program only when a declared
517/// code is left to it — not covered by `/ToUnicode` or `/Differences`
518/// (docling-parse reads the program lazily, on the first code that needs it).
519#[cfg(feature = "render")]
520fn builtin_encoding(doc: &Document, fdict: &Dictionary, font: &Font) -> HashMap<u8, String> {
521    let mut codes: Vec<u32> = font.widths.keys().copied().collect();
522    if codes.is_empty() {
523        codes.extend(0..=255);
524    }
525    let needed = codes.iter().any(|&c| {
526        c <= 255 && !font.to_unicode.contains_key(&c) && !font.diff_codes.contains(&(c as u8))
527    });
528    if !needed {
529        return HashMap::new();
530    }
531    crate::render::font::LoadedFont::load(doc, fdict).builtin_encoding_text()
532}
533
534/// Without the renderer there is no program to read: the font keeps the
535/// encoding tables' reading.
536#[cfg(not(feature = "render"))]
537fn builtin_encoding(_doc: &Document, _fdict: &Dictionary, _font: &Font) -> HashMap<u8, String> {
538    HashMap::new()
539}
540
541/// The ink extents of `font`'s declared codes ([`Font::ink_y`]), read from
542/// its embedded program; nothing past a CID font's first 4096 codes.
543#[cfg(feature = "render")]
544fn ink_extents(doc: &Document, fdict: &Dictionary, font: &Font) -> HashMap<u32, (f64, f64)> {
545    const MAX_CODES: usize = 4096;
546    let loaded = crate::render::font::LoadedFont::load(doc, fdict);
547    if !loaded.has_program() {
548        return HashMap::new();
549    }
550    let mut codes: Vec<u32> = font.widths.keys().copied().collect();
551    if !font.two_byte {
552        codes.extend(0..=255);
553        codes.sort_unstable();
554        codes.dedup();
555    }
556    codes
557        .into_iter()
558        .take(MAX_CODES)
559        .filter_map(|c| Some((c, loaded.glyph_ink_y(c)??)))
560        .collect()
561}
562
563/// Without the renderer there is no program to read: the font keeps its
564/// loose box (docling-parse's own fallback when FreeType resolves no glyph).
565#[cfg(not(feature = "render"))]
566fn ink_extents(_doc: &Document, _fdict: &Dictionary, _font: &Font) -> HashMap<u32, (f64, f64)> {
567    HashMap::new()
568}
569
570/// The codes `font` leaves unmapped whose embedded glyph paints nothing
571/// ([`Font::blank_codes`]). The program is loaded only when the font has
572/// unmapped codes among those it declares widths for; without the
573/// renderer (the wasm text-layer build) there is no program to ask, and
574/// every unmapped glyph reads U+FFFD.
575#[cfg(feature = "render")]
576fn blank_codes(doc: &Document, fdict: &Dictionary, font: &Font) -> HashSet<u32> {
577    // A CID font can declare tens of thousands of widths: past this many
578    // unmapped codes nothing is classified (the glyphs read U+FFFD).
579    const MAX_CANDIDATES: usize = 4096;
580    let unmapped: Vec<u32> = font
581        .widths
582        .keys()
583        .copied()
584        .filter(|&c| font.decode_code(c).0.as_deref() == Some("\u{FFFD}"))
585        .take(MAX_CANDIDATES + 1)
586        .collect();
587    if unmapped.is_empty() || unmapped.len() > MAX_CANDIDATES {
588        return HashSet::new();
589    }
590    let loaded = crate::render::font::LoadedFont::load(doc, fdict);
591    unmapped
592        .into_iter()
593        .filter(|&c| loaded.glyph_has_ink(c) == Some(false))
594        .collect()
595}
596
597#[cfg(not(feature = "render"))]
598fn blank_codes(_doc: &Document, _fdict: &Dictionary, _font: &Font) -> HashSet<u32> {
599    HashSet::new()
600}
601
602#[derive(Clone, Copy)]
603enum CidFallback {
604    None,
605    UnicodeCodes { utf16: bool },
606    Collection(&'static [u8]),
607}
608
609/// A Unicode CMap code retains its original bytes for ToUnicode lookups;
610/// a four-byte UTF-16 code is a packed surrogate pair, not a scalar value.
611pub(crate) fn unicode_cmap_char(code: u32) -> Option<char> {
612    if code <= 0xffff {
613        char::from_u32(code)
614    } else {
615        let units = [(code >> 16) as u16, code as u16];
616        let mut chars = char::decode_utf16(units);
617        let ch = chars.next()?.ok()?;
618        chars.next().is_none().then_some(ch)
619    }
620}
621
622#[cfg(not(feature = "cid-collections"))]
623fn collection_table(_ordering: &[u8]) -> Option<&'static [u8]> {
624    None
625}
626
627#[cfg(feature = "cid-collections")]
628fn collection_table(ordering: &[u8]) -> Option<&'static [u8]> {
629    macro_rules! table {
630        ($file:literal) => {
631            include_bytes!(concat!(env!("OUT_DIR"), "/", $file))
632        };
633    }
634    Some(match ordering {
635        b"Japan1" => table!("Adobe-Japan1-UCS2"),
636        b"GB1" => table!("Adobe-GB1-UCS2"),
637        b"CNS1" => table!("Adobe-CNS1-UCS2"),
638        b"Korea1" => table!("Adobe-Korea1-UCS2"),
639        b"KR" => table!("Adobe-KR-UCS2"),
640        _ => return None,
641    })
642}
643
644/// The text of `cid` in a table build.rs wrote: the CID count, then three
645/// bytes per CID (0 for none, a code point, or 0x110000 + the offset of a
646/// length-prefixed UTF-8 string after the entries).
647pub(crate) fn collection_text(table: &[u8], cid: u32) -> Option<String> {
648    let n = u32::from_le_bytes(table.get(..4)?.try_into().ok()?) as usize;
649    let at = 4 + 3 * usize::try_from(cid).ok().filter(|&c| c < n)?;
650    let &[a, b, c] = table.get(at..at + 3)? else {
651        return None;
652    };
653    match u32::from_be_bytes([0, a, b, c]) {
654        0 => None,
655        v @ ..0x110000 => char::from_u32(v).map(String::from),
656        v => {
657            let at = 4 + 3 * n + (v - 0x110000) as usize;
658            let len = *table.get(at)? as usize;
659            std::str::from_utf8(table.get(at + 1..at + 1 + len)?)
660                .ok()
661                .map(str::to_owned)
662        }
663    }
664}
665
666fn cid_fallback(doc: &Document, fdict: &Dictionary) -> CidFallback {
667    let encoding = fdict
668        .get(b"Encoding")
669        .ok()
670        .and_then(|o| deref(doc, o))
671        .and_then(|o| o.as_name().ok())
672        .unwrap_or(&[]);
673    if encoding.starts_with(b"Uni")
674        && (encoding.windows(4).any(|w| w == b"UCS2") || encoding.windows(5).any(|w| w == b"UTF16"))
675    {
676        return CidFallback::UnicodeCodes {
677            utf16: encoding.windows(5).any(|w| w == b"UTF16"),
678        };
679    }
680    if encoding != b"Identity-H" && encoding != b"Identity-V" {
681        // Other CMaps need code → CID tables that are not bundled.
682        return CidFallback::None;
683    }
684    cid_collection(doc, fdict).map_or(CidFallback::None, CidFallback::Collection)
685}
686
687/// The CID → Unicode table of the character collection a Type0 font uses.
688pub(crate) fn cid_collection(doc: &Document, fdict: &Dictionary) -> Option<&'static [u8]> {
689    let Object::Array(descendants) = deref(doc, fdict.get(b"DescendantFonts").ok()?)? else {
690        return None;
691    };
692    let descendant = as_dict(doc, descendants.first()?)?;
693    let info = as_dict(doc, descendant.get(b"CIDSystemInfo").ok()?)?;
694    let registry = deref(doc, info.get(b"Registry").ok()?)?.as_str().ok()?;
695    let ordering = deref(doc, info.get(b"Ordering").ok()?)?.as_str().ok()?;
696    (registry == b"Adobe").then(|| collection_table(ordering))?
697}
698
699/// A simple font's `/Encoding` `/Differences` array, dereferenced.
700fn differences_array(doc: &Document, fdict: &Dictionary) -> Option<Vec<Object>> {
701    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))?;
702    let diffs = enc.as_dict().ok()?.get(b"Differences").ok()?;
703    match deref(doc, diffs)? {
704        Object::Array(a) => Some(a.clone()),
705        _ => None,
706    }
707}
708
709/// A ToUnicode CMap copied from the *original* font while the glyphs were
710/// re-encoded: the encrypted `2206.01062_pg3` fixture (a Quartz re-save of
711/// pdfTeX output) keeps the T1-encoding CMap, keyed on the old codes, next
712/// to a `/Differences [33 /f_f_i /f_i /f_f /f_l]` that names the ligatures at
713/// new ones, so the CMap maps code 33 to `!` — its own identity — and
714/// `difficulties` read `di ! culties`, in docling's pdfium chain too. A
715/// ToUnicode entry that is the code's identity, for a code whose
716/// `/Differences` name resolves to something else, is that stale copy: the
717/// name wins. A CMap that disagrees any other way is left alone — ToUnicode
718/// is the producer's word on what the glyph means.
719///
720/// The name also wins over a ToUnicode value that says nothing (#1121):
721/// docling-parse's `has_to_unicode` discards an empty value and a single
722/// NUL, U+FFFD or Unicode noncharacter for a code `/Differences` names, and
723/// resolves the name instead. A figure embedded in 1503.04203 re-encodes
724/// its TeX glyphs to code 33 with `/Differences [33 /minus]` and a CMap
725/// `<21> <0000>` — the glyph's old code written as its Unicode — so
726/// `Q = −1/2` read `Q = � 12` and `√Δ²` read `p�2`.
727fn reconcile_stale_tounicode(to_unicode: &mut HashMap<u32, String>, diffs: &[Object]) {
728    let says_nothing = |value: &str| {
729        let mut chars = value.chars();
730        match (chars.next(), chars.next()) {
731            (None, _) => true,
732            (Some(c), None) => {
733                let cp = c as u32;
734                cp == 0 || cp == 0xFFFD || (0xFDD0..=0xFDEF).contains(&cp) || cp & 0xFFFE == 0xFFFE
735            }
736            _ => false,
737        }
738    };
739    let mut code = 0u8;
740    for el in diffs {
741        match el {
742            Object::Integer(i) => code = *i as u8,
743            Object::Name(name) => {
744                if let Some(ch) = glyph_name_to_char(name) {
745                    let identity = (code as char).to_string();
746                    let current = to_unicode.get(&u32::from(code));
747                    if (ch.to_string() != identity && current == Some(&identity))
748                        || current.is_some_and(|v| says_nothing(v))
749                    {
750                        to_unicode.insert(u32::from(code), ch.to_string());
751                    }
752                }
753                code = code.wrapping_add(1);
754            }
755            _ => {}
756        }
757    }
758}
759
760/// Collect `/Differences` entries whose glyph name is a GID placeholder
761/// (`g115`, `cid42`, `glyph7`, `index9`, `SM590000`) with no Unicode mapping
762/// (subsetted fonts with no ToUnicode); [`unknown_name_text`] decides what
763/// each reads as. The placeholder-name restriction keeps real Adobe glyph
764/// names on the normal path so this never invents garbage on the clean files.
765fn differences_gid_names(doc: &Document, fdict: &Dictionary) -> HashMap<u8, String> {
766    let mut map = HashMap::new();
767    let Some(Object::Dictionary(enc)) = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))
768    else {
769        return map;
770    };
771    let Some(Object::Array(diffs)) = enc.get(b"Differences").ok().and_then(|o| deref(doc, o))
772    else {
773        return map;
774    };
775    let mut code = 0u8;
776    for el in diffs {
777        match el {
778            Object::Integer(i) => code = *i as u8,
779            Object::Name(name) => {
780                if glyph_name_to_char(name).is_none() && is_gid_name(name) {
781                    map.insert(code, String::from_utf8_lossy(name).into_owned());
782                }
783                code = code.wrapping_add(1);
784            }
785            _ => {}
786        }
787    }
788    map
789}
790
791/// Parse the embedded Type1 font program's built-in `/Encoding` vector
792/// (`dup <code> /<glyphname> put` entries in the clear-text header before
793/// `eexec`) into `code → char`. This is how docling recovers glyphs from
794/// standard TeX math fonts (CMMI/CMSY/…) that carry no PDF `/Encoding` and no
795/// ToUnicode — e.g. CMMI's `dup 21 /lambda` or CMSY's `dup 20 /lessequal`.
796/// Only `FontFile` (Type1) is parsed; CFF (`FontFile3`) and TrueType
797/// (`FontFile2`) store their encoding in a binary table and are left alone.
798fn type1_program_encoding(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
799    let mut map = HashMap::new();
800    let Some(desc) = fdict
801        .get(b"FontDescriptor")
802        .ok()
803        .and_then(|o| deref(doc, o))
804        .and_then(|o| o.as_dict().ok())
805    else {
806        return map;
807    };
808    let Some(data) = desc
809        .get(b"FontFile")
810        .ok()
811        .and_then(|o| deref(doc, o))
812        .and_then(|o| o.as_stream().ok())
813        .and_then(|s| s.decompressed_content().ok())
814    else {
815        return map;
816    };
817    // The clear-text header (PostScript) ends at `eexec`; the rest is encrypted.
818    let head_end = data
819        .windows(5)
820        .position(|w| w == b"eexec")
821        .unwrap_or(data.len());
822    let head = String::from_utf8_lossy(&data[..head_end]);
823    // Scan for `dup <code> /<name> put` tokens.
824    let toks: Vec<&str> = head.split_whitespace().collect();
825    for w in toks.windows(4) {
826        if w[0] == "dup" && w[3] == "put" {
827            if let (Ok(code), Some(name)) = (w[1].parse::<u32>(), w[2].strip_prefix('/')) {
828                if code <= 255 {
829                    if let Some(ch) = glyph_name_to_char(name.as_bytes()) {
830                        map.insert(code as u8, ch);
831                    }
832                }
833            }
834        }
835    }
836    map
837}
838
839/// The reading text docling-parse 7.24 keeps for a `/Differences` glyph name
840/// no table resolves (`resolve_unknown_name`): the name without any
841/// `.suffix` — `None` when that is a bare glyph index
842/// (`(gid|glyph|g|cid|index)\d+`, any case), which names a glyph of the
843/// font program but carries no text (docling-parse#238), so it is unmapped.
844fn unknown_name_text(name: &str) -> Option<String> {
845    let stem = name.rsplit_once('.').map_or(name, |(s, _)| s);
846    (!is_index_name(stem)).then(|| stem.to_string())
847}
848
849/// docling-parse's index-style glyph name, `(gid|glyph|g|cid|index)` then
850/// hexadecimal digits, case-insensitive (`re_gid`): `/gid00043`, `/g43`,
851/// `/cid43`, `/index43` from the subset generators, and since docling-parse
852/// 7.25 (#389 in docling-parse#431, #1113) Skia/PDF's `g%X` — `/g15C`,
853/// `/gA1` from Chrome's print-to-PDF and Notion exports, which read as
854/// their base encoding's letters before. The name points into the font
855/// program and carries no text.
856fn is_index_name(name: &str) -> bool {
857    let lower = name.to_ascii_lowercase();
858    ["gid", "glyph", "g", "cid", "index"].iter().any(|p| {
859        lower
860            .strip_prefix(p)
861            .is_some_and(|d| !d.is_empty() && d.bytes().all(|b| b.is_ascii_hexdigit()))
862    })
863}
864
865/// A glyph name that is a synthetic placeholder, not a real Adobe name:
866/// `g115`, `cid42`, `glyph7`, `index9`, `G12`, or a short-prefix code name like
867/// `SM590000` (IBM BookMaster). These carry no Unicode meaning (docling-parse
868/// keeps the code-like names as text, `SM590000`, and reads the glyph indexes
869/// as unmapped, see [`unknown_name_text`]). `afii####` / `uni####` are real Adobe names
870/// and excluded. The restriction keeps genuine glyph names on the Unicode path.
871fn is_gid_name(name: &[u8]) -> bool {
872    let Ok(s) = std::str::from_utf8(name) else {
873        return false;
874    };
875    if s.starts_with("afii") || s.starts_with("uni") {
876        return false;
877    }
878    if is_index_name(s) {
879        return true;
880    }
881    // Short alpha prefix (≤3 letters) followed by a run of ≥3 digits — synthetic
882    // code names like `SM590000`, distinct from real Adobe names (whole words or
883    // letter+`.suffix` variants).
884    let alpha = s.bytes().take_while(|b| b.is_ascii_alphabetic()).count();
885    let digits = s.len() - alpha;
886    (1..=3).contains(&alpha)
887        && digits >= 3
888        && s.as_bytes()[alpha..].iter().all(|b| b.is_ascii_digit())
889}
890
891/// The font's ascent and descent (1000-unit em) for its loose char boxes,
892/// and whether docling-parse 7.24 rejects its descriptor metrics
893/// (`invalid_vertical_metrics`, `init_ascent_and_descent`) — then it boxes
894/// each glyph by its embedded outline instead ([`Font::ink_y`]):
895///
896/// - an `/Ascent` or `/Descent` beyond ±8000 (`MAX_PLAUSIBLE_VERTICAL_METRIC`)
897///   is ignored for the `/FontBBox` edge, when that is usable;
898/// - metrics that do not straddle the baseline (`descent <= 0 <= ascent`,
899///   `descent < ascent` — `/Ascent 0 /Descent 0`, or a math font's two
900///   positive values) give way to a valid `/FontBBox`.
901fn font_ascent_descent(doc: &Document, fdict: &Dictionary, two_byte: bool) -> (f64, f64, bool) {
902    // For Type0, the descriptor lives on the descendant CIDFont.
903    let descr_owner = if two_byte {
904        fdict
905            .get(b"DescendantFonts")
906            .ok()
907            .and_then(|o| deref(doc, o))
908            .and_then(|o| match o {
909                Object::Array(a) => a.first(),
910                _ => None,
911            })
912            .and_then(|o| as_dict(doc, o))
913    } else {
914        Some(fdict)
915    };
916    let fd = descr_owner
917        .and_then(|d| d.get(b"FontDescriptor").ok())
918        .and_then(|o| as_dict(doc, o));
919    let read = |key: &[u8]| fd.and_then(|d| d.get(key).ok()).and_then(num);
920    let plausible = |v: f64| v.is_finite() && v.abs() <= 8000.0;
921    let bbox = fd
922        .and_then(|d| d.get(b"FontBBox").ok())
923        .and_then(|o| deref(doc, o))
924        .and_then(|o| match o {
925            Object::Array(a) if a.len() == 4 => {
926                let v: Vec<f64> = a.iter().filter_map(num).collect();
927                (v.len() == 4).then(|| [v[0], v[1], v[2], v[3]])
928            }
929            _ => None,
930        });
931    let usable = |v: f64| v.abs() > 1e-3 && plausible(v);
932    let mut invalid = false;
933    let asc = match read(b"Ascent") {
934        Some(v) if plausible(v) => v,
935        Some(_) => {
936            invalid = true;
937            bbox.map(|b| b[3]).filter(|&v| usable(v)).unwrap_or(750.0)
938        }
939        None => 750.0,
940    };
941    let desc = match read(b"Descent") {
942        Some(v) if plausible(v) => v,
943        Some(_) => {
944            invalid = true;
945            bbox.map(|b| b[1]).filter(|&v| usable(v)).unwrap_or(-250.0)
946        }
947        None => -250.0,
948    };
949    if !(desc <= 0.0 && asc >= 0.0 && desc < asc) {
950        if let Some(b) = bbox.filter(|b| {
951            b[1] <= 0.0 && b[3] >= 0.0 && b[1] < b[3] && plausible(b[1]) && plausible(b[3])
952        }) {
953            return (b[3], b[1], true);
954        }
955    }
956    // Some subsetted fonts carry a degenerate FontDescriptor (`/Ascent 0
957    // /Descent 0`) and no usable FontBBox either: the real metrics live in
958    // the font program. A zero-height loose box gives the line cells zero
959    // area and the layout's region/text assignment drops them (2305's
960    // References list lost every prose line, keeping only the URLs), so
961    // fall back to typical text metrics.
962    if asc - desc <= 1.0 {
963        return (750.0, -250.0, invalid);
964    }
965    (asc, desc, invalid)
966}
967
968/// The `/BaseFont` name with any `ABCDEF+` subset prefix stripped (#187).
969fn base_font_name(fdict: &Dictionary) -> Option<Vec<u8>> {
970    let name = fdict.get(b"BaseFont").ok()?.as_name().ok()?;
971    let stripped = match name.iter().position(|&b| b == b'+') {
972        Some(i) if i == 6 => &name[i + 1..],
973        _ => name,
974    };
975    Some(stripped.to_vec())
976}
977
978/// Simple-font widths: `/FirstChar` + `/Widths` array, `/MissingWidth` default.
979fn simple_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
980    let mut map = HashMap::new();
981    let first = fdict
982        .get(b"FirstChar")
983        .ok()
984        .and_then(|o| o.as_i64().ok())
985        .unwrap_or(0) as u32;
986    if let Some(Object::Array(arr)) = fdict.get(b"Widths").ok().and_then(|o| deref(doc, o)) {
987        for (i, w) in arr.iter().enumerate() {
988            if let Some(w) = num(w) {
989                map.insert(first + i as u32, w);
990            }
991        }
992    }
993    let dw = fdict
994        .get(b"FontDescriptor")
995        .ok()
996        .and_then(|o| as_dict(doc, o))
997        .and_then(|d| d.get(b"MissingWidth").ok())
998        .and_then(num)
999        .unwrap_or(0.0);
1000    (map, dw)
1001}
1002
1003/// CIDFont widths: the `/W` array on the descendant font (`/DW` default = 1000).
1004fn cid_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
1005    let mut map = HashMap::new();
1006    let Some(desc) = fdict
1007        .get(b"DescendantFonts")
1008        .ok()
1009        .and_then(|o| deref(doc, o))
1010        .and_then(|o| match o {
1011            Object::Array(a) => a.first(),
1012            _ => None,
1013        })
1014        .and_then(|o| as_dict(doc, o))
1015    else {
1016        return (map, 1000.0);
1017    };
1018    let dw = desc.get(b"DW").ok().and_then(num).unwrap_or(1000.0);
1019    if let Some(Object::Array(w)) = desc.get(b"W").ok().and_then(|o| deref(doc, o)) {
1020        let mut i = 0;
1021        while i < w.len() {
1022            let c = w.get(i).and_then(num);
1023            match (c, w.get(i + 1)) {
1024                // `c [w1 w2 ...]`: consecutive CIDs starting at c.
1025                (Some(c), Some(Object::Array(list))) => {
1026                    for (k, wv) in list.iter().enumerate() {
1027                        if let Some(wv) = num(wv) {
1028                            map.insert(c as u32 + k as u32, wv);
1029                        }
1030                    }
1031                    i += 2;
1032                }
1033                // `c_first c_last w`: a run all of width w.
1034                (Some(c1), Some(o2)) => {
1035                    if let (Some(c2), Some(wv)) = (num(o2), w.get(i + 2).and_then(num)) {
1036                        for cid in c1 as u32..=c2 as u32 {
1037                            map.insert(cid, wv);
1038                        }
1039                    }
1040                    i += 3;
1041                }
1042                _ => break,
1043            }
1044        }
1045    }
1046    (map, dw)
1047}
1048
1049fn num(o: &Object) -> Option<f64> {
1050    match o {
1051        Object::Integer(i) => Some(*i as f64),
1052        Object::Real(r) => Some(real(*r)),
1053        _ => None,
1054    }
1055}
1056
1057/// lopdf keeps a real as an `f32`; docling-parse reads the same token as a
1058/// double. The shortest decimal that round-trips the `f32` is the token the
1059/// file wrote (for any real written with the precision `f32` holds), so
1060/// `90.144 + 1.44` meets `91.584` here as it does there.
1061fn real(r: f32) -> f64 {
1062    use std::fmt::Write as _;
1063    let mut buf = String::with_capacity(16);
1064    let _ = write!(buf, "{r}");
1065    buf.parse().unwrap_or(r as f64)
1066}
1067
1068/// Decode a PDF string literal into codes, retaining packed UTF-16 pairs
1069/// as the original four-byte keys used by ToUnicode.
1070fn codes(font: &Font, bytes: &[u8]) -> Vec<u32> {
1071    if font.two_byte {
1072        let mut units = bytes
1073            .chunks(2)
1074            .map(|c| {
1075                if c.len() == 2 {
1076                    ((c[0] as u32) << 8) | c[1] as u32
1077                } else {
1078                    c[0] as u32
1079                }
1080            })
1081            .peekable();
1082        let utf16 = matches!(font.cid_fallback, CidFallback::UnicodeCodes { utf16: true });
1083        let mut out = Vec::with_capacity(bytes.len() / 2 + 1);
1084        while let Some(mut code) = units.next() {
1085            if utf16 && (0xd800..=0xdbff).contains(&code) {
1086                if let Some(low) = units.next_if(|u| (0xdc00..=0xdfff).contains(u)) {
1087                    code = (code << 16) | low;
1088                }
1089            }
1090            out.push(code);
1091        }
1092        out
1093    } else {
1094        bytes.iter().map(|&b| b as u32).collect()
1095    }
1096}
1097
1098/// A page's display box, in PDF user space: the `/CropBox` clipped to the
1099/// `/MediaBox`, both inherited through the page tree and normalized — a
1100/// missing or empty MediaBox is US Letter, an empty CropBox is the MediaBox
1101/// (pdfium's `CPDF_Page::UpdateDimensions`). pdfium reports the page size from
1102/// this box, renders exactly it, and translates every content coordinate so
1103/// its lower-left corner is the origin (`m_PageMatrix`); docling's backends
1104/// inherit that frame, so text cells, `prov` boxes and destinations all count
1105/// from the CropBox corner, not the MediaBox one. The parser used to flip
1106/// glyphs with the MediaBox *height* and no translation at all, so a page
1107/// whose boxes do not start at (0, 0) — a trimmed book page with
1108/// `MediaBox [-56 -58 576 723]` / `CropBox [1 -0.6 519 666]`, or a LaTeX
1109/// figure cropped to `[156 147 637 391]` — had its text displaced against the
1110/// rendered bitmap by the box offset, the bottom lines pushed past the page
1111/// edge and clamped to `t = b`.
1112#[derive(Debug, Clone, Copy, PartialEq)]
1113pub(crate) struct PageBox {
1114    /// Left edge, user space.
1115    pub l: f32,
1116    /// Bottom edge, user space.
1117    pub b: f32,
1118    pub w: f32,
1119    pub h: f32,
1120}
1121
1122impl PageBox {
1123    /// Top edge, user space — the y that becomes `0` in the y-down frame.
1124    pub fn top(&self) -> f32 {
1125        self.b + self.h
1126    }
1127}
1128
1129/// A page-tree rect attribute (`/MediaBox`, `/CropBox`), inherited from the
1130/// nearest ancestor that sets it, as normalized `(l, b, r, t)`.
1131fn inherited_rect(
1132    doc: &Document,
1133    page_id: lopdf::ObjectId,
1134    key: &[u8],
1135) -> Option<(f32, f32, f32, f32)> {
1136    let mut id = page_id;
1137    for _ in 0..32 {
1138        let dict = doc.get_object(id).ok()?.as_dict().ok()?;
1139        if let Some(Object::Array(a)) = dict.get(key).ok().and_then(|o| deref(doc, o)) {
1140            let v: Vec<f32> = a.iter().filter_map(|o| num(o).map(|x| x as f32)).collect();
1141            if v.len() == 4 && v.iter().all(|x| x.is_finite()) {
1142                return Some((
1143                    v[0].min(v[2]),
1144                    v[1].min(v[3]),
1145                    v[0].max(v[2]),
1146                    v[1].max(v[3]),
1147                ));
1148            }
1149            return None;
1150        }
1151        id = dict.get(b"Parent").ok()?.as_reference().ok()?;
1152    }
1153    None
1154}
1155
1156pub(crate) fn page_box(doc: &Document, page_id: lopdf::ObjectId) -> PageBox {
1157    let nonempty = |r: &(f32, f32, f32, f32)| r.2 > r.0 && r.3 > r.1;
1158    let media = inherited_rect(doc, page_id, b"MediaBox")
1159        .filter(nonempty)
1160        .unwrap_or((0.0, 0.0, 612.0, 792.0));
1161    let crop = inherited_rect(doc, page_id, b"CropBox")
1162        .map(|c| {
1163            (
1164                c.0.max(media.0),
1165                c.1.max(media.1),
1166                c.2.min(media.2),
1167                c.3.min(media.3),
1168            )
1169        })
1170        .filter(nonempty)
1171        .unwrap_or(media);
1172    PageBox {
1173        l: crop.0,
1174        b: crop.1,
1175        w: crop.2 - crop.0,
1176        h: crop.3 - crop.1,
1177    }
1178}
1179
1180/// Page size (width, height) in PDF points — the display box's, like pdfium's
1181/// `FPDF_GetPageWidthF/HeightF`.
1182fn page_size(doc: &Document, page_id: lopdf::ObjectId) -> (f32, f32) {
1183    let pb = page_box(doc, page_id);
1184    (pb.w, pb.h)
1185}
1186
1187/// Localize where a page's text is lost, for the `text_layer` diagnostic.
1188/// Extraction can come up empty at three different points — no content stream
1189/// reached the parser, the stream did not decode into operators, or it ran but
1190/// produced no glyphs (fonts/encodings) — and from the outside all three look
1191/// the same. Report them per page.
1192pub fn content_diagnosis(bytes: &[u8]) -> String {
1193    let Some(doc) = load_document(bytes) else {
1194        return "document does not load".into();
1195    };
1196    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1197    pages.sort_by_key(|(n, _)| *n);
1198    let mut out = String::new();
1199    let mut caches = DocCaches::default();
1200    for (n, pid) in pages.into_iter().take(4) {
1201        let content_bytes = doc.get_page_content(pid);
1202        let ops = lopdf::content::Content::decode(&content_bytes)
1203            .map(|c| c.operations.len())
1204            .ok();
1205        let res = page_res(&doc, pid);
1206        let fonts = res.map(|r| fonts_from_res(&doc, r, &mut caches).len());
1207        let glyphs = page_glyphs_cached(&doc, pid, &mut caches).len();
1208        out.push_str(&format!(
1209            "\n   page {n}: content {} B, ops {}, resources {}, fonts {}, glyphs {}",
1210            content_bytes.len(),
1211            ops.map_or("UNDECODABLE".to_string(), |n| n.to_string()),
1212            if res.is_some() { "ok" } else { "MISSING" },
1213            fonts.map_or("-".to_string(), |n| n.to_string()),
1214            glyphs,
1215        ));
1216    }
1217    out
1218}
1219
1220/// Is this "text layer" a vestige rather than the document's text?
1221///
1222/// Scanned forms often carry a handful of typed-in strings — a date filled
1223/// into three form fields, say — on top of pages that are otherwise images.
1224/// Treating that as a real text layer is the worst of both worlds: the text
1225/// path proudly extracts thirteen characters, and no OCR ever runs on the
1226/// letter the pages actually show. The reported form did exactly this (3
1227/// lines, 13 chars, 3 pages).
1228///
1229/// The rule is deliberately tight so genuinely sparse *digital* documents are
1230/// not misrouted into OCR: only a document averaging at most one line per page
1231/// **and** totalling fewer than 32 characters is called vestigial.
1232pub fn text_layer_is_vestigial(pages: &[crate::pdfium_backend::PdfPage]) -> bool {
1233    let mut tally = TextLayerTally::default();
1234    pages.iter().for_each(|p| tally.add(p));
1235    tally.is_vestigial()
1236}
1237
1238/// [`text_layer_is_vestigial`]'s counts, accumulated a page at a time so a
1239/// streaming caller need not keep the pages around to decide.
1240#[derive(Debug, Default, Clone, Copy)]
1241pub struct TextLayerTally {
1242    pages: usize,
1243    lines: usize,
1244    chars: usize,
1245}
1246
1247impl TextLayerTally {
1248    pub fn add(&mut self, page: &crate::pdfium_backend::PdfPage) {
1249        self.pages += 1;
1250        self.lines += page.cells.len();
1251        self.chars += page
1252            .cells
1253            .iter()
1254            .map(|c| c.text.chars().count())
1255            .sum::<usize>();
1256    }
1257
1258    /// The verdict over the pages added so far.
1259    pub fn is_vestigial(&self) -> bool {
1260        self.lines == 0 || (self.lines <= self.pages && self.chars < 32)
1261    }
1262
1263    /// Whether the pages added so far already rule vestigial out for a
1264    /// document of `total_pages`, whatever the remaining pages hold — both
1265    /// counts only grow, so a caller may stop parsing pages it does not need.
1266    pub fn proves_text(&self, total_pages: usize) -> bool {
1267        self.lines > 0 && (self.lines > total_pages || self.chars >= 32)
1268    }
1269}
1270
1271/// Why the cross-reference repair did or did not fire, for the `text_layer`
1272/// diagnostic. A PDF that will not load is indistinguishable from a scan in
1273/// production (both convert to nothing), so the reason has to be askable.
1274pub fn xref_repair_status(bytes: &[u8]) -> String {
1275    if Document::load_mem(bytes).is_ok() {
1276        return "loads unaided; no repair needed".into();
1277    }
1278    match pad_short_xref_entries(bytes) {
1279        Ok(fixed) => match Document::load_mem(&fixed) {
1280            Ok(_) => "repaired: cross-reference entries padded to 20 bytes".into(),
1281            Err(e) => format!("padded the entries, but it still will not load: {e}"),
1282        },
1283        Err(why) => format!("repair declined — {why}"),
1284    }
1285}
1286
1287/// Load a PDF, repairing the one malformation that otherwise costs us the whole
1288/// document: **19-byte cross-reference entries**.
1289///
1290/// The spec fixes an xref entry at 20 bytes — `nnnnnnnnnn ggggg n` plus a
1291/// *two*-byte EOL. Some generators (an Austrian telecom's invoices, for one)
1292/// emit a bare LF instead, making each entry 19 bytes. lopdf rejects the file
1293/// outright (`invalid file trailer`) where pdfium reads it happily, so a
1294/// perfectly good text layer looked to the browser exactly like a scan and cost
1295/// ten seconds of OCR.
1296///
1297/// Padding is only attempted when it cannot move anything the xref points at:
1298/// a single `xref` section that begins after the last object. The repair then
1299/// has to prove itself — the padded bytes are used only if they load — so a
1300/// mis-repair degrades to today's behaviour rather than to silent garbage.
1301pub(crate) fn load_document(bytes: &[u8]) -> Option<Document> {
1302    open_document(bytes, None).ok()
1303}
1304
1305/// Why [`open_document`] could not hand back a readable document.
1306#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1307pub(crate) enum OpenError {
1308    /// lopdf cannot read the file even after the repairs.
1309    Unreadable,
1310    /// The file is encrypted and the password given (or none) does not open it.
1311    Password,
1312}
1313
1314/// Load `bytes` with the document's password. lopdf decrypts while reading —
1315/// with `password`, or the empty user password most "protected" PDFs carry
1316/// (what every viewer opens silently) — so every reader downstream sees plain
1317/// streams; a file the password does not open loads with its streams still
1318/// encrypted, which is reported as [`OpenError::Password`] rather than handed
1319/// on as a document whose every stream decodes to nothing.
1320pub(crate) fn open_document(bytes: &[u8], password: Option<&str>) -> Result<Document, OpenError> {
1321    let Some(doc) = load_document_raw(bytes, password) else {
1322        // lopdf refuses to load at all under a *wrong* password (a missing
1323        // one loads the file with its streams still encrypted); tell the two
1324        // apart by loading without it.
1325        if password.is_some() && load_document_raw(bytes, None).is_some_and(|d| d.is_encrypted()) {
1326            return Err(OpenError::Password);
1327        }
1328        return Err(OpenError::Unreadable);
1329    };
1330    if doc.is_encrypted() {
1331        return Err(OpenError::Password);
1332    }
1333    Ok(doc)
1334}
1335
1336fn load_options(password: Option<&str>) -> lopdf::LoadOptions {
1337    lopdf::LoadOptions {
1338        password: password.map(str::to_string),
1339        ..lopdf::LoadOptions::default()
1340    }
1341}
1342
1343fn load_document_raw(bytes: &[u8], password: Option<&str>) -> Option<Document> {
1344    // Try progressively more repair, and accept a candidate only once the pages
1345    // actually carry content — a document whose streams were dropped still
1346    // "loads", so loading alone is not evidence the repair helped. A
1347    // well-formed file returns on the first attempt and pays for nothing.
1348    let mut fallback = None;
1349    if let Some(doc) = best_effort_load(bytes, password, &mut fallback) {
1350        return Some(doc);
1351    }
1352    let xref_fixed = pad_short_xref_entries(bytes).ok();
1353    if let Some(fixed) = &xref_fixed {
1354        if let Some(doc) = best_effort_load(fixed, password, &mut fallback) {
1355            return Some(doc);
1356        }
1357    }
1358    // Both defects can coexist, and the second only becomes visible once the
1359    // first is repaired, so build on whatever the previous step produced.
1360    let lengths_fixed = fix_stream_lengths(xref_fixed.as_deref().unwrap_or(bytes));
1361    if let Some(doc) = best_effort_load(&lengths_fixed, password, &mut fallback) {
1362        return Some(doc);
1363    }
1364    fallback
1365}
1366
1367/// Load `data`, returning it only when its pages carry content; a document that
1368/// merely parses is remembered as the fallback for when nothing does better.
1369fn best_effort_load(
1370    data: &[u8],
1371    password: Option<&str>,
1372    fallback: &mut Option<Document>,
1373) -> Option<Document> {
1374    match Document::load_mem_with_options(data, load_options(password)) {
1375        Ok(doc) if has_page_content(&doc) => Some(doc),
1376        Ok(doc) => {
1377            fallback.get_or_insert(doc);
1378            None
1379        }
1380        Err(_) => None,
1381    }
1382}
1383
1384/// Does any page actually hand us a content stream? A document whose streams
1385/// were dropped still parses — it simply has nothing to read — so this is what
1386/// tells a successful repair from a pointless one.
1387fn has_page_content(doc: &Document) -> bool {
1388    doc.get_pages()
1389        .into_values()
1390        .take(4)
1391        .any(|pid| !doc.get_page_content(pid).is_empty())
1392}
1393
1394/// Correct `/Length` values that disagree with where `endstream` actually is.
1395///
1396/// The same generator that writes short xref entries also overstates its
1397/// content-stream lengths by a byte or two. lopdf trusts `/Length`, reads past
1398/// the data, fails to find `endstream` there and drops the stream — the object
1399/// comes back as a bare dictionary, so the page has no content at all and the
1400/// document looks like a scan. pdfium instead trusts `endstream`, which is what
1401/// this does.
1402///
1403/// The rewrite is length-preserving: the corrected number is written over the
1404/// old digits and padded with spaces, so every byte offset in the file — and
1405/// therefore the whole cross-reference table — stays valid.
1406fn fix_stream_lengths(bytes: &[u8]) -> Vec<u8> {
1407    let mut out = bytes.to_vec();
1408    let mut i = 0;
1409    while let Some(rel) = find(&out[i..], b"stream") {
1410        let kw = i + rel;
1411        i = kw + 6;
1412        // Skip `endstream` (the keyword we are measuring *to*).
1413        if kw >= 3 && &out[kw - 3..kw] == b"end" {
1414            continue;
1415        }
1416        // The stream data starts after the EOL that follows the keyword.
1417        let mut data = kw + 6;
1418        if out.get(data..data + 2) == Some(b"\r\n".as_slice()) {
1419            data += 2;
1420        } else if matches!(out.get(data), Some(b'\n' | b'\r')) {
1421            data += 1;
1422        }
1423        let Some(end) = find(&out[data..], b"endstream").map(|r| data + r) else {
1424            continue;
1425        };
1426        // `/Length <digits>` in the dictionary just before the keyword.
1427        let dict_start = out[..kw].iter().rposition(|&c| c == b'<').unwrap_or(0);
1428        let Some(lrel) = find(&out[dict_start..kw], b"/Length") else {
1429            continue;
1430        };
1431        let mut d = dict_start + lrel + 7;
1432        while matches!(out.get(d), Some(b' ')) {
1433            d += 1;
1434        }
1435        let digits = out[d..].iter().take_while(|c| c.is_ascii_digit()).count();
1436        if digits == 0 {
1437            continue;
1438        }
1439        let declared: usize = match std::str::from_utf8(&out[d..d + digits])
1440            .ok()
1441            .and_then(|s| s.parse().ok())
1442        {
1443            Some(v) => v,
1444            None => continue,
1445        };
1446        let actual = end - data;
1447        // Only shrink, and only when the new value fits the space the old one
1448        // occupied — growing the number would move every following byte.
1449        let replacement = actual.to_string();
1450        if actual == declared || replacement.len() > digits {
1451            continue;
1452        }
1453        out[d..d + digits].fill(b' ');
1454        out[d..d + replacement.len()].copy_from_slice(replacement.as_bytes());
1455    }
1456    out
1457}
1458
1459fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
1460    haystack.windows(needle.len()).position(|w| w == needle)
1461}
1462
1463/// Rewrite a classic cross-reference table's entries to the spec's 20 bytes,
1464/// or `None` when the file's shape makes that unsafe (see [`load_document`]).
1465pub(crate) fn pad_short_xref_entries(bytes: &[u8]) -> Result<Vec<u8>, &'static str> {
1466    // Exactly one xref section, and it must start after every object, so that
1467    // growing it shifts nothing the table's offsets refer to.
1468    let is_boundary = |i: usize| i == 0 || matches!(bytes[i - 1], b'\n' | b'\r');
1469    let mut starts = (0..bytes.len().saturating_sub(4))
1470        .filter(|&i| &bytes[i..i + 4] == b"xref" && is_boundary(i));
1471    let xref_at = starts
1472        .next()
1473        .ok_or("no classic `xref` section (an xref stream?)")?;
1474    if starts.next().is_some() {
1475        return Err("more than one xref section (incremental update)");
1476    }
1477    let last_obj = bytes
1478        .windows(3)
1479        .rposition(|w| w == b"obj")
1480        .ok_or("no objects found")?;
1481    if last_obj > xref_at {
1482        return Err("an object follows the xref — padding would move it");
1483    }
1484
1485    let mut out = bytes[..xref_at].to_vec();
1486    out.extend_from_slice(b"xref\n");
1487    let mut i = xref_at + 4;
1488    let skip_ws = |i: &mut usize| {
1489        while matches!(bytes.get(*i), Some(b'\r' | b'\n' | b' ')) {
1490            *i += 1;
1491        }
1492    };
1493    loop {
1494        skip_ws(&mut i);
1495        // Either the next subsection header ("first count") or the trailer.
1496        if bytes[i..].starts_with(b"trailer") {
1497            out.extend_from_slice(&bytes[i..]);
1498            return Ok(out);
1499        }
1500        let header_end = i + bytes[i..]
1501            .iter()
1502            .position(|c| matches!(c, b'\n' | b'\r'))
1503            .ok_or("subsection header runs off the end")?;
1504        let header = std::str::from_utf8(&bytes[i..header_end])
1505            .map_err(|_| "subsection header is not text")?
1506            .trim();
1507        let mut parts = header.split_whitespace();
1508        let count: usize = parts
1509            .nth(1)
1510            .and_then(|c| c.parse().ok())
1511            .ok_or("unparseable subsection header")?;
1512        if parts.next().is_some() || count == 0 {
1513            return Err("unexpected subsection header shape");
1514        }
1515        out.extend_from_slice(header.as_bytes());
1516        out.push(b'\n');
1517        i = header_end;
1518        for _ in 0..count {
1519            skip_ws(&mut i);
1520            // `nnnnnnnnnn ggggg n` — the 18 bytes before whatever EOL follows.
1521            let entry = bytes.get(i..i + 18).ok_or("xref entry runs off the end")?;
1522            let well_formed = entry[..10].iter().all(u8::is_ascii_digit)
1523                && entry[10] == b' '
1524                && entry[11..16].iter().all(u8::is_ascii_digit)
1525                && entry[16] == b' '
1526                && matches!(entry[17], b'n' | b'f');
1527            if !well_formed {
1528                return Err("xref entry is not `nnnnnnnnnn ggggg n`");
1529            }
1530            out.extend_from_slice(entry);
1531            out.extend_from_slice(b" \n"); // the spec's 2-byte EOL -> 20 bytes
1532            i += 18;
1533        }
1534    }
1535}
1536
1537/// Debug: raw glyph stream `(ch, ll, lr, lb, lt)` (native coords) for page
1538/// `index`, before the sanitizer. For comparing char cells to docling-parse.
1539pub fn debug_glyphs(bytes: &[u8], index: usize) -> Vec<(char, f32, f32, f32, f32)> {
1540    let Some(doc) = load_document(bytes) else {
1541        return Vec::new();
1542    };
1543    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1544    pages.sort_by_key(|(n, _)| *n);
1545    let Some((_, pid)) = pages.get(index) else {
1546        return Vec::new();
1547    };
1548    page_glyphs(&doc, *pid)
1549        .into_iter()
1550        .map(|g| (g.ch, g.ll, g.lr, g.lb, g.lt))
1551        .collect()
1552}
1553
1554/// Public entry: per-page (width, height, line cells) for a PDF, via the Rust
1555/// text parser + the docling-parse line sanitizer. Used by the pipeline and the
1556/// `textparse_dump` example.
1557pub fn pdf_textlines(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1558    let Some(doc) = load_document(bytes) else {
1559        return Vec::new();
1560    };
1561    let mut caches = DocCaches::default();
1562    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1563    pages.sort_by_key(|(n, _)| *n);
1564    pages
1565        .into_iter()
1566        .map(|(_, pid)| {
1567            let (w, h) = page_size(&doc, pid);
1568            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1569            let cells = crate::dp_lines::line_cells(&glyphs, h, true);
1570            (w, h, cells)
1571        })
1572        .collect()
1573}
1574
1575/// Debug/diagnostic entry: per-page (width, height, word cells) for a PDF, via
1576/// the Rust parser glyphs run through the docling-parse word grouping. Used to
1577/// compare parser word cells against docling-parse's `word_cells` oracle (roadmap
1578/// item 6).
1579pub fn pdf_words(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1580    let Some(doc) = load_document(bytes) else {
1581        return Vec::new();
1582    };
1583    let mut caches = DocCaches::default();
1584    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1585    pages.sort_by_key(|(n, _)| *n);
1586    pages
1587        .into_iter()
1588        .map(|(_, pid)| {
1589            let (w, h) = page_size(&doc, pid);
1590            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1591            let cells = crate::dp_lines::word_cells(&glyphs, h, true);
1592            (w, h, cells)
1593        })
1594        .collect()
1595}
1596
1597/// One page's text cells from the pure-Rust parser: prose line cells, per-word
1598/// cells, and code line cells — all from a single glyph parse. Replaces the
1599/// pdfium text path (roadmap item 6) when the parser drop is enabled.
1600#[derive(Default)]
1601pub struct PageParserCells {
1602    pub prose: Vec<crate::pdfium_backend::TextCell>,
1603    pub words: Vec<crate::pdfium_backend::TextCell>,
1604    pub code: Vec<crate::pdfium_backend::TextCell>,
1605    /// Drawn checkbox squares (#609), in the cells' frame.
1606    pub checkboxes: Vec<crate::checkbox::CheckBox>,
1607    /// Image placements (#999), in the cells' frame — see
1608    /// [`crate::pdfium_backend::PdfPage::bitmaps`].
1609    pub bitmaps: Vec<[f32; 4]>,
1610    /// The page's shapes for the reading-order separators (#1092).
1611    pub shapes: PageShapes,
1612}
1613
1614/// The parser text layer, driven one page at a time: the document is loaded
1615/// (and repaired, see [`load_document`]) once, the font/form caches persist
1616/// across pages, and each page's glyphs are parsed only when asked for.
1617///
1618/// The eager whole-document walk this replaces ran *before* the first page
1619/// was rendered, so on a long PDF it was a serial prefix the page-worker pool
1620/// sat idle through — 6.2 s on the 1913-page .NET reference, in front of a
1621/// pipeline that otherwise overlaps parsing with inference — and a `--pages`
1622/// window still paid for every page in the file. Pulling pages on demand
1623/// keeps the parse on the producer thread but interleaved with rendering,
1624/// and skips unselected pages entirely. Output per page is unchanged: same
1625/// glyph walk, same shared caches, same contraction.
1626pub struct PageTextParser {
1627    doc: Document,
1628    caches: DocCaches,
1629    /// Page object ids in document order (page 1 first).
1630    pages: Vec<lopdf::ObjectId>,
1631    /// How words split where no space glyph does (#921); from
1632    /// `DOCLING_RS_PDF_WORD_GAP` unless [`Self::set_word_gap`] says.
1633    word_gap: crate::dp_lines::WordGap,
1634    /// Whether [`Self::text_page`]'s page keeps typographic punctuation as
1635    /// drawn (#921); `DOCLING_RS_PDF_KEEP_TYPOGRAPHY` unless
1636    /// [`Self::set_keep_typography`] says.
1637    keep_typography: bool,
1638}
1639
1640impl PageTextParser {
1641    /// Load the document; `None` when lopdf cannot read it at all.
1642    pub fn open(bytes: &[u8]) -> Option<Self> {
1643        Self::open_with_password(bytes, None)
1644    }
1645
1646    /// [`open`](Self::open) with the document's password (an encrypted file
1647    /// the password does not open reads as `None` here; the pipeline reports
1648    /// it through [`crate::pdf_meta::PdfMeta::open_with_password`] first).
1649    pub fn open_with_password(bytes: &[u8], password: Option<&str>) -> Option<Self> {
1650        let doc = open_document(bytes, password).ok()?;
1651        let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1652        pages.sort_by_key(|(n, _)| *n);
1653        Some(Self {
1654            doc,
1655            caches: DocCaches::default(),
1656            pages: pages.into_iter().map(|(_, pid)| pid).collect(),
1657            word_gap: crate::dp_lines::WordGap::from_env(),
1658            keep_typography: crate::assemble::keep_typography_from_env(),
1659        })
1660    }
1661
1662    /// The word-gap rule the line and word cells are built with (#921);
1663    /// `None` keeps the environment's (`DOCLING_RS_PDF_WORD_GAP`, else
1664    /// `adaptive`).
1665    pub fn set_word_gap(&mut self, gap: Option<crate::dp_lines::WordGap>) {
1666        if let Some(g) = gap {
1667            self.word_gap = g;
1668        }
1669    }
1670
1671    /// Whether the pages [`Self::text_page`] builds keep typographic
1672    /// punctuation as drawn (#921); `None` keeps the environment's
1673    /// (`DOCLING_RS_PDF_KEEP_TYPOGRAPHY`, else folding).
1674    pub fn set_keep_typography(&mut self, keep: Option<bool>) {
1675        if let Some(k) = keep {
1676            self.keep_typography = k;
1677        }
1678    }
1679
1680    /// The glyph boxes and font styles of the 0-based page `index` — the
1681    /// heading-hierarchy stage's style signal (#302): every non-space glyph's
1682    /// box (font ascent + descent at its size — the font-size proxy pdfium's
1683    /// loose char box also gave) in top-left coordinates, with the weight
1684    /// class and slant its `/BaseFont` name declares. Empty for a page
1685    /// without a text layer (a scan), and the stage falls back to its other
1686    /// signals.
1687    pub(crate) fn glyph_styles(
1688        &mut self,
1689        index: usize,
1690    ) -> Vec<crate::heading_hierarchy::GlyphStyle> {
1691        let Some(&pid) = self.pages.get(index) else {
1692            return Vec::new();
1693        };
1694        let (_w, h) = page_size(&self.doc, pid);
1695        let glyphs = page_glyphs_cached(&self.doc, pid, &mut self.caches);
1696        // Font hash → style, over the fonts the walk just parsed (an inline,
1697        // uncached font dictionary reads as unstyled).
1698        let styles: HashMap<u64, crate::font_style::FontStyle> = self
1699            .caches
1700            .fonts
1701            .values()
1702            .map(|f| (f.hash, f.style))
1703            .collect();
1704        glyphs
1705            .iter()
1706            .filter(|g| !g.ch.is_whitespace() && g.ll.is_finite())
1707            .map(|g| {
1708                let st = styles.get(&g.font).copied().unwrap_or_default();
1709                crate::heading_hierarchy::GlyphStyle {
1710                    l: g.ll,
1711                    t: h - g.lt,
1712                    r: g.lr,
1713                    b: h - g.lb,
1714                    height: g.height(),
1715                    weight_cls: crate::font_style::weight_class(st.weight),
1716                    italic: st.italic,
1717                    styled: st.known,
1718                }
1719            })
1720            .collect()
1721    }
1722
1723    /// The characters of the 0-based page `index` as the script-position
1724    /// stage measures them (#1005, docling's `read_page_chars`): every
1725    /// visible, non-blank glyph with its font-metric box, baseline and drawn
1726    /// size in the unrotated page frame (y up), and beside it the same box in
1727    /// the display frame (`/Rotate` applied, top-left origin: the frame the
1728    /// items' boxes live in). Invisible text (`Tr` 3 / 7, an OCR layer over a
1729    /// scan) is left out, as docling leaves it: it follows the scanned lines,
1730    /// not a typeset baseline.
1731    pub(crate) fn script_chars(
1732        &mut self,
1733        index: usize,
1734    ) -> Vec<(crate::script_position::ScriptChar, [f32; 4])> {
1735        let Some(&pid) = self.pages.get(index) else {
1736            return Vec::new();
1737        };
1738        let (w, h) = page_size(&self.doc, pid);
1739        let rotation = page_rotation(&self.doc, pid);
1740        page_glyphs_cached(&self.doc, pid, &mut self.caches)
1741            .iter()
1742            .filter(|g| !g.invisible && !g.ch.is_whitespace() && g.ll.is_finite())
1743            .filter_map(|g| {
1744                let [_, y, size] = g.pen?;
1745                let c = crate::script_position::ScriptChar {
1746                    text: g.ch,
1747                    left: g.ll,
1748                    bottom: g.lb,
1749                    right: g.lr,
1750                    top: g.lt,
1751                    baseline: y,
1752                    size,
1753                    horizontal: g.quad.is_none(),
1754                };
1755                let (l, t, r, b) = crate::pdfium_backend::to_display_frame(
1756                    (g.ll, h - g.lt, g.lr, h - g.lb),
1757                    rotation,
1758                    w,
1759                    h,
1760                );
1761                Some((c, [l, t, r, b]))
1762            })
1763            .collect()
1764    }
1765
1766    /// Prose, word and code cells of the 0-based page `index` — empty for an
1767    /// index the parser's page tree doesn't have (a damaged file whose page
1768    /// tree disagrees with the object model's count).
1769    pub fn cells(&mut self, index: usize) -> PageParserCells {
1770        let Some(&pid) = self.pages.get(index) else {
1771            return PageParserCells::default();
1772        };
1773        let (_w, h) = page_size(&self.doc, pid);
1774        let (glyphs, inks, bitmaps, shapes) =
1775            page_glyphs_and_inks(&self.doc, pid, &mut self.caches);
1776        let (prose, words) =
1777            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1778        PageParserCells {
1779            prose,
1780            words,
1781            code: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1782            checkboxes: crate::checkbox::find(&inks, h),
1783            bitmaps,
1784            shapes,
1785        }
1786    }
1787
1788    /// Pages in the document's page tree.
1789    pub fn page_count(&self) -> usize {
1790        self.pages.len()
1791    }
1792
1793    /// The 0-based page `index` as the text-layer-only conversion sees it —
1794    /// one element of [`pdf_text_pages`], parsed on demand so a caller can
1795    /// drop each page once it is assembled instead of holding the whole
1796    /// document's cells. A blank page (no cells) for an index the page tree
1797    /// doesn't have.
1798    pub fn text_page(&mut self, index: usize) -> crate::pdfium_backend::PdfPage {
1799        let (w, h, (glyphs, inks, _, shapes)) = match self.pages.get(index) {
1800            Some(&pid) => {
1801                let (w, h) = page_size(&self.doc, pid);
1802                (w, h, page_glyphs_and_inks(&self.doc, pid, &mut self.caches))
1803            }
1804            None => (0.0, 0.0, Default::default()),
1805        };
1806        let (mut prose, mut words) =
1807            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1808        drop_overpainted_cells(&mut prose);
1809        drop_overpainted_cells(&mut words);
1810        crate::pdfium_backend::PdfPage {
1811            #[cfg(feature = "ocr-prep")]
1812            image_layout: None,
1813            width: w,
1814            height: h,
1815            // Cells are native PDF points; there is no rendered bitmap.
1816            scale: 1.0,
1817            cells: prose,
1818            code_cells: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1819            word_cells: words,
1820            checkboxes: crate::checkbox::find(&inks, h),
1821            bitmaps: Vec::new(),
1822            shapes,
1823            #[cfg(feature = "ocr-prep")]
1824            image: image::RgbImage::new(1, 1),
1825            links: Vec::new(),
1826            rotation: 0,
1827            keep_typography: self.keep_typography,
1828            #[cfg(feature = "render")]
1829            embedded_images: Vec::new(),
1830            #[cfg(feature = "render")]
1831            pending_image: None,
1832        }
1833    }
1834}
1835
1836/// Full parser text layer: prose + word + code cells per page, glyphs parsed once.
1837/// `prose`/`words` come from the docling-parse contraction ([`crate::dp_lines`]);
1838/// `code` splits only at the parser's own space glyphs (monospace keeps its
1839/// source spacing). The eager form of [`PageTextParser`].
1840pub fn pdf_all_cells(bytes: &[u8]) -> Vec<PageParserCells> {
1841    let Some(mut parser) = PageTextParser::open(bytes) else {
1842        return Vec::new();
1843    };
1844    (0..parser.pages.len()).map(|i| parser.cells(i)).collect()
1845}
1846
1847/// Whole pages for the text-layer-only conversion ([`crate::convert_text_layer`]):
1848/// the parser's prose/word/code cells plus page geometry, assembled into
1849/// [`PdfPage`]s with no rendered image and no link annotations. Everything here
1850/// is pure Rust (lopdf), so it compiles without the `ml` feature — including on
1851/// wasm32. A page the parser can't read (no text layer) comes back with empty
1852/// cells; there is no pdfium fallback on this path.
1853pub fn pdf_text_pages(bytes: &[u8]) -> Vec<crate::pdfium_backend::PdfPage> {
1854    let Some(mut parser) = PageTextParser::open(bytes) else {
1855        return Vec::new();
1856    };
1857    (0..parser.page_count())
1858        .map(|i| parser.text_page(i))
1859        .collect()
1860}
1861
1862/// Drop line cells that are *painted over each other* — glyphs used as artwork.
1863///
1864/// Some generators draw their logo with a symbol font: on the reporting
1865/// invoice, a `TeleLogo` Type1 paints the T-Mobile mark by stacking the glyphs
1866/// encoded as `"` and `==` on top of one another, and the flat text-layer
1867/// output opened with that garbage. Nothing in the font metadata gives it away
1868/// (the *text* fonts in the same file are also flagged symbolic, and the logo
1869/// font names its glyphs `quotedbl` &c.), but the geometry does: two cells with
1870/// different text where one lies inside the other on the same line is
1871/// physically impossible for prose — ink from two words never occupies the
1872/// same box. Both cells of such a pair are paint, not text.
1873///
1874/// Containment (not mere overlap) keeps this narrow: adjacent words touch but
1875/// never contain each other, and a same-text near-duplicate (double-draw faux
1876/// bold) is left alone for the sanitizer's usual handling. Applied on the
1877/// flat/browser path only — the ML pipeline's text layer is byte-pinned by the
1878/// PDF corpus, and there the layout model already sinks logo marks into
1879/// `picture` regions.
1880fn drop_overpainted_cells(cells: &mut Vec<crate::pdfium_backend::TextCell>) {
1881    let mut paint = vec![false; cells.len()];
1882    for i in 0..cells.len() {
1883        for j in 0..cells.len() {
1884            if i == j || cells[i].text == cells[j].text {
1885                continue;
1886            }
1887            let (a, b) = (&cells[i], &cells[j]);
1888            // Same line band: the vertical overlap covers most of the shorter.
1889            let vo = (a.b.min(b.b) - a.t.max(b.t)).max(0.0);
1890            if vo < 0.6 * (a.b - a.t).min(b.b - b.t) {
1891                continue;
1892            }
1893            // … and the two sit at one height. The overlap alone says nothing
1894            // when the font's vertical metrics are inflated (#1129: a face
1895            // whose ascent and descent make every cell 55 pt tall on lines
1896            // 11.7 pt apart — each line overlaps the next by four fifths, and
1897            // a short line under a long one read as paint: 37 of the page's
1898            // 44 lines went, and its table with them). A cell's height is the
1899            // font's claim; its width per character is what was drawn, and
1900            // lines of prose are never set closer than that, so a shift of
1901            // more than half of it is another line, however tall the boxes.
1902            let shift = ((a.t + a.b) - (b.t + b.b)).abs() / 2.0;
1903            if shift > 0.5 * a.mean_advance().min(b.mean_advance()) {
1904                continue;
1905            }
1906            // `a` horizontally inside `b` (with a small tolerance).
1907            let ho = (a.r.min(b.r) - a.l.max(b.l)).max(0.0);
1908            if ho >= 0.8 * (a.r - a.l) && (a.r - a.l) <= (b.r - b.l) {
1909                paint[i] = true;
1910                paint[j] = true;
1911            }
1912        }
1913    }
1914    let mut keep = paint.iter().map(|p| !p);
1915    cells.retain(|_| keep.next().unwrap());
1916}
1917
1918/// The text-state scalars inherited by a Form XObject when it is invoked via
1919/// `Do` (the PDF graphics state includes the text parameters, but not the text
1920/// matrices, which a form re-establishes inside its own `BT`/`ET`).
1921#[derive(Clone, Copy)]
1922struct TextState {
1923    tc: f64,
1924    tw: f64,
1925    th: f64,
1926    tl: f64,
1927    trise: f64,
1928    fsize: f64,
1929    /// The text rendering mode (`Tr`).
1930    tr: i64,
1931}
1932
1933impl TextState {
1934    const INIT: TextState = TextState {
1935        tc: 0.0,
1936        tw: 0.0,
1937        th: 1.0,
1938        tl: 0.0,
1939        trise: 0.0,
1940        fsize: 0.0,
1941        tr: 0,
1942    };
1943}
1944
1945/// pdfium's `CPDF_Page::GetPageRotation` — the inherited `/Rotate`, in
1946/// quarter turns, as degrees (`pdf_meta`'s rule).
1947fn page_rotation(doc: &Document, pid: lopdf::ObjectId) -> u16 {
1948    let mut id = pid;
1949    for _ in 0..32 {
1950        let Ok(dict) = doc.get_dictionary(id) else {
1951            return 0;
1952        };
1953        if let Some(obj) = dict.get(b"Rotate").ok().and_then(|o| deref(doc, o)) {
1954            let quarter = (num(obj).unwrap_or(0.0) as i64 / 90).rem_euclid(4);
1955            return (quarter * 90) as u16;
1956        }
1957        match dict.get(b"Parent").ok().and_then(|o| o.as_reference().ok()) {
1958            Some(parent) => id = parent,
1959            None => return 0,
1960        }
1961    }
1962    0
1963}
1964
1965/// The effective `/Resources` dictionary for a page: its own (inline or via
1966/// reference), else the nearest ancestor's in the page tree — ISO 32000-1
1967/// 7.7.3.4 makes `/Resources` inheritable. Walked by hand rather than through
1968/// lopdf's `get_page_resources`, which collects an ancestor's `/Resources`
1969/// only when it is an indirect reference: an *inline* dictionary on the
1970/// `/Pages` node — the R&OS php pdf writer puts the one font there and gives
1971/// each page only `/Type`, `/Parent` and `/Contents` (#1130) — was dropped,
1972/// every font went missing and the whole text layer read as empty. A
1973/// `/Resources` that does not resolve to a dictionary is skipped in favour of
1974/// the next ancestor's (pdf.js and poppler read such files the same way).
1975pub(crate) fn page_res(doc: &Document, page_id: lopdf::ObjectId) -> Option<&Dictionary> {
1976    let mut id = page_id;
1977    // The page-tree depth lopdf itself tolerates; also ends a `/Parent` cycle.
1978    for _ in 0..32 {
1979        let dict = doc.get_object(id).ok()?.as_dict().ok()?;
1980        if let Some(res) = dict.get(b"Resources").ok().and_then(|o| as_dict(doc, o)) {
1981            return Some(res);
1982        }
1983        id = dict.get(b"Parent").ok()?.as_reference().ok()?;
1984    }
1985    None
1986}
1987
1988/// Build the code→[`Font`] map for a resources dictionary's `/Font` sub-dict,
1989/// reusing the per-document cache for fonts referenced indirectly (the common
1990/// case — the same font objects recur on every page).
1991fn fonts_from_res(
1992    doc: &Document,
1993    res: &Dictionary,
1994    caches: &mut DocCaches,
1995) -> HashMap<Vec<u8>, Arc<Font>> {
1996    let mut map = HashMap::new();
1997    let font_dict = res
1998        .get(b"Font")
1999        .ok()
2000        .and_then(|o| deref(doc, o))
2001        .and_then(|o| o.as_dict().ok());
2002    if let Some(fd) = font_dict {
2003        for (name, value) in fd.iter() {
2004            let font = match value {
2005                Object::Reference(id) => {
2006                    let key = (*id, name.clone());
2007                    if let Some(f) = caches.fonts.get(&key) {
2008                        Arc::clone(f)
2009                    } else if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
2010                        let f = Arc::new(parse_font(doc, name, fdict));
2011                        caches.fonts.insert(key, Arc::clone(&f));
2012                        f
2013                    } else {
2014                        continue;
2015                    }
2016                }
2017                _ => {
2018                    if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
2019                        Arc::new(parse_font(doc, name, fdict))
2020                    } else {
2021                        continue;
2022                    }
2023                }
2024            };
2025            map.insert(name.clone(), font);
2026        }
2027    }
2028    map
2029}
2030
2031/// Extract every glyph on a page as a native-coordinate [`Glyph`].
2032/// [`PageTextParser::glyph_styles`] for the given **1-based** pages of a
2033/// document, keyed by page number — a separate, on-demand pass (no
2034/// rendering), so the extraction pipeline stays byte-identical whether or not
2035/// the heading-hierarchy stage runs. Empty when lopdf cannot open the file.
2036pub(crate) fn glyph_styles(
2037    bytes: &[u8],
2038    pages: &[usize],
2039) -> HashMap<usize, Vec<crate::heading_hierarchy::GlyphStyle>> {
2040    let mut out = HashMap::new();
2041    let Some(mut parser) = PageTextParser::open(bytes) else {
2042        return out;
2043    };
2044    for &page_no in pages {
2045        if page_no == 0 {
2046            continue;
2047        }
2048        let styles = parser.glyph_styles(page_no - 1);
2049        if !styles.is_empty() {
2050            out.insert(page_no, styles);
2051        }
2052    }
2053    out
2054}
2055
2056pub(crate) fn page_glyphs(doc: &Document, page_id: lopdf::ObjectId) -> Vec<Glyph> {
2057    page_glyphs_cached(doc, page_id, &mut DocCaches::default())
2058}
2059
2060/// [`page_glyphs`] with an explicit per-document cache, so a multi-page walk
2061/// parses each font / decodes each form once instead of once per page.
2062fn page_glyphs_cached(
2063    doc: &Document,
2064    page_id: lopdf::ObjectId,
2065    caches: &mut DocCaches,
2066) -> Vec<Glyph> {
2067    page_glyphs_and_inks(doc, page_id, caches).0
2068}
2069
2070/// [`page_glyphs_cached`] plus the small painted path pieces the same walk
2071/// passed — what [`crate::checkbox::find`] looks for checkbox squares in
2072/// (#609) — the page's image placements (#999, `[l, t, r, b]` top-left
2073/// page points clipped to the page, the cells' frame), and its shapes for the
2074/// reading-order separators (#1092, [`PageShapes`]).
2075fn page_glyphs_and_inks(
2076    doc: &Document,
2077    page_id: lopdf::ObjectId,
2078    caches: &mut DocCaches,
2079) -> (
2080    Vec<Glyph>,
2081    Vec<crate::checkbox::Ink>,
2082    Vec<[f32; 4]>,
2083    PageShapes,
2084) {
2085    let mut out = Vec::new();
2086    let mut inks = Vec::new();
2087    // get_page_content returns the assembled content-stream bytes directly
2088    // (an empty Vec when the page has none).
2089    let content_bytes = doc.get_page_content(page_id);
2090    let Ok(content) = lopdf::content::Content::decode(&content_bytes) else {
2091        return (out, inks, Vec::new(), PageShapes::default());
2092    };
2093    let mut bitmaps = Vec::new();
2094    let mut page_shapes = PageShapes::default();
2095    if let Some(res) = page_res(doc, page_id) {
2096        // pdfium's page matrix: user space translated so the display box's
2097        // lower-left corner is the origin (see [`PageBox`]).
2098        let pb = page_box(doc, page_id);
2099        let base = Mat {
2100            e: -(pb.l as f64),
2101            f: -(pb.b as f64),
2102            ..Mat::ID
2103        };
2104        let mut shapes = Shapes::default();
2105        run_content(
2106            doc,
2107            res,
2108            &content,
2109            base,
2110            TextState::INIT,
2111            0,
2112            caches,
2113            &mut out,
2114            &mut inks,
2115            ShapeGs::INIT,
2116            &mut shapes,
2117        );
2118        out.retain(|g| on_page(g, pb.w, pb.h));
2119        drop_overprinted_spaces(&mut out);
2120        drop_shadow_runs(&mut out);
2121        let (w, h) = (f64::from(pb.w), f64::from(pb.h));
2122        inks.retain(|ink| match *ink {
2123            crate::checkbox::Ink::Bitmap { l, b, r, t } => {
2124                let (l, r, b, t) = (l.max(0.0), r.min(w), b.max(0.0), t.min(h));
2125                if r > l && t > b {
2126                    bitmaps.push([l as f32, (h - t) as f32, r as f32, (h - b) as f32]);
2127                }
2128                false
2129            }
2130            _ => true,
2131        });
2132        let top_left =
2133            |[l, b, r, t]: [f64; 4]| [l as f32, (h - t) as f32, r as f32, (h - b) as f32];
2134        page_shapes.lines = shapes.lines.into_iter().map(top_left).collect();
2135        if shapes.boxes.len() <= MAX_SHAPE_BOXES {
2136            page_shapes.boxes = connected_boxes(&shapes.boxes)
2137                .into_iter()
2138                .map(top_left)
2139                .collect();
2140        } else {
2141            docling_core::debug_log!(
2142                "textparse: {} painted paths on the page; their boxes are not merged",
2143                shapes.boxes.len()
2144            );
2145        }
2146    }
2147    (out, inks, bitmaps, page_shapes)
2148}
2149
2150/// A page's shapes for the reading-order separators (#1092), in the cells'
2151/// frame (`[l, t, r, b]`, top-left page points): docling-parse's
2152/// `get_shape_lines` and `get_connected_shape_bounding_boxes`.
2153#[derive(Default, Clone, Debug, PartialEq)]
2154pub struct PageShapes {
2155    pub lines: Vec<[f32; 4]>,
2156    pub boxes: Vec<[f32; 4]>,
2157}
2158
2159/// Drop a run of glyphs painted again over itself — a text shadow. A word
2160/// processor exports a heading with a shadow effect as the same run drawn
2161/// twice: the shadow, then the text over it, shifted by a fraction of a point
2162/// (0.5–0.8 pt up and left, black under red, in the manuals that showed it).
2163/// docling-parse keeps both copies and reads `Safety instructions Safety
2164/// instructions`; pdfium's text page, behind docling's pypdfium2 backend,
2165/// drops the second, and the page shows the heading once. A deliberate
2166/// divergence (`docs/MIGRATION.md` §4 item 25): a glyph that repeats an
2167/// earlier one — same character, font and size, its pen shifted from the
2168/// earlier pen by at most 0.15 em on each axis, and by less than the earlier
2169/// glyph's step to its own successor — starts a copy when the glyphs after it
2170/// repeat the earlier one's successors by the same offset and size, and the
2171/// whole stretch that keeps the offset is dropped. Two glyphs at least: a lone
2172/// mark stamped twice is left alone, and so is the per-glyph re-stamp the
2173/// sanitizer already folds (`dp_lines`, the overprint duplicate). Text that
2174/// really repeats sits an advance apart or more — a doubled letter (`ll`,
2175/// 0.22 em in Helvetica), a word set twice (`very very`), the same text on
2176/// the next line or in the next table cell — and is kept, and so is a run of
2177/// one character set tighter than a shadow's shift (dot leaders squeezed by
2178/// `Tc`): its glyph two on is farther than the next one. A line stamped two
2179/// or three times at a small offset to fake bold reads once the same way. The
2180/// original must start within the 1024 glyphs before its copy, so a shadowed
2181/// block longer than that is not caught; a shadowed heading is drawn right
2182/// after itself. A rotated run, whose pen positions are not compared here, is
2183/// left alone.
2184fn drop_shadow_runs(glyphs: &mut Vec<Glyph>) {
2185    // How far back a copy's original may start.
2186    const LOOKBACK: usize = 1024;
2187    // The largest shift of a copy, per axis, in ems of the glyph's size.
2188    const NEAR_EM: f32 = 0.15;
2189    // How much the shift and the size may vary along a copy, in ems.
2190    const SAME_EM: f32 = 0.01;
2191    let pen = |g: &Glyph| if g.quad.is_none() { g.pen } else { None };
2192    let n = glyphs.len();
2193    let mut keep = vec![true; n];
2194    // The glyphs a copy may start on, by character and font, in stream
2195    // order: kept (not part of a copy), upright, sized and not whitespace. A
2196    // copy repeats both, so a glyph walks only its own list back through the
2197    // window; scanning the whole window nearly doubled a `--text-layer-only`
2198    // read of pages without a shadow.
2199    let mut earlier: HashMap<(char, u64), Vec<usize>> = HashMap::new();
2200    let mut j = 0;
2201    while j < n {
2202        let b = &glyphs[j];
2203        let start = pen(b).filter(|p| p[2] > 0.0 && !b.ch.is_whitespace());
2204        let mut copy = 0;
2205        if let Some([xj, yj, sj]) = start {
2206            let lo = j.saturating_sub(LOOKBACK);
2207            let candidates = earlier.entry((b.ch, b.font)).or_default();
2208            for &i in candidates.iter().rev().take_while(|&&i| i >= lo) {
2209                let Some([xi, yi, si]) = pen(&glyphs[i]) else {
2210                    continue;
2211                };
2212                if (si - sj).abs() > SAME_EM * sj {
2213                    continue;
2214                }
2215                let (dx, dy) = (xj - xi, yj - yi);
2216                if dx.abs() > NEAR_EM * sj || dy.abs() > NEAR_EM * sj {
2217                    continue;
2218                }
2219                // A shadow is shifted: the characters one glyph expands to
2220                // (a code-like name read as `SM590000`, a decomposed
2221                // ligature) share the glyph's pen and are not a copy.
2222                if dx.abs() + dy.abs() <= SAME_EM * sj {
2223                    continue;
2224                }
2225                // And it lies over its original, nearer to it than the
2226                // original's next glyph is: one character set tighter than
2227                // half the largest shift (dot leaders squeezed by `Tc`) meets
2228                // its own successors at one offset two glyphs on.
2229                if let Some([xn, yn, _]) = glyphs.get(i + 1).and_then(pen) {
2230                    let step = (xn - xi).abs().max((yn - yi).abs());
2231                    if step > SAME_EM * sj && dx.abs().max(dy.abs()) >= step {
2232                        continue;
2233                    }
2234                }
2235                // The copy goes on while each next glyph repeats the
2236                // original's next one by the same shift; the original
2237                // ends before the copy starts.
2238                let mut m = 1;
2239                while j + m < n && i + m < j {
2240                    let (a, b) = (&glyphs[i + m], &glyphs[j + m]);
2241                    let (Some([xa, ya, sa]), Some([xb, yb, sb])) = (pen(a), pen(b)) else {
2242                        break;
2243                    };
2244                    if a.ch != b.ch
2245                        || a.font != b.font
2246                        || (sa - sb).abs() > SAME_EM * sj
2247                        || ((xb - xa) - dx).abs() > SAME_EM * sj
2248                        || ((yb - ya) - dy).abs() > SAME_EM * sj
2249                    {
2250                        break;
2251                    }
2252                    m += 1;
2253                }
2254                if m >= 2 {
2255                    copy = m;
2256                    break;
2257                }
2258            }
2259            if copy == 0 {
2260                candidates.push(j);
2261            }
2262        }
2263        if copy > 0 {
2264            keep[j..j + copy].iter_mut().for_each(|k| *k = false);
2265            j += copy;
2266        } else {
2267            j += 1;
2268        }
2269    }
2270    let mut it = keep.into_iter();
2271    glyphs.retain(|_| it.next().unwrap_or(true));
2272}
2273
2274/// How many marked-content spans may be open for one's `/ActualText` to be
2275/// read (see [`apply_actual_text`]); real documents nest a few.
2276const MAX_ACTUAL_TEXT_DEPTH: usize = 64;
2277
2278/// Put a marked-content span's `/ActualText` in place of the glyphs drawn
2279/// inside it, as docling-parse does since 7.9 (`apply_actual_text`,
2280/// docling-parse#300, on by default; PDF 32000-1 §14.9.4). A writer draws a
2281/// glyph cluster whose characters the font cannot spell this way: a base
2282/// letter and a zero-advance accent whose code collides with a Latin letter
2283/// (`Oreši` read as `OresMi`, docling#3783), or the Arabic marks of a
2284/// LibreOffice font subset whose `/ToUnicode` skips them, so the byte reads as
2285/// WinAnsi (`منعًا` as `منع8ا`). docling-parse's rules, cell for cell: a span
2286/// with no glyphs (over graphics) is left alone, and so is a text implausibly
2287/// long for it (more than 8 characters and 4 per cell: `/Alt` written into
2288/// the wrong key); an empty text removes the glyphs; as many characters as
2289/// cells replace them one by one, keeping each cell's geometry; any other
2290/// count puts the whole text on the span's first cell, over the union of the
2291/// span's boxes, its baseline running to the cell that reaches furthest along
2292/// it (an accent drawn after its base letter trails behind it). Nested spans
2293/// apply innermost first, so the outer text wins. docling-parse has a cell per
2294/// character code; a code whose `/ToUnicode` spells several characters is
2295/// several glyphs here, all on the code's pen, and counts as one cell.
2296///
2297/// Each `EMC` rewrites the glyphs under its span, so a crafted nest of spans
2298/// over the same glyphs would cost glyphs × depth: past
2299/// [`MAX_ACTUAL_TEXT_DEPTH`] open spans, a span still pairs with its `EMC` but
2300/// its `/ActualText` is not read.
2301fn apply_actual_text(out: &mut Vec<Glyph>, begin: usize, text: &str) {
2302    let begin = begin.min(out.len());
2303    let mut cells: Vec<std::ops::Range<usize>> = Vec::new();
2304    for i in begin..out.len() {
2305        match cells.last_mut() {
2306            Some(cell) if out[i].pen.is_some() && out[i].pen == out[cell.end - 1].pen => {
2307                cell.end = i + 1;
2308            }
2309            _ => cells.push(i..i + 1),
2310        }
2311    }
2312    if cells.is_empty() {
2313        return;
2314    }
2315    let chars: Vec<char> = text.chars().collect();
2316    if chars.len() > (4 * cells.len()).max(8) {
2317        return;
2318    }
2319    if chars.is_empty() {
2320        out.truncate(begin);
2321        return;
2322    }
2323    if chars.len() == cells.len() {
2324        // Each cell's first glyph takes its character, the rest of the cell
2325        // goes: one pass, writing behind the cells read.
2326        for (w, (cell, &ch)) in cells.iter().zip(&chars).enumerate() {
2327            out[begin + w] = Glyph {
2328                ch,
2329                ..out[cell.start]
2330            };
2331        }
2332        out.truncate(begin + cells.len());
2333        return;
2334    }
2335    // The leading edge (quad corners 0 and 3) stays the first cell's; the
2336    // trailing one (1 and 2) is the cell's that reaches furthest along the
2337    // baseline, projected on it so a rotated run keeps its direction.
2338    let quad_of = |g: &Glyph| {
2339        g.quad
2340            .unwrap_or([g.ll, g.lb, g.lr, g.lb, g.lr, g.lt, g.ll, g.lt])
2341    };
2342    let mut merged = Glyph { ..out[begin] };
2343    let mut quad = quad_of(&merged);
2344    let dir = (quad[2] - quad[0], quad[3] - quad[1]);
2345    let along = |q: &[f32; 8]| q[2] * dir.0 + q[3] * dir.1;
2346    let mut reach = along(&quad);
2347    for g in &out[begin + 1..] {
2348        merged.l = merged.l.min(g.l);
2349        merged.b = merged.b.min(g.b);
2350        merged.r = merged.r.max(g.r);
2351        merged.t = merged.t.max(g.t);
2352        merged.ll = merged.ll.min(g.ll);
2353        merged.lb = merged.lb.min(g.lb);
2354        merged.lr = merged.lr.max(g.lr);
2355        merged.lt = merged.lt.max(g.lt);
2356        let q = quad_of(g);
2357        if along(&q) > reach {
2358            reach = along(&q);
2359            quad[2..6].copy_from_slice(&q[2..6]);
2360        }
2361    }
2362    if merged.quad.is_some() {
2363        merged.quad = Some(quad);
2364    }
2365    out.truncate(begin);
2366    // The characters of one cell share its box, as a ligature's do.
2367    out.extend(chars.iter().map(|&ch| Glyph { ch, ..merged }));
2368}
2369
2370/// Drop a space glyph the next glyph is drawn over (#992). A typesetter
2371/// that sets a superscript footnote marker as `( 1 )` — space glyphs around
2372/// the raised digit — and then cancels each space's advance with a negative
2373/// `TJ` offset (Arbortext on EUR-Lex: the `1` starts 0.05 pt *before* its
2374/// space begins) renders `(1)`; the spaces exist in the content stream
2375/// only. docling-parse keeps them as characters and its line cell reads
2376/// `( 1 )`; pdftotext and pdfium, which break words by geometry alone,
2377/// read `(1)`, and so does the page. A deliberate divergence
2378/// (`docs/MIGRATION.md` §4 item 22): a space whose successor in stream
2379/// order sits on the same line and starts within the first 30 % of the
2380/// space's advance — the pen went back to (or before) where the space
2381/// began, so it left no gap — is not a character of the text. A space
2382/// followed by a glyph on another line (a trailing space before the
2383/// cursor jumps to the next line's left margin, far to the left of it) or
2384/// by one that starts past that mark (a kerned pair, a justified word gap
2385/// pulled in a little) is kept; the word-gap rules then see the geometry
2386/// as before.
2387fn drop_overprinted_spaces(glyphs: &mut Vec<Glyph>) {
2388    if glyphs.len() < 2 {
2389        return;
2390    }
2391    let overprinted = |sp: &Glyph, next: &Glyph| {
2392        if sp.ch != ' ' || next.ch == ' ' {
2393            return false;
2394        }
2395        // Upright glyphs only: a rotated run's loose boxes are the extents
2396        // of its quads, every glyph of a 90° line shares one left edge and
2397        // the advance runs along y, so the test below would read every
2398        // space of a landscape slide drawn through a 90° `cm` as
2399        // overprinted (`Nullammollis`, odf_presentation_02).
2400        if sp.quad.is_some() || next.quad.is_some() {
2401            return false;
2402        }
2403        let width = sp.lr - sp.ll;
2404        if !width.is_finite() || width <= 0.0 || !next.ll.is_finite() {
2405            return false;
2406        }
2407        // Same line: the loose boxes overlap vertically (a raised digit's box
2408        // still overlaps its neighbours' em boxes).
2409        let same_line = next.lb < sp.lt && next.lt > sp.lb;
2410        same_line && next.ll >= sp.ll - 0.1 * width && next.ll <= sp.ll + 0.3 * width
2411    };
2412    let mut keep = vec![true; glyphs.len()];
2413    for i in 0..glyphs.len() - 1 {
2414        if overprinted(&glyphs[i], &glyphs[i + 1]) {
2415            keep[i] = false;
2416        }
2417    }
2418    let mut it = keep.into_iter();
2419    glyphs.retain(|_| it.next().unwrap_or(true));
2420}
2421
2422/// Whether a glyph is on the page (#529): docling-parse keeps a character
2423/// only when its whole box lies inside the display box — the CropBox, or
2424/// the MediaBox without one — edges included, so a FrameMaker print slug
2425/// drawn beside the CropBox or a tiled page's neighbouring text beyond the
2426/// MediaBox never becomes a cell, and a line crossing the edge is cut at the
2427/// last glyph that fits (`Crossing the rig`). The box is docling-parse's
2428/// char box — the advance by the font's ascent/descent, the loose box here
2429/// (its axis-aligned extent for rotated text) — in the frame `page_glyphs`
2430/// already moved to the display box's corner, so the page is `[0, w] × [0,
2431/// h]`. A glyph without a finite box is kept, as before. (A standard-14 font
2432/// without a FontDescriptor gets the 750 / −250 default ascent / descent
2433/// here where docling-parse reads the AFM's — Helvetica's 718 / −207 — so
2434/// for those a glyph within ~0.04 em of the top or bottom edge can fall the
2435/// other way; horizontally the advance is the same.)
2436fn on_page(g: &Glyph, w: f32, h: f32) -> bool {
2437    // f32 noise only: docling-parse already drops a glyph 0.01 pt over.
2438    const EPS: f32 = 1e-3;
2439    let (l, b, r, t) = if [g.ll, g.lb, g.lr, g.lt].iter().all(|v| v.is_finite()) {
2440        (g.ll, g.lb, g.lr, g.lt)
2441    } else {
2442        (g.l, g.b, g.r, g.t)
2443    };
2444    if ![l, b, r, t].iter().all(|v| v.is_finite()) {
2445        return true;
2446    }
2447    l >= -EPS && b >= -EPS && r <= w + EPS && t <= h + EPS
2448}
2449
2450/// Run a content stream's operators, emitting glyphs into `out`. Recurses into
2451/// Form XObjects on `Do` (bulk body text in heavy PDFs lives inside a form, not
2452/// the page content stream). `res` is the resources dict in scope (the page's,
2453/// or the form's own); `base_ctm` is the CTM at the point of invocation.
2454#[allow(clippy::too_many_arguments)]
2455fn run_content(
2456    doc: &Document,
2457    res: &Dictionary,
2458    content: &lopdf::content::Content,
2459    base_ctm: Mat,
2460    init: TextState,
2461    depth: u32,
2462    caches: &mut DocCaches,
2463    out: &mut Vec<Glyph>,
2464    inks: &mut Vec<crate::checkbox::Ink>,
2465    shape_init: ShapeGs,
2466    shapes: &mut Shapes,
2467) {
2468    let fonts = fonts_from_res(doc, res, caches);
2469    let xobjects = res
2470        .get(b"XObject")
2471        .ok()
2472        .and_then(|o| deref(doc, o))
2473        .and_then(|o| o.as_dict().ok());
2474
2475    // Graphics + text state. `q`/`Q` save and restore the whole graphics state,
2476    // which includes the text parameters (Tc, Tw, Tz, TL, Tfs, Trise, font) —
2477    // *not* the text matrix (that is reset by BT). Saving only the CTM let a Tc
2478    // set inside a `q…Q` block leak out and drift every later glyph.
2479    #[allow(clippy::type_complexity)]
2480    let mut gstate_stack: Vec<(Mat, f64, f64, f64, f64, f64, f64, i64, Option<&Arc<Font>>)> =
2481        Vec::new();
2482    let mut ctm = base_ctm;
2483    let mut tm = Mat::ID;
2484    let mut tlm = Mat::ID;
2485    let mut font: Option<&Arc<Font>> = None;
2486    let mut fsize = init.fsize;
2487    let mut tc = init.tc; // char spacing
2488    let mut tw = init.tw; // word spacing
2489    let mut th = init.th; // horizontal scale (Tz/100)
2490    let mut tl = init.tl; // leading
2491    let mut trise = init.trise;
2492    let mut tr = init.tr; // rendering mode
2493
2494    let op_f = |operands: &[Object], i: usize| operands.get(i).and_then(num).unwrap_or(0.0);
2495    // The path under construction, already in page space (a path is built
2496    // under one CTM — `cm` is not allowed inside it), for the checkbox
2497    // squares (#609). Text extraction never reads it.
2498    let mut path: Vec<PathPiece> = Vec::new();
2499    let (mut cur, mut start) = ((0.0, 0.0), (0.0, 0.0));
2500    // Whether the fill colour is (near) white, saved/restored with `q`/`Q`:
2501    // a white-filled stroked box is a checkbox outline, a coloured one a
2502    // chart legend's swatch. The initial fill colour is black.
2503    let mut fill_light = false;
2504    let mut fill_stack: Vec<bool> = Vec::new();
2505    // Open marked-content sequences (`BMC`/`BDC` … `EMC`): where each one's
2506    // glyphs start in `out`, and its inline `/ActualText`, if any.
2507    let mut marked: Vec<(usize, Option<String>)> = Vec::new();
2508    // The reading-order shapes (#1092): docling-parse's view of the path
2509    // under construction, its graphics state and the pending clip.
2510    let mut shape_gs = shape_init;
2511    let mut shape_stack: Vec<ShapeGs> = Vec::new();
2512    let mut shape_path: Vec<ShapeSub> = Vec::new();
2513    let mut clip_pending = false;
2514    let ext_gstates = res
2515        .get(b"ExtGState")
2516        .ok()
2517        .and_then(|o| deref(doc, o))
2518        .and_then(|o| o.as_dict().ok());
2519    let colorspaces = res
2520        .get(b"ColorSpace")
2521        .ok()
2522        .and_then(|o| deref(doc, o))
2523        .and_then(|o| o.as_dict().ok());
2524
2525    for op in &content.operations {
2526        let operands = &op.operands;
2527        let shown = out.len();
2528        match op.operator.as_str() {
2529            "q" => {
2530                gstate_stack.push((ctm, tc, tw, th, tl, trise, fsize, tr, font));
2531                fill_stack.push(fill_light);
2532                shape_stack.push(shape_gs.clone());
2533            }
2534            "BMC" => marked.push((out.len(), None)),
2535            // Only an inline property list is read: a named one (`/OC /MC0`)
2536            // would need the `/Properties` resource, as in docling-parse.
2537            "BDC" => {
2538                let actual = operands
2539                    .get(1)
2540                    .filter(|_| marked.len() < MAX_ACTUAL_TEXT_DEPTH)
2541                    .and_then(|o| o.as_dict().ok())
2542                    .and_then(|d| d.get(b"ActualText").ok())
2543                    .and_then(|o| match o {
2544                        Object::String(bytes, _) => Some(crate::info::decode_text(bytes)),
2545                        _ => None,
2546                    });
2547                marked.push((out.len(), actual));
2548            }
2549            "EMC" => {
2550                if let Some((begin, Some(text))) = marked.pop() {
2551                    apply_actual_text(out, begin, &text);
2552                }
2553                // Shows no text: the invisible-mode stamp below has nothing
2554                // new to mark, and the span may have shortened `out`.
2555                continue;
2556            }
2557            "Q" => {
2558                fill_light = fill_stack.pop().unwrap_or(false);
2559                if let Some(gs) = shape_stack.pop() {
2560                    shape_gs = gs;
2561                }
2562                if let Some((c, a, b, h, l, r, fs, mode, f)) = gstate_stack.pop() {
2563                    ctm = c;
2564                    tc = a;
2565                    tw = b;
2566                    th = h;
2567                    tl = l;
2568                    trise = r;
2569                    fsize = fs;
2570                    tr = mode;
2571                    font = f;
2572                }
2573            }
2574            "cm" => {
2575                let m = Mat {
2576                    a: op_f(operands, 0),
2577                    b: op_f(operands, 1),
2578                    c: op_f(operands, 2),
2579                    d: op_f(operands, 3),
2580                    e: op_f(operands, 4),
2581                    f: op_f(operands, 5),
2582                };
2583                ctm = m.then(ctm);
2584            }
2585            "BT" => {
2586                tm = Mat::ID;
2587                tlm = Mat::ID;
2588            }
2589            "ET" => {}
2590            "Tf" => {
2591                if let Some(Object::Name(n)) = operands.first() {
2592                    font = fonts.get(n.as_slice());
2593                }
2594                fsize = op_f(operands, 1);
2595            }
2596            "Td" => {
2597                tlm = Mat {
2598                    a: 1.0,
2599                    b: 0.0,
2600                    c: 0.0,
2601                    d: 1.0,
2602                    e: op_f(operands, 0),
2603                    f: op_f(operands, 1),
2604                }
2605                .then(tlm);
2606                tm = tlm;
2607            }
2608            "TD" => {
2609                tl = -op_f(operands, 1);
2610                tlm = Mat {
2611                    a: 1.0,
2612                    b: 0.0,
2613                    c: 0.0,
2614                    d: 1.0,
2615                    e: op_f(operands, 0),
2616                    f: op_f(operands, 1),
2617                }
2618                .then(tlm);
2619                tm = tlm;
2620            }
2621            "Tm" => {
2622                tlm = Mat {
2623                    a: op_f(operands, 0),
2624                    b: op_f(operands, 1),
2625                    c: op_f(operands, 2),
2626                    d: op_f(operands, 3),
2627                    e: op_f(operands, 4),
2628                    f: op_f(operands, 5),
2629                };
2630                tm = tlm;
2631            }
2632            "T*" => {
2633                tlm = Mat {
2634                    a: 1.0,
2635                    b: 0.0,
2636                    c: 0.0,
2637                    d: 1.0,
2638                    e: 0.0,
2639                    f: -tl,
2640                }
2641                .then(tlm);
2642                tm = tlm;
2643            }
2644            "Tc" => tc = op_f(operands, 0),
2645            "Tw" => tw = op_f(operands, 0),
2646            "Tz" => th = op_f(operands, 0) / 100.0,
2647            "TL" => tl = op_f(operands, 0),
2648            "Ts" => trise = op_f(operands, 0),
2649            "Tr" => tr = op_f(operands, 0) as i64,
2650            "Tj" | "'" | "\"" => {
2651                if op.operator == "'" || op.operator == "\"" {
2652                    // move to next line first
2653                    tlm = Mat {
2654                        a: 1.0,
2655                        b: 0.0,
2656                        c: 0.0,
2657                        d: 1.0,
2658                        e: 0.0,
2659                        f: -tl,
2660                    }
2661                    .then(tlm);
2662                    tm = tlm;
2663                }
2664                if op.operator == "\"" {
2665                    // `aw ac string "` sets word- and char-spacing before
2666                    // showing the string (PDF 32000-1 §9.4.3), persisting after.
2667                    tw = op_f(operands, 0);
2668                    tc = op_f(operands, 1);
2669                }
2670                if let (Some(f), Some(Object::String(s, _))) = (font, operands.last()) {
2671                    show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out);
2672                }
2673            }
2674            "TJ" => {
2675                if let (Some(f), Some(Object::Array(arr))) = (font, operands.first()) {
2676                    for el in arr {
2677                        match el {
2678                            Object::String(s, _) => {
2679                                show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out)
2680                            }
2681                            other => {
2682                                if let Some(adj) = num(other) {
2683                                    // negative number moves text right (PDF: subtract)
2684                                    let tx = -adj / 1000.0 * fsize * th;
2685                                    tm = Mat {
2686                                        a: 1.0,
2687                                        b: 0.0,
2688                                        c: 0.0,
2689                                        d: 1.0,
2690                                        e: tx,
2691                                        f: 0.0,
2692                                    }
2693                                    .then(tm);
2694                                }
2695                            }
2696                        }
2697                    }
2698                }
2699            }
2700            "m" => {
2701                cur = ctm.apply(op_f(operands, 0), op_f(operands, 1));
2702                start = cur;
2703                shape_path.push(ShapeSub::at(cur));
2704            }
2705            "l" => {
2706                let p = ctm.apply(op_f(operands, 0), op_f(operands, 1));
2707                path.push(PathPiece::Line(cur, p));
2708                cur = p;
2709                if let Some(sub) = shape_path.last_mut() {
2710                    sub.lines.push((sub.cur, p));
2711                    sub.pts.push(p);
2712                    sub.cur = p;
2713                }
2714            }
2715            "c" | "v" | "y" => {
2716                let pts: Vec<(f64, f64)> = (0..operands.len() / 2)
2717                    .map(|k| ctm.apply(op_f(operands, 2 * k), op_f(operands, 2 * k + 1)))
2718                    .collect();
2719                if let Some(&end) = pts.last() {
2720                    path.push(PathPiece::Curve(
2721                        std::iter::once(cur).chain(pts.iter().copied()).collect(),
2722                    ));
2723                    cur = end;
2724                    if let Some(sub) = shape_path.last_mut() {
2725                        sub.pts.extend(pts.iter().copied());
2726                        sub.cur = end;
2727                    }
2728                }
2729            }
2730            "h" => {
2731                if cur != start {
2732                    path.push(PathPiece::Line(cur, start));
2733                }
2734                cur = start;
2735                // docling-parse closes the subpath and opens the next one at
2736                // its start, where a following `l` continues.
2737                if let Some(sub) = shape_path.last_mut() {
2738                    sub.closed = true;
2739                    sub.pts.push(sub.start);
2740                    let at = sub.start;
2741                    shape_path.push(ShapeSub::at(at));
2742                }
2743            }
2744            "re" => {
2745                let (x, y, w, h) = (
2746                    op_f(operands, 0),
2747                    op_f(operands, 1),
2748                    op_f(operands, 2),
2749                    op_f(operands, 3),
2750                );
2751                path.push(PathPiece::Rect([
2752                    ctm.apply(x, y),
2753                    ctm.apply(x + w, y),
2754                    ctm.apply(x + w, y + h),
2755                    ctm.apply(x, y + h),
2756                ]));
2757                cur = ctm.apply(x, y);
2758                start = cur;
2759                // `m` + three `l` + `h`, as docling-parse builds it.
2760                let c = [
2761                    ctm.apply(x, y),
2762                    ctm.apply(x + w, y),
2763                    ctm.apply(x + w, y + h),
2764                    ctm.apply(x, y + h),
2765                ];
2766                shape_path.push(ShapeSub {
2767                    start: c[0],
2768                    cur: c[3],
2769                    pts: vec![c[0], c[1], c[2], c[3], c[0]],
2770                    lines: vec![(c[0], c[1]), (c[1], c[2]), (c[2], c[3])],
2771                    closed: true,
2772                });
2773                shape_path.push(ShapeSub::at(c[0]));
2774            }
2775            "S" | "s" | "f" | "F" | "f*" | "B" | "B*" | "b" | "b*" => {
2776                let op = op.operator.as_str();
2777                if matches!(op, "s" | "b" | "b*") && cur != start {
2778                    path.push(PathPiece::Line(cur, start));
2779                }
2780                let stroke = matches!(op, "S" | "s" | "B" | "B*" | "b" | "b*");
2781                let fill = !matches!(op, "S" | "s");
2782                paint_path(&path, stroke, fill, fill_light, inks);
2783                path.clear();
2784                // `s`, `f`, `f*`, `b`, `b*` close the last subpath first.
2785                if !matches!(op, "S" | "B" | "B*") {
2786                    if let Some(sub) = shape_path.last_mut() {
2787                        sub.closed = true;
2788                    }
2789                }
2790                let scale = (ctm.a * ctm.d - ctm.b * ctm.c).abs().sqrt();
2791                record_shape(
2792                    &shape_path,
2793                    stroke,
2794                    fill,
2795                    &shape_gs,
2796                    if scale > 0.0 { scale } else { 1.0 },
2797                    shapes,
2798                );
2799                // A pending `W` takes effect after the painting operator.
2800                if std::mem::take(&mut clip_pending) {
2801                    capture_clip(&shape_path, &mut shape_gs.clips);
2802                }
2803                shape_path.clear();
2804            }
2805            "n" => {
2806                path.clear();
2807                if std::mem::take(&mut clip_pending) {
2808                    capture_clip(&shape_path, &mut shape_gs.clips);
2809                }
2810                shape_path.clear();
2811            }
2812            "W" | "W*" => clip_pending = true,
2813            "w" => shape_gs.line_width = op_f(operands, 0),
2814            "gs" => {
2815                let params = operands
2816                    .first()
2817                    .and_then(|o| o.as_name().ok())
2818                    .and_then(|n| ext_gstates?.get(n).ok())
2819                    .and_then(|o| deref(doc, o))
2820                    .and_then(|o| o.as_dict().ok());
2821                if let Some(params) = params {
2822                    let get = |k: &[u8]| params.get(k).ok().and_then(num);
2823                    if let Some(lw) = get(b"LW") {
2824                        shape_gs.line_width = lw;
2825                    }
2826                    if let Some(a) = get(b"CA") {
2827                        shape_gs.stroke_alpha = a;
2828                    }
2829                    if let Some(a) = get(b"ca") {
2830                        shape_gs.fill_alpha = a;
2831                    }
2832                }
2833            }
2834            // The fill colour, by its components: gray `g`, `rg`, CMYK `k`,
2835            // and `sc`/`scn` by operand count (a pattern name is not light).
2836            "g" | "rg" | "k" | "sc" | "scn" => {
2837                // docling-parse: a colour resolves a pattern fill; a pattern
2838                // name in `scn` makes it one.
2839                if matches!(op.operator.as_str(), "scn")
2840                    && matches!(operands.last(), Some(Object::Name(_)))
2841                {
2842                    shape_gs.fill_pattern = true;
2843                } else if !operands.is_empty() {
2844                    shape_gs.fill_pattern = false;
2845                }
2846                let v: Vec<f64> = operands
2847                    .iter()
2848                    .map(num)
2849                    .collect::<Option<_>>()
2850                    .unwrap_or_default();
2851                fill_light = match v.as_slice() {
2852                    [g] => *g >= 0.9,
2853                    [r, g, b] => r.min(*g).min(*b) >= 0.9,
2854                    [c, m, y, k] => c.max(*m).max(*y).max(*k) <= 0.1,
2855                    _ => false,
2856                };
2857            }
2858            "cs" => {
2859                fill_light = false;
2860                // A Pattern space carries no colour until `scn` names one.
2861                let pattern = match operands.first() {
2862                    Some(Object::Name(n)) if n.as_slice() == b"Pattern" => true,
2863                    Some(Object::Name(n)) => colorspaces
2864                        .and_then(|cs| cs.get(n).ok())
2865                        .and_then(|o| deref(doc, o))
2866                        .is_some_and(|o| match o {
2867                            Object::Name(f) => f.as_slice() == b"Pattern",
2868                            Object::Array(a) => a
2869                                .first()
2870                                .and_then(|f| f.as_name().ok())
2871                                .is_some_and(|f| f == b"Pattern"),
2872                            _ => false,
2873                        }),
2874                    _ => false,
2875                };
2876                if pattern {
2877                    shape_gs.fill_pattern = true;
2878                }
2879            }
2880            // An inline image: the unit square under the CTM (#999).
2881            "BI" => inks.push(image_ink(ctm)),
2882            "Do" => {
2883                // Invoke a Form XObject: bulk body text in many PDFs lives inside
2884                // a form, reached only here. An image XObject carries no text,
2885                // only its placement (#999).
2886                if depth >= 8 {
2887                    continue;
2888                }
2889                let Some(Object::Name(n)) = operands.first() else {
2890                    continue;
2891                };
2892                let obj = xobjects.and_then(|d| d.get(n.as_slice()).ok());
2893                let form_id = match obj {
2894                    Some(Object::Reference(id)) => Some(*id),
2895                    _ => None,
2896                };
2897                let stream = obj
2898                    .and_then(|o| deref(doc, o))
2899                    .and_then(|o| o.as_stream().ok());
2900                let Some(stream) = stream else { continue };
2901                let is_form = stream
2902                    .dict
2903                    .get(b"Subtype")
2904                    .ok()
2905                    .and_then(|o| o.as_name().ok())
2906                    == Some(b"Form".as_slice());
2907                if !is_form {
2908                    let is_image = stream
2909                        .dict
2910                        .get(b"Subtype")
2911                        .ok()
2912                        .and_then(|o| o.as_name().ok())
2913                        == Some(b"Image".as_slice());
2914                    if is_image {
2915                        inks.push(image_ink(ctm));
2916                    }
2917                    continue;
2918                }
2919                // Decode a reused form's content once per document
2920                // (headers/footers and bulk body text invoke the same form on
2921                // every page); see `DocCaches::forms_seen` for single-use ones.
2922                let cached = form_id.and_then(|id| caches.forms.get(&id).cloned());
2923                let form_content = match cached {
2924                    Some(c) => c,
2925                    None => {
2926                        let Ok(data) = stream.decompressed_content() else {
2927                            continue;
2928                        };
2929                        let Ok(c) = lopdf::content::Content::decode(&data) else {
2930                            continue;
2931                        };
2932                        let c = Arc::new(c);
2933                        if let Some(id) = form_id {
2934                            if !caches.forms_seen.insert(id) {
2935                                caches.forms.insert(id, Arc::clone(&c));
2936                            }
2937                        }
2938                        c
2939                    }
2940                };
2941                // The form's /Matrix maps form space into the CTM at invocation.
2942                let form_mat = match stream.dict.get(b"Matrix").ok() {
2943                    Some(Object::Array(a)) if a.len() == 6 => {
2944                        let v: Vec<f64> = a.iter().filter_map(num).collect();
2945                        if v.len() == 6 {
2946                            Mat {
2947                                a: v[0],
2948                                b: v[1],
2949                                c: v[2],
2950                                d: v[3],
2951                                e: v[4],
2952                                f: v[5],
2953                            }
2954                        } else {
2955                            Mat::ID
2956                        }
2957                    }
2958                    _ => Mat::ID,
2959                };
2960                // The form's own /Resources, falling back to the inherited ones.
2961                let form_res = stream
2962                    .dict
2963                    .get(b"Resources")
2964                    .ok()
2965                    .and_then(|o| deref(doc, o))
2966                    .and_then(|o| o.as_dict().ok())
2967                    .unwrap_or(res);
2968                let state = TextState {
2969                    tc,
2970                    tw,
2971                    th,
2972                    tl,
2973                    trise,
2974                    fsize,
2975                    tr,
2976                };
2977                run_content(
2978                    doc,
2979                    form_res,
2980                    &form_content,
2981                    form_mat.then(ctm),
2982                    state,
2983                    depth + 1,
2984                    caches,
2985                    out,
2986                    inks,
2987                    shape_gs.clone(),
2988                    shapes,
2989                );
2990            }
2991            _ => {}
2992        }
2993        // Invisible text (`Tr` 3, or 7: clip only) — an OCR layer over a scan
2994        // follows the scanned lines, not a typeset baseline (#1005).
2995        if matches!(tr, 3 | 7) {
2996            for g in &mut out[shown..] {
2997                g.invisible = true;
2998            }
2999        }
3000    }
3001}
3002
3003/// An image placement's box — the unit square under `ctm`, y-up page
3004/// points — as an [`Ink::Bitmap`](crate::checkbox::Ink::Bitmap).
3005fn image_ink(ctm: Mat) -> crate::checkbox::Ink {
3006    let pts = [(0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0)].map(|(x, y)| ctm.apply(x, y));
3007    let (mut l, mut b, mut r, mut t) = (
3008        f64::INFINITY,
3009        f64::INFINITY,
3010        f64::NEG_INFINITY,
3011        f64::NEG_INFINITY,
3012    );
3013    for (x, y) in pts {
3014        (l, b, r, t) = (l.min(x), b.min(y), r.max(x), t.max(y));
3015    }
3016    crate::checkbox::Ink::Bitmap { l, b, r, t }
3017}
3018
3019/// The page's vector shapes as docling-parse's page decoder reports them to
3020/// docling's reading order (#1092): `get_shape_lines` — every stroked,
3021/// axis-aligned straight segment, clipped — and the boxes
3022/// `get_connected_shape_bounding_boxes` merges — every visible painted path,
3023/// its stroke's half width included, clipped. `[l, b, r, t]`, page points,
3024/// bottom-left origin, in the frame `run_content` walks (the display box's
3025/// lower-left corner at the origin, before `/Rotate`).
3026#[derive(Default, Clone, Debug, PartialEq)]
3027pub(crate) struct Shapes {
3028    pub lines: Vec<[f64; 4]>,
3029    pub boxes: Vec<[f64; 4]>,
3030}
3031
3032/// More painted paths than this on a page and their boxes are not merged —
3033/// docling-parse's merge is quadratic in them, and a page that dense (a map,
3034/// a seat plan, text drawn as outlines) yields no trustworthy rule anyway;
3035/// the stroked lines still count.
3036const MAX_SHAPE_BOXES: usize = 10_000;
3037
3038/// One subpath of the shape path under construction, page space: what
3039/// docling-parse's `page_item<PAGE_SHAPE>` keeps of it.
3040struct ShapeSub {
3041    start: (f64, f64),
3042    cur: (f64, f64),
3043    /// Every point the path passes or is controlled by — a curve's control
3044    /// points included, as docling-parse's segment data has them.
3045    pts: Vec<(f64, f64)>,
3046    /// Its straight segments, in order.
3047    lines: Vec<((f64, f64), (f64, f64))>,
3048    closed: bool,
3049}
3050
3051impl ShapeSub {
3052    fn at(p: (f64, f64)) -> Self {
3053        ShapeSub {
3054            start: p,
3055            cur: p,
3056            pts: vec![p],
3057            lines: Vec::new(),
3058            closed: false,
3059        }
3060    }
3061
3062    /// `keep_shape`: two points at least, spanning more than a speck (the
3063    /// box diagonal, squared, over 1e-3).
3064    fn kept(&self) -> bool {
3065        let Some((l, b, r, t)) = bbox_of(&self.pts) else {
3066            return false;
3067        };
3068        self.pts.len() >= 2 && (r - l).powi(2) + (t - b).powi(2) >= 1e-3
3069    }
3070}
3071
3072fn bbox_of(pts: &[(f64, f64)]) -> Option<(f64, f64, f64, f64)> {
3073    let (&first, rest) = pts.split_first()?;
3074    Some(rest.iter().fold(
3075        (first.0, first.1, first.0, first.1),
3076        |(l, b, r, t), &(x, y)| (l.min(x), b.min(y), r.max(x), t.max(y)),
3077    ))
3078}
3079
3080/// The graphics state docling-parse's shapes read: line width (user space),
3081/// stroke and fill alpha (`CA` / `ca`), whether the fill colour is a pattern
3082/// not yet resolved to a paint, and the clip paths' boxes. Saved and
3083/// restored with `q` / `Q`.
3084#[derive(Clone)]
3085struct ShapeGs {
3086    line_width: f64,
3087    stroke_alpha: f64,
3088    fill_alpha: f64,
3089    fill_pattern: bool,
3090    clips: Vec<[f64; 4]>,
3091}
3092
3093impl ShapeGs {
3094    const INIT: ShapeGs = ShapeGs {
3095        line_width: 1.0,
3096        stroke_alpha: 1.0,
3097        fill_alpha: 1.0,
3098        fill_pattern: false,
3099        clips: Vec::new(),
3100    };
3101}
3102
3103/// docling-parse's `bbox_intersects`: the boxes share area (a degenerate box
3104/// lying inside the other counts).
3105fn boxes_intersect(a: [f64; 4], b: [f64; 4]) -> bool {
3106    a[0] < b[2] && a[2] > b[0] && a[1] < b[3] && a[3] > b[1]
3107}
3108
3109/// `clipped_bbox`: a box cut to every clip path's box in turn; `None` once
3110/// one of them misses it.
3111fn clipped(mut bbox: [f64; 4], clips: &[[f64; 4]]) -> Option<[f64; 4]> {
3112    for c in clips {
3113        if !boxes_intersect(bbox, *c) {
3114            return None;
3115        }
3116        bbox = [
3117            bbox[0].max(c[0]),
3118            bbox[1].max(c[1]),
3119            bbox[2].min(c[2]),
3120            bbox[3].min(c[3]),
3121        ];
3122    }
3123    Some(bbox)
3124}
3125
3126/// `get_shape_lines`' `maybe_add_line` + `clip_axis_aligned_segment`: a
3127/// straight segment that is horizontal or vertical within 1e-3 (not both: a
3128/// point), cut to every clip path's box along its axis and dropped when one
3129/// misses it.
3130fn shape_line(a: (f64, f64), z: (f64, f64), clips: &[[f64; 4]], out: &mut Vec<[f64; 4]>) {
3131    const TOL: f64 = 1e-3;
3132    let horizontal = (z.1 - a.1).abs() <= TOL;
3133    let vertical = (z.0 - a.0).abs() <= TOL;
3134    if horizontal == vertical {
3135        return;
3136    }
3137    let mut bbox = [a.0.min(z.0), a.1.min(z.1), a.0.max(z.0), a.1.max(z.1)];
3138    for c in clips {
3139        if horizontal {
3140            if a.1 < c[1] - TOL || a.1 > c[3] + TOL {
3141                return;
3142            }
3143            bbox[0] = bbox[0].max(c[0]);
3144            bbox[2] = bbox[2].min(c[2]);
3145            if bbox[0] > bbox[2] + TOL {
3146                return;
3147            }
3148        } else {
3149            if a.0 < c[0] - TOL || a.0 > c[2] + TOL {
3150                return;
3151            }
3152            bbox[1] = bbox[1].max(c[1]);
3153            bbox[3] = bbox[3].min(c[3]);
3154            if bbox[1] > bbox[3] + TOL {
3155                return;
3156            }
3157        }
3158    }
3159    out.push(bbox);
3160}
3161
3162/// `capture_pending_clip`: each subpath of the clipping path with a segment
3163/// (two points or more) bounds what later operators paint — appended, which
3164/// approximates the intersection the spec asks for, as docling-parse does.
3165fn capture_clip(subs: &[ShapeSub], clips: &mut Vec<[f64; 4]>) {
3166    for sub in subs.iter().filter(|s| s.pts.len() >= 2) {
3167        if let Some((l, b, r, t)) = bbox_of(&sub.pts) {
3168            clips.push([l, b, r, t]);
3169        }
3170    }
3171}
3172
3173/// docling-parse's `register_paths` for the reading-order geometry: a painted
3174/// path's kept subpaths become shape lines (when its stroke shows) and a
3175/// shape box (when anything of it shows). A fill whose colour is an
3176/// unresolved pattern paints nothing, so a fill-only path is no shape and a
3177/// fill-and-stroke one keeps its stroke. `scale` maps the user-space line
3178/// width to page space (the CTM's `sqrt|det|`; a hairline stays 0).
3179fn record_shape(
3180    subs: &[ShapeSub],
3181    stroke: bool,
3182    fill: bool,
3183    gs: &ShapeGs,
3184    scale: f64,
3185    shapes: &mut Shapes,
3186) {
3187    let kept: Vec<&ShapeSub> = subs.iter().filter(|s| s.kept()).collect();
3188    if kept.is_empty() {
3189        return;
3190    }
3191    let fill = fill && !gs.fill_pattern;
3192    if gs.fill_pattern && !stroke {
3193        return;
3194    }
3195    let strokes_visible = stroke && gs.stroke_alpha > 0.0;
3196    let visible = strokes_visible || (fill && gs.fill_alpha > 0.0);
3197    if strokes_visible {
3198        for sub in &kept {
3199            for &(a, z) in &sub.lines {
3200                shape_line(a, z, &gs.clips, &mut shapes.lines);
3201            }
3202            if sub.closed {
3203                shape_line(sub.cur, sub.start, &gs.clips, &mut shapes.lines);
3204            }
3205        }
3206    }
3207    if visible {
3208        let pts: Vec<(f64, f64)> = kept.iter().flat_map(|s| s.pts.iter().copied()).collect();
3209        if let Some((l, b, r, t)) = bbox_of(&pts) {
3210            let pad = if strokes_visible {
3211                (gs.line_width * scale).max(0.0) * 0.5
3212            } else {
3213                0.0
3214            };
3215            if let Some(bbox) = clipped([l - pad, b - pad, r + pad, t + pad], &gs.clips) {
3216                shapes.boxes.push(bbox);
3217            }
3218        }
3219    }
3220}
3221
3222/// `get_connected_shape_bounding_boxes`: each box in turn (unless already
3223/// taken) grows by every untaken box its growing union touches, until none
3224/// does — docling-parse's greedy pass, so a component finished early keeps a
3225/// box a later one would also have reached. Tolerance 0, as docling calls
3226/// it: two `re` that meet in the file meet here because [`real`] reads their
3227/// operands as the doubles docling-parse reads.
3228fn connected_boxes(boxes: &[[f64; 4]]) -> Vec<[f64; 4]> {
3229    let touches =
3230        |a: [f64; 4], b: [f64; 4]| a[0] <= b[2] && a[2] >= b[0] && a[1] <= b[3] && a[3] >= b[1];
3231    let mut taken = vec![false; boxes.len()];
3232    let mut out = Vec::new();
3233    for i in 0..boxes.len() {
3234        if taken[i] {
3235            continue;
3236        }
3237        taken[i] = true;
3238        let mut component = boxes[i];
3239        let mut changed = true;
3240        while changed {
3241            changed = false;
3242            for j in 0..boxes.len() {
3243                if !taken[j] && touches(component, boxes[j]) {
3244                    let b = boxes[j];
3245                    component = [
3246                        component[0].min(b[0]),
3247                        component[1].min(b[1]),
3248                        component[2].max(b[2]),
3249                        component[3].max(b[3]),
3250                    ];
3251                    taken[j] = true;
3252                    changed = true;
3253                }
3254            }
3255        }
3256        out.push(component);
3257    }
3258    out
3259}
3260
3261/// One piece of a path under construction, page space (see `run_content`).
3262enum PathPiece {
3263    Line((f64, f64), (f64, f64)),
3264    /// An `re`: its four corners, counter-clockwise from `(x, y)`.
3265    Rect([(f64, f64); 4]),
3266    /// A Bézier segment: its start and control/end points (their hull).
3267    Curve(Vec<(f64, f64)>),
3268}
3269
3270/// Record a painted path's pieces for the checkbox finder (#609): a stroked
3271/// line is a [`Seg`](crate::checkbox::Ink::Seg), a stroked rectangle that
3272/// stays axis-aligned a [`Rect`](crate::checkbox::Ink::Rect) (one filled
3273/// white too — still an outline), and a coloured fill or a curve only its
3274/// bounding box, a [`Blot`](crate::checkbox::Ink::Blot): a possible tick
3275/// mark, or a swatch. White fills draw nothing and are dropped. Only pieces small
3276/// enough to be a checkbox edge or a mark inside one are kept.
3277fn paint_path(
3278    path: &[PathPiece],
3279    stroke: bool,
3280    fill: bool,
3281    fill_light: bool,
3282    inks: &mut Vec<crate::checkbox::Ink>,
3283) {
3284    use crate::checkbox::Ink;
3285    let bbox = |pts: &mut dyn Iterator<Item = (f64, f64)>| {
3286        pts.fold(
3287            (
3288                f64::INFINITY,
3289                f64::INFINITY,
3290                f64::NEG_INFINITY,
3291                f64::NEG_INFINITY,
3292            ),
3293            |(l, b, r, t), (x, y)| (l.min(x), b.min(y), r.max(x), t.max(y)),
3294        )
3295    };
3296    let mut keep = |ink: Ink| {
3297        if ink.worth_keeping() {
3298            inks.push(ink);
3299        }
3300    };
3301    if fill && !fill_light {
3302        // A coloured fill is a blot — a tick mark, or (the size of a square)
3303        // a legend swatch, which `checkbox::find` then rules out. A white
3304        // fill is no ink on paper: it only leaves the stroke, if any, below.
3305        let (l, b, r, t) = bbox(&mut path.iter().flat_map(|p| match p {
3306            PathPiece::Line(a, z) => vec![*a, *z],
3307            PathPiece::Rect(c) => c.to_vec(),
3308            PathPiece::Curve(c) => c.clone(),
3309        }));
3310        if l.is_finite() {
3311            keep(Ink::Blot { l, b, r, t });
3312        }
3313        return;
3314    }
3315    if !stroke {
3316        return;
3317    }
3318    for piece in path {
3319        match piece {
3320            PathPiece::Line(a, z) => keep(Ink::Seg {
3321                x0: a.0,
3322                y0: a.1,
3323                x1: z.0,
3324                y1: z.1,
3325            }),
3326            PathPiece::Rect(c) => {
3327                let axis = ((c[0].1 - c[1].1).abs() < 1e-6 && (c[1].0 - c[2].0).abs() < 1e-6)
3328                    || ((c[0].0 - c[1].0).abs() < 1e-6 && (c[1].1 - c[2].1).abs() < 1e-6);
3329                if axis {
3330                    let (l, b, r, t) = bbox(&mut c.iter().copied());
3331                    keep(Ink::Rect { l, b, r, t });
3332                } else {
3333                    for k in 0..4 {
3334                        let (a, z) = (c[k], c[(k + 1) % 4]);
3335                        keep(Ink::Seg {
3336                            x0: a.0,
3337                            y0: a.1,
3338                            x1: z.0,
3339                            y1: z.1,
3340                        });
3341                    }
3342                }
3343            }
3344            PathPiece::Curve(c) => {
3345                let (l, b, r, t) = bbox(&mut c.iter().copied());
3346                keep(Ink::Blot { l, b, r, t });
3347            }
3348        }
3349    }
3350}
3351
3352#[allow(clippy::too_many_arguments)]
3353fn show_text(
3354    font: &Font,
3355    bytes: &[u8],
3356    fsize: f64,
3357    tc: f64,
3358    tw: f64,
3359    th: f64,
3360    trise: f64,
3361    tm: &mut Mat,
3362    ctm: Mat,
3363    out: &mut Vec<Glyph>,
3364) {
3365    for code in codes(font, bytes) {
3366        let (text, w, blank) = font.decode_code(code);
3367        let w0 = w / 1000.0; // advance in text-space (em) units
3368                             // The glyph→user transform: scale glyph space by font size, then Tm, CTM.
3369        let scale = Mat {
3370            a: fsize * th,
3371            b: 0.0,
3372            c: 0.0,
3373            d: fsize,
3374            e: 0.0,
3375            f: trise,
3376        };
3377        let trm = scale.then(*tm).then(ctm);
3378        // Box in glyph space (1000-unit em): x 0..w, y descent..ascent.
3379        // A blank unmapped glyph keeps its advance but no ink height
3380        // (docling-parse collapses its ascent and descent).
3381        let (desc, asc) = if blank {
3382            (0.0, 0.0)
3383        } else if let Some(&(lo, hi)) = font.ink_y.get(&code) {
3384            (lo / 1000.0, hi / 1000.0)
3385        } else {
3386            (font.descent / 1000.0, font.ascent / 1000.0)
3387        };
3388        let (x0, y0) = trm.apply(0.0, desc);
3389        let (x1, y1) = trm.apply(w0, desc);
3390        let (x2, y2) = trm.apply(w0, asc);
3391        let (x3, y3) = trm.apply(0.0, asc);
3392        // Upright = the baseline runs left-to-right along +x. Only then is the
3393        // loose box the rectangle spanned by the baseline's x and the em's y
3394        // (a synthetic oblique's skew is ignored, as it always was). A rotated
3395        // matrix — the `0 s -s 0 tx ty Tm` landscape pages are built with
3396        // (#528) — collapsed that rectangle to zero width, and every such run
3397        // vanished; those glyphs keep their real quad (docling-parse's char
3398        // rect) and its extent instead.
3399        let upright = trm.a > 0.0 && trm.b.abs() <= 1e-6 * trm.a;
3400        let (left, bot, right, top, quad) = if upright {
3401            (x0.min(x1), y0.min(y3), x0.max(x1), y0.max(y3), None)
3402        } else {
3403            let (xs, ys) = ([x0, x1, x2, x3], [y0, y1, y2, y3]);
3404            let fold = |v: [f64; 4], f: fn(f64, f64) -> f64| v.into_iter().reduce(f).unwrap();
3405            let q = [x0, y0, x1, y1, x2, y2, x3, y3].map(|v| v as f32);
3406            (
3407                fold(xs, f64::min),
3408                fold(ys, f64::min),
3409                fold(xs, f64::max),
3410                fold(ys, f64::max),
3411                Some(q),
3412            )
3413        };
3414        // The pen's origin (rise included) and the font size's extent from
3415        // it, as docling-parse maps them (`map_text_point(0, rise)` and
3416        // `(0, rise + font_size)`).
3417        let (ox, oy) = trm.apply(0.0, 0.0);
3418        let (sx, sy) = trm.apply(0.0, 1.0);
3419        let pen = Some([ox as f32, oy as f32, (sx - ox).hypot(sy - oy) as f32]);
3420        if let Some(s) = text {
3421            // A run may map one code to multiple chars (ligature/fraction); share box.
3422            for ch in s.chars() {
3423                // A NUL from the `/ToUnicode` is an unmapped glyph too
3424                // (docling#5070): U+FFFD.
3425                let ch = if ch == '\u{0}' { '\u{FFFD}' } else { ch };
3426                out.push(Glyph {
3427                    ch,
3428                    l: left as f32,
3429                    b: bot as f32,
3430                    r: right as f32,
3431                    t: top as f32,
3432                    ll: left as f32,
3433                    lb: bot as f32,
3434                    lr: right as f32,
3435                    lt: top as f32,
3436                    font: font.hash,
3437                    quad,
3438                    pen,
3439                    math_font: font.math,
3440                    invisible: false,
3441                });
3442            }
3443        }
3444        // Advance the text matrix. Word spacing applies to single-byte code 32.
3445        let is_space = !font.two_byte && code == 32;
3446        let tx = (w0 * fsize + tc + if is_space { tw } else { 0.0 }) * th;
3447        *tm = Mat {
3448            a: 1.0,
3449            b: 0.0,
3450            c: 0.0,
3451            d: 1.0,
3452            e: tx,
3453            f: 0.0,
3454        }
3455        .then(*tm);
3456    }
3457}
3458
3459/// Build a simple font's code→char table from its `/Encoding`: the base
3460/// encoding (WinAnsi / MacRoman) plus any `/Differences` overrides (glyph names
3461/// resolved through a small Adobe-glyph-name subset).
3462fn simple_encoding_table(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
3463    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o));
3464    let base_name = match enc {
3465        Some(Object::Name(n)) => n.clone(),
3466        Some(Object::Dictionary(d)) => d
3467            .get(b"BaseEncoding")
3468            .ok()
3469            .and_then(|o| o.as_name().ok())
3470            .map(|n| n.to_vec())
3471            .unwrap_or_default(),
3472        _ => Vec::new(),
3473    };
3474    let face = || {
3475        let font_name = sanitizer_font_name(doc, fdict, false);
3476        let base = fdict.get(b"BaseFont").ok().and_then(|o| o.as_name().ok());
3477        let names: Vec<&[u8]> = std::iter::once(font_name.as_slice()).chain(base).collect();
3478        crate::symbol_fonts::symbol_face(&names)
3479    };
3480    let mut m = if base_name == b"MacRomanEncoding" {
3481        macroman_table()
3482    } else if let Some(face) = base_name.is_empty().then(face).flatten() {
3483        // A font matched to Symbol or ZapfDingbats that names no encoding:
3484        // the face's own (docling-parse's base-font AFM, a "FontSpecific"
3485        // scheme), never the Latin tables — a non-embedded ZapfDingbats'
3486        // `l` is `●` (docling-parse#431, #1113). An embedded program's
3487        // builtin encoding still comes first ([`Font::builtin`]).
3488        crate::symbol_fonts::face_table(face)
3489    } else if base_name.is_empty() {
3490        // No PDF /Encoding at all: the font's *built-in* encoding applies. For
3491        // the standard TeX math fonts that is their fixed TeX layout — falling
3492        // back to StandardEncoding read CMSY's braces as `f`/`g`, `→` as `!`,
3493        // `∈` as `2` (2203's `{ahn,…}` author line). The font program (often
3494        // CFF, which this parser does not read) carries the same mapping;
3495        // docling-parse decodes it from there.
3496        tex_math_builtin(fdict).unwrap_or_else(winansi_table)
3497    } else if base_name == b"WinAnsiEncoding" {
3498        let mut m = winansi_table();
3499        // PDF 32000-1:2008 Annex D.2, note 3 to the Latin set: "In
3500        // WinAnsiEncoding, all unused codes greater than 40 [octal] map to the
3501        // bullet character" — pdfium and pdf.js read them so. ReportLab draws
3502        // its default list bullet as `\177` in a non-embedded WinAnsi
3503        // Helvetica, which read U+FFFD before (#1102). Only a font that names
3504        // WinAnsiEncoding gets them: the no-/Encoding fallback above stands
3505        // in for a built-in encoding, where these codes stay undefined.
3506        for b in [0x7Fu8, 0x81, 0x8D, 0x8F, 0x90, 0x9D] {
3507            m.insert(b, '\u{2022}');
3508        }
3509        m
3510    } else {
3511        winansi_table()
3512    };
3513    // Apply /Differences: `code /glyphname /glyphname ... code ...`.
3514    if let Some(Object::Dictionary(d)) = enc {
3515        if let Some(Object::Array(diffs)) = d.get(b"Differences").ok().and_then(|o| deref(doc, o)) {
3516            let mut code = 0u8;
3517            for el in diffs {
3518                match el {
3519                    Object::Integer(i) => code = *i as u8,
3520                    Object::Name(name) => {
3521                        if let Some(ch) = glyph_name_to_char(name) {
3522                            m.insert(code, ch);
3523                        }
3524                        code = code.wrapping_add(1);
3525                    }
3526                    _ => {}
3527                }
3528            }
3529        }
3530    }
3531    m
3532}
3533
3534/// The fixed built-in encodings of the standard TeX math fonts (TeXbook
3535/// Appendix F), keyed off the base font name: `CMSY*` (symbols; `CMBSY` is its
3536/// bold) and `CMMI*` (math italic). These fonts ship no PDF `/Encoding` and no
3537/// ToUnicode, and their program is usually CFF — without this table the codes
3538/// fell through to StandardEncoding and rendered as the wrong ASCII.
3539fn tex_math_builtin(fdict: &Dictionary) -> Option<HashMap<u8, char>> {
3540    const CMSY: [char; 128] = [
3541        '−', '·', '×', '∗', '÷', '⋄', '±', '∓', '⊕', '⊖', '⊗', '⊘', '⊙', '◯', '∘', '•', '≍', '≡',
3542        '⊆', '⊇', '≤', '≥', '≼', '≽', '∼', '≈', '⊂', '⊃', '≪', '≫', '≺', '≻', '←', '→', '↑', '↓',
3543        '↔', '↗', '↘', '≃', '⇐', '⇒', '⇑', '⇓', '⇔', '↖', '↙', '∝', '′', '∞', '∈', '∋', '△', '▽',
3544        '\u{338}', '↦', '∀', '∃', '¬', '∅', 'ℜ', 'ℑ', '⊤', '⊥', 'ℵ', 'A', 'B', 'C', 'D', 'E', 'F',
3545        'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
3546        'Y', 'Z', '∪', '∩', '⊎', '∧', '∨', '⊢', '⊣', '⌊', '⌋', '⌈', '⌉', '{', '}', '⟨', '⟩', '|',
3547        '∥', '↕', '⇕', '\\', '≀', '√', '∐', '∇', '∫', '⊔', '⊓', '⊑', '⊒', '§', '†', '‡', '¶', '♣',
3548        '♢', '♡', '♠',
3549    ];
3550    const CMMI: [char; 128] = [
3551        'Γ', 'Δ', 'Θ', 'Λ', 'Ξ', 'Π', 'Σ', 'Υ', 'Φ', 'Ψ', 'Ω', 'α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η',
3552        'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'π', 'ρ', 'σ', 'τ', 'υ', 'φ', 'χ', 'ψ', 'ω', 'ϵ', 'ϑ',
3553        'ϖ', 'ϱ', 'ς', 'ϕ', '↼', '↽', '⇀', '⇁', '↩', '↪', '▷', '◁', '0', '1', '2', '3', '4', '5',
3554        '6', '7', '8', '9', '.', ',', '<', '/', '>', '⋆', '∂', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
3555        'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y',
3556        'Z', '♭', '♮', '♯', '⌣', '⌢', 'ℓ', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
3557        'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'ı', 'ȷ', '℘',
3558        '\u{20d7}', '⁀',
3559    ];
3560    let name = base_font_name(fdict)?;
3561    let up = name.to_ascii_uppercase();
3562    let table: &[char; 128] = if up.starts_with(b"CMSY") || up.starts_with(b"CMBSY") {
3563        &CMSY
3564    } else if up.starts_with(b"CMMI") {
3565        &CMMI
3566    } else {
3567        return None;
3568    };
3569    Some(
3570        table
3571            .iter()
3572            .enumerate()
3573            .map(|(i, &c)| (i as u8, c))
3574            .collect(),
3575    )
3576}
3577
3578/// The Adobe Glyph List's names for Latin-1 Supplement and Latin Extended-A
3579/// (plus `Euro`, `florin`, the comma-accent pairs): what a Type1 font's
3580/// `/Differences` or builtin encoding calls an accented or stroked letter.
3581/// 2203.01017v2 names `Ł` as `/Lslash` in a NimbusRomNo9L subset with no
3582/// ToUnicode, and the resolver, which knew only the ASCII, ligature and symbol
3583/// names, dropped the glyph outright (`Ł ukasz Kaiser` read `ukasz Kaiser`).
3584/// Sorted by name for the binary search in [`glyph_name_to_char`].
3585const AGL_LATIN: &[(&str, char)] = &[
3586    ("AE", '\u{00C6}'),
3587    ("Aacute", '\u{00C1}'),
3588    ("Abreve", '\u{0102}'),
3589    ("Acircumflex", '\u{00C2}'),
3590    ("Adieresis", '\u{00C4}'),
3591    ("Agrave", '\u{00C0}'),
3592    ("Amacron", '\u{0100}'),
3593    ("Aogonek", '\u{0104}'),
3594    ("Aring", '\u{00C5}'),
3595    ("Atilde", '\u{00C3}'),
3596    ("Cacute", '\u{0106}'),
3597    ("Ccaron", '\u{010C}'),
3598    ("Ccedilla", '\u{00C7}'),
3599    ("Ccircumflex", '\u{0108}'),
3600    ("Cdotaccent", '\u{010A}'),
3601    ("Dcaron", '\u{010E}'),
3602    ("Dcroat", '\u{0110}'),
3603    ("Eacute", '\u{00C9}'),
3604    ("Ebreve", '\u{0114}'),
3605    ("Ecaron", '\u{011A}'),
3606    ("Ecircumflex", '\u{00CA}'),
3607    ("Edieresis", '\u{00CB}'),
3608    ("Edotaccent", '\u{0116}'),
3609    ("Egrave", '\u{00C8}'),
3610    ("Emacron", '\u{0112}'),
3611    ("Eng", '\u{014A}'),
3612    ("Eogonek", '\u{0118}'),
3613    ("Eth", '\u{00D0}'),
3614    ("Euro", '\u{20AC}'),
3615    ("Gbreve", '\u{011E}'),
3616    ("Gcircumflex", '\u{011C}'),
3617    ("Gcommaaccent", '\u{0122}'),
3618    ("Gdotaccent", '\u{0120}'),
3619    ("Hbar", '\u{0126}'),
3620    ("Hcircumflex", '\u{0124}'),
3621    ("IJ", '\u{0132}'),
3622    ("Iacute", '\u{00CD}'),
3623    ("Ibreve", '\u{012C}'),
3624    ("Icircumflex", '\u{00CE}'),
3625    ("Idieresis", '\u{00CF}'),
3626    ("Idotaccent", '\u{0130}'),
3627    ("Igrave", '\u{00CC}'),
3628    ("Imacron", '\u{012A}'),
3629    ("Iogonek", '\u{012E}'),
3630    ("Itilde", '\u{0128}'),
3631    ("Jcircumflex", '\u{0134}'),
3632    ("Kcommaaccent", '\u{0136}'),
3633    ("Lacute", '\u{0139}'),
3634    ("Lcaron", '\u{013D}'),
3635    ("Lcommaaccent", '\u{013B}'),
3636    ("Ldot", '\u{013F}'),
3637    ("Lslash", '\u{0141}'),
3638    ("Nacute", '\u{0143}'),
3639    ("Ncaron", '\u{0147}'),
3640    ("Ncommaaccent", '\u{0145}'),
3641    ("Ntilde", '\u{00D1}'),
3642    ("OE", '\u{0152}'),
3643    ("Oacute", '\u{00D3}'),
3644    ("Obreve", '\u{014E}'),
3645    ("Ocircumflex", '\u{00D4}'),
3646    ("Odieresis", '\u{00D6}'),
3647    ("Ograve", '\u{00D2}'),
3648    ("Ohungarumlaut", '\u{0150}'),
3649    ("Omacron", '\u{014C}'),
3650    ("Oslash", '\u{00D8}'),
3651    ("Otilde", '\u{00D5}'),
3652    ("Racute", '\u{0154}'),
3653    ("Rcaron", '\u{0158}'),
3654    ("Rcommaaccent", '\u{0156}'),
3655    ("Sacute", '\u{015A}'),
3656    ("Scaron", '\u{0160}'),
3657    ("Scedilla", '\u{015E}'),
3658    ("Scircumflex", '\u{015C}'),
3659    ("Scommaaccent", '\u{0218}'),
3660    ("Tbar", '\u{0166}'),
3661    ("Tcaron", '\u{0164}'),
3662    ("Tcommaaccent", '\u{0162}'),
3663    ("Thorn", '\u{00DE}'),
3664    ("Uacute", '\u{00DA}'),
3665    ("Ubreve", '\u{016C}'),
3666    ("Ucircumflex", '\u{00DB}'),
3667    ("Udieresis", '\u{00DC}'),
3668    ("Ugrave", '\u{00D9}'),
3669    ("Uhungarumlaut", '\u{0170}'),
3670    ("Umacron", '\u{016A}'),
3671    ("Uogonek", '\u{0172}'),
3672    ("Uring", '\u{016E}'),
3673    ("Utilde", '\u{0168}'),
3674    ("Wcircumflex", '\u{0174}'),
3675    ("Yacute", '\u{00DD}'),
3676    ("Ycircumflex", '\u{0176}'),
3677    ("Ydieresis", '\u{0178}'),
3678    ("Zacute", '\u{0179}'),
3679    ("Zcaron", '\u{017D}'),
3680    ("Zdotaccent", '\u{017B}'),
3681    ("aacute", '\u{00E1}'),
3682    ("abreve", '\u{0103}'),
3683    ("acircumflex", '\u{00E2}'),
3684    ("acute", '\u{00B4}'),
3685    ("adieresis", '\u{00E4}'),
3686    ("ae", '\u{00E6}'),
3687    ("agrave", '\u{00E0}'),
3688    ("amacron", '\u{0101}'),
3689    ("aogonek", '\u{0105}'),
3690    ("aring", '\u{00E5}'),
3691    ("atilde", '\u{00E3}'),
3692    ("brokenbar", '\u{00A6}'),
3693    ("cacute", '\u{0107}'),
3694    ("ccaron", '\u{010D}'),
3695    ("ccedilla", '\u{00E7}'),
3696    ("ccircumflex", '\u{0109}'),
3697    ("cdotaccent", '\u{010B}'),
3698    ("cedilla", '\u{00B8}'),
3699    ("cent", '\u{00A2}'),
3700    ("currency", '\u{00A4}'),
3701    ("dcaron", '\u{010F}'),
3702    ("dcroat", '\u{0111}'),
3703    ("dieresis", '\u{00A8}'),
3704    ("dotlessi", '\u{0131}'),
3705    ("eacute", '\u{00E9}'),
3706    ("ebreve", '\u{0115}'),
3707    ("ecaron", '\u{011B}'),
3708    ("ecircumflex", '\u{00EA}'),
3709    ("edieresis", '\u{00EB}'),
3710    ("edotaccent", '\u{0117}'),
3711    ("egrave", '\u{00E8}'),
3712    ("emacron", '\u{0113}'),
3713    ("eng", '\u{014B}'),
3714    ("eogonek", '\u{0119}'),
3715    ("eth", '\u{00F0}'),
3716    ("exclamdown", '\u{00A1}'),
3717    ("florin", '\u{0192}'),
3718    ("gbreve", '\u{011F}'),
3719    ("gcircumflex", '\u{011D}'),
3720    ("gcommaaccent", '\u{0123}'),
3721    ("gdotaccent", '\u{0121}'),
3722    ("germandbls", '\u{00DF}'),
3723    ("guillemotleft", '\u{00AB}'),
3724    ("guillemotright", '\u{00BB}'),
3725    ("hbar", '\u{0127}'),
3726    ("hcircumflex", '\u{0125}'),
3727    ("iacute", '\u{00ED}'),
3728    ("ibreve", '\u{012D}'),
3729    ("icircumflex", '\u{00EE}'),
3730    ("idieresis", '\u{00EF}'),
3731    ("igrave", '\u{00EC}'),
3732    ("ij", '\u{0133}'),
3733    ("imacron", '\u{012B}'),
3734    ("iogonek", '\u{012F}'),
3735    ("itilde", '\u{0129}'),
3736    ("jcircumflex", '\u{0135}'),
3737    ("kcommaaccent", '\u{0137}'),
3738    ("kgreenlandic", '\u{0138}'),
3739    ("lacute", '\u{013A}'),
3740    ("lcaron", '\u{013E}'),
3741    ("lcommaaccent", '\u{013C}'),
3742    ("ldot", '\u{0140}'),
3743    ("logicalnot", '\u{00AC}'),
3744    ("longs", '\u{017F}'),
3745    ("lslash", '\u{0142}'),
3746    ("macron", '\u{00AF}'),
3747    ("nacute", '\u{0144}'),
3748    ("napostrophe", '\u{0149}'),
3749    ("ncaron", '\u{0148}'),
3750    ("ncommaaccent", '\u{0146}'),
3751    ("ntilde", '\u{00F1}'),
3752    ("oacute", '\u{00F3}'),
3753    ("obreve", '\u{014F}'),
3754    ("ocircumflex", '\u{00F4}'),
3755    ("odieresis", '\u{00F6}'),
3756    ("oe", '\u{0153}'),
3757    ("ograve", '\u{00F2}'),
3758    ("ohungarumlaut", '\u{0151}'),
3759    ("omacron", '\u{014D}'),
3760    ("onehalf", '\u{00BD}'),
3761    ("onequarter", '\u{00BC}'),
3762    ("onesuperior", '\u{00B9}'),
3763    ("ordfeminine", '\u{00AA}'),
3764    ("ordmasculine", '\u{00BA}'),
3765    ("oslash", '\u{00F8}'),
3766    ("otilde", '\u{00F5}'),
3767    ("paragraph", '\u{00B6}'),
3768    ("questiondown", '\u{00BF}'),
3769    ("racute", '\u{0155}'),
3770    ("rcaron", '\u{0159}'),
3771    ("rcommaaccent", '\u{0157}'),
3772    ("sacute", '\u{015B}'),
3773    ("scaron", '\u{0161}'),
3774    ("scedilla", '\u{015F}'),
3775    ("scircumflex", '\u{015D}'),
3776    ("scommaaccent", '\u{0219}'),
3777    ("section", '\u{00A7}'),
3778    ("sfthyphen", '\u{00AD}'),
3779    ("sterling", '\u{00A3}'),
3780    ("tbar", '\u{0167}'),
3781    ("tcaron", '\u{0165}'),
3782    ("tcommaaccent", '\u{0163}'),
3783    ("thorn", '\u{00FE}'),
3784    ("threequarters", '\u{00BE}'),
3785    ("threesuperior", '\u{00B3}'),
3786    ("twosuperior", '\u{00B2}'),
3787    ("uacute", '\u{00FA}'),
3788    ("ubreve", '\u{016D}'),
3789    ("ucircumflex", '\u{00FB}'),
3790    ("udieresis", '\u{00FC}'),
3791    ("ugrave", '\u{00F9}'),
3792    ("uhungarumlaut", '\u{0171}'),
3793    ("umacron", '\u{016B}'),
3794    ("uogonek", '\u{0173}'),
3795    ("uring", '\u{016F}'),
3796    ("utilde", '\u{0169}'),
3797    ("wcircumflex", '\u{0175}'),
3798    ("yacute", '\u{00FD}'),
3799    ("ycircumflex", '\u{0177}'),
3800    ("ydieresis", '\u{00FF}'),
3801    ("yen", '\u{00A5}'),
3802    ("zacute", '\u{017A}'),
3803    ("zcaron", '\u{017E}'),
3804    ("zdotaccent", '\u{017C}'),
3805];
3806
3807/// docling-parse 7.25's reading text of a PostScript glyph name in a
3808/// program's builtin encoding (`builtin_font_encoding::glyph_name_to_utf8`,
3809/// #1113): `.notdef` and an index-style name (`g12`, `gid00012`, `cid12`,
3810/// `index12`) carry none; a stylistic `.suffix` is dropped; then the glyph
3811/// tables (the symbol fonts' names, [`glyph_name_to_char`]'s), a `uniXXXX`
3812/// sequence (a ligature's code points), a `uXXXX`–`uXXXXXX` name.
3813pub(crate) fn builtin_glyph_text(name: &str) -> Option<String> {
3814    if name.is_empty() || name == ".notdef" {
3815        return None;
3816    }
3817    let name = match name.find('.') {
3818        Some(dot) if dot > 0 => &name[..dot],
3819        _ => name,
3820    };
3821    let lower = name.to_ascii_lowercase();
3822    let index_name = ["gid", "glyph", "g", "cid", "index"].iter().any(|p| {
3823        lower
3824            .strip_prefix(p)
3825            .is_some_and(|d| !d.is_empty() && d.bytes().all(|b| b.is_ascii_digit()))
3826    });
3827    if index_name {
3828        return None;
3829    }
3830    if let Some(c) = crate::symbol_fonts::symbol_glyph_char(name) {
3831        return Some(c.to_string());
3832    }
3833    let hex_points = |hex: &str, width: usize| -> Option<String> {
3834        (!hex.is_empty()
3835            && hex.len().is_multiple_of(width)
3836            && hex.bytes().all(|b| b.is_ascii_hexdigit()))
3837        .then(|| {
3838            (0..hex.len())
3839                .step_by(width)
3840                .map(|i| {
3841                    u32::from_str_radix(&hex[i..i + width], 16)
3842                        .ok()
3843                        .and_then(char::from_u32)
3844                })
3845                .collect::<Option<String>>()
3846        })
3847        .flatten()
3848    };
3849    if let Some(hex) = name.strip_prefix("uni") {
3850        if let Some(t) = hex_points(hex, 4) {
3851            return Some(t);
3852        }
3853    }
3854    if let Some(hex) = name.strip_prefix('u') {
3855        if (4..=6).contains(&hex.len()) {
3856            if let Some(t) = hex_points(hex, hex.len()) {
3857                return Some(t);
3858            }
3859        }
3860    }
3861    glyph_name_to_char(name.as_bytes()).map(String::from)
3862}
3863
3864/// Resolve an Adobe glyph name to Unicode: `uniXXXX`, single ASCII letters, the
3865/// digit/punctuation names from the Adobe Glyph List, and common typographic
3866/// names. A `.suffix` (`one.taboldstyle`, `a.sc`) is stripped and the base name
3867/// retried — docling renders these as the base character.
3868pub(crate) fn glyph_name_to_char(name: &[u8]) -> Option<char> {
3869    let s = std::str::from_utf8(name).ok()?;
3870    // `uniXXXX`. A name that merely starts with `uni` (`union`) goes on to
3871    // the tables: it used to end here, unresolved, for want of four digits.
3872    if let Some(cp) = s
3873        .strip_prefix("uni")
3874        .and_then(|hex| hex.get(0..4))
3875        .and_then(|hex| u32::from_str_radix(hex, 16).ok())
3876    {
3877        return char::from_u32(cp);
3878    }
3879    // Single ASCII letter names (`A`, `m`) map to themselves.
3880    if s.len() == 1 {
3881        let b = s.as_bytes()[0];
3882        if b.is_ascii_alphabetic() {
3883            return Some(b as char);
3884        }
3885    }
3886    let resolved = match s {
3887        "space" => ' ',
3888        "exclam" => '!',
3889        "quotedbl" => '"',
3890        "numbersign" => '#',
3891        "dollar" => '$',
3892        "percent" => '%',
3893        "ampersand" => '&',
3894        "quotesingle" => '\'',
3895        "parenleft" => '(',
3896        "parenright" => ')',
3897        "asterisk" => '*',
3898        "plus" => '+',
3899        "comma" => ',',
3900        "hyphen" => '-',
3901        "period" => '.',
3902        "slash" => '/',
3903        "zero" => '0',
3904        "one" => '1',
3905        "two" => '2',
3906        "three" => '3',
3907        "four" => '4',
3908        "five" => '5',
3909        "six" => '6',
3910        "seven" => '7',
3911        "eight" => '8',
3912        "nine" => '9',
3913        "colon" => ':',
3914        "semicolon" => ';',
3915        "less" => '<',
3916        "equal" => '=',
3917        "greater" => '>',
3918        "question" => '?',
3919        "at" => '@',
3920        "bracketleft" => '[',
3921        "backslash" => '\\',
3922        "bracketright" => ']',
3923        "asciicircum" => '^',
3924        "underscore" => '_',
3925        "grave" => '`',
3926        "braceleft" => '{',
3927        "bar" => '|',
3928        "braceright" => '}',
3929        "asciitilde" => '~',
3930        "bullet" => '\u{2022}',
3931        "periodcentered" => '\u{00B7}',
3932        "endash" => '\u{2013}',
3933        "emdash" => '\u{2014}',
3934        "quoteright" => '\u{2019}',
3935        "quoteleft" => '\u{2018}',
3936        "quotedblleft" => '\u{201C}',
3937        "quotedblright" => '\u{201D}',
3938        "quotedblbase" => '\u{201E}',
3939        "quotesinglbase" => '\u{201A}',
3940        // Latin f-ligatures named in `/Differences` (e.g. 2305's body font). These
3941        // map to the presentation-form code points, which `decompose_ligatures`
3942        // then spells back out (`ff`→"ff") — without them the glyph decodes to
3943        // nothing and the sanitizer fills the gap with a space (`di erences`).
3944        "ff" => '\u{FB00}',
3945        "fi" => '\u{FB01}',
3946        "fl" => '\u{FB02}',
3947        "ffi" => '\u{FB03}',
3948        "ffl" => '\u{FB04}',
3949        "ft" => '\u{FB05}',
3950        "st" => '\u{FB06}',
3951        "degree" => '\u{00B0}',
3952        "trademark" => '\u{2122}',
3953        "registered" => '\u{00AE}',
3954        "copyright" => '\u{00A9}',
3955        "ellipsis" => '\u{2026}',
3956        "minus" => '\u{2212}',
3957        "fraction" => '\u{2044}',
3958        "nbspace" => '\u{00A0}',
3959        // Greek + math glyph names (standard Adobe Glyph List). Standard TeX math
3960        // fonts (CMMI/CMSY/…) name their glyphs this way in the embedded font
3961        // program's `/Encoding`; without these a `λ`/`≤` decodes to nothing and is
3962        // dropped from body text (`and λ set to 0.5` → `and set to 0.5`).
3963        "alpha" => '\u{03B1}',
3964        "beta" => '\u{03B2}',
3965        "gamma" => '\u{03B3}',
3966        "delta" => '\u{03B4}',
3967        "epsilon" | "epsilon1" => '\u{03B5}',
3968        "zeta" => '\u{03B6}',
3969        "eta" => '\u{03B7}',
3970        "theta" | "theta1" => '\u{03B8}',
3971        "iota" => '\u{03B9}',
3972        "kappa" => '\u{03BA}',
3973        "lambda" => '\u{03BB}',
3974        "mu" => '\u{03BC}',
3975        "nu" => '\u{03BD}',
3976        "xi" => '\u{03BE}',
3977        "omicron" => '\u{03BF}',
3978        "pi" | "pi1" => '\u{03C0}',
3979        "rho" | "rho1" => '\u{03C1}',
3980        "sigma" => '\u{03C3}',
3981        "sigma1" => '\u{03C2}',
3982        "tau" => '\u{03C4}',
3983        "upsilon" => '\u{03C5}',
3984        "phi" | "phi1" => '\u{03C6}',
3985        "chi" => '\u{03C7}',
3986        "psi" => '\u{03C8}',
3987        "omega" | "omega1" => '\u{03C9}',
3988        "Gamma" => '\u{0393}',
3989        "Delta" => '\u{0394}',
3990        "Theta" => '\u{0398}',
3991        "Lambda" => '\u{039B}',
3992        "Xi" => '\u{039E}',
3993        "Pi" => '\u{03A0}',
3994        "Sigma" => '\u{03A3}',
3995        "Upsilon" => '\u{03A5}',
3996        "Phi" => '\u{03A6}',
3997        "Psi" => '\u{03A8}',
3998        "Omega" => '\u{03A9}',
3999        "lessequal" => '\u{2264}',
4000        "greaterequal" => '\u{2265}',
4001        "notequal" => '\u{2260}',
4002        "approxequal" => '\u{2248}',
4003        "equivalence" => '\u{2261}',
4004        "element" => '\u{2208}',
4005        "plusminus" => '\u{00B1}',
4006        "multiply" => '\u{00D7}',
4007        "divide" => '\u{00F7}',
4008        "infinity" => '\u{221E}',
4009        "partialdiff" => '\u{2202}',
4010        "gradient" => '\u{2207}',
4011        "summation" => '\u{2211}',
4012        "product" => '\u{220F}',
4013        "integral" => '\u{222B}',
4014        "radical" => '\u{221A}',
4015        "proportional" => '\u{221D}',
4016        "arrowright" => '\u{2192}',
4017        "arrowleft" => '\u{2190}',
4018        "arrowup" => '\u{2191}',
4019        "arrowdown" => '\u{2193}',
4020        "arrowboth" => '\u{2194}',
4021        "arrowdblright" => '\u{21D2}',
4022        "logicaland" => '\u{2227}',
4023        "logicalor" => '\u{2228}',
4024        "intersection" => '\u{2229}',
4025        "union" => '\u{222A}',
4026        "similar" => '\u{223C}',
4027        "congruent" => '\u{2245}',
4028        "dotmath" => '\u{22C5}',
4029        "asteriskmath" => '\u{2217}',
4030        _ => {
4031            if let Ok(i) = AGL_LATIN.binary_search_by_key(&s, |(n, _)| n) {
4032                return Some(AGL_LATIN[i].1);
4033            }
4034            // The math names the arms above leave out (#1121): the TeX math
4035            // fonts' own (`parenleftbig`, `radicalbig`, `integraldisplay`,
4036            // `prime`), then the Symbol font's — Glyph List names such as
4037            // `universal`, `existential`, `arrowup`, which a Latin Modern
4038            // math font's `/Differences` uses. The arms above keep their
4039            // readings where docling-parse's lists differ (`Delta` is the
4040            // Greek letter here, the Glyph List's `∆` increment sign there).
4041            // Last, the Adobe Glyph List in full (the `glyph-list` feature):
4042            // every name none of the above reads — the `afii` names of a
4043            // Cyrillic, Hebrew or Arabic font, the rest of the symbols.
4044            if let Some(ch) = crate::glyph_names::lookup(s)
4045                .or_else(|| crate::symbol_fonts::symbol_glyph_char(s))
4046                .or_else(|| crate::adobe_glyph_list::lookup(s))
4047            {
4048                return Some(ch);
4049            }
4050            // Strip an AGL `.suffix` (oldstyle/small-cap variant) and retry.
4051            if let Some((base, _)) = s.split_once('.') {
4052                if !base.is_empty() {
4053                    return glyph_name_to_char(base.as_bytes());
4054                }
4055            }
4056            // An AGL ligature name joins its components with `_` (`f_f_i`,
4057            // `f_l` — what a re-encoded Libertine subset calls them); joined,
4058            // the presentation-form arms above resolve them.
4059            if s.contains('_') {
4060                let joined: String = s.chars().filter(|&c| c != '_').collect();
4061                if !joined.is_empty() && joined.len() < s.len() {
4062                    return glyph_name_to_char(joined.as_bytes());
4063                }
4064            }
4065            return None;
4066        }
4067    };
4068    Some(resolved)
4069}
4070
4071/// Minimal WinAnsiEncoding (Latin-1-ish) for simple fonts lacking ToUnicode.
4072fn winansi_table() -> HashMap<u8, char> {
4073    let mut m = HashMap::new();
4074    for b in 0x20u8..=0x7e {
4075        m.insert(b, b as char);
4076    }
4077    // High range: Windows-1252 printable points that differ from Latin-1.
4078    let extra: &[(u8, char)] = &[
4079        (0x91, '\u{2018}'),
4080        (0x92, '\u{2019}'),
4081        (0x93, '\u{201C}'),
4082        (0x94, '\u{201D}'),
4083        (0x95, '\u{2022}'),
4084        (0x96, '\u{2013}'),
4085        (0x97, '\u{2014}'),
4086        (0x85, '\u{2026}'),
4087        (0xA0, '\u{00A0}'),
4088        // The rest of Windows-1252's 0x80–0x9F row. Left out, the German
4089        // opening quote `„` (0x84) vanished from the text layer while its
4090        // closing `“` (0x93) stayed (#922); `€ ‚ ƒ † ‡ ˆ ‰ Š ‹ Œ Ž ˜ ™ š › œ ž Ÿ`
4091        // were lost the same way. 0x81/0x8D/0x8F/0x90/0x9D (and 0x7F) are
4092        // unused; a font naming WinAnsiEncoding reads them as bullets
4093        // (`simple_encoding_table`, #1102).
4094        (0x80, '\u{20AC}'),
4095        (0x82, '\u{201A}'),
4096        (0x83, '\u{0192}'),
4097        (0x84, '\u{201E}'),
4098        (0x86, '\u{2020}'),
4099        (0x87, '\u{2021}'),
4100        (0x88, '\u{02C6}'),
4101        (0x89, '\u{2030}'),
4102        (0x8A, '\u{0160}'),
4103        (0x8B, '\u{2039}'),
4104        (0x8C, '\u{0152}'),
4105        (0x8E, '\u{017D}'),
4106        (0x98, '\u{02DC}'),
4107        (0x99, '\u{2122}'),
4108        (0x9A, '\u{0161}'),
4109        (0x9B, '\u{203A}'),
4110        (0x9C, '\u{0153}'),
4111        (0x9E, '\u{017E}'),
4112        (0x9F, '\u{0178}'),
4113    ];
4114    for &(b, c) in extra {
4115        m.insert(b, c);
4116    }
4117    for b in 0xA1u8..=0xFF {
4118        m.entry(b).or_insert(b as char);
4119    }
4120    m
4121}
4122
4123/// MacRomanEncoding: ASCII plus the whole high range, 0x80–0xFF, as Mac OS
4124/// Roman maps it (#744) — the table pdfium, pdftotext and pdf.js decode with.
4125/// Two codes follow the PDF reference instead of Apple's later revisions:
4126/// 0xDB is `currency` (¤, not €), and 0xF0 (`apple`, a private-use glyph) is
4127/// left unmapped. 0xCA, the PDF table's second `space`, reads as a no-break
4128/// space.
4129fn macroman_table() -> HashMap<u8, char> {
4130    #[rustfmt::skip]
4131    const HIGH: [char; 128] = [
4132        'Ä', 'Å', 'Ç', 'É', 'Ñ', 'Ö', 'Ü', 'á',  // 0x80
4133        'à', 'â', 'ä', 'ã', 'å', 'ç', 'é', 'è',  // 0x88
4134        'ê', 'ë', 'í', 'ì', 'î', 'ï', 'ñ', 'ó',  // 0x90
4135        'ò', 'ô', 'ö', 'õ', 'ú', 'ù', 'û', 'ü',  // 0x98
4136        '†', '°', '¢', '£', '§', '•', '¶', 'ß',  // 0xA0
4137        '®', '©', '™', '´', '¨', '≠', 'Æ', 'Ø',  // 0xA8
4138        '∞', '±', '≤', '≥', '¥', 'µ', '∂', '∑',  // 0xB0
4139        '∏', 'π', '∫', 'ª', 'º', 'Ω', 'æ', 'ø',  // 0xB8
4140        '¿', '¡', '¬', '√', 'ƒ', '≈', '∆', '«',  // 0xC0
4141        '»', '…', '\u{00A0}', 'À', 'Ã', 'Õ', 'Œ', 'œ',  // 0xC8
4142        '–', '—', '“', '”', '‘', '’', '÷', '◊',  // 0xD0
4143        'ÿ', 'Ÿ', '⁄', '¤', '‹', '›', 'fi', 'fl',  // 0xD8
4144        '‡', '·', '‚', '„', '‰', 'Â', 'Ê', 'Á',  // 0xE0
4145        'Ë', 'È', 'Í', 'Î', 'Ï', 'Ì', 'Ó', 'Ô',  // 0xE8
4146        '\0', 'Ò', 'Ú', 'Û', 'Ù', 'ı', 'ˆ', '˜',  // 0xF0
4147        '¯', '˘', '˙', '˚', '¸', '˝', '˛', 'ˇ',  // 0xF8
4148    ];
4149    let mut m = HashMap::new();
4150    for b in 0x20u8..=0x7e {
4151        m.insert(b, b as char);
4152    }
4153    for (b, &c) in (0x80u8..=0xFF).zip(HIGH.iter()) {
4154        if c != '\0' {
4155            m.insert(b, c);
4156        }
4157    }
4158    m
4159}
4160
4161#[cfg(test)]
4162mod glyph_names {
4163    /// `/Lslash` and the other Latin-1 / Latin Extended-A glyph names a Type1
4164    /// `/Differences` uses resolve (2203.01017v2's `Łukasz`); the table the
4165    /// lookup bisects is sorted and the ligature / symbol arms still win.
4166    #[test]
4167    fn latin_glyph_names_resolve() {
4168        for w in super::AGL_LATIN.windows(2) {
4169            assert!(
4170                w[0].0 < w[1].0,
4171                "AGL_LATIN not sorted at {:?} / {:?}",
4172                w[0].0,
4173                w[1].0
4174            );
4175        }
4176        for (name, ch) in [
4177            ("Lslash", 'Ł'),
4178            ("lslash", 'ł'),
4179            ("Aacute", 'Á'),
4180            ("aacute", 'á'),
4181            ("Scaron", 'Š'),
4182            ("germandbls", 'ß'),
4183            ("oe", 'œ'),
4184            ("Ydieresis", 'Ÿ'),
4185            ("Euro", '€'),
4186            ("Lslash.sc", 'Ł'),
4187            ("fi", '\u{FB01}'),
4188            ("Omega", '\u{03A9}'),
4189        ] {
4190            assert_eq!(
4191                super::glyph_name_to_char(name.as_bytes()),
4192                Some(ch),
4193                "{name}"
4194            );
4195        }
4196        assert_eq!(super::glyph_name_to_char(b"g123"), None);
4197        // AGL ligature names: components joined with `_`.
4198        assert_eq!(super::glyph_name_to_char(b"f_f_i"), Some('\u{FB03}'));
4199        assert_eq!(super::glyph_name_to_char(b"f_l"), Some('\u{FB02}'));
4200        assert_eq!(super::glyph_name_to_char(b"a_b"), None);
4201    }
4202
4203    /// The stale-CMap case: ToUnicode maps a code to its own identity while
4204    /// `/Differences` names a ligature there — the name wins; a CMap that
4205    /// maps the code anywhere else, or a code with no entry, is untouched.
4206    #[test]
4207    fn a_stale_tounicode_yields_to_differences() {
4208        use lopdf::Object;
4209        use std::collections::HashMap;
4210        let mut tu: HashMap<u32, String> = HashMap::new();
4211        tu.insert(33, "!".into()); // identity — stale
4212        tu.insert(34, "x".into()); // the producer meant it
4213        let diffs = vec![
4214            Object::Integer(33),
4215            Object::Name(b"f_f_i".to_vec()),
4216            Object::Name(b"f_i".to_vec()),
4217            Object::Name(b"f_f".to_vec()),
4218            Object::Integer(65),
4219            Object::Name(b"A".to_vec()), // identity on both sides: nothing to do
4220        ];
4221        super::reconcile_stale_tounicode(&mut tu, &diffs);
4222        assert_eq!(tu.get(&33).map(String::as_str), Some("\u{FB03}"));
4223        assert_eq!(tu.get(&34).map(String::as_str), Some("x"));
4224        assert_eq!(tu.get(&35), None);
4225        assert_eq!(tu.get(&65), None);
4226    }
4227
4228    /// #1121: a ToUnicode value that says nothing — empty, a NUL, U+FFFD, a
4229    /// noncharacter — yields to the `/Differences` name (docling-parse's
4230    /// `has_to_unicode`). 1503.04203's embedded figure maps its re-encoded
4231    /// TeX glyphs to `<0000>`: `/minus` and `/Delta` at code 33. A value that
4232    /// says something stays, and so does one whose name does not resolve.
4233    #[test]
4234    fn a_tounicode_value_that_says_nothing_yields_to_differences() {
4235        use lopdf::Object;
4236        use std::collections::HashMap;
4237        let name = |n: &str| Object::Name(n.as_bytes().to_vec());
4238        let mut tu: HashMap<u32, String> = HashMap::new();
4239        tu.insert(33, "\u{0}".into());
4240        tu.insert(34, "\u{FFFD}".into());
4241        tu.insert(35, "\u{FFFF}".into());
4242        tu.insert(36, String::new());
4243        tu.insert(37, "\u{0}".into()); // the name is unknown: left alone
4244        tu.insert(38, "\u{0}\u{0}".into()); // not a single code point
4245        tu.insert(39, "−".into()); // says something
4246        let diffs = vec![
4247            Object::Integer(33),
4248            name("minus"),
4249            name("Delta"),
4250            name("radicalbig"),
4251            name("parenleftbig"),
4252            name("nosuchglyphname"),
4253            name("plus"),
4254            name("hyphen"),
4255        ];
4256        super::reconcile_stale_tounicode(&mut tu, &diffs);
4257        let got = |c: u32| tu.get(&c).map(String::as_str);
4258        assert_eq!(got(33), Some("−"));
4259        assert_eq!(got(34), Some("Δ"));
4260        assert_eq!(got(35), Some("√"));
4261        assert_eq!(got(36), Some("("));
4262        assert_eq!(got(37), Some("\u{0}"));
4263        assert_eq!(got(38), Some("\u{0}\u{0}"));
4264        assert_eq!(got(39), Some("−"));
4265    }
4266
4267    /// #1121: the TeX delimiter and operator names resolve through the glyph
4268    /// rules, the Symbol font's Glyph List names through its table; the
4269    /// names this resolver already knew keep their reading, and a name that
4270    /// only starts with `uni` is no `uniXXXX`.
4271    #[test]
4272    fn tex_glyph_names_resolve() {
4273        let c = |n: &str| super::glyph_name_to_char(n.as_bytes());
4274        assert_eq!(c("parenleftbig"), Some('('));
4275        assert_eq!(c("parenrightBigg"), Some(')'));
4276        assert_eq!(c("integraldisplay"), Some('∫'));
4277        assert_eq!(c("trianglerightsld"), Some('▶'));
4278        assert_eq!(c("prime"), Some('′'));
4279        assert_eq!(c("universal"), Some('∀'));
4280        assert_eq!(c("existential"), Some('∃'));
4281        assert_eq!(c("arrowup"), Some('↑'));
4282        assert_eq!(c("union"), Some('∪'));
4283        assert_eq!(c("uni2212"), Some('−'));
4284        assert_eq!(c("minus"), Some('−'));
4285        assert_eq!(c("Delta"), Some('Δ'), "the Greek letter, not the list's ∆");
4286        assert_eq!(c("angbracketleft"), Some('⟨'));
4287        assert_eq!(c("nosuchglyphname"), None);
4288    }
4289}
4290
4291#[cfg(test)]
4292mod page_box_frame {
4293    use super::*;
4294    use lopdf::dictionary;
4295
4296    /// One page, `boxes` spliced into the page dictionary verbatim, one text
4297    /// run at user-space `(x, y)`.
4298    fn pdf(boxes: &str, x: f32, y: f32) -> Vec<u8> {
4299        pdf_content(
4300            boxes,
4301            &format!("BT /F1 12 Tf {x} {y} Td (First printing) Tj ET\n"),
4302        )
4303    }
4304
4305    /// One page, `boxes` spliced into the page dictionary, `content` as its
4306    /// content stream (font `/F1` = Helvetica).
4307    fn pdf_content(boxes: &str, content: &str) -> Vec<u8> {
4308        let objs: Vec<String> = vec![
4309            "<</Type/Catalog/Pages 2 0 R>>".into(),
4310            format!("<</Type/Pages/Kids[3 0 R]/Count 1{boxes}>>"),
4311            "<</Type/Page/Parent 2 0 R/Contents 4 0 R/Resources<</Font<</F1 5 0 R>>>>>>".into(),
4312            format!("<</Length {}>>stream\n{content}endstream", content.len()),
4313            "<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".into(),
4314        ];
4315        let mut out = b"%PDF-1.4\n".to_vec();
4316        let mut offsets = Vec::new();
4317        for (i, body) in objs.iter().enumerate() {
4318            offsets.push(out.len());
4319            out.extend_from_slice(format!("{} 0 obj{body}endobj\n", i + 1).as_bytes());
4320        }
4321        let xref_at = out.len();
4322        out.extend_from_slice(
4323            format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes(),
4324        );
4325        for off in &offsets {
4326            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
4327        }
4328        out.extend_from_slice(
4329            format!(
4330                "trailer<</Size {}/Root 1 0 R>>\nstartxref\n{xref_at}\n%%EOF\n",
4331                objs.len() + 1
4332            )
4333            .as_bytes(),
4334        );
4335        out
4336    }
4337
4338    /// One page whose `/Contents` is an array of separate streams.
4339    fn pdf_content_streams(contents: &[&[u8]]) -> Vec<u8> {
4340        let mut doc = Document::with_version("1.5");
4341        let streams = contents
4342            .iter()
4343            .map(|content| {
4344                Object::Reference(
4345                    doc.add_object(lopdf::Stream::new(Dictionary::new(), content.to_vec())),
4346                )
4347            })
4348            .collect::<Vec<_>>();
4349        let font_id = doc.add_object(dictionary! {
4350            "Type" => "Font",
4351            "Subtype" => "Type1",
4352            "BaseFont" => "Helvetica",
4353        });
4354        let resources_id = doc.add_object(dictionary! {
4355            "Font" => dictionary! { "F1" => Object::Reference(font_id) },
4356        });
4357        let pages_id = doc.new_object_id();
4358        let page_id = doc.add_object(dictionary! {
4359            "Type" => "Page",
4360            "Parent" => Object::Reference(pages_id),
4361            "MediaBox" => vec![0.into(), 0.into(), 300.into(), 400.into()],
4362            "Contents" => streams,
4363            "Resources" => Object::Reference(resources_id),
4364        });
4365        doc.objects.insert(
4366            pages_id,
4367            Object::Dictionary(dictionary! {
4368                "Type" => "Pages",
4369                "Kids" => vec![Object::Reference(page_id)],
4370                "Count" => 1,
4371            }),
4372        );
4373        let catalog_id = doc.add_object(dictionary! {
4374            "Type" => "Catalog",
4375            "Pages" => Object::Reference(pages_id),
4376        });
4377        doc.trailer.set("Root", Object::Reference(catalog_id));
4378        let mut out = Vec::new();
4379        doc.save_to(&mut out).expect("save synthetic PDF");
4380        out
4381    }
4382
4383    fn only_page(bytes: &[u8]) -> (PageBox, Vec<Glyph>) {
4384        let doc = load_document(bytes).expect("loads");
4385        let pid = *doc.get_pages().values().next().expect("one page");
4386        (page_box(&doc, pid), page_glyphs(&doc, pid))
4387    }
4388
4389    /// The trimmed-book-page shape: MediaBox and CropBox both away from the
4390    /// origin (inherited from `/Pages`). The display box is the CropBox, and a
4391    /// glyph's coordinates count from its lower-left corner — the same numbers
4392    /// the same text gets on a page whose CropBox *is* `[0 0 w h]`.
4393    #[test]
4394    fn glyphs_count_from_the_cropbox_corner_like_pdfium() {
4395        let (pb, shifted) = only_page(&pdf(
4396            "/MediaBox[-56.505 -58.25 576.303 723.31]/CropBox[1.095 -0.65 518.703 665.71]",
4397            37.0 + 1.095,
4398            58.0 - 0.65,
4399        ));
4400        assert!((pb.l - 1.095).abs() < 1e-3 && (pb.b + 0.65).abs() < 1e-3);
4401        assert!(
4402            (pb.w - 517.608).abs() < 1e-3 && (pb.h - 666.36).abs() < 1e-3,
4403            "{pb:?}"
4404        );
4405        let (pb0, plain) = only_page(&pdf("/MediaBox[0 0 517.608 666.36]", 37.0, 58.0));
4406        assert!((pb0.w - pb.w).abs() < 1e-3 && (pb0.h - pb.h).abs() < 1e-3);
4407        assert_eq!(shifted.len(), plain.len());
4408        assert!(!plain.is_empty());
4409        for (a, b) in shifted.iter().zip(&plain) {
4410            assert!(
4411                (a.l - b.l).abs() < 1e-3 && (a.b - b.b).abs() < 1e-3,
4412                "{:?} vs {:?}",
4413                (a.l, a.b),
4414                (b.l, b.b)
4415            );
4416        }
4417        assert!((plain[0].l - 37.0).abs() < 1e-3, "{}", plain[0].l);
4418    }
4419
4420    fn text(glyphs: &[Glyph]) -> String {
4421        glyphs.iter().map(|g| g.ch).collect()
4422    }
4423
4424    /// #529: text drawn outside the display box — a print slug beside the
4425    /// CropBox, a tiled page's neighbour beyond the MediaBox — is dropped,
4426    /// glyph by glyph like docling-parse, instead of being clamped onto the
4427    /// page edge; a line crossing the edge keeps the glyphs that fit.
4428    #[test]
4429    fn glyphs_outside_the_display_box_are_dropped() {
4430        let (_, g) = only_page(&pdf_content(
4431            "/MediaBox[0 0 400 400]/CropBox[100 100 400 400]",
4432            "BT /F1 14 Tf 120 300 Td (Visible.) Tj ET\nBT /F1 8 Tf 5 40 Td (Slug) Tj ET\n",
4433        ));
4434        assert_eq!(text(&g), "Visible.");
4435        let (_, g) = only_page(&pdf_content(
4436            "/MediaBox[0 0 300 400]",
4437            "BT /F1 14 Tf 40 300 Td (On page) Tj ET\nBT /F1 14 Tf 400 250 Td (Beyond) Tj ET\n",
4438        ));
4439        assert_eq!(text(&g), "On page");
4440        // docling-parse: "Crossing the right edge" on a 300 pt page → "Crossing the rig".
4441        let (_, g) = only_page(&pdf_content(
4442            "/MediaBox[0 0 300 400]",
4443            "BT /F1 14 Tf 200 250 Td (Crossing the right edge) Tj ET\n",
4444        ));
4445        assert_eq!(text(&g), "Crossing the rig");
4446    }
4447
4448    #[test]
4449    fn leading_comment_content_stream_does_not_hide_later_text() {
4450        // The comment has its own EOL; lopdf inserts another while assembling
4451        // the streams, which is the boundary its 0.44 parser stopped at.
4452        let (_, glyphs) = only_page(&pdf_content_streams(&[
4453            b"% CANON_PFINF_TYPE2_TEXTOFF\n",
4454            b"BT /F1 14 Tf 40 300 Td (Second stream text) Tj ET",
4455        ]));
4456        assert_eq!(text(&glyphs), "Second stream text");
4457    }
4458
4459    /// The containment test is docling-parse's: the whole char box (advance
4460    /// × the font's ascent / descent) inside the display box, edges
4461    /// included — a Helvetica `W` at 50 pt (47.2 wide) ending exactly on the
4462    /// right edge stays, 0.01 pt further it goes; likewise on the left.
4463    #[test]
4464    fn a_glyph_on_the_edge_stays_one_over_it_goes() {
4465        let w_at = |x: f32, y: f32| {
4466            let (_, g) = only_page(&pdf_content(
4467                "/MediaBox[0 0 300 400]",
4468                &format!("BT /F1 50 Tf {x} {y} Td (W) Tj ET\n"),
4469            ));
4470            text(&g)
4471        };
4472        assert_eq!(w_at(252.8, 200.0), "W");
4473        assert_eq!(w_at(252.81, 200.0), "");
4474        assert_eq!(w_at(0.0, 200.0), "W");
4475        assert_eq!(w_at(-0.01, 200.0), "");
4476        // Vertically the box is the font's ascent / descent: a baseline far
4477        // enough up keeps the descender on the page, one at 0 does not.
4478        assert_eq!(w_at(100.0, 20.0), "W");
4479        assert_eq!(w_at(100.0, 0.0), "");
4480        assert_eq!(w_at(100.0, 380.0), "");
4481    }
4482
4483    /// pdfium's fallbacks: no MediaBox → Letter; a CropBox is clipped to the
4484    /// MediaBox, and one that misses it entirely is ignored.
4485    #[test]
4486    fn page_box_follows_pdfium_fallbacks() {
4487        let (pb, _) = only_page(&pdf("", 10.0, 10.0));
4488        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 612.0, 792.0));
4489        let (pb, _) = only_page(&pdf(
4490            "/MediaBox[0 0 500 700]/CropBox[-100 100 600 900]",
4491            10.0,
4492            10.0,
4493        ));
4494        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 100.0, 500.0, 600.0));
4495        let (pb, _) = only_page(&pdf(
4496            "/MediaBox[0 0 500 700]/CropBox[800 800 900 900]",
4497            10.0,
4498            10.0,
4499        ));
4500        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
4501        // Reversed corners normalize.
4502        let (pb, _) = only_page(&pdf("/MediaBox[500 700 0 0]", 10.0, 10.0));
4503        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
4504    }
4505}
4506
4507/// #1130: `/Resources` inherited from the page tree (ISO 32000-1 7.7.3.4) —
4508/// above all an *inline* dictionary on a `/Pages` node, which lopdf's
4509/// `get_page_resources` never returns (it collects an ancestor's only when it
4510/// is a reference). The R&OS php pdf writer files in Govdocs1 (448692.pdf)
4511/// carry their one font that way; the text layer read as empty and the wasm
4512/// `DigitalConverter` refused the file as a scan.
4513#[cfg(test)]
4514mod inherited_resources {
4515    use super::*;
4516    use lopdf::dictionary;
4517
4518    /// Root `/Pages` (holding `root_res` inline and the MediaBox) → an
4519    /// intermediate `/Pages` (holding `mid_res`, if any) → one page with only
4520    /// `/Type`, `/Parent`, `/Contents` and, optionally, its own resources.
4521    fn pdf(root_res: Object, mid_res: Option<Object>, page_res: Option<Object>) -> Vec<u8> {
4522        let mut doc = Document::with_version("1.3");
4523        let content = doc.add_object(lopdf::Stream::new(
4524            Dictionary::new(),
4525            b"BT /F1 12 Tf 72 700 Td (Inherited resources) Tj ET".to_vec(),
4526        ));
4527        let root_id = doc.new_object_id();
4528        let mid_id = doc.new_object_id();
4529        let mut page = dictionary! {
4530            "Type" => "Page",
4531            "Parent" => Object::Reference(mid_id),
4532            "Contents" => vec![Object::Reference(content)],
4533        };
4534        if let Some(r) = page_res {
4535            page.set("Resources", r);
4536        }
4537        let page_id = doc.add_object(page);
4538        let mut mid = dictionary! {
4539            "Type" => "Pages",
4540            "Parent" => Object::Reference(root_id),
4541            "Kids" => vec![Object::Reference(page_id)],
4542            "Count" => 1,
4543        };
4544        if let Some(r) = mid_res {
4545            mid.set("Resources", r);
4546        }
4547        doc.objects.insert(mid_id, Object::Dictionary(mid));
4548        doc.objects.insert(
4549            root_id,
4550            Object::Dictionary(dictionary! {
4551                "Type" => "Pages",
4552                "Kids" => vec![Object::Reference(mid_id)],
4553                "Count" => 1,
4554                "Resources" => root_res,
4555                "MediaBox" => vec![0.into(), 0.into(), 595.into(), 842.into()],
4556            }),
4557        );
4558        let catalog = doc.add_object(dictionary! {
4559            "Type" => "Catalog",
4560            "Pages" => Object::Reference(root_id),
4561        });
4562        doc.trailer.set("Root", Object::Reference(catalog));
4563        let mut out = Vec::new();
4564        doc.save_to(&mut out).expect("save synthetic PDF");
4565        out
4566    }
4567
4568    fn fonts(base: &str) -> Object {
4569        Object::Dictionary(dictionary! {
4570            "Font" => dictionary! { "F1" => dictionary! {
4571                "Type" => "Font", "Subtype" => "Type1", "BaseFont" => base,
4572            }},
4573        })
4574    }
4575
4576    fn page_text(bytes: &[u8]) -> String {
4577        let pages = pdf_text_pages(bytes);
4578        assert_eq!(pages.len(), 1);
4579        pages[0]
4580            .cells
4581            .iter()
4582            .map(|c| c.text.as_str())
4583            .collect::<Vec<_>>()
4584            .join(" ")
4585    }
4586
4587    #[test]
4588    fn inline_resources_on_an_ancestor_pages_node_are_inherited() {
4589        let bytes = pdf(fonts("Helvetica"), None, None);
4590        assert_eq!(page_text(&bytes), "Inherited resources");
4591    }
4592
4593    /// The nearest node that sets `/Resources` wins: the page's own over the
4594    /// intermediate node's over the root's — and one that does not resolve to
4595    /// a dictionary is passed over for the next ancestor's.
4596    #[test]
4597    fn the_nearest_resources_dictionary_wins() {
4598        let doc_of = |b: &[u8]| load_document(b).expect("load");
4599        let font_of = |bytes: &[u8]| {
4600            let doc = doc_of(bytes);
4601            let pid = *doc.get_pages().values().next().unwrap();
4602            let res = page_res(&doc, pid).expect("resources");
4603            let f1 = res
4604                .get(b"Font")
4605                .ok()
4606                .and_then(|o| o.as_dict().ok())
4607                .and_then(|f| f.get(b"F1").ok())
4608                .and_then(|o| o.as_dict().ok())
4609                .and_then(|f| f.get(b"BaseFont").ok())
4610                .and_then(|o| o.as_name().ok())
4611                .map(|n| String::from_utf8_lossy(n).into_owned());
4612            f1.unwrap_or_default()
4613        };
4614        assert_eq!(font_of(&pdf(fonts("Root"), None, None)), "Root");
4615        assert_eq!(
4616            font_of(&pdf(fonts("Root"), Some(fonts("Mid")), None)),
4617            "Mid"
4618        );
4619        assert_eq!(
4620            font_of(&pdf(fonts("Root"), Some(fonts("Mid")), Some(fonts("Page")))),
4621            "Page"
4622        );
4623        assert_eq!(
4624            font_of(&pdf(fonts("Root"), Some(Object::Null), None)),
4625            "Root"
4626        );
4627    }
4628}
4629
4630#[cfg(test)]
4631mod xref_repair {
4632    /// Build a tiny one-page PDF whose cross-reference entries are either the
4633    /// spec's 20 bytes (`two_byte_eol`) or the 19-byte form some generators
4634    /// emit — everything else about the two files is identical.
4635    fn pdf_with_xref(two_byte_eol: bool) -> Vec<u8> {
4636        let content = b"BT /F1 12 Tf 72 700 Td (Invoice 922769430725) Tj ET\n";
4637        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
4638        let objs: Vec<Vec<u8>> = vec![
4639            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
4640            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
4641            b"<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
4642               /Resources<</Font<</F1 5 0 R>>>>>>"
4643                .to_vec(),
4644            [stream.as_slice(), content.as_slice(), b"endstream"].concat(),
4645            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".to_vec(),
4646        ];
4647
4648        let mut out = b"%PDF-1.4\n".to_vec();
4649        let mut offsets = Vec::new();
4650        for (i, body) in objs.iter().enumerate() {
4651            offsets.push(out.len());
4652            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
4653            out.extend_from_slice(body);
4654            out.extend_from_slice(b"endobj\n");
4655        }
4656        let xref_at = out.len();
4657        let eol: &[u8] = if two_byte_eol { b" \n" } else { b"\n" };
4658        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
4659        out.extend_from_slice(b"0000000000 65535 f");
4660        out.extend_from_slice(eol);
4661        for off in &offsets {
4662            out.extend_from_slice(format!("{off:010} 00000 n").as_bytes());
4663            out.extend_from_slice(eol);
4664        }
4665        out.extend_from_slice(
4666            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
4667        );
4668        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
4669        out
4670    }
4671
4672    /// A 19-byte cross-reference entry (a bare LF where the spec wants a
4673    /// two-byte EOL) used to make lopdf reject the whole file. Lopdf 0.45
4674    /// accepts it directly, so exercise the fallback itself and prove it still
4675    /// produces the exact 20-byte entries lopdf 0.44 needed.
4676    #[test]
4677    fn short_xref_entries_are_padded_for_fallback() {
4678        let good = pdf_with_xref(true);
4679        let broken = pdf_with_xref(false);
4680        assert!(
4681            broken.len() < good.len(),
4682            "the broken file is the shorter one"
4683        );
4684        assert!(
4685            lopdf::Document::load_mem(&good).is_ok(),
4686            "the control file must load unaided"
4687        );
4688        assert!(
4689            lopdf::Document::load_mem(&broken).is_ok(),
4690            "lopdf 0.45 should accept 19-byte entries"
4691        );
4692
4693        let repaired = super::pad_short_xref_entries(&broken).expect("repair short entries");
4694        assert_eq!(repaired, good, "repair should recreate the valid xref");
4695        assert!(
4696            lopdf::Document::load_mem(&repaired).is_ok(),
4697            "the bytes produced for lopdf 0.44 must load"
4698        );
4699
4700        let cells = |b: &[u8]| -> Vec<String> {
4701            super::pdf_textlines(b)
4702                .into_iter()
4703                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
4704                .collect()
4705        };
4706        let from_good = cells(&good);
4707        assert!(
4708            from_good.iter().any(|t| t.contains("922769430725")),
4709            "control text: {from_good:?}"
4710        );
4711        assert_eq!(
4712            cells(&repaired),
4713            from_good,
4714            "repair must match the good parse"
4715        );
4716    }
4717
4718    /// The same generator overstates `/Length`. Lopdf 0.45 recovers at
4719    /// `endstream`, so exercise our fallback directly and prove it still writes
4720    /// the exact length lopdf 0.44 needed without moving any bytes.
4721    #[test]
4722    fn overstated_stream_length_is_fixed_for_fallback() {
4723        let good = pdf_with_xref(true);
4724        // Inflate the content stream's /Length by one, exactly as the invoice
4725        // that prompted this does.
4726        let broken = {
4727            let at = good
4728                .windows(8)
4729                .position(|w| w == b"/Length ")
4730                .expect("a /Length")
4731                + 8;
4732            let digits = good[at..].iter().take_while(|c| c.is_ascii_digit()).count();
4733            let n: usize = std::str::from_utf8(&good[at..at + digits])
4734                .unwrap()
4735                .parse()
4736                .unwrap();
4737            let inflated = (n + 1).to_string();
4738            assert_eq!(inflated.len(), digits, "keep the digit count");
4739            let mut b = good.clone();
4740            b[at..at + digits].copy_from_slice(inflated.as_bytes());
4741            b
4742        };
4743        assert_eq!(broken.len(), good.len(), "the defect must not move bytes");
4744        // Lopdf 0.45 keeps the stream despite the overstated length.
4745        let raw = lopdf::Document::load_mem(&broken).expect("still loads");
4746        assert!(
4747            raw.get_pages()
4748                .into_values()
4749                .any(|p| !raw.get_page_content(p).is_empty()),
4750            "lopdf 0.45 should recover content at endstream"
4751        );
4752
4753        let repaired = super::fix_stream_lengths(&broken);
4754        assert_eq!(repaired.len(), broken.len(), "repair must preserve offsets");
4755        assert_eq!(repaired, good, "repair should restore the true length");
4756        let repaired_doc = lopdf::Document::load_mem(&repaired).expect("repaired PDF loads");
4757        assert!(
4758            repaired_doc
4759                .get_pages()
4760                .into_values()
4761                .any(|p| !repaired_doc.get_page_content(p).is_empty()),
4762            "the bytes produced for lopdf 0.44 must retain page content"
4763        );
4764
4765        // Ours recovers the same text the well-formed file gives.
4766        let text = |b: &[u8]| -> Vec<String> {
4767            super::pdf_textlines(b)
4768                .into_iter()
4769                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
4770                .collect()
4771        };
4772        let expected = text(&good);
4773        assert!(!expected.is_empty(), "control must produce text");
4774        assert_eq!(text(&repaired), expected);
4775    }
4776
4777    /// The repair only fires where padding cannot move an object: it declines a
4778    /// file whose xref precedes an object (an incremental update), rather than
4779    /// shifting every offset the table records.
4780    #[test]
4781    fn repair_declines_when_padding_would_move_objects() {
4782        let mut incremental = pdf_with_xref(false);
4783        incremental.extend_from_slice(b"6 0 obj<</Type/Whatever>>endobj\n");
4784        let declined = super::pad_short_xref_entries(&incremental).unwrap_err();
4785        assert!(
4786            declined.contains("object follows the xref"),
4787            "reason: {declined}"
4788        );
4789    }
4790}
4791
4792/// #187: standard-14 fonts referenced without an embedded program (and thus
4793/// usually without `/Widths` or a `/FontDescriptor`) must decode with the
4794/// built-in Adobe Core 14 metrics instead of collapsing every cell to zero
4795/// width — the failure mode where a valid text layer was silently dropped
4796/// while pdfium read the same file fine.
4797#[cfg(test)]
4798mod base14_fonts {
4799    /// A one-page PDF whose single `Tj` uses `fontdict` (no embedded program).
4800    fn pdf_with_font(fontdict: &[u8], text: &[u8]) -> Vec<u8> {
4801        let content = [b"BT /F1 12 Tf 72 700 Td (".as_slice(), text, b") Tj ET\n"].concat();
4802        pdf_with_content(fontdict, &content)
4803    }
4804
4805    /// A one-page A4 PDF drawing `content` with `fontdict` as `/F1`.
4806    pub(super) fn pdf_with_content(fontdict: &[u8], content: &[u8]) -> Vec<u8> {
4807        pdf_with_fonts(&[fontdict], content)
4808    }
4809
4810    /// A one-page A4 PDF drawing `content` with `fontdicts[i]` bound as
4811    /// `/F{i+1}` (objects 5, 6, …).
4812    fn pdf_with_fonts(fontdicts: &[&[u8]], content: &[u8]) -> Vec<u8> {
4813        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
4814        let font_res: String = fontdicts
4815            .iter()
4816            .enumerate()
4817            .map(|(i, _)| format!("/F{} {} 0 R", i + 1, i + 5))
4818            .collect();
4819        let mut objs: Vec<Vec<u8>> = vec![
4820            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
4821            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
4822            format!(
4823                "<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
4824                 /Resources<</Font<<{font_res}>>>>>>"
4825            )
4826            .into_bytes(),
4827            [stream.as_slice(), content, b"endstream"].concat(),
4828        ];
4829        objs.extend(fontdicts.iter().map(|d| d.to_vec()));
4830        let mut out = b"%PDF-1.4\n".to_vec();
4831        let mut offsets = Vec::new();
4832        for (i, body) in objs.iter().enumerate() {
4833            offsets.push(out.len());
4834            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
4835            out.extend_from_slice(body);
4836            out.extend_from_slice(b"endobj\n");
4837        }
4838        let xref_at = out.len();
4839        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
4840        out.extend_from_slice(b"0000000000 65535 f \n");
4841        for off in &offsets {
4842            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
4843        }
4844        out.extend_from_slice(
4845            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
4846        );
4847        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
4848        out
4849    }
4850
4851    /// The parsed cells of the only page.
4852    pub(super) fn cells(pdf: &[u8]) -> Vec<crate::pdfium_backend::TextCell> {
4853        super::pdf_textlines(pdf)
4854            .into_iter()
4855            .flat_map(|(_, _, c)| c)
4856            .collect()
4857    }
4858
4859    /// Text drawn with a rotated text matrix reads as one line in its own
4860    /// reading order, boxed by the run's axis-aligned extent (#528). The
4861    /// landscape case is `0 s -s 0 tx ty Tm` on a `/Rotate 90` page; every
4862    /// such run used to collapse to zero width and vanish (90°/270°/tilted)
4863    /// or read backwards (180°). Expected boxes are docling-parse 7.22's line
4864    /// cells for the same runs (y-up, PDF points).
4865    #[test]
4866    fn rotated_text_matrix_reads_in_order() {
4867        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
4868        const TEXT: &str = "Upright text on a rotated page.";
4869        for (tm, [l, b, r, t]) in [
4870            ("0 14 -14 0 200 100", [189.9, 100.0, 202.9, 289.1]), // 90°
4871            ("-14 0 0 -14 350 300", [160.9, 289.9, 350.0, 302.9]), // 180°
4872            ("0 -14 14 0 200 500", [197.1, 310.9, 210.1, 500.0]), // 270°
4873            (
4874                "9.8995 9.8995 -9.8995 9.8995 100 100",
4875                [92.9, 98.0, 235.8, 240.8],
4876            ), // 45°
4877        ] {
4878            let content = format!("BT /F1 1 Tf {tm} Tm ({TEXT}) Tj ET\n");
4879            let pdf = pdf_with_content(HELV, content.as_bytes());
4880            let cs = cells(&pdf);
4881            assert_eq!(cs.len(), 1, "{tm}: {cs:?}");
4882            let c = &cs[0];
4883            assert_eq!(c.text, TEXT, "{tm}");
4884            // `cells` is top-left-origin on the 842 pt page. The descriptor-
4885            // less base-14 face gets the parser's 1-em box where docling-parse
4886            // reads Helvetica's AFM (718/−207) — the same ≤ 0.6 pt
4887            // across-the-baseline offset upright text has — so allow 1 pt.
4888            let got = [c.l, 842.0 - c.b, c.r, 842.0 - c.t];
4889            for (g, w) in got.iter().zip([l, b, r, t]) {
4890                assert!(
4891                    (g - w).abs() < 1.0,
4892                    "{tm}: box {got:?} vs docling-parse {:?}",
4893                    [l, b, r, t]
4894                );
4895            }
4896            let words: Vec<String> = super::pdf_words(&pdf)
4897                .into_iter()
4898                .flat_map(|(_, _, c)| c)
4899                .map(|c| c.text)
4900                .collect();
4901            assert_eq!(words, TEXT.split(' ').collect::<Vec<_>>(), "{tm}");
4902        }
4903    }
4904
4905    /// A rotated run past the page edge (a cover's spine title) is dropped
4906    /// by the on-page test (#529) — measured on the quad's extent, not the
4907    /// zero-width box rotated glyphs used to get — instead of clamping onto
4908    /// the edge as a zero-width cell. One inside the page is kept.
4909    #[test]
4910    fn rotated_text_off_the_page_is_dropped() {
4911        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
4912        let off = pdf_with_content(
4913            HELV,
4914            b"BT /F1 1 Tf 0 14 -14 0 -4 200 Tm (Spine title) Tj ET\n",
4915        );
4916        assert!(cells(&off).is_empty(), "{:?}", cells(&off));
4917        let on = pdf_with_content(
4918            HELV,
4919            b"BT /F1 1 Tf 0 14 -14 0 30 200 Tm (Margin note) Tj ET\n",
4920        );
4921        assert_eq!(cells(&on).len(), 1);
4922    }
4923
4924    /// Upright text keeps the plain loose rectangle — no quad, and the
4925    /// sanitizer path is the one every pinned PDF baseline was made with.
4926    #[test]
4927    fn upright_glyphs_carry_no_quad() {
4928        let only_page = |pdf: &[u8]| {
4929            let doc = super::load_document(pdf).expect("loads");
4930            let pid = *doc.get_pages().values().next().expect("one page");
4931            super::page_glyphs(&doc, pid)
4932        };
4933        let pdf = pdf_with_font(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", b"Plain");
4934        let glyphs = only_page(&pdf);
4935        assert!(!glyphs.is_empty());
4936        assert!(glyphs.iter().all(|g| g.quad.is_none() && g.lr > g.ll));
4937        // A 90° glyph's style height is its font size, not its advance.
4938        let pdf = pdf_with_content(
4939            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>",
4940            b"BT /F1 1 Tf 0 14 -14 0 200 100 Tm (W) Tj ET\n",
4941        );
4942        let glyphs = only_page(&pdf);
4943        let g = &glyphs[0];
4944        assert!(g.quad.is_some());
4945        // 14 pt × the face's 1-em box (no descriptor: ascent − descent = 1).
4946        assert!((g.height() - 14.0).abs() < 0.05, "{}", g.height());
4947    }
4948
4949    /// #609: ReportLab's chart axis sets each tick label with the *same* `Tm`
4950    /// inside its own `q … cm … Q`, so only the CTM stacks `3`…`8` one under
4951    /// another at one x. Each tick is its own line cell with its own box (as
4952    /// docling-parse reads it), not one `345678` cell boxed like the first.
4953    #[test]
4954    fn ticks_stacked_by_cm_are_separate_line_cells() {
4955        let mut content = String::new();
4956        for (k, tick) in "345678".chars().enumerate() {
4957            let y = 300.0 + 25.92 * k as f64;
4958            content.push_str(&format!(
4959                "q 1 0 0 1 125 {y} cm BT /F1 10 Tf 1 0 0 1 -5 -4 Tm ({tick}) Tj ET Q\n"
4960            ));
4961        }
4962        let pdf = pdf_with_content(
4963            b"<</Type/Font/Subtype/Type1/BaseFont/Times-Roman>>",
4964            content.as_bytes(),
4965        );
4966        let cells = cells(&pdf);
4967        let texts: Vec<&str> = cells.iter().map(|c| c.text.as_str()).collect();
4968        assert_eq!(texts, ["3", "4", "5", "6", "7", "8"]);
4969        for (k, c) in cells.iter().enumerate() {
4970            // Top-left y of a tick whose baseline is 4 pt under its cm origin.
4971            let base = 842.0 - (300.0 + 25.92 * k as f32 - 4.0);
4972            assert!((c.l - 120.0).abs() < 0.1, "{k}: l {}", c.l);
4973            assert!(
4974                c.t < base && base - c.t < 10.0,
4975                "{k}: t {} base {base}",
4976                c.t
4977            );
4978        }
4979    }
4980
4981    /// #609: the path walk finds drawn checkbox squares — ReportLab's four
4982    /// `m … l S` edges under a `cm`, and a stroked `re` with a tick inside
4983    /// (checked) — in the text cells' top-left frame, and a stroked box
4984    /// filled white; a filled bar, a big frame and a colour-filled legend
4985    /// swatch are not checkboxes.
4986    #[test]
4987    fn drawn_checkbox_squares_are_found() {
4988        let content = b"q 1 0 0 1 119.52 449.04 cm \
4989            n 0 12.96 m 12.96 12.96 l S n 0 0 m 12.96 0 l S \
4990            n 0 0 m 0 12.96 l S n 12.96 0 m 12.96 12.96 l S Q\n\
4991            200 400 10 10 re S 202 405 m 204.5 402 l 208.5 408.5 l S\n\
4992            300 400 12 12 re f 50 50 100 100 re S\n\
4993            q 0.2 0.4 0.8 rg 400 400 12 12 re B Q q 1 g 500 400 12 12 re B Q\n";
4994        let pdf = pdf_with_content(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", content);
4995        let mut parser = super::PageTextParser::open(&pdf).expect("parses");
4996        let mut boxes = parser.cells(0).checkboxes;
4997        boxes.sort_by(|a, b| a.l.total_cmp(&b.l));
4998        let got: Vec<([i32; 4], bool)> = boxes
4999            .iter()
5000            .map(|c| {
5001                let r = |v: f32| (v * 100.0).round() as i32;
5002                ([r(c.l), r(c.t), r(c.r), r(c.b)], c.checked)
5003            })
5004            .collect();
5005        // Page height 842: y-up 449.04..462.0 → top-left 380.0..392.96.
5006        assert_eq!(
5007            got,
5008            [
5009                ([11952, 38000, 13248, 39296], false),
5010                ([20000, 43200, 21000, 44200], true),
5011                ([50000, 43000, 51200, 44200], false),
5012            ]
5013        );
5014    }
5015
5016    /// Every standard-14 alias/style decodes with real (positive-width) boxes.
5017    #[test]
5018    fn standard14_faces_get_builtin_widths() {
5019        for fontdict in [
5020            // ReportLab's default: base-14 Helvetica, WinAnsi, nothing else.
5021            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>".as_slice(),
5022            // No /Encoding at all (StandardEncoding-ish default).
5023            b"<</Type/Font/Subtype/Type1/BaseFont/Times-BoldItalic>>",
5024            // Substitution aliases + a subset prefix.
5025            b"<</Type/Font/Subtype/TrueType/BaseFont/Arial,Bold>>",
5026            b"<</Type/Font/Subtype/Type1/BaseFont/ABCDEF+Courier-Oblique>>",
5027        ] {
5028            let pdf = pdf_with_font(fontdict, b"Words have width now");
5029            let cs = cells(&pdf);
5030            let text: String = cs
5031                .iter()
5032                .map(|c| c.text.as_str())
5033                .collect::<Vec<_>>()
5034                .join(" ");
5035            assert!(
5036                text.contains("Words have width now"),
5037                "{}: text lost: {text:?}",
5038                String::from_utf8_lossy(fontdict)
5039            );
5040            assert!(
5041                cs.iter().all(|c| c.r > c.l),
5042                "{}: zero-width cells: {cs:?}",
5043                String::from_utf8_lossy(fontdict)
5044            );
5045        }
5046    }
5047
5048    /// MacRomanEncoding without a ToUnicode CMap (#744) — Word for Mac's
5049    /// Quartz output: the whole high range decodes, Danish æ ø å, é, ü, ß
5050    /// and § included, through the plain encoding name and as the base of a
5051    /// `/Differences` dictionary alike.
5052    #[test]
5053    fn macroman_fonts_keep_accented_letters() {
5054        let sentence = b"Bl\x8cb\xbergr\xbfd p\x8c \xbfen \xd0 caf\x8e M\x9fller, Stra\xa7e \xa4 3";
5055        for (fontdict, want) in [
5056            (
5057                b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/MacRomanEncoding>>"
5058                    .as_slice(),
5059                "Blåbærgrød på øen – café Müller, Straße § 3",
5060            ),
5061            (
5062                b"<</Type/Font/Subtype/TrueType/BaseFont/ABCDEF+Arial\
5063                   /Encoding<</Type/Encoding/BaseEncoding/MacRomanEncoding/Differences[51/four]>>>>",
5064                "Blåbærgrød på øen – café Müller, Straße § 4",
5065            ),
5066        ] {
5067            let text: String = cells(&pdf_with_font(fontdict, sentence))
5068                .iter()
5069                .map(|c| c.text.as_str())
5070                .collect::<Vec<_>>()
5071                .join(" ");
5072            assert_eq!(text, want, "{}", String::from_utf8_lossy(fontdict));
5073        }
5074
5075        // The table itself: all of 0x80–0xFF but the Apple logo, ¤ at 0xDB.
5076        let table = super::macroman_table();
5077        assert_eq!(
5078            (0x80u8..=0xFF).filter(|b| table.contains_key(b)).count(),
5079            127
5080        );
5081        assert_eq!(table.get(&0xF0), None);
5082        assert_eq!(table[&0xDB], '¤');
5083        assert_eq!(
5084            [0x80, 0x8C, 0xAE, 0xAF, 0xBE, 0xBF, 0xFF].map(|b| table[&b]),
5085            ['Ä', 'å', 'Æ', 'Ø', 'æ', 'ø', 'ˇ']
5086        );
5087    }
5088
5089    /// WinAnsiEncoding's unused codes above octal 40 are bullets (PDF
5090    /// 32000-1 Annex D.2) — ReportLab's default list bullet is `\177` in a
5091    /// non-embedded WinAnsi Helvetica and read U+FFFD (#1102). A font
5092    /// without an `/Encoding` keeps them undefined.
5093    #[test]
5094    fn winansi_unused_codes_are_bullets() {
5095        let read = |fontdict: &[u8], text: &[u8]| -> String {
5096            cells(&pdf_with_font(fontdict, text))
5097                .iter()
5098                .map(|c| c.text.as_str())
5099                .collect::<Vec<_>>()
5100                .join(" ")
5101        };
5102        let winansi = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
5103        assert_eq!(read(winansi, b"\\177 Cut costs"), "\u{2022} Cut costs");
5104        for code in [0x7Fu8, 0x81, 0x8D, 0x8F, 0x90, 0x9D] {
5105            assert_eq!(
5106                read(winansi, &[b'x', code, b'x']),
5107                "x\u{2022}x",
5108                "{code:#x}"
5109            );
5110        }
5111        let dict = b"<</Type/Font/Subtype/TrueType/BaseFont/Arial\
5112                     /Encoding<</Type/Encoding/BaseEncoding/WinAnsiEncoding>>>>";
5113        assert_eq!(read(dict, b"\\177 Cut costs"), "\u{2022} Cut costs");
5114        let bare = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
5115        assert!(!read(bare, b"x\\177x").contains('\u{2022}'));
5116    }
5117
5118    /// Unmapped glyphs read U+FFFD (#1003, docling#5070): a ToUnicode NUL —
5119    /// docling's `test_threaded_page_backend_replaces_nul_with_replacement_char`
5120    /// case, `mg.mL \x00 1` — and a bare glyph-index `/Differences` name,
5121    /// while an unknown code-like name keeps its text without the slash
5122    /// (docling-parse 7.24's `SM590000`).
5123    #[test]
5124    fn unmapped_glyphs_read_as_the_replacement_char() {
5125        let cmap = b"/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n\
5126            1 begincodespacerange <00> <FF> endcodespacerange\n\
5127            1 beginbfchar <7e> <0000> endbfchar\n\
5128            endcmap CMapName currentdict /CMap defineresource pop end end";
5129        let stream = [
5130            format!("<</Length {}>>stream\n", cmap.len()).as_bytes(),
5131            cmap.as_slice(),
5132            b"\nendstream",
5133        ]
5134        .concat();
5135        let pdf = pdf_with_fonts(
5136            &[
5137                b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/ToUnicode 6 0 R\
5138                   /Encoding<</Type/Encoding/Differences[1/g115 2/SM590000]>>>>",
5139                &stream,
5140            ],
5141            b"BT /F1 12 Tf 72 700 Td (mg.mL ~ 1 \x01 x \x02 y) Tj ET\n",
5142        );
5143        let text: String = cells(&pdf)
5144            .iter()
5145            .map(|c| c.text.as_str())
5146            .collect::<Vec<_>>()
5147            .join(" ");
5148        assert_eq!(text, "mg.mL \u{FFFD} 1 \u{FFFD} x SM590000 y");
5149    }
5150
5151    /// redp5110's subset font names every glyph by index (`/g115/g3 /g40…`):
5152    /// the painted ones read U+FFFD, the blank `/g3` program is a space, so
5153    /// the line keeps its words (docling-parse 7.24's line cell).
5154    #[cfg(feature = "render")]
5155    #[test]
5156    fn a_blank_glyph_program_is_a_space() {
5157        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
5158            .join("../../tests/data/pdf/sources/redp5110_sampled.pdf");
5159        let Ok(bytes) = std::fs::read(path) else {
5160            return;
5161        };
5162        let lines: Vec<String> = super::pdf_textlines(&bytes)
5163            .into_iter()
5164            .flat_map(|(_, _, c)| c)
5165            .map(|c| c.text)
5166            .collect();
5167        let want = "\u{FFFD} \u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD} \
5168                    \u{FFFD}\u{FFFD}\u{FFFD} ";
5169        assert!(
5170            lines.iter().any(|l| l.starts_with(want)),
5171            "no `{want}` line in {:?}",
5172            lines
5173                .iter()
5174                .filter(|l| l.contains('\u{FFFD}'))
5175                .collect::<Vec<_>>()
5176        );
5177    }
5178
5179    /// 2305.03393's figure-caption font (`SFRM0900`) declares `/Ascent 0
5180    /// /Descent 0`, which docling-parse 7.24 rejects: its glyphs are boxed by
5181    /// their outline, so the underscores of `"PMC2807444_006_00.png"` — ink
5182    /// under the baseline — are line cells of their own, as docling-parse's
5183    /// are (same boxes to 0.01 pt), not part of the file name.
5184    #[cfg(feature = "render")]
5185    #[test]
5186    fn an_invalid_metric_font_is_boxed_by_its_ink() {
5187        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
5188            .join("../../tests/data/pdf/sources/2305.03393v1.pdf");
5189        let Ok(bytes) = std::fs::read(path) else {
5190            return;
5191        };
5192        let pages = super::pdf_textlines(&bytes);
5193        let line = |text: &str| {
5194            pages[9]
5195                .2
5196                .iter()
5197                .find(|c| c.text == text)
5198                .unwrap_or_else(|| panic!("no `{text}` line cell"))
5199                .clone()
5200        };
5201        let name = line("\"PMC2807444");
5202        let under: Vec<_> = pages[9].2.iter().filter(|c| c.text == "_").collect();
5203        assert_eq!(under.len(), 2, "two `_` cells");
5204        for u in under {
5205            assert!(
5206                u.t > name.b,
5207                "`_` under the digits' ink: {:?} vs {:?}",
5208                u,
5209                name
5210            );
5211            assert!(
5212                (u.b - u.t - 0.403).abs() < 0.01,
5213                "an outline-tight box: {u:?}"
5214            );
5215        }
5216    }
5217
5218    /// docling-parse's descriptor checks (`init_ascent_and_descent`): metrics
5219    /// that do not straddle the baseline give way to a valid `/FontBBox` and
5220    /// mark the font for outline boxes; an implausible one is ignored for its
5221    /// `/FontBBox` edge; plain metrics are kept.
5222    #[test]
5223    fn descriptor_metrics_follow_docling_parse() {
5224        use lopdf::{dictionary, Object};
5225        let font = |asc: Option<i64>, desc: Option<i64>, bbox: bool| {
5226            let mut fd = dictionary! { "Type" => "FontDescriptor", "FontName" => "X" };
5227            if let Some(a) = asc {
5228                fd.set("Ascent", a);
5229            }
5230            if let Some(d) = desc {
5231                fd.set("Descent", d);
5232            }
5233            if bbox {
5234                let b: Vec<Object> = [-196, -322, 1502, 937].map(Object::from).to_vec();
5235                fd.set("FontBBox", b);
5236            }
5237            let f = dictionary! {
5238                "Type" => "Font", "Subtype" => "Type1", "BaseFont" => "X",
5239                "FontDescriptor" => fd,
5240            };
5241            super::font_ascent_descent(&lopdf::Document::with_version("1.5"), &f, false)
5242        };
5243        assert_eq!(font(Some(0), Some(0), true), (937.0, -322.0, true));
5244        assert_eq!(font(Some(700), Some(200), true), (937.0, -322.0, true));
5245        assert_eq!(font(Some(90000), Some(-250), true), (937.0, -250.0, true));
5246        assert_eq!(font(Some(718), Some(-207), true), (718.0, -207.0, false));
5247        assert_eq!(font(Some(0), Some(0), false), (750.0, -250.0, false));
5248        assert_eq!(font(None, None, true), (750.0, -250.0, false));
5249    }
5250
5251    /// redp5110 p. 14 draws its navigation arrows in `SymbolMT`, whose
5252    /// `/ToUnicode` maps them into the private-use symbol charset
5253    /// (`U+F0AE`): read through Symbol's encoding they are `→`, as
5254    /// docling-parse 7.25 reads them (#1113).
5255    #[test]
5256    fn symbol_private_use_values_read_as_the_glyphs() {
5257        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
5258            .join("../../tests/data/pdf/sources/redp5110_sampled.pdf");
5259        let Ok(bytes) = std::fs::read(path) else {
5260            return;
5261        };
5262        let lines: Vec<String> = super::pdf_textlines(&bytes)
5263            .into_iter()
5264            .flat_map(|(_, _, c)| c)
5265            .map(|c| c.text)
5266            .collect();
5267        assert!(
5268            lines
5269                .iter()
5270                .any(|l| l.ends_with("click Schemas → HR_SCHEMA→")),
5271            "{:?}",
5272            lines
5273                .iter()
5274                .filter(|l| l.contains("Schemas"))
5275                .collect::<Vec<_>>()
5276        );
5277        assert!(lines.iter().all(|l| !l.contains('\u{F0AE}')));
5278    }
5279
5280    /// 2203.01017's p. 4 table marks its datasets with an embedded Type 1
5281    /// `Dingbats` — symbolic, no `/Encoding`, no `/ToUnicode` — whose
5282    /// program encodes code `3` as `/a19` and `7` as `/a23`: `✓` and `✗`
5283    /// through the builtin encoding, as docling-parse 7.25 reads them
5284    /// (#1113), where the Latin tables read the digits.
5285    #[cfg(feature = "render")]
5286    #[test]
5287    fn a_symbolic_font_reads_through_its_builtin_encoding() {
5288        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
5289            .join("../../tests/data/pdf/sources/2203.01017v2.pdf");
5290        let Ok(bytes) = std::fs::read(path) else {
5291            return;
5292        };
5293        let pages = super::pdf_textlines(&bytes);
5294        let text: Vec<&str> = pages[3].2.iter().map(|c| c.text.as_str()).collect();
5295        let page = text.join("\n");
5296        assert_eq!(page.matches('✓').count(), 11, "{text:?}");
5297        assert_eq!(page.matches('✗').count(), 1, "{text:?}");
5298        assert!(text.contains(&"✓ ✗ 145k JPEG"), "{text:?}");
5299    }
5300
5301    /// docling-parse 7.25's `glyph_name_to_utf8`.
5302    #[test]
5303    fn builtin_glyph_names_read_as_docling_parse() {
5304        let t = |n: &str| super::builtin_glyph_text(n);
5305        assert_eq!(t("a19").as_deref(), Some("✓"));
5306        assert_eq!(t("arrowright").as_deref(), Some("→"));
5307        assert_eq!(t("A").as_deref(), Some("A"));
5308        assert_eq!(t("one.oldstyle").as_deref(), Some("1"), "suffix dropped");
5309        assert_eq!(t("uni00660069").as_deref(), Some("fi"), "a uni sequence");
5310        assert_eq!(t("u1F600").as_deref(), Some("😀"));
5311        assert_eq!(t("g12"), None, "an index name");
5312        assert_eq!(t("GID00012"), None);
5313        assert_eq!(t("cid7"), None);
5314        assert_eq!(t(".notdef"), None);
5315        assert_eq!(t(""), None);
5316        assert_eq!(t("nosuchglyphname"), None);
5317    }
5318
5319    /// The page's image placements reach the cells (#999): the scan under
5320    /// the #999 fixture's one native word is the whole page.
5321    #[test]
5322    fn image_placements_are_the_pages_bitmaps() {
5323        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
5324            .join("../docling/tests/fixtures/partial_text_layer/scan_with_native_word.pdf");
5325        let bytes = std::fs::read(path).expect("fixture");
5326        let mut parser = super::PageTextParser::open(&bytes).expect("parser");
5327        let page = parser.cells(0);
5328        assert_eq!(page.bitmaps, vec![[0.0, 0.0, 612.0, 792.0]]);
5329        assert_eq!(page.words.len(), 1);
5330        assert!(page.checkboxes.is_empty());
5331    }
5332
5333    /// A `bfrange` whose destination is a surrogate pair steps its last code
5334    /// unit (#1003: equations.pdf's `H𝛂`, `<1d59> <1d5a> <d835dec2>`); a
5335    /// single-unit base keeps counting in code points.
5336    #[test]
5337    fn surrogate_pair_bfrange_maps_each_code() {
5338        let map = super::parse_tounicode(
5339            b"1 beginbfrange <1d59> <1d5a> <d835dec2> endbfrange\n\
5340              1 beginbfrange <0010> <0011> <0041> endbfrange",
5341        );
5342        assert_eq!(map.get(&0x1d59).map(String::as_str), Some("\u{1D6C2}"));
5343        assert_eq!(map.get(&0x1d5a).map(String::as_str), Some("\u{1D6C3}"));
5344        assert_eq!(map.get(&0x11).map(String::as_str), Some("B"));
5345    }
5346
5347    /// Which unknown glyph names are text and which are bare indexes
5348    /// (docling-parse's `resolve_unknown_name`; Skia's hexadecimal `g%X`
5349    /// since 7.25, #1113).
5350    #[test]
5351    fn unknown_glyph_names_follow_docling_parse() {
5352        for name in [
5353            "g115", "gid00043", "Glyph7", "cid42", "index9", "g3.alt", "g15C", "gA1", "G1F",
5354        ] {
5355            assert_eq!(super::unknown_name_text(name), None, "{name}");
5356            assert!(
5357                super::is_gid_name(name.split('.').next().unwrap().as_bytes()),
5358                "{name}"
5359            );
5360        }
5361        assert!(!super::is_gid_name(b"glyphs"), "not hexadecimal");
5362        assert!(!super::is_gid_name(b"uni0041"));
5363        assert_eq!(
5364            super::unknown_name_text("SM590000").as_deref(),
5365            Some("SM590000")
5366        );
5367        assert_eq!(
5368            super::unknown_name_text("SM590000.sc").as_deref(),
5369            Some("SM590000")
5370        );
5371    }
5372
5373    /// An explicit `/Widths` array always wins over the built-in metrics, and a
5374    /// non-standard face without `/Widths` stays as before (no invented boxes).
5375    #[test]
5376    fn explicit_widths_win_and_unknown_faces_are_untouched() {
5377        // Helvetica with explicit 100/1000-em widths: the word's box must be
5378        // ~4×100 units at 12pt = 4.8pt wide — far narrower than the ~2.7×
5379        // wider built-in Helvetica advances would make it.
5380        let explicit = pdf_with_font(
5381            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/FirstChar 65\
5382               /Widths[100 100 100 100]/Encoding/WinAnsiEncoding>>",
5383            b"ABBA",
5384        );
5385        let builtin = pdf_with_font(
5386            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>",
5387            b"ABBA",
5388        );
5389        let w = |pdf: &[u8]| {
5390            let cs = cells(pdf);
5391            assert_eq!(cs.len(), 1, "one word cell: {cs:?}");
5392            cs[0].r - cs[0].l
5393        };
5394        let (we, wb) = (w(&explicit), w(&builtin));
5395        assert!(
5396            (we - 4.8).abs() < 0.1,
5397            "explicit widths must win: got {we}, want 4×100×12/1000"
5398        );
5399        assert!(
5400            wb > 2.0 * we,
5401            "built-in Helvetica is much wider: {wb} vs {we}"
5402        );
5403
5404        // An unknown face with no /Widths: still parses (text kept), but no
5405        // built-in table applies — the old zero-width behavior is preserved
5406        // rather than inventing Helvetica metrics for an arbitrary font.
5407        let unknown = pdf_with_font(
5408            b"<</Type/Font/Subtype/Type1/BaseFont/FancyCorp-Display>>",
5409            b"Mystery",
5410        );
5411        let cs = cells(&unknown);
5412        let text: String = cs.iter().map(|c| c.text.as_str()).collect();
5413        assert!(text.contains("Mystery"), "text still decodes: {cs:?}");
5414    }
5415
5416    /// #1121: a TeX math font names its glyphs at the low char codes —
5417    /// `/parenleftbig` at 0, `/parenrightbig` at 1 in Latin Modern's math
5418    /// extension font (1805.06723) — with no ToUnicode. The names are not
5419    /// Adobe Glyph List names; unresolved, the codes read U+FFFD
5420    /// (`rt�A(M)�`). They read as docling-parse reads them, and a font whose
5421    /// CMap maps the code to NUL reads its `/Differences` name.
5422    #[test]
5423    fn tex_math_glyphs_at_low_codes_decode() {
5424        let font = b"<</Type/Font/Subtype/Type1/BaseFont/ABCDEF+LMMathExtension10-Regular\
5425            /FirstChar 0/LastChar 90/Widths[458 458 0 0 0 0 0 0 0 0 0 0 0 0 0 0 597 597 \
5426            0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 \
5427            0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 472]\
5428            /Encoding<</Type/Encoding/Differences[0/parenleftbig/parenrightbig \
5429            16/parenleftBig/parenrightBig 90/integraldisplay]>>>>";
5430        let text = |bytes: &[u8]| -> String {
5431            cells(&pdf_with_font(font, bytes))
5432                .iter()
5433                .map(|c| c.text.as_str())
5434                .collect()
5435        };
5436        assert_eq!(text(b"\\000\\001"), "()");
5437        assert_eq!(text(b"\\020\\021Z"), "()∫");
5438        assert!(!text(b"\\000\\001\\020\\021Z").contains('\u{FFFD}'));
5439    }
5440
5441    /// The Adobe Glyph List in full (the `glyph-list` feature): a Cyrillic
5442    /// font re-encoded with `/Differences` and no ToUnicode names its glyphs
5443    /// `afii10017`, `afii10018`, …. Unresolved, each read the Latin letter of
5444    /// the base encoding at its code — plausible text in the wrong script.
5445    #[cfg(feature = "glyph-list")]
5446    #[test]
5447    fn glyph_list_names_decode_a_reencoded_cyrillic_font() {
5448        let font = b"<</Type/Font/Subtype/Type1/BaseFont/ABCDEF+SchoolBookC\
5449            /FirstChar 65/LastChar 68/Widths[700 650 650 600]\
5450            /Encoding<</Type/Encoding/BaseEncoding/WinAnsiEncoding\
5451            /Differences[65/afii10017/afii10018/afii10019/afii10020]>>>>";
5452        let text: String = cells(&pdf_with_font(font, b"ABCD"))
5453            .iter()
5454            .map(|c| c.text.as_str())
5455            .collect();
5456        assert_eq!(text, "АБВГ");
5457    }
5458
5459    /// A non-embedded ZapfDingbats or Symbol that names no encoding reads
5460    /// through the face's own (docling-parse's base-font AFM; its
5461    /// `annots_01`: `l` is `●`, #1113) and advances by its AFM widths — no
5462    /// Latin letters, no zero-width cells.
5463    #[test]
5464    fn non_embedded_symbol_faces_read_through_their_encoding() {
5465        let dingbats = pdf_with_font(
5466            b"<</Type/Font/Subtype/Type1/BaseFont/ZapfDingbats/Name/F3>>",
5467            b"l",
5468        );
5469        let cs = cells(&dingbats);
5470        assert_eq!(cs.len(), 1, "{cs:?}");
5471        assert_eq!(cs[0].text, "●");
5472        assert!(
5473            (cs[0].r - cs[0].l - 791.0 * 12.0 / 1000.0).abs() < 0.1,
5474            "{cs:?}"
5475        );
5476        let symbol = pdf_with_font(b"<</Type/Font/Subtype/Type1/BaseFont/Symbol>>", b"abg");
5477        let text: String = cells(&symbol).iter().map(|c| c.text.as_str()).collect();
5478        assert_eq!(text, "αβγ");
5479        // A declared encoding keeps its table (multi_page's `•`, a
5480        // deliberate divergence from docling-parse's FontSpecific reading).
5481        let declared = pdf_with_font(
5482            b"<</Type/Font/Subtype/Type1/BaseFont/Symbol/Encoding/WinAnsiEncoding>>",
5483            b"a",
5484        );
5485        let text: String = cells(&declared).iter().map(|c| c.text.as_str()).collect();
5486        assert_eq!(text, "a");
5487    }
5488
5489    /// #728: one face bound under two resource keys — `/F1` Helvetica in
5490    /// WinAnsi, `/F2` the same Helvetica re-encoded so code 0x81 is the `fi`
5491    /// glyph — with every fi-word drawn in `/F2` right after an `/F1` run.
5492    /// docling-parse names a font by `/FontDescriptor/FontName` → `/Name` →
5493    /// `/BaseFont`, never by the resource key, so both resources are the same
5494    /// font and the line reads `financial`. Keyed by resource key the `fi`
5495    /// ligature (font-neutral in `applicable_for_merge`) joined the `/F1`
5496    /// run and the font barrier then split it from `nancial`.
5497    #[test]
5498    fn same_face_under_two_resource_keys_is_one_font_for_the_sanitizer() {
5499        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
5500        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding<</Type/Encoding\
5501                   /BaseEncoding/WinAnsiEncoding/Differences[129/fi]>>>>";
5502        let content = b"BT 12 TL 72 720 Td \
5503            /F1 12 Tf (The ) Tj /F2 12 Tf (\\201nancial position) Tj \
5504            /F1 12 Tf ( of the company is shown in ) Tj /F2 12 Tf (\\201gure 3) Tj \
5505            /F1 12 Tf ( with the ) Tj /F2 12 Tf (\\201rst results.) Tj \
5506            T* T* /F2 12 Tf (Same words in /F2 only: \\201nancial, \\201gure, \\201rst.) Tj ET\n";
5507        let cs = cells(&pdf_with_fonts(&[f1, f2], content));
5508        let lines: Vec<&str> = cs.iter().map(|c| c.text.as_str()).collect();
5509        assert_eq!(
5510            lines,
5511            [
5512                "The financial position of the company is shown in figure 3 with the first results.",
5513                "Same words in /F2 only: financial, figure, first.",
5514            ],
5515            "{cs:?}"
5516        );
5517    }
5518
5519    /// Under `legacy` the font barrier itself is untouched: two *different*
5520    /// faces (regular and bold Helvetica, which docling-parse names apart) set
5521    /// flush against each other stay separate line cells, as
5522    /// `enforce_same_font` demands. docling-parse 7 dropped the barrier — its
5523    /// line runs follow geometry alone — so under the default `adaptive` the
5524    /// flush pair is one line and one word.
5525    #[test]
5526    fn different_faces_keep_the_font_barrier_only_under_legacy() {
5527        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
5528        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica-Bold/Encoding/WinAnsiEncoding>>";
5529        let content = b"BT 72 720 Td /F2 12 Tf (Total:) Tj /F1 12 Tf (42) Tj ET\n";
5530        let pdf = pdf_with_fonts(&[f1, f2], content);
5531        let lines = |gap| {
5532            let mut p = super::PageTextParser::open(&pdf).expect("opens");
5533            p.set_word_gap(Some(gap));
5534            p.cells(0)
5535                .prose
5536                .into_iter()
5537                .map(|c| c.text)
5538                .collect::<Vec<_>>()
5539        };
5540        assert_eq!(lines(crate::dp_lines::WordGap::Legacy), ["Total:", "42"]);
5541        assert_eq!(lines(crate::dp_lines::WordGap::Adaptive), ["Total:42"]);
5542    }
5543
5544    /// #992: a space glyph whose advance the next `TJ` offset takes back —
5545    /// EUR-Lex's superscript footnote marker `( 1 )`, set with spaces the
5546    /// digit is then drawn over — is not a character of the text: the page
5547    /// shows `(1)`, pdftotext and pdfium read `(1)`, and so does the line
5548    /// cell here (docling-parse keeps the spaces, `( 1 )`). A space the next
5549    /// glyph only kerns into (−100/1000 em, well past the space's start) is
5550    /// a real gap and stays, as does one before a jump to the next line.
5551    #[test]
5552    fn a_space_the_next_glyph_is_drawn_over_is_not_a_character() {
5553        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
5554        let lines = |content: &[u8]| {
5555            let pdf = pdf_with_content(f1, content);
5556            super::PageTextParser::open(&pdf)
5557                .expect("opens")
5558                .cells(0)
5559                .prose
5560                .into_iter()
5561                .map(|c| c.text)
5562                .collect::<Vec<_>>()
5563        };
5564        // Helvetica's space is 278/1000 em: an offset of 300 moves the pen
5565        // back past where the space began, 100 leaves two thirds of it.
5566        assert_eq!(
5567            lines(b"BT /F1 12 Tf 72 700 Td [(EG \\() ( ) 300 (1) ( ) 300 (\\) erf)] TJ ET\n"),
5568            ["EG (1) erf"]
5569        );
5570        assert_eq!(
5571            lines(b"BT /F1 12 Tf 72 700 Td [(EG \\() ( ) 100 (1) ( ) 100 (\\) erf)] TJ ET\n"),
5572            ["EG ( 1 ) erf"]
5573        );
5574        // A trailing space, then the next line starts at the left margin.
5575        assert_eq!(
5576            lines(b"BT /F1 12 Tf 72 700 Td (first line ) Tj 0 -14 Td (second) Tj ET\n"),
5577            ["first line", "second"]
5578        );
5579        // A line drawn rotated 90° (a landscape slide on a portrait page,
5580        // #528): its glyphs' loose boxes stack on one left edge, which is
5581        // not an overprint — the space stays.
5582        assert_eq!(
5583            lines(b"BT 0 1 -1 0 300 100 Tm /F1 12 Tf (Nullam mollis) Tj ET\n"),
5584            ["Nullam mollis"]
5585        );
5586    }
5587
5588    /// A text shadow: a word processor exports a heading with a shadow
5589    /// effect as the same run drawn twice — the shadow in black, then the
5590    /// text in colour over it, shifted up and left by a fraction of a point.
5591    /// pdfium's text page reads the heading once and so does the line cell
5592    /// here (docling-parse reads `Safety instructions Safety instructions`).
5593    /// #1092: docling-parse's view of a page's shapes — a stroked straight
5594    /// rule is a shape line and a shape box, a thin filled box only a box, a
5595    /// curve or a diagonal no line, a path outside the clip nothing, and a
5596    /// fill in an unresolved Pattern space nothing at all. In the cells'
5597    /// frame: top-left, on the 595 × 842 test page.
5598    #[test]
5599    fn the_page_shapes_follow_docling_parse() {
5600        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
5601        let shapes = |content: &[u8]| {
5602            let pdf = pdf_with_content(f1, content);
5603            super::PageTextParser::open(&pdf)
5604                .expect("opens")
5605                .cells(0)
5606                .shapes
5607        };
5608        let s = shapes(b"2 w 72 400 m 540 400 l S\n");
5609        assert_eq!(s.lines, vec![[72.0, 442.0, 540.0, 442.0]]);
5610        // The stroke's half width pads the box.
5611        assert_eq!(s.boxes, vec![[71.0, 441.0, 541.0, 443.0]]);
5612
5613        let s = shapes(b"72 300 468 1 re f\n");
5614        assert!(s.lines.is_empty(), "a fill strokes no line");
5615        assert_eq!(s.boxes, vec![[72.0, 541.0, 540.0, 542.0]]);
5616
5617        // A stroked rectangle: four edges, `h`'s closing one included.
5618        let s = shapes(b"100 100 200 50 re S\n");
5619        assert_eq!(s.lines.len(), 4);
5620
5621        let s = shapes(b"72 400 m 100 450 l 200 300 300 500 400 400 c S\n");
5622        assert!(s.lines.is_empty(), "neither a diagonal nor a curve");
5623        assert_eq!(s.boxes.len(), 1);
5624
5625        let s = shapes(b"q 0 0 300 842 re W n 72 400 m 540 400 l S 400 100 m 400 700 l S Q\n");
5626        assert_eq!(
5627            s.lines,
5628            vec![[72.0, 442.0, 300.0, 442.0]],
5629            "cut to the clip"
5630        );
5631        assert_eq!(s.boxes.len(), 1, "the vertical rule is clipped away");
5632
5633        let s = shapes(b"/Pattern cs /P0 scn 72 300 468 1 re f\n");
5634        assert!(s.lines.is_empty() && s.boxes.is_empty());
5635
5636        // Two touching boxes merge into one component.
5637        let s = shapes(b"72 300 100 1 re f 172 300 100 1 re f 400 600 10 10 re f\n");
5638        assert_eq!(
5639            s.boxes,
5640            vec![[72.0, 541.0, 272.0, 542.0], [400.0, 232.0, 410.0, 242.0]]
5641        );
5642
5643        // A table rule a word processor draws piece by piece, cell borders
5644        // and their corner squares: `90.144 + 1.44` is `91.584` in the file
5645        // and in docling-parse's doubles, not in lopdf's `f32` reals cast to
5646        // `f64`, where every such joint missed by a hair and the rule fell
5647        // into pieces: `real` reads the operands as the file wrote them. A
5648        // gap the producer left (`147.5 + 159.86` = 307.36, then 307.37)
5649        // still parts two components, as in docling-parse.
5650        let s = shapes(
5651            b"56.64 690.22 33.504 1.44 re f 90.144 690.22 1.44 1.44 re f \
5652              91.584 690.22 54.48 1.44 re f 146.06 690.22 1.44 1.44 re f \
5653              147.5 690.22 159.86 1.44 re f 307.37 690.22 1.44 1.44 re f \
5654              308.81 690.22 50.304 1.44 re f\n",
5655        );
5656        let spans: Vec<(f32, f32)> = s.boxes.iter().map(|b| (b[0], b[2])).collect();
5657        assert_eq!(spans, vec![(56.64, 307.36), (307.37, 359.114)]);
5658    }
5659
5660    /// A transparent stroke or fill is no shape (`CA` / `ca` 0).
5661    #[test]
5662    fn transparent_paths_are_no_shapes() {
5663        let rule = |closed| super::ShapeSub {
5664            start: (0.0, 0.0),
5665            cur: (100.0, 0.0),
5666            pts: vec![(0.0, 0.0), (100.0, 0.0)],
5667            lines: vec![((0.0, 0.0), (100.0, 0.0))],
5668            closed,
5669        };
5670        let gs = |stroke_alpha, fill_alpha| super::ShapeGs {
5671            stroke_alpha,
5672            fill_alpha,
5673            ..super::ShapeGs::INIT
5674        };
5675        let mut shapes = super::Shapes::default();
5676        super::record_shape(&[rule(false)], true, false, &gs(0.0, 1.0), 1.0, &mut shapes);
5677        assert_eq!(shapes, super::Shapes::default());
5678        super::record_shape(&[rule(true)], false, true, &gs(1.0, 0.0), 1.0, &mut shapes);
5679        assert_eq!(shapes, super::Shapes::default());
5680        super::record_shape(&[rule(false)], true, false, &gs(0.5, 0.0), 1.0, &mut shapes);
5681        assert_eq!(shapes.lines.len(), 1);
5682    }
5683
5684    /// Text that really repeats sits an advance apart or more and stays: a
5685    /// word set twice, a doubled letter, the same words on the next line.
5686    #[test]
5687    fn a_run_drawn_again_over_itself_is_read_once() {
5688        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica-Bold/Encoding/WinAnsiEncoding>>";
5689        let lines = |content: &[u8]| {
5690            let pdf = pdf_with_content(f1, content);
5691            super::PageTextParser::open(&pdf)
5692                .expect("opens")
5693                .cells(0)
5694                .prose
5695                .into_iter()
5696                .map(|c| c.text)
5697                .collect::<Vec<_>>()
5698        };
5699        // The shadow, then the text 0.5 pt up and left of it, at 18 pt.
5700        assert_eq!(
5701            lines(
5702                b"BT 0 g /F1 18 Tf 1 0 0 1 72 720 Tm (Safety instructions) Tj ET\n\
5703                  BT 0.8 0 0 rg /F1 18 Tf 1 0 0 1 71.5 720.5 Tm (Safety instructions) Tj ET\n"
5704            ),
5705            ["Safety instructions"]
5706        );
5707        // 0.8 pt at 14 pt, and the line under it untouched.
5708        assert_eq!(
5709            lines(
5710                b"BT /F1 14 Tf 1 0 0 1 72 656 Tm (Maintenance) Tj ET\n\
5711                  BT /F1 14 Tf 1 0 0 1 71.2 656.8 Tm (Maintenance) Tj ET\n\
5712                  BT /F1 11 Tf 1 0 0 1 72 632 Tm (Clean the filter) Tj ET\n"
5713            ),
5714            ["Maintenance", "Clean the filter"]
5715        );
5716        // A word set twice, a doubled letter, the same words a line apart.
5717        assert_eq!(
5718            lines(b"BT /F1 12 Tf 72 700 Td (very very small) Tj ET\n"),
5719            ["very very small"]
5720        );
5721        assert_eq!(
5722            lines(b"BT /F1 12 Tf 72 700 Td (Hello, all) Tj ET\n"),
5723            ["Hello, all"]
5724        );
5725        assert_eq!(
5726            lines(b"BT /F1 12 Tf 72 700 Td (Total) Tj 0 -14 Td (Total) Tj ET\n"),
5727            ["Total", "Total"]
5728        );
5729        // Dot leaders squeezed by `Tc` to 0.05, 0.07 and 0.1 em: two dots on,
5730        // a dot sits within a shadow's shift and the dots after it repeat by
5731        // the same offset, but the next dot is nearer still.
5732        for tc in ["-2.28", "-2.08", "-1.78"] {
5733            let content = format!(
5734                "BT /F1 10 Tf 72 700 Td (Contents) Tj {tc} Tc (..........) Tj 0 Tc ( 5) Tj ET\n"
5735            );
5736            assert_eq!(
5737                lines(content.as_bytes()),
5738                ["Contents.......... 5"],
5739                "Tc {tc}"
5740            );
5741        }
5742    }
5743
5744    /// docling-parse's `test_unit_actual_text.py` (docling-parse#300): a
5745    /// marked-content span's `/ActualText` replaces the glyphs drawn in it.
5746    #[test]
5747    fn actual_text_replaces_the_glyphs_of_its_span() {
5748        let helvetica = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
5749        // A non-embedded TrueType whose only code, 0x4D (`M` in WinAnsi), is a
5750        // zero-advance caron: drawn after `s` it makes `š`.
5751        let caron = b"<</Type/Font/Subtype/TrueType/BaseFont/SynthCaron/FirstChar 77/LastChar 77\
5752            /Widths[0]/Encoding/WinAnsiEncoding/FontDescriptor<</Type/FontDescriptor\
5753            /FontName/SynthCaron/Flags 32/FontBBox[-540 -200 0 800]/ItalicAngle 0/Ascent 800\
5754            /Descent -200/CapHeight 700/StemV 80>>>>";
5755        // Code 1 is the `fi` ligature: two glyphs on one pen, one cell.
5756        let ligature = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica\
5757            /Encoding<</Type/Encoding/BaseEncoding/WinAnsiEncoding/Differences[1/fi]>>>>";
5758        let text = |content: &str| {
5759            let pdf = pdf_with_fonts(&[helvetica, caron, ligature], content.as_bytes());
5760            super::PageTextParser::open(&pdf)
5761                .expect("opens")
5762                .cells(0)
5763                .prose
5764                .into_iter()
5765                .map(|c| c.text)
5766                .collect::<String>()
5767        };
5768        // Unmarked, the caron reads as its WinAnsi letter.
5769        assert_eq!(
5770            text("BT /F1 24 Tf 40 50 Td (Ore) Tj (s) Tj /F2 24 Tf <4D> Tj /F1 24 Tf (i) Tj ET"),
5771            "OresMi"
5772        );
5773        // Two cells, one character: the accent composes onto its base letter.
5774        assert_eq!(
5775            text(
5776                "BT /F1 24 Tf 40 50 Td (Ore) Tj /Span <</ActualText <FEFF0161>>> BDC \
5777                  (s) Tj /F2 24 Tf <4D> Tj EMC /F1 24 Tf (i) Tj ET"
5778            ),
5779            "Oreši"
5780        );
5781        // As many characters as cells: one for one.
5782        assert_eq!(
5783            text("BT /F1 24 Tf 40 50 Td /Span <</ActualText (ab)>> BDC (xy) Tj EMC (z) Tj ET"),
5784            "abz"
5785        );
5786        // A cell of two glyphs takes one character.
5787        assert_eq!(
5788            text("BT /F3 24 Tf 40 50 Td <01> Tj /F1 24 Tf (a) Tj ET"),
5789            "fia"
5790        );
5791        assert_eq!(
5792            text(
5793                "BT /F3 24 Tf 40 50 Td /Span <</ActualText (XY)>> BDC <01> Tj \
5794                  /F1 24 Tf (a) Tj EMC (z) Tj ET"
5795            ),
5796            "XYz"
5797        );
5798        // An empty text declares the glyphs not text (a visual line-break
5799        // hyphen); its advance may still read as a gap.
5800        let hyphen = text(
5801            "BT /F1 24 Tf 40 50 Td (exam) Tj /Span <</ActualText <FEFF>>> BDC (-) Tj EMC \
5802             (ple) Tj ET",
5803        );
5804        assert!(!hyphen.contains('-'), "{hyphen}");
5805        assert_eq!(hyphen.replace(' ', ""), "example");
5806        // `/Alt` written into the wrong key: implausibly long, ignored.
5807        assert_eq!(
5808            text(
5809                "BT /F1 24 Tf 40 50 Td /Span <</ActualText (A long description that belongs \
5810                  in Alt not here)>> BDC (ab) Tj EMC ET"
5811            ),
5812            "ab"
5813        );
5814        // Over graphics only: nothing to replace.
5815        assert_eq!(
5816            text(
5817                "BT /F1 24 Tf 40 50 Td (x) Tj ET /Span <</ActualText (ghost)>> BDC \
5818                  10 10 50 50 re f EMC"
5819            ),
5820            "x"
5821        );
5822        // A named property list is not resolved.
5823        assert_eq!(
5824            text("BT /F1 24 Tf 40 50 Td /OC /MC0 BDC (ok) Tj EMC ET"),
5825            "ok"
5826        );
5827        // Nested spans: the outer text describes the whole run.
5828        assert_eq!(
5829            text(
5830                "BT /F1 24 Tf 40 50 Td /Span <</ActualText (AB)>> BDC (a) Tj \
5831                  /Span <</ActualText (Z)>> BDC (b) Tj EMC EMC ET"
5832            ),
5833            "AB"
5834        );
5835        // A span nested past the depth limit keeps its glyphs.
5836        let nested = |depth: usize| {
5837            text(&format!(
5838                "BT /F1 24 Tf 40 50 Td {}/Span <</ActualText (z)>> BDC (a) Tj EMC{} ET",
5839                "/X BMC ".repeat(depth),
5840                " EMC".repeat(depth)
5841            ))
5842        };
5843        assert_eq!(nested(super::MAX_ACTUAL_TEXT_DEPTH - 1), "z");
5844        assert_eq!(nested(super::MAX_ACTUAL_TEXT_DEPTH), "a");
5845    }
5846
5847    /// One for one, each character keeps its cell's box, and a cell of
5848    /// several glyphs on one pen (a ligature's) becomes its first glyph.
5849    #[test]
5850    fn actual_text_one_for_one_keeps_each_cells_box() {
5851        let glyph = |ch, x: f32, pen: f32| super::Glyph {
5852            ch,
5853            l: x,
5854            b: 0.0,
5855            r: x + 5.0,
5856            t: 10.0,
5857            ll: x,
5858            lb: -2.0,
5859            lr: x + 5.0,
5860            lt: 10.0,
5861            font: 0,
5862            quad: None,
5863            pen: Some([pen, 0.0, 10.0]),
5864            math_font: false,
5865            invisible: false,
5866        };
5867        let mut out = vec![
5868            glyph('o', 0.0, 0.0),
5869            glyph('f', 10.0, 10.0),
5870            glyph('i', 15.0, 10.0),
5871            glyph('a', 20.0, 20.0),
5872        ];
5873        super::apply_actual_text(&mut out, 1, "XY");
5874        let got: Vec<(char, f32)> = out.iter().map(|g| (g.ch, g.l)).collect();
5875        assert_eq!(got, [('o', 0.0), ('X', 10.0), ('Y', 20.0)]);
5876    }
5877
5878    /// The identity follows docling-parse's precedence: a `/FontDescriptor/FontName`
5879    /// wins over `/BaseFont`, so two dictionaries that share a `/BaseFont` but
5880    /// describe different programs are two fonts, and two that share the
5881    /// descriptor name are one whatever their `/BaseFont` says.
5882    #[test]
5883    fn math_faces_are_named_like_docling_parse_names_them() {
5884        for name in [
5885            "ABCDEF+CMR7",
5886            "CMMI10",
5887            "MSBM10",
5888            "Cambria-Math",
5889            "Symbol",
5890            "MTExtra",
5891        ] {
5892            assert!(super::is_math_font_name(name.as_bytes()), "{name}");
5893        }
5894        for name in ["Helvetica", "ABCDEF+TimesNewRomanPSMT", "Arial-BoldMT"] {
5895            assert!(!super::is_math_font_name(name.as_bytes()), "{name}");
5896        }
5897    }
5898
5899    #[test]
5900    fn sanitizer_font_name_follows_docling_parse_precedence() {
5901        use lopdf::{dictionary, Document, Object};
5902        let mut doc = Document::with_version("1.4");
5903        let fd_a =
5904            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "AAAAAA+Face" });
5905        let fd_b =
5906            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "BBBBBB+Face" });
5907        let with = |fd: Option<lopdf::ObjectId>, base: &str| {
5908            let mut d = dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => base };
5909            if let Some(id) = fd {
5910                d.set("FontDescriptor", Object::Reference(id));
5911            }
5912            d
5913        };
5914        let name = |d: &lopdf::Dictionary| {
5915            String::from_utf8(super::sanitizer_font_name(&doc, d, false)).unwrap()
5916        };
5917        assert_eq!(name(&with(Some(fd_a), "Face")), "AAAAAA+Face");
5918        assert_eq!(name(&with(Some(fd_b), "Face")), "BBBBBB+Face");
5919        assert_eq!(name(&with(None, "ABCDEF+Face")), "ABCDEF+Face");
5920        assert_eq!(
5921            name(&dictionary! { "Type" => "Font", "Name" => "F7" }),
5922            "F7"
5923        );
5924        assert_eq!(name(&dictionary! { "Type" => "Font" }), "null");
5925    }
5926}
5927
5928#[cfg(test)]
5929mod overpainted {
5930    use crate::pdfium_backend::TextCell;
5931
5932    fn cell(text: &str, l: f32, t: f32, r: f32, b: f32) -> TextCell {
5933        TextCell {
5934            text: text.into(),
5935            l,
5936            t,
5937            r,
5938            b,
5939        }
5940    }
5941
5942    /// The reporting invoice's logo: a `"` painted inside a `==` on one band —
5943    /// artwork drawn with glyphs. Both cells go; the real text on the next
5944    /// band stays.
5945    #[test]
5946    fn stacked_logo_glyphs_are_dropped() {
5947        let mut cells = vec![
5948            cell("\"", 72.7, 21.5, 86.4, 31.5),
5949            cell("==", 59.4, 21.5, 99.6, 31.5),
5950            cell("Herr", 65.2, 151.3, 81.7, 161.3),
5951        ];
5952        super::drop_overpainted_cells(&mut cells);
5953        assert_eq!(cells.len(), 1, "cells: {cells:?}");
5954        assert_eq!(cells[0].text, "Herr");
5955    }
5956
5957    /// Adjacent words on a line touch but never contain each other — prose is
5958    /// untouched, and so is a same-text near-duplicate (double-drawn faux
5959    /// bold), which is not evidence of artwork.
5960    #[test]
5961    fn prose_and_double_draw_are_kept() {
5962        let mut cells = vec![
5963            cell("Telefon", 354.3, 133.2, 381.5, 143.2),
5964            cell("0676/2000", 387.3, 133.2, 428.7, 143.2),
5965            cell("Bold", 100.0, 50.0, 130.0, 60.0),
5966            cell("Bold", 100.3, 50.0, 130.3, 60.0),
5967        ];
5968        super::drop_overpainted_cells(&mut cells);
5969        assert_eq!(cells.len(), 4);
5970    }
5971
5972    /// #1129: Cambria's regular face declares an ascent of 3117 and a descent
5973    /// of -2464, so each of its lines sits in a box 5.6 em tall and overlaps
5974    /// the next line by four fifths. A short line under a long one was "one
5975    /// cell inside another on the same band" — paint — and both went: these
5976    /// are the issue page's cells, of whose 44 lines 7 were left. Lines one
5977    /// pitch apart are lines; the stacked logo, at one height, still goes.
5978    #[test]
5979    fn lines_of_a_tall_box_font_are_not_paint() {
5980        let mut cells = vec![
5981            cell(
5982                "17. Carefully pour off the supernatant from both tubes. Invert the",
5983                72.0,
5984                77.1,
5985                514.4,
5986                132.7,
5987            ),
5988            cell(
5989                "tubes and tap them gently on the surface of a clean paper towel.",
5990                72.0,
5991                88.8,
5992                453.0,
5993                144.4,
5994            ),
5995            cell("“Evidence A” DNA*", 77.6, 412.5, 150.6, 462.7),
5996            cell("“Evidence B” DNA*", 77.6, 423.0, 150.5, 473.2),
5997            cell("enzyme mixture*", 77.6, 444.1, 143.9, 494.4),
5998            // The same face painting a mark: two glyph runs at one height.
5999            cell("\"", 72.7, 600.0, 86.4, 655.6),
6000            cell("==", 59.4, 600.0, 99.6, 655.6),
6001        ];
6002        super::drop_overpainted_cells(&mut cells);
6003        let texts: Vec<&str> = cells.iter().map(|c| c.text.as_str()).collect();
6004        assert_eq!(texts.len(), 5, "{texts:?}");
6005        assert!(
6006            !texts.contains(&"==") && !texts.contains(&"\""),
6007            "{texts:?}"
6008        );
6009    }
6010
6011    /// The same through the parser: a paragraph and its short last line in a
6012    /// font whose descriptor declares Cambria's metrics. Every line and every
6013    /// word of the text-layer page (the browser's and `convert_text_layer`'s)
6014    /// is there; before, the page came back empty.
6015    #[test]
6016    fn a_tall_box_font_keeps_its_lines_on_the_text_layer_page() {
6017        let widths = "500 ".repeat(91);
6018        let font = format!(
6019            "<</Type/Font/Subtype/TrueType/BaseFont/Cambria/Encoding/WinAnsiEncoding\
6020             /FirstChar 32/LastChar 122/Widths[{widths}]\
6021             /FontDescriptor<</Type/FontDescriptor/FontName/Cambria/Flags 32\
6022             /FontBBox[-1475 -2464 2868 3117]/Ascent 3117/Descent -2464\
6023             /CapHeight 667/ItalicAngle 0/StemV 80>>>>"
6024        );
6025        let content = b"BT /F1 10 Tf 72 700 Td \
6026            (Carefully pour off the supernatant from both of the tubes) Tj \
6027            0 -12 Td (and tap them gently on the surface of a clean towel) Tj \
6028            0 -12 Td (to drain them.) Tj ET\n";
6029        let pdf = super::base14_fonts::pdf_with_content(font.as_bytes(), content);
6030        let page = super::pdf_text_pages(&pdf).remove(0);
6031        assert!(
6032            page.cells.iter().all(|c| c.b - c.t > 50.0),
6033            "the boxes are the font's 5.6 em: {:?}",
6034            page.cells
6035        );
6036        let lines: Vec<&str> = page.cells.iter().map(|c| c.text.as_str()).collect();
6037        assert_eq!(
6038            lines,
6039            [
6040                "Carefully pour off the supernatant from both of the tubes",
6041                "and tap them gently on the surface of a clean towel",
6042                "to drain them.",
6043            ]
6044        );
6045        assert_eq!(page.word_cells.len(), 24, "{:?}", page.word_cells);
6046    }
6047}
6048
6049#[cfg(test)]
6050mod vestigial_layer {
6051    use crate::pdfium_backend::{PdfPage, TextCell};
6052
6053    fn page_with(texts: &[&str]) -> PdfPage {
6054        let cells = texts
6055            .iter()
6056            .enumerate()
6057            .map(|(i, t)| TextCell {
6058                text: t.to_string(),
6059                l: 10.0,
6060                t: 10.0 + 12.0 * i as f32,
6061                r: 90.0,
6062                b: 20.0 + 12.0 * i as f32,
6063            })
6064            .collect();
6065        PdfPage::from_cells(595.0, 842.0, 1.0, cells)
6066    }
6067
6068    /// The reported scanned form: three typed-in field values ("03", "05",
6069    /// "2025") over three image pages. That must read as *no usable layer*,
6070    /// so the browser routes the document to OCR instead of extracting
6071    /// thirteen characters and skipping the letter entirely.
6072    #[test]
6073    fn typed_in_form_fields_are_not_a_text_layer() {
6074        let pages = vec![
6075            page_with(&["03", "05", "2025"]),
6076            page_with(&[]),
6077            page_with(&[]),
6078        ];
6079        assert!(super::text_layer_is_vestigial(&pages));
6080        assert!(super::text_layer_is_vestigial(&[page_with(&[])]));
6081    }
6082
6083    /// A short but genuine digital document — one page, a few real lines —
6084    /// keeps the fast text path.
6085    #[test]
6086    fn sparse_but_real_documents_pass() {
6087        let one_pager = vec![page_with(&[
6088            "Confidential briefing",
6089            "Prepared for the board meeting",
6090            "Do not distribute",
6091        ])];
6092        assert!(!super::text_layer_is_vestigial(&one_pager));
6093    }
6094}
6095
6096/// #994: decoded Form XObject content is cached per document only from a
6097/// form's second sighting, so one-form-per-page documents do not hold every
6098/// page's decoded form until the document ends.
6099#[cfg(test)]
6100mod form_cache {
6101    use lopdf::content::{Content, Operation};
6102    use lopdf::{dictionary, Document, Object, Stream};
6103
6104    fn form(doc: &mut Document, text: &str) -> lopdf::ObjectId {
6105        let ops = vec![
6106            Operation::new("BT", vec![]),
6107            Operation::new("Tf", vec!["F1".into(), 12.into()]),
6108            Operation::new("Td", vec![40.into(), 500.into()]),
6109            Operation::new("Tj", vec![Object::string_literal(text)]),
6110            Operation::new("ET", vec![]),
6111        ];
6112        let data = Content { operations: ops }.encode().unwrap();
6113        doc.add_object(Stream::new(
6114            dictionary! { "Type" => "XObject", "Subtype" => "Form",
6115            "BBox" => vec![0.into(), 0.into(), 595.into(), 842.into()] },
6116            data,
6117        ))
6118    }
6119
6120    #[test]
6121    fn single_use_forms_are_not_retained_shared_ones_are() {
6122        let mut doc = Document::with_version("1.5");
6123        let pages_id = doc.new_object_id();
6124        let font = doc.add_object(
6125            dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => "Helvetica" },
6126        );
6127        let header = form(&mut doc, "Header");
6128        let mut kids = Vec::new();
6129        let mut own_forms = Vec::new();
6130        for p in 0..3 {
6131            let own = form(&mut doc, &format!("Body{p}"));
6132            own_forms.push(own);
6133            let content = doc.add_object(Stream::new(
6134                dictionary! {},
6135                b"q /Hd Do Q q /Fx Do Q".to_vec(),
6136            ));
6137            kids.push(Object::Reference(doc.add_object(dictionary! {
6138                "Type" => "Page", "Parent" => pages_id,
6139                "MediaBox" => vec![0.into(), 0.into(), 595.into(), 842.into()],
6140                "Contents" => content,
6141                "Resources" => dictionary! {
6142                    "Font" => dictionary! { "F1" => font },
6143                    "XObject" => dictionary! { "Hd" => header, "Fx" => own },
6144                },
6145            })));
6146        }
6147        doc.objects.insert(
6148            pages_id,
6149            Object::Dictionary(dictionary! { "Type" => "Pages", "Kids" => kids, "Count" => 3 }),
6150        );
6151        let catalog = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages_id });
6152        doc.trailer.set("Root", catalog);
6153
6154        let mut caches = super::DocCaches::default();
6155        let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
6156        pages.sort_by_key(|(n, _)| *n);
6157        for (i, (_, pid)) in pages.iter().enumerate() {
6158            let text: String = super::page_glyphs_cached(&doc, *pid, &mut caches)
6159                .iter()
6160                .map(|g| g.ch)
6161                .collect();
6162            // Both forms' text is still read, cached or not.
6163            assert!(text.contains("Header"), "page {i}: {text:?}");
6164            assert!(text.contains(&format!("Body{i}")), "page {i}: {text:?}");
6165        }
6166        // The shared header form is cached from its second page on; the
6167        // per-page forms, each used once, are not held.
6168        assert!(caches.forms.contains_key(&header));
6169        for own in &own_forms {
6170            assert!(!caches.forms.contains_key(own));
6171        }
6172        assert_eq!(caches.forms.len(), 1);
6173    }
6174}
6175
6176#[cfg(test)]
6177mod cid_collections {
6178    use super::base14_fonts::{cells, pdf_with_content};
6179
6180    fn type0(encoding: &str) -> Vec<u8> {
6181        format!(
6182            "<</Type/Font/Subtype/Type0/BaseFont/Test-Font-Light/Encoding/{encoding}\
6183             /DescendantFonts[<</Type/Font/Subtype/CIDFontType0/BaseFont/Test-Font-Light\
6184             /CIDSystemInfo<</Registry(Adobe)/Ordering(Japan1)/Supplement 7>>/DW 1000>>]>>"
6185        )
6186        .into_bytes()
6187    }
6188
6189    fn text(fontdict: &[u8], hex: &str) -> String {
6190        let content = format!("BT /F1 12 Tf 72 700 Td <{hex}> Tj ET\n");
6191        cells(&pdf_with_content(fontdict, content.as_bytes()))
6192            .iter()
6193            .map(|c| c.text.as_str())
6194            .collect()
6195    }
6196
6197    #[test]
6198    #[cfg(feature = "cid-collections")]
6199    fn identity_codes_read_through_the_collection() {
6200        assert_eq!(text(&type0("Identity-H"), "0CD40E8A07A0"), "日本語");
6201    }
6202
6203    #[test]
6204    fn unicode_keyed_cmaps_read_the_code() {
6205        assert_eq!(text(&type0("UniJIS-UCS2-H"), "65E5672C8A9E"), "日本語");
6206    }
6207
6208    #[test]
6209    #[cfg(feature = "cid-collections")]
6210    fn collection_ranges_keep_supplementary_characters() {
6211        assert_eq!(text(&type0("Identity-H"), "0CD44507450807A0"), "日𢭏𢭐語");
6212        let text = |o: &[u8], cid| super::collection_text(super::collection_table(o).unwrap(), cid);
6213        assert_eq!(text(b"Japan1", 0x2714).as_deref(), Some("🄐"));
6214        assert_eq!(text(b"CNS1", 0x3a32).as_deref(), Some("𡤃"));
6215    }
6216
6217    #[test]
6218    #[cfg(feature = "cid-collections")]
6219    fn collection_entries_are_plain_characters() {
6220        assert_eq!(text(&type0("Identity-H"), "046D"), "逢");
6221        let text = |o: &[u8], cid| super::collection_text(super::collection_table(o).unwrap(), cid);
6222        assert_eq!(text(b"Korea1", 8752), None);
6223        assert_eq!(text(b"KR", 11704), None);
6224        assert_eq!(text(b"Japan1", 0x1ffd).as_deref(), Some("sec"));
6225    }
6226
6227    #[test]
6228    fn tounicode_ranges_increment_the_utf16_string() {
6229        let map = super::parse_tounicode(
6230            b"3 beginbfrange\n\
6231              <01> <02> <d842dfb7>\n\
6232              <03> <04> <00660069>\n\
6233              <05> <06> <0061>\n\
6234              endbfrange",
6235        );
6236        assert_eq!(map[&1], "𠮷");
6237        assert_eq!(map[&2], "𠮸");
6238        assert_eq!(map[&3], "fi");
6239        assert_eq!(map[&4], "fj");
6240        assert_eq!(map[&5], "a");
6241        assert_eq!(map[&6], "b");
6242    }
6243
6244    #[test]
6245    fn tounicode_skips_lone_surrogates_overflows_and_huge_ranges() {
6246        let map = super::parse_tounicode(
6247            b"2 beginbfchar\n<01> <d842>\n<02> <0041>\nendbfchar\n\
6248              4 beginbfrange\n\
6249              <10> <11> [<dc00> <0042>]\n\
6250              <20> <22> <0041fffe>\n\
6251              <30> <31> <d842dfff>\n\
6252              <00010000> <ffffffff> <>\n\
6253              endbfrange",
6254        );
6255        assert!(!map.contains_key(&0x01));
6256        assert_eq!(map[&0x02], "A");
6257        assert!(!map.contains_key(&0x10));
6258        assert_eq!(map[&0x11], "B");
6259        assert_eq!(map[&0x20], "A\u{fffe}");
6260        assert_eq!(map[&0x21], "A\u{ffff}");
6261        assert!(!map.contains_key(&0x22));
6262        assert_eq!(map[&0x30], "\u{20bff}");
6263        assert!(!map.contains_key(&0x31));
6264        assert_eq!(map[&0x1ffff], "");
6265        assert_eq!(map.len(), 5 + 0x10000);
6266    }
6267
6268    #[test]
6269    fn utf16_pairs_are_one_glyph_and_one_advance() {
6270        let font = type0("UniJIS-UTF16-H");
6271        assert_eq!(text(&font, "65E5D842DFB78A9E"), "日𠮷語");
6272        let content = b"BT /F1 12 Tf 72 700 Td <65E5D842DFB78A9E> Tj ET\n";
6273        let pdf = pdf_with_content(&font, content);
6274        let doc = lopdf::Document::load_mem(&pdf).unwrap();
6275        let glyphs = super::page_glyphs(&doc, doc.get_pages()[&1]);
6276        assert_eq!(glyphs.len(), 3);
6277        assert!((glyphs[2].l - glyphs[0].l - 24.0).abs() < 0.001);
6278    }
6279
6280    #[test]
6281    fn utf16_codes_preserve_tounicode_keys_and_identity_cids() {
6282        use lopdf::{dictionary, Document};
6283        let doc = Document::with_version("1.4");
6284        let dict = dictionary! {"Subtype" => "Type0", "Encoding" => "UniJIS-UTF16-H"};
6285        let mut font = super::parse_font(&doc, b"F1", &dict);
6286        assert_eq!(
6287            super::codes(&font, &[0xd8, 0x42, 0xdf, 0xb7]),
6288            vec![0xd842dfb7]
6289        );
6290        font.to_unicode.insert(0xd842dfb7, "override".into());
6291        assert_eq!(font.decode_code(0xd842dfb7).0.as_deref(), Some("override"));
6292        assert_eq!(
6293            super::codes(&font, &[0xd8, 0x42, 0x65, 0xe5]),
6294            vec![0xd842, 0x65e5]
6295        );
6296        let dict = dictionary! {"Subtype" => "Type0", "Encoding" => "Identity-H"};
6297        let font = super::parse_font(&doc, b"F1", &dict);
6298        assert_eq!(
6299            super::codes(&font, &[0xd8, 0x42, 0xdf, 0xb7]),
6300            vec![0xd842, 0xdfb7]
6301        );
6302    }
6303}