Skip to main content

docling_pdf/
textparse.rs

1//! Pure-Rust PDF text extraction (replacing pdfium's glyph layer).
2//!
3//! pdfium reports *rendered* glyph boxes, which diverge from docling's
4//! `docling-parse` C++ parser at exactly the points that drive conformance:
5//! generated spaces get a zero-width box, combining diacritics get a real-width
6//! box, and ligature/fraction glyphs land at different x. This module instead
7//! reconstructs each glyph's box from the **font's own advance widths** and the
8//! PDF text/graphics matrices — the same information docling-parse uses — so a
9//! space is as wide as the font says and a combining mark has zero advance.
10//!
11//! The output is the same [`Glyph`] stream pdfium produces (native PDF
12//! coordinates, y-up), fed straight into the existing docling-parse line
13//! sanitizer ([`crate::dp_lines`]). Only the digital text layer is handled here;
14//! pages without one still fall back to OCR upstream.
15
16use std::collections::{HashMap, HashSet};
17use std::sync::Arc;
18
19use lopdf::{Dictionary, Document, Object};
20
21use crate::pdfium_backend::Glyph;
22use crate::tounicode::parse_tounicode;
23
24/// Per-document caches for the content-stream interpreter. Fonts are indirect
25/// objects shared by many pages, but were fully re-parsed — ToUnicode CMap
26/// decompression + tokenization, embedded Type1 program scan, width tables —
27/// for **every page and every Form XObject invocation**; decoded form content
28/// streams were likewise re-inflated on every `Do`. Cached per document,
29/// keyed by the referenced object id (fonts also by resource name — the
30/// `/Font` entry a page binds the object to). Inline (non-reference) dicts
31/// are rare and stay uncached.
32#[derive(Default)]
33struct DocCaches {
34    fonts: HashMap<(lopdf::ObjectId, Vec<u8>), Arc<Font>>,
35    forms: HashMap<lopdf::ObjectId, Arc<lopdf::content::Content>>,
36    /// Forms decoded once and not (yet) cached (#994). A form enters
37    /// [`Self::forms`] on its *second* sighting: a shared header/footer form
38    /// is decoded twice and then hits the cache on every later page, while
39    /// a generator's one-form-per-page body (each page's chart in its own
40    /// form, used once) is dropped with its page instead of every decoded
41    /// `Content` — many times its stream's size — living until the document
42    /// ends. Caching on first use made peak RSS grow ~45 MB per page on such
43    /// files (7 GiB at 60 pages of the issue's repro, flat ~0.6 GiB now).
44    /// Only ids are kept here, so the set itself stays tiny.
45    forms_seen: HashSet<lopdf::ObjectId>,
46}
47
48/// A 2×3 affine matrix `[a b c d e f]`: maps `(x,y)` → `(a·x+c·y+e, b·x+d·y+f)`.
49#[derive(Clone, Copy)]
50struct Mat {
51    a: f64,
52    b: f64,
53    c: f64,
54    d: f64,
55    e: f64,
56    f: f64,
57}
58
59impl Mat {
60    const ID: Mat = Mat {
61        a: 1.0,
62        b: 0.0,
63        c: 0.0,
64        d: 1.0,
65        e: 0.0,
66        f: 0.0,
67    };
68
69    /// `self ∘ m`: the matrix that applies `self` first, then `m`.
70    fn then(self, m: Mat) -> Mat {
71        Mat {
72            a: self.a * m.a + self.b * m.c,
73            b: self.a * m.b + self.b * m.d,
74            c: self.c * m.a + self.d * m.c,
75            d: self.c * m.b + self.d * m.d,
76            e: self.e * m.a + self.f * m.c + m.e,
77            f: self.e * m.b + self.f * m.d + m.f,
78        }
79    }
80
81    fn apply(self, x: f64, y: f64) -> (f64, f64) {
82        (
83            self.a * x + self.c * y + self.e,
84            self.b * x + self.d * y + self.f,
85        )
86    }
87}
88
89/// A parsed font: how to turn raw string bytes into (unicode, advance) pairs.
90struct Font {
91    /// 2-byte codes (Type0 / Identity-H) vs 1-byte (simple fonts).
92    two_byte: bool,
93    /// code → Unicode string (from ToUnicode; may be multi-char, e.g. ligatures).
94    to_unicode: HashMap<u32, String>,
95    /// For codes ToUnicode lacks (ISO 32000-1, 9.10.2).
96    cid_fallback: CidFallback,
97    /// code → glyph advance, in 1000-unit glyph space.
98    widths: HashMap<u32, f64>,
99    default_width: f64,
100    /// 1-byte fallback decoding when ToUnicode lacks a code (WinAnsi-ish).
101    simple_encoding: Option<HashMap<u8, char>>,
102    /// code → raw `/Differences` glyph name, for placeholder names (`g115`,
103    /// `SM590000`) that have no Unicode mapping; they override the base
104    /// encoding (see [`unknown_name_text`]).
105    fallback_names: HashMap<u8, String>,
106    /// The unmapped codes whose embedded glyph paints nothing — semantic
107    /// spaces (docling-parse 7.24's blank glyph programs: the redp5110
108    /// list's `/g3`). Every other unmapped code reads U+FFFD.
109    blank_codes: HashSet<u32>,
110    /// code → char from the embedded Type1 font program's own `/Encoding` vector,
111    /// used only as a last resort for glyphs the base encoding leaves unmapped
112    /// (standard TeX math fonts: `λ`, `≤`, …).
113    program_encoding: HashMap<u8, char>,
114    /// A symbolic simple font without a declared base encoding: the text of
115    /// each code its embedded program's builtin encoding resolves
116    /// (docling-parse 7.25, #1113 — 2203.01017's `Dingbats` check marks are
117    /// `a19` / `a23`, `✓` / `✗`, not the `3` / `7` the Latin tables read).
118    /// It outranks every encoding table but a `/Differences` entry
119    /// ([`Font::diff_codes`]); empty for every other font and in a build
120    /// without the renderer.
121    builtin: HashMap<u8, String>,
122    /// The codes `/Differences` names.
123    diff_codes: HashSet<u8>,
124    ascent: f64,
125    descent: f64,
126    /// code → its embedded glyph's ink extent `(y_min, y_max)` in 1000-unit
127    /// em, for a font whose descriptor metrics docling-parse rejects
128    /// ([`font_ascent_descent`]'s third value): docling-parse 7.24 boxes such
129    /// a font's glyphs by their outline's control box, so a glyph drawn
130    /// under the baseline — 2305.03393's `_` in `PMC5406406_003_01.png` —
131    /// is a cell of its own, not part of the word. Empty for every other
132    /// font, a glyph with no ink, and a build without the renderer.
133    ink_y: HashMap<u32, (f64, f64)>,
134    hash: u64,
135    /// Whether the font's docling-parse name marks a math face
136    /// ([`is_math_font_name`]) — for the word contraction's local
137    /// attachments (#921).
138    math: bool,
139    /// Weight and slant read from the `/BaseFont` name (the heading
140    /// hierarchy's style signal, #302).
141    style: crate::font_style::FontStyle,
142}
143
144impl Font {
145    /// The text of `code` and its advance; the flag marks a blank unmapped
146    /// glyph, whose box has no height.
147    ///
148    /// A code no source maps to Unicode is visibly painted but carries no
149    /// text: docling-parse 7.24 emits U+FFFD for it, and docling turns a NUL
150    /// into U+FFFD too (docling#5070), so the loss stays visible and counted
151    /// by the text-quality check instead of silently dropping the glyph
152    /// (`mL⁻¹` reading `mL1`). A glyph whose embedded program paints nothing
153    /// is a space — a word boundary — instead.
154    fn decode_code(&self, code: u32) -> (Option<String>, f64, bool) {
155        let w = self
156            .widths
157            .get(&code)
158            .copied()
159            .unwrap_or(self.default_width);
160        if let Some(s) = self.to_unicode.get(&code) {
161            return (Some(decompose_ligatures(s)), w, false);
162        }
163        let unmapped = |font: &Font| {
164            let blank = font.blank_codes.contains(&code);
165            (
166                Some(if blank { " " } else { "\u{FFFD}" }.to_string()),
167                w,
168                blank,
169            )
170        };
171        if !self.two_byte {
172            // A placeholder `/Differences` name (no Unicode) overrides the
173            // base encoding. docling-parse 7.24 reads a bare glyph index
174            // (`g115`, `gid00043`) as unmapped and any other unknown name as
175            // its text (`SM590000`, the redp5110 bullets).
176            if let Some(name) = self.fallback_names.get(&(code as u8)) {
177                return match unknown_name_text(name) {
178                    Some(text) => (Some(text), w, false),
179                    None => unmapped(self),
180                };
181            }
182            // docling-parse 7.25's `get_correct_character`: after
183            // `/Differences` and `/ToUnicode`, a symbolic font's builtin
184            // encoding — "the producer's own statement of what each code
185            // draws" — before any base-font or Latin table.
186            if !self.diff_codes.contains(&(code as u8)) {
187                if let Some(text) = self.builtin.get(&(code as u8)) {
188                    return (Some(decompose_ligatures(text)), w, false);
189                }
190            }
191            if let Some(enc) = &self.simple_encoding {
192                if let Some(&ch) = enc.get(&(code as u8)) {
193                    return (Some(decompose_ligatures(&ch.to_string())), w, false);
194                }
195            }
196            // Last resort: the embedded Type1 font program's own `/Encoding`
197            // vector (`dup N /glyphname put`). Standard TeX math fonts (CMMI, CMSY,
198            // …) ship no PDF `/Encoding` and no ToUnicode, so a glyph like `λ`
199            // (CMMI code 21 → `/lambda`) or `≤` (CMSY code 20 → `/lessequal`) has
200            // no other mapping and would otherwise be silently dropped. docling
201            // recovers these from the same font program. This only fills codes the
202            // base encoding left unmapped, so it never changes an existing decode.
203            if let Some(&ch) = self.program_encoding.get(&(code as u8)) {
204                return (Some(decompose_ligatures(&ch.to_string())), w, false);
205            }
206        } else {
207            match self.cid_fallback {
208                CidFallback::None => {}
209                CidFallback::UnicodeCodes { .. } => {
210                    if let Some(ch) = unicode_cmap_char(code).filter(|c| !c.is_control()) {
211                        return (Some(ch.to_string()), w, false);
212                    }
213                }
214                CidFallback::Collection(table) => {
215                    if let Some(s) = collection_text(table, code) {
216                        return (Some(decompose_ligatures(&s)), w, false);
217                    }
218                }
219            }
220        }
221        unmapped(self)
222    }
223}
224
225/// Spell out Latin presentation-form ligatures (`fi`→`fi`, `ffi`→`ffi`, …) the way
226/// docling does, so `configuration`/`difficult` don't keep the ligature glyph.
227/// The chars share the ligature's box, so the line sanitizer recomposes them.
228fn decompose_ligatures(s: &str) -> String {
229    if !s.chars().any(|c| ('\u{FB00}'..='\u{FB06}').contains(&c)) {
230        return s.to_string();
231    }
232    s.chars()
233        .map(|c| {
234            match c {
235                '\u{FB00}' => "ff",
236                '\u{FB01}' => "fi",
237                '\u{FB02}' => "fl",
238                '\u{FB03}' => "ffi",
239                '\u{FB04}' => "ffl",
240                // U+FB05 is LATIN SMALL LIGATURE LONG S T (`ſt`): `st`, as
241                // docling's ligature map and Unicode's decomposition (#1001).
242                '\u{FB05}' => "st",
243                '\u{FB06}' => "st",
244                _ => return c.to_string(),
245            }
246            .to_string()
247        })
248        .collect()
249}
250
251fn hash_name(name: &[u8]) -> u64 {
252    use std::hash::{Hash, Hasher};
253    let mut h = std::collections::hash_map::DefaultHasher::new();
254    name.hash(&mut h);
255    h.finish()
256}
257
258/// Resolve a possibly-indirect object to a dictionary.
259fn as_dict<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Dictionary> {
260    match obj {
261        Object::Dictionary(d) => Some(d),
262        Object::Reference(id) => doc.get_object(*id).ok().and_then(|o| o.as_dict().ok()),
263        _ => None,
264    }
265}
266
267fn deref<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Object> {
268    match obj {
269        Object::Reference(id) => doc.get_object(*id).ok(),
270        other => Some(other),
271    }
272}
273
274/// The font identity the line sanitizer's `enforce_same_font` compares —
275/// docling-parse's `page_font::init_font_name`: `/FontDescriptor/FontName`
276/// (the descendant CIDFont's for a Type0), else `/Name`, else `/BaseFont`
277/// (its `base_font`), else the literal `"null"`. The resource key (`/F1`,
278/// `/F2`) is deliberately **not** part of it: a producer that references one
279/// face twice — the same Helvetica under two keys, one re-encoded so a code
280/// maps to the `fi` glyph — draws one word across the two resources, and
281/// docling-parse reads both as the same font. Keyed by resource name (as
282/// this port was until #728) the `fi` ligature, which bridges fonts, glued
283/// itself to the preceding run while the font barrier then cut it off from
284/// its own word (`fi nancial`); docling's `financial` needs the two
285/// resources to compare equal.
286/// docling-parse's `has_math_positioning_semantics`, the font-name half: a
287/// TeX math or symbol face (`cmmi`, `cmsy`, `cmex`, `cmr`, `cmbx`, `msam`,
288/// `msbm`) or any name saying `math`, `symbol` or `mtextra`.
289fn is_math_font_name(name: &[u8]) -> bool {
290    let lower = name.to_ascii_lowercase();
291    [
292        "cmmi", "cmsy", "cmex", "cmr", "cmbx", "msam", "msbm", "math", "symbol", "mtextra",
293    ]
294    .iter()
295    .any(|t| lower.windows(t.len()).any(|w| w == t.as_bytes()))
296}
297
298fn sanitizer_font_name(doc: &Document, fdict: &Dictionary, two_byte: bool) -> Vec<u8> {
299    let descriptor_font_name = |owner: &Dictionary| {
300        owner
301            .get(b"FontDescriptor")
302            .ok()
303            .and_then(|o| as_dict(doc, o))
304            .and_then(|fd| fd.get(b"FontName").ok())
305            .and_then(|o| deref(doc, o))
306            .and_then(|o| o.as_name().ok().map(<[u8]>::to_vec))
307    };
308    if let Some(n) = descriptor_font_name(fdict) {
309        return n;
310    }
311    if two_byte {
312        let descendant = fdict
313            .get(b"DescendantFonts")
314            .ok()
315            .and_then(|o| deref(doc, o))
316            .and_then(|o| match o {
317                Object::Array(a) => a.first(),
318                _ => None,
319            })
320            .and_then(|o| as_dict(doc, o));
321        if let Some(n) = descendant.and_then(descriptor_font_name) {
322            return n;
323        }
324    }
325    if let Some(n) = fdict
326        .get(b"Name")
327        .ok()
328        .and_then(|o| deref(doc, o))
329        .and_then(|o| o.as_name().ok())
330    {
331        return n.to_vec();
332    }
333    // docling-parse's `base_font` keeps the subset prefix (`ABCDEF+Helvetica`):
334    // two subsets of one face are two fonts there, and so here.
335    if let Some(n) = fdict.get(b"BaseFont").ok().and_then(|o| o.as_name().ok()) {
336        return n.to_vec();
337    }
338    b"null".to_vec()
339}
340
341/// Parse one font dictionary into a [`Font`].
342fn parse_font(doc: &Document, name: &[u8], fdict: &Dictionary) -> Font {
343    let subtype: &[u8] = fdict
344        .get(b"Subtype")
345        .ok()
346        .and_then(|o| o.as_name().ok())
347        .unwrap_or(&[]);
348    let two_byte = subtype == b"Type0".as_slice();
349
350    let mut to_unicode = fdict
351        .get(b"ToUnicode")
352        .ok()
353        .and_then(|o| deref(doc, o))
354        .and_then(|o| o.as_stream().ok())
355        .and_then(|s| s.decompressed_content().ok())
356        .map(|data| parse_tounicode(&data))
357        .unwrap_or_default();
358    if !two_byte {
359        if let Some(diffs) = differences_array(doc, fdict) {
360            reconcile_stale_tounicode(&mut to_unicode, &diffs);
361        }
362    }
363    let sanitizer_name = sanitizer_font_name(doc, fdict, two_byte);
364    // Symbol / ZapfDingbats mapped into the private-use "symbol charset"
365    // (`U+F000` + byte): docling-parse 7.25 reads the glyph's text through
366    // the face's own encoding instead (docling-parse#431, #1113 — redp5110's
367    // `U+F0AE` is `→`, a Word bullet `U+F0B7` is `•`).
368    let raw_base = fdict.get(b"BaseFont").ok().and_then(|o| o.as_name().ok());
369    let names: Vec<&[u8]> = std::iter::once(sanitizer_name.as_slice())
370        .chain(raw_base)
371        .collect();
372    if let Some(face) = crate::symbol_fonts::symbol_face(&names) {
373        for value in to_unicode.values_mut() {
374            if let Some(ch) = crate::symbol_fonts::fold_private_use(face, value) {
375                *value = ch.to_string();
376            }
377        }
378    }
379
380    let cid_fallback = if two_byte {
381        cid_fallback(doc, fdict)
382    } else {
383        CidFallback::None
384    };
385
386    let (mut widths, mut default_width) = if two_byte {
387        cid_widths(doc, fdict)
388    } else {
389        simple_widths(doc, fdict)
390    };
391
392    let simple_encoding = if two_byte {
393        None
394    } else {
395        Some(simple_encoding_table(doc, fdict))
396    };
397
398    // A standard-14 font referenced without an embedded program usually ships
399    // no `/Widths` and no `/FontDescriptor` either (ReportLab's default, #187).
400    // Every advance then resolved to 0, the cells collapsed to zero width, and
401    // the page's whole text layer was silently dropped — while pdfium, with
402    // its built-in metrics, reads the same file fine. Fill the widths from the
403    // built-in Adobe Core 14 AFM tables via the font's own code→char decode
404    // (base encoding + `/Differences`), so an explicit `/Widths` always wins.
405    if !two_byte && widths.is_empty() && default_width == 0.0 {
406        if let Some(std14) = base_font_name(fdict).and_then(|n| crate::std14::widths_for(&n)) {
407            if let Some(enc) = &simple_encoding {
408                for (&code, &ch) in enc {
409                    if let Some(w) = std14.width(ch) {
410                        widths.insert(u32::from(code), w);
411                    }
412                }
413            }
414            // Codes the table misses still advance a typical width instead of
415            // stacking at x=0 (the failure mode this whole branch fixes).
416            default_width = 500.0;
417        } else if let Some(face) = crate::symbol_fonts::symbol_face(&[base_font_name(fdict)
418            .as_deref()
419            .unwrap_or_default()])
420        {
421            // The two symbol faces are no std14 text faces: their AFM
422            // advances by code, as docling-parse's base font reads them.
423            for code in 0..=255u8 {
424                if let Some(w) = crate::symbol_fonts::face_width(face, code) {
425                    widths.insert(u32::from(code), w);
426                }
427            }
428            default_width = 500.0;
429        }
430    }
431    let fallback_names = if two_byte {
432        HashMap::new()
433    } else {
434        differences_gid_names(doc, fdict)
435    };
436    let program_encoding = if two_byte {
437        HashMap::new()
438    } else {
439        type1_program_encoding(doc, fdict)
440    };
441
442    let (ascent, descent, ink_boxes) = font_ascent_descent(doc, fdict, two_byte);
443    let mut font = Font {
444        two_byte,
445        to_unicode,
446        cid_fallback,
447        widths,
448        default_width,
449        simple_encoding,
450        fallback_names,
451        blank_codes: HashSet::new(),
452        program_encoding,
453        builtin: HashMap::new(),
454        diff_codes: differences_codes(doc, fdict),
455        ascent,
456        descent,
457        ink_y: HashMap::new(),
458        hash: hash_name(&sanitizer_name),
459        math: is_math_font_name(&sanitizer_name),
460        style: crate::font_style::parse_font_style(&String::from_utf8_lossy(
461            &base_font_name(fdict).unwrap_or_else(|| name.to_vec()),
462        )),
463    };
464    font.blank_codes = blank_codes(doc, fdict, &font);
465    if !two_byte && subtype != b"Type3".as_slice() && uses_builtin_encoding(doc, fdict) {
466        font.builtin = builtin_encoding(doc, fdict, &font);
467    }
468    if ink_boxes {
469        font.ink_y = ink_extents(doc, fdict, &font);
470    }
471    font
472}
473
474/// docling-parse 7.25's `uses_builtin_encoding` for a simple, non-Type 3
475/// font: the descriptor flags it symbolic (bit 3) and the PDF declares no
476/// base encoding — no `/Encoding` name, no `/BaseEncoding` in an `/Encoding`
477/// dictionary (a bare `/Differences` overrides only the codes it lists,
478/// 9.6.6.1).
479fn uses_builtin_encoding(doc: &Document, fdict: &Dictionary) -> bool {
480    let symbolic = fdict
481        .get(b"FontDescriptor")
482        .ok()
483        .and_then(|o| as_dict(doc, o))
484        .and_then(|fd| fd.get(b"Flags").ok())
485        .and_then(|o| deref(doc, o))
486        .and_then(num)
487        .is_some_and(|f| (f as i64) & 4 != 0);
488    let declared_base = match fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o)) {
489        Some(Object::Name(_)) => true,
490        Some(Object::Dictionary(d)) => d.get(b"BaseEncoding").is_ok(),
491        _ => false,
492    };
493    symbolic && !declared_base
494}
495
496/// The codes a simple font's `/Differences` names.
497fn differences_codes(doc: &Document, fdict: &Dictionary) -> HashSet<u8> {
498    let mut codes = HashSet::new();
499    let mut code = 0i64;
500    for el in differences_array(doc, fdict).unwrap_or_default() {
501        match el {
502            Object::Integer(i) => code = i,
503            Object::Real(r) => code = r as i64,
504            Object::Name(_) => {
505                if (0..=255).contains(&code) {
506                    codes.insert(code as u8);
507                }
508                code += 1;
509            }
510            _ => {}
511        }
512    }
513    codes
514}
515
516/// [`Font::builtin`]: read from the embedded program only when a declared
517/// code is left to it — not covered by `/ToUnicode` or `/Differences`
518/// (docling-parse reads the program lazily, on the first code that needs it).
519#[cfg(feature = "render")]
520fn builtin_encoding(doc: &Document, fdict: &Dictionary, font: &Font) -> HashMap<u8, String> {
521    let mut codes: Vec<u32> = font.widths.keys().copied().collect();
522    if codes.is_empty() {
523        codes.extend(0..=255);
524    }
525    let needed = codes.iter().any(|&c| {
526        c <= 255 && !font.to_unicode.contains_key(&c) && !font.diff_codes.contains(&(c as u8))
527    });
528    if !needed {
529        return HashMap::new();
530    }
531    crate::render::font::LoadedFont::load(doc, fdict).builtin_encoding_text()
532}
533
534/// Without the renderer there is no program to read: the font keeps the
535/// encoding tables' reading.
536#[cfg(not(feature = "render"))]
537fn builtin_encoding(_doc: &Document, _fdict: &Dictionary, _font: &Font) -> HashMap<u8, String> {
538    HashMap::new()
539}
540
541/// The ink extents of `font`'s declared codes ([`Font::ink_y`]), read from
542/// its embedded program; nothing past a CID font's first 4096 codes.
543#[cfg(feature = "render")]
544fn ink_extents(doc: &Document, fdict: &Dictionary, font: &Font) -> HashMap<u32, (f64, f64)> {
545    const MAX_CODES: usize = 4096;
546    let loaded = crate::render::font::LoadedFont::load(doc, fdict);
547    if !loaded.has_program() {
548        return HashMap::new();
549    }
550    let mut codes: Vec<u32> = font.widths.keys().copied().collect();
551    if !font.two_byte {
552        codes.extend(0..=255);
553        codes.sort_unstable();
554        codes.dedup();
555    }
556    codes
557        .into_iter()
558        .take(MAX_CODES)
559        .filter_map(|c| Some((c, loaded.glyph_ink_y(c)??)))
560        .collect()
561}
562
563/// Without the renderer there is no program to read: the font keeps its
564/// loose box (docling-parse's own fallback when FreeType resolves no glyph).
565#[cfg(not(feature = "render"))]
566fn ink_extents(_doc: &Document, _fdict: &Dictionary, _font: &Font) -> HashMap<u32, (f64, f64)> {
567    HashMap::new()
568}
569
570/// The codes `font` leaves unmapped whose embedded glyph paints nothing
571/// ([`Font::blank_codes`]). The program is loaded only when the font has
572/// unmapped codes among those it declares widths for; without the
573/// renderer (the wasm text-layer build) there is no program to ask, and
574/// every unmapped glyph reads U+FFFD.
575#[cfg(feature = "render")]
576fn blank_codes(doc: &Document, fdict: &Dictionary, font: &Font) -> HashSet<u32> {
577    // A CID font can declare tens of thousands of widths: past this many
578    // unmapped codes nothing is classified (the glyphs read U+FFFD).
579    const MAX_CANDIDATES: usize = 4096;
580    let unmapped: Vec<u32> = font
581        .widths
582        .keys()
583        .copied()
584        .filter(|&c| font.decode_code(c).0.as_deref() == Some("\u{FFFD}"))
585        .take(MAX_CANDIDATES + 1)
586        .collect();
587    if unmapped.is_empty() || unmapped.len() > MAX_CANDIDATES {
588        return HashSet::new();
589    }
590    let loaded = crate::render::font::LoadedFont::load(doc, fdict);
591    unmapped
592        .into_iter()
593        .filter(|&c| loaded.glyph_has_ink(c) == Some(false))
594        .collect()
595}
596
597#[cfg(not(feature = "render"))]
598fn blank_codes(_doc: &Document, _fdict: &Dictionary, _font: &Font) -> HashSet<u32> {
599    HashSet::new()
600}
601
602#[derive(Clone, Copy)]
603enum CidFallback {
604    None,
605    UnicodeCodes { utf16: bool },
606    Collection(&'static [u8]),
607}
608
609/// A Unicode CMap code retains its original bytes for ToUnicode lookups;
610/// a four-byte UTF-16 code is a packed surrogate pair, not a scalar value.
611pub(crate) fn unicode_cmap_char(code: u32) -> Option<char> {
612    if code <= 0xffff {
613        char::from_u32(code)
614    } else {
615        let units = [(code >> 16) as u16, code as u16];
616        let mut chars = char::decode_utf16(units);
617        let ch = chars.next()?.ok()?;
618        chars.next().is_none().then_some(ch)
619    }
620}
621
622#[cfg(not(feature = "cid-collections"))]
623fn collection_table(_ordering: &[u8]) -> Option<&'static [u8]> {
624    None
625}
626
627#[cfg(feature = "cid-collections")]
628fn collection_table(ordering: &[u8]) -> Option<&'static [u8]> {
629    macro_rules! table {
630        ($file:literal) => {
631            include_bytes!(concat!(env!("OUT_DIR"), "/", $file))
632        };
633    }
634    Some(match ordering {
635        b"Japan1" => table!("Adobe-Japan1-UCS2"),
636        b"GB1" => table!("Adobe-GB1-UCS2"),
637        b"CNS1" => table!("Adobe-CNS1-UCS2"),
638        b"Korea1" => table!("Adobe-Korea1-UCS2"),
639        b"KR" => table!("Adobe-KR-UCS2"),
640        _ => return None,
641    })
642}
643
644/// The text of `cid` in a table build.rs wrote: the CID count, then three
645/// bytes per CID (0 for none, a code point, or 0x110000 + the offset of a
646/// length-prefixed UTF-8 string after the entries).
647pub(crate) fn collection_text(table: &[u8], cid: u32) -> Option<String> {
648    let n = u32::from_le_bytes(table.get(..4)?.try_into().ok()?) as usize;
649    let at = 4 + 3 * usize::try_from(cid).ok().filter(|&c| c < n)?;
650    let &[a, b, c] = table.get(at..at + 3)? else {
651        return None;
652    };
653    match u32::from_be_bytes([0, a, b, c]) {
654        0 => None,
655        v @ ..0x110000 => char::from_u32(v).map(String::from),
656        v => {
657            let at = 4 + 3 * n + (v - 0x110000) as usize;
658            let len = *table.get(at)? as usize;
659            std::str::from_utf8(table.get(at + 1..at + 1 + len)?)
660                .ok()
661                .map(str::to_owned)
662        }
663    }
664}
665
666fn cid_fallback(doc: &Document, fdict: &Dictionary) -> CidFallback {
667    let encoding = fdict
668        .get(b"Encoding")
669        .ok()
670        .and_then(|o| deref(doc, o))
671        .and_then(|o| o.as_name().ok())
672        .unwrap_or(&[]);
673    if encoding.starts_with(b"Uni")
674        && (encoding.windows(4).any(|w| w == b"UCS2") || encoding.windows(5).any(|w| w == b"UTF16"))
675    {
676        return CidFallback::UnicodeCodes {
677            utf16: encoding.windows(5).any(|w| w == b"UTF16"),
678        };
679    }
680    if encoding != b"Identity-H" && encoding != b"Identity-V" {
681        // Other CMaps need code → CID tables that are not bundled.
682        return CidFallback::None;
683    }
684    cid_collection(doc, fdict).map_or(CidFallback::None, CidFallback::Collection)
685}
686
687/// The CID → Unicode table of the character collection a Type0 font uses.
688pub(crate) fn cid_collection(doc: &Document, fdict: &Dictionary) -> Option<&'static [u8]> {
689    let Object::Array(descendants) = deref(doc, fdict.get(b"DescendantFonts").ok()?)? else {
690        return None;
691    };
692    let descendant = as_dict(doc, descendants.first()?)?;
693    let info = as_dict(doc, descendant.get(b"CIDSystemInfo").ok()?)?;
694    let registry = deref(doc, info.get(b"Registry").ok()?)?.as_str().ok()?;
695    let ordering = deref(doc, info.get(b"Ordering").ok()?)?.as_str().ok()?;
696    (registry == b"Adobe").then(|| collection_table(ordering))?
697}
698
699/// A simple font's `/Encoding` `/Differences` array, dereferenced.
700fn differences_array(doc: &Document, fdict: &Dictionary) -> Option<Vec<Object>> {
701    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))?;
702    let diffs = enc.as_dict().ok()?.get(b"Differences").ok()?;
703    match deref(doc, diffs)? {
704        Object::Array(a) => Some(a.clone()),
705        _ => None,
706    }
707}
708
709/// A ToUnicode CMap copied from the *original* font while the glyphs were
710/// re-encoded: the encrypted `2206.01062_pg3` fixture (a Quartz re-save of
711/// pdfTeX output) keeps the T1-encoding CMap, keyed on the old codes, next
712/// to a `/Differences [33 /f_f_i /f_i /f_f /f_l]` that names the ligatures at
713/// new ones, so the CMap maps code 33 to `!` — its own identity — and
714/// `difficulties` read `di ! culties`, in docling's pdfium chain too. A
715/// ToUnicode entry that is the code's identity, for a code whose
716/// `/Differences` name resolves to something else, is that stale copy: the
717/// name wins. A CMap that disagrees any other way is left alone — ToUnicode
718/// is the producer's word on what the glyph means.
719///
720/// The name also wins over a ToUnicode value that says nothing (#1121):
721/// docling-parse's `has_to_unicode` discards an empty value and a single
722/// NUL, U+FFFD or Unicode noncharacter for a code `/Differences` names, and
723/// resolves the name instead. A figure embedded in 1503.04203 re-encodes
724/// its TeX glyphs to code 33 with `/Differences [33 /minus]` and a CMap
725/// `<21> <0000>` — the glyph's old code written as its Unicode — so
726/// `Q = −1/2` read `Q = � 12` and `√Δ²` read `p�2`.
727fn reconcile_stale_tounicode(to_unicode: &mut HashMap<u32, String>, diffs: &[Object]) {
728    let says_nothing = |value: &str| {
729        let mut chars = value.chars();
730        match (chars.next(), chars.next()) {
731            (None, _) => true,
732            (Some(c), None) => {
733                let cp = c as u32;
734                cp == 0 || cp == 0xFFFD || (0xFDD0..=0xFDEF).contains(&cp) || cp & 0xFFFE == 0xFFFE
735            }
736            _ => false,
737        }
738    };
739    let mut code = 0u8;
740    for el in diffs {
741        match el {
742            Object::Integer(i) => code = *i as u8,
743            Object::Name(name) => {
744                if let Some(ch) = glyph_name_to_char(name) {
745                    let identity = (code as char).to_string();
746                    let current = to_unicode.get(&u32::from(code));
747                    if (ch.to_string() != identity && current == Some(&identity))
748                        || current.is_some_and(|v| says_nothing(v))
749                    {
750                        to_unicode.insert(u32::from(code), ch.to_string());
751                    }
752                }
753                code = code.wrapping_add(1);
754            }
755            _ => {}
756        }
757    }
758}
759
760/// Collect `/Differences` entries whose glyph name is a GID placeholder
761/// (`g115`, `cid42`, `glyph7`, `index9`, `SM590000`) with no Unicode mapping
762/// (subsetted fonts with no ToUnicode); [`unknown_name_text`] decides what
763/// each reads as. The placeholder-name restriction keeps real Adobe glyph
764/// names on the normal path so this never invents garbage on the clean files.
765fn differences_gid_names(doc: &Document, fdict: &Dictionary) -> HashMap<u8, String> {
766    let mut map = HashMap::new();
767    let Some(Object::Dictionary(enc)) = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))
768    else {
769        return map;
770    };
771    let Some(Object::Array(diffs)) = enc.get(b"Differences").ok().and_then(|o| deref(doc, o))
772    else {
773        return map;
774    };
775    let mut code = 0u8;
776    for el in diffs {
777        match el {
778            Object::Integer(i) => code = *i as u8,
779            Object::Name(name) => {
780                if glyph_name_to_char(name).is_none() && is_gid_name(name) {
781                    map.insert(code, String::from_utf8_lossy(name).into_owned());
782                }
783                code = code.wrapping_add(1);
784            }
785            _ => {}
786        }
787    }
788    map
789}
790
791/// Parse the embedded Type1 font program's built-in `/Encoding` vector
792/// (`dup <code> /<glyphname> put` entries in the clear-text header before
793/// `eexec`) into `code → char`. This is how docling recovers glyphs from
794/// standard TeX math fonts (CMMI/CMSY/…) that carry no PDF `/Encoding` and no
795/// ToUnicode — e.g. CMMI's `dup 21 /lambda` or CMSY's `dup 20 /lessequal`.
796/// Only `FontFile` (Type1) is parsed; CFF (`FontFile3`) and TrueType
797/// (`FontFile2`) store their encoding in a binary table and are left alone.
798fn type1_program_encoding(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
799    let mut map = HashMap::new();
800    let Some(desc) = fdict
801        .get(b"FontDescriptor")
802        .ok()
803        .and_then(|o| deref(doc, o))
804        .and_then(|o| o.as_dict().ok())
805    else {
806        return map;
807    };
808    let Some(data) = desc
809        .get(b"FontFile")
810        .ok()
811        .and_then(|o| deref(doc, o))
812        .and_then(|o| o.as_stream().ok())
813        .and_then(|s| s.decompressed_content().ok())
814    else {
815        return map;
816    };
817    // The clear-text header (PostScript) ends at `eexec`; the rest is encrypted.
818    let head_end = data
819        .windows(5)
820        .position(|w| w == b"eexec")
821        .unwrap_or(data.len());
822    let head = String::from_utf8_lossy(&data[..head_end]);
823    // Scan for `dup <code> /<name> put` tokens.
824    let toks: Vec<&str> = head.split_whitespace().collect();
825    for w in toks.windows(4) {
826        if w[0] == "dup" && w[3] == "put" {
827            if let (Ok(code), Some(name)) = (w[1].parse::<u32>(), w[2].strip_prefix('/')) {
828                if code <= 255 {
829                    if let Some(ch) = glyph_name_to_char(name.as_bytes()) {
830                        map.insert(code as u8, ch);
831                    }
832                }
833            }
834        }
835    }
836    map
837}
838
839/// The reading text docling-parse 7.24 keeps for a `/Differences` glyph name
840/// no table resolves (`resolve_unknown_name`): the name without any
841/// `.suffix` — `None` when that is a bare glyph index
842/// (`(gid|glyph|g|cid|index)\d+`, any case), which names a glyph of the
843/// font program but carries no text (docling-parse#238), so it is unmapped.
844fn unknown_name_text(name: &str) -> Option<String> {
845    let stem = name.rsplit_once('.').map_or(name, |(s, _)| s);
846    (!is_index_name(stem)).then(|| stem.to_string())
847}
848
849/// docling-parse's index-style glyph name, `(gid|glyph|g|cid|index)` then
850/// hexadecimal digits, case-insensitive (`re_gid`): `/gid00043`, `/g43`,
851/// `/cid43`, `/index43` from the subset generators, and since docling-parse
852/// 7.25 (#389 in docling-parse#431, #1113) Skia/PDF's `g%X` — `/g15C`,
853/// `/gA1` from Chrome's print-to-PDF and Notion exports, which read as
854/// their base encoding's letters before. The name points into the font
855/// program and carries no text.
856fn is_index_name(name: &str) -> bool {
857    let lower = name.to_ascii_lowercase();
858    ["gid", "glyph", "g", "cid", "index"].iter().any(|p| {
859        lower
860            .strip_prefix(p)
861            .is_some_and(|d| !d.is_empty() && d.bytes().all(|b| b.is_ascii_hexdigit()))
862    })
863}
864
865/// A glyph name that is a synthetic placeholder, not a real Adobe name:
866/// `g115`, `cid42`, `glyph7`, `index9`, `G12`, or a short-prefix code name like
867/// `SM590000` (IBM BookMaster). These carry no Unicode meaning (docling-parse
868/// keeps the code-like names as text, `SM590000`, and reads the glyph indexes
869/// as unmapped, see [`unknown_name_text`]). `afii####` / `uni####` are real Adobe names
870/// and excluded. The restriction keeps genuine glyph names on the Unicode path.
871fn is_gid_name(name: &[u8]) -> bool {
872    let Ok(s) = std::str::from_utf8(name) else {
873        return false;
874    };
875    if s.starts_with("afii") || s.starts_with("uni") {
876        return false;
877    }
878    if is_index_name(s) {
879        return true;
880    }
881    // Short alpha prefix (≤3 letters) followed by a run of ≥3 digits — synthetic
882    // code names like `SM590000`, distinct from real Adobe names (whole words or
883    // letter+`.suffix` variants).
884    let alpha = s.bytes().take_while(|b| b.is_ascii_alphabetic()).count();
885    let digits = s.len() - alpha;
886    (1..=3).contains(&alpha)
887        && digits >= 3
888        && s.as_bytes()[alpha..].iter().all(|b| b.is_ascii_digit())
889}
890
891/// The font's ascent and descent (1000-unit em) for its loose char boxes,
892/// and whether docling-parse 7.24 rejects its descriptor metrics
893/// (`invalid_vertical_metrics`, `init_ascent_and_descent`) — then it boxes
894/// each glyph by its embedded outline instead ([`Font::ink_y`]):
895///
896/// - an `/Ascent` or `/Descent` beyond ±8000 (`MAX_PLAUSIBLE_VERTICAL_METRIC`)
897///   is ignored for the `/FontBBox` edge, when that is usable;
898/// - metrics that do not straddle the baseline (`descent <= 0 <= ascent`,
899///   `descent < ascent` — `/Ascent 0 /Descent 0`, or a math font's two
900///   positive values) give way to a valid `/FontBBox`.
901fn font_ascent_descent(doc: &Document, fdict: &Dictionary, two_byte: bool) -> (f64, f64, bool) {
902    // For Type0, the descriptor lives on the descendant CIDFont.
903    let descr_owner = if two_byte {
904        fdict
905            .get(b"DescendantFonts")
906            .ok()
907            .and_then(|o| deref(doc, o))
908            .and_then(|o| match o {
909                Object::Array(a) => a.first(),
910                _ => None,
911            })
912            .and_then(|o| as_dict(doc, o))
913    } else {
914        Some(fdict)
915    };
916    let fd = descr_owner
917        .and_then(|d| d.get(b"FontDescriptor").ok())
918        .and_then(|o| as_dict(doc, o));
919    let read = |key: &[u8]| fd.and_then(|d| d.get(key).ok()).and_then(num);
920    let plausible = |v: f64| v.is_finite() && v.abs() <= 8000.0;
921    let bbox = fd
922        .and_then(|d| d.get(b"FontBBox").ok())
923        .and_then(|o| deref(doc, o))
924        .and_then(|o| match o {
925            Object::Array(a) if a.len() == 4 => {
926                let v: Vec<f64> = a.iter().filter_map(num).collect();
927                (v.len() == 4).then(|| [v[0], v[1], v[2], v[3]])
928            }
929            _ => None,
930        });
931    let usable = |v: f64| v.abs() > 1e-3 && plausible(v);
932    let mut invalid = false;
933    let asc = match read(b"Ascent") {
934        Some(v) if plausible(v) => v,
935        Some(_) => {
936            invalid = true;
937            bbox.map(|b| b[3]).filter(|&v| usable(v)).unwrap_or(750.0)
938        }
939        None => 750.0,
940    };
941    let desc = match read(b"Descent") {
942        Some(v) if plausible(v) => v,
943        Some(_) => {
944            invalid = true;
945            bbox.map(|b| b[1]).filter(|&v| usable(v)).unwrap_or(-250.0)
946        }
947        None => -250.0,
948    };
949    if !(desc <= 0.0 && asc >= 0.0 && desc < asc) {
950        if let Some(b) = bbox.filter(|b| {
951            b[1] <= 0.0 && b[3] >= 0.0 && b[1] < b[3] && plausible(b[1]) && plausible(b[3])
952        }) {
953            return (b[3], b[1], true);
954        }
955    }
956    // Some subsetted fonts carry a degenerate FontDescriptor (`/Ascent 0
957    // /Descent 0`) and no usable FontBBox either: the real metrics live in
958    // the font program. A zero-height loose box gives the line cells zero
959    // area and the layout's region/text assignment drops them (2305's
960    // References list lost every prose line, keeping only the URLs), so
961    // fall back to typical text metrics.
962    if asc - desc <= 1.0 {
963        return (750.0, -250.0, invalid);
964    }
965    (asc, desc, invalid)
966}
967
968/// The `/BaseFont` name with any `ABCDEF+` subset prefix stripped (#187).
969fn base_font_name(fdict: &Dictionary) -> Option<Vec<u8>> {
970    let name = fdict.get(b"BaseFont").ok()?.as_name().ok()?;
971    let stripped = match name.iter().position(|&b| b == b'+') {
972        Some(i) if i == 6 => &name[i + 1..],
973        _ => name,
974    };
975    Some(stripped.to_vec())
976}
977
978/// Simple-font widths: `/FirstChar` + `/Widths` array, `/MissingWidth` default.
979fn simple_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
980    let mut map = HashMap::new();
981    let first = fdict
982        .get(b"FirstChar")
983        .ok()
984        .and_then(|o| o.as_i64().ok())
985        .unwrap_or(0) as u32;
986    if let Some(Object::Array(arr)) = fdict.get(b"Widths").ok().and_then(|o| deref(doc, o)) {
987        for (i, w) in arr.iter().enumerate() {
988            if let Some(w) = num(w) {
989                map.insert(first + i as u32, w);
990            }
991        }
992    }
993    let dw = fdict
994        .get(b"FontDescriptor")
995        .ok()
996        .and_then(|o| as_dict(doc, o))
997        .and_then(|d| d.get(b"MissingWidth").ok())
998        .and_then(num)
999        .unwrap_or(0.0);
1000    (map, dw)
1001}
1002
1003/// CIDFont widths: the `/W` array on the descendant font (`/DW` default = 1000).
1004fn cid_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
1005    let mut map = HashMap::new();
1006    let Some(desc) = fdict
1007        .get(b"DescendantFonts")
1008        .ok()
1009        .and_then(|o| deref(doc, o))
1010        .and_then(|o| match o {
1011            Object::Array(a) => a.first(),
1012            _ => None,
1013        })
1014        .and_then(|o| as_dict(doc, o))
1015    else {
1016        return (map, 1000.0);
1017    };
1018    let dw = desc.get(b"DW").ok().and_then(num).unwrap_or(1000.0);
1019    if let Some(Object::Array(w)) = desc.get(b"W").ok().and_then(|o| deref(doc, o)) {
1020        let mut i = 0;
1021        while i < w.len() {
1022            let c = w.get(i).and_then(num);
1023            match (c, w.get(i + 1)) {
1024                // `c [w1 w2 ...]`: consecutive CIDs starting at c.
1025                (Some(c), Some(Object::Array(list))) => {
1026                    for (k, wv) in list.iter().enumerate() {
1027                        if let Some(wv) = num(wv) {
1028                            map.insert(c as u32 + k as u32, wv);
1029                        }
1030                    }
1031                    i += 2;
1032                }
1033                // `c_first c_last w`: a run all of width w.
1034                (Some(c1), Some(o2)) => {
1035                    if let (Some(c2), Some(wv)) = (num(o2), w.get(i + 2).and_then(num)) {
1036                        for cid in c1 as u32..=c2 as u32 {
1037                            map.insert(cid, wv);
1038                        }
1039                    }
1040                    i += 3;
1041                }
1042                _ => break,
1043            }
1044        }
1045    }
1046    (map, dw)
1047}
1048
1049fn num(o: &Object) -> Option<f64> {
1050    match o {
1051        Object::Integer(i) => Some(*i as f64),
1052        Object::Real(r) => Some(real(*r)),
1053        _ => None,
1054    }
1055}
1056
1057/// lopdf keeps a real as an `f32`; docling-parse reads the same token as a
1058/// double. The shortest decimal that round-trips the `f32` is the token the
1059/// file wrote (for any real written with the precision `f32` holds), so
1060/// `90.144 + 1.44` meets `91.584` here as it does there.
1061fn real(r: f32) -> f64 {
1062    use std::fmt::Write as _;
1063    let mut buf = String::with_capacity(16);
1064    let _ = write!(buf, "{r}");
1065    buf.parse().unwrap_or(r as f64)
1066}
1067
1068/// Decode a PDF string literal into codes, retaining packed UTF-16 pairs
1069/// as the original four-byte keys used by ToUnicode.
1070fn codes(font: &Font, bytes: &[u8]) -> Vec<u32> {
1071    if font.two_byte {
1072        let mut units = bytes
1073            .chunks(2)
1074            .map(|c| {
1075                if c.len() == 2 {
1076                    ((c[0] as u32) << 8) | c[1] as u32
1077                } else {
1078                    c[0] as u32
1079                }
1080            })
1081            .peekable();
1082        let utf16 = matches!(font.cid_fallback, CidFallback::UnicodeCodes { utf16: true });
1083        let mut out = Vec::with_capacity(bytes.len() / 2 + 1);
1084        while let Some(mut code) = units.next() {
1085            if utf16 && (0xd800..=0xdbff).contains(&code) {
1086                if let Some(low) = units.next_if(|u| (0xdc00..=0xdfff).contains(u)) {
1087                    code = (code << 16) | low;
1088                }
1089            }
1090            out.push(code);
1091        }
1092        out
1093    } else {
1094        bytes.iter().map(|&b| b as u32).collect()
1095    }
1096}
1097
1098/// A page's display box, in PDF user space: the `/CropBox` clipped to the
1099/// `/MediaBox`, both inherited through the page tree and normalized — a
1100/// missing or empty MediaBox is US Letter, an empty CropBox is the MediaBox
1101/// (pdfium's `CPDF_Page::UpdateDimensions`). pdfium reports the page size from
1102/// this box, renders exactly it, and translates every content coordinate so
1103/// its lower-left corner is the origin (`m_PageMatrix`); docling's backends
1104/// inherit that frame, so text cells, `prov` boxes and destinations all count
1105/// from the CropBox corner, not the MediaBox one. The parser used to flip
1106/// glyphs with the MediaBox *height* and no translation at all, so a page
1107/// whose boxes do not start at (0, 0) — a trimmed book page with
1108/// `MediaBox [-56 -58 576 723]` / `CropBox [1 -0.6 519 666]`, or a LaTeX
1109/// figure cropped to `[156 147 637 391]` — had its text displaced against the
1110/// rendered bitmap by the box offset, the bottom lines pushed past the page
1111/// edge and clamped to `t = b`.
1112#[derive(Debug, Clone, Copy, PartialEq)]
1113pub(crate) struct PageBox {
1114    /// Left edge, user space.
1115    pub l: f32,
1116    /// Bottom edge, user space.
1117    pub b: f32,
1118    pub w: f32,
1119    pub h: f32,
1120}
1121
1122impl PageBox {
1123    /// Top edge, user space — the y that becomes `0` in the y-down frame.
1124    pub fn top(&self) -> f32 {
1125        self.b + self.h
1126    }
1127}
1128
1129/// A page-tree rect attribute (`/MediaBox`, `/CropBox`), inherited from the
1130/// nearest ancestor that sets it, as normalized `(l, b, r, t)`.
1131fn inherited_rect(
1132    doc: &Document,
1133    page_id: lopdf::ObjectId,
1134    key: &[u8],
1135) -> Option<(f32, f32, f32, f32)> {
1136    let mut id = page_id;
1137    for _ in 0..32 {
1138        let dict = doc.get_object(id).ok()?.as_dict().ok()?;
1139        if let Some(Object::Array(a)) = dict.get(key).ok().and_then(|o| deref(doc, o)) {
1140            let v: Vec<f32> = a.iter().filter_map(|o| num(o).map(|x| x as f32)).collect();
1141            if v.len() == 4 && v.iter().all(|x| x.is_finite()) {
1142                return Some((
1143                    v[0].min(v[2]),
1144                    v[1].min(v[3]),
1145                    v[0].max(v[2]),
1146                    v[1].max(v[3]),
1147                ));
1148            }
1149            return None;
1150        }
1151        id = dict.get(b"Parent").ok()?.as_reference().ok()?;
1152    }
1153    None
1154}
1155
1156pub(crate) fn page_box(doc: &Document, page_id: lopdf::ObjectId) -> PageBox {
1157    let nonempty = |r: &(f32, f32, f32, f32)| r.2 > r.0 && r.3 > r.1;
1158    let media = inherited_rect(doc, page_id, b"MediaBox")
1159        .filter(nonempty)
1160        .unwrap_or((0.0, 0.0, 612.0, 792.0));
1161    let crop = inherited_rect(doc, page_id, b"CropBox")
1162        .map(|c| {
1163            (
1164                c.0.max(media.0),
1165                c.1.max(media.1),
1166                c.2.min(media.2),
1167                c.3.min(media.3),
1168            )
1169        })
1170        .filter(nonempty)
1171        .unwrap_or(media);
1172    PageBox {
1173        l: crop.0,
1174        b: crop.1,
1175        w: crop.2 - crop.0,
1176        h: crop.3 - crop.1,
1177    }
1178}
1179
1180/// Page size (width, height) in PDF points — the display box's, like pdfium's
1181/// `FPDF_GetPageWidthF/HeightF`.
1182fn page_size(doc: &Document, page_id: lopdf::ObjectId) -> (f32, f32) {
1183    let pb = page_box(doc, page_id);
1184    (pb.w, pb.h)
1185}
1186
1187/// Localize where a page's text is lost, for the `text_layer` diagnostic.
1188/// Extraction can come up empty at three different points — no content stream
1189/// reached the parser, the stream did not decode into operators, or it ran but
1190/// produced no glyphs (fonts/encodings) — and from the outside all three look
1191/// the same. Report them per page.
1192pub fn content_diagnosis(bytes: &[u8]) -> String {
1193    let Some(doc) = load_document(bytes) else {
1194        return "document does not load".into();
1195    };
1196    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1197    pages.sort_by_key(|(n, _)| *n);
1198    let mut out = String::new();
1199    let mut caches = DocCaches::default();
1200    for (n, pid) in pages.into_iter().take(4) {
1201        let content_bytes = doc.get_page_content(pid);
1202        let ops = lopdf::content::Content::decode(&content_bytes)
1203            .map(|c| c.operations.len())
1204            .ok();
1205        let res = page_res(&doc, pid);
1206        let fonts = res.map(|r| fonts_from_res(&doc, r, &mut caches).len());
1207        let glyphs = page_glyphs_cached(&doc, pid, &mut caches).len();
1208        out.push_str(&format!(
1209            "\n   page {n}: content {} B, ops {}, resources {}, fonts {}, glyphs {}",
1210            content_bytes.len(),
1211            ops.map_or("UNDECODABLE".to_string(), |n| n.to_string()),
1212            if res.is_some() { "ok" } else { "MISSING" },
1213            fonts.map_or("-".to_string(), |n| n.to_string()),
1214            glyphs,
1215        ));
1216    }
1217    out
1218}
1219
1220/// Is this "text layer" a vestige rather than the document's text?
1221///
1222/// Scanned forms often carry a handful of typed-in strings — a date filled
1223/// into three form fields, say — on top of pages that are otherwise images.
1224/// Treating that as a real text layer is the worst of both worlds: the text
1225/// path proudly extracts thirteen characters, and no OCR ever runs on the
1226/// letter the pages actually show. The reported form did exactly this (3
1227/// lines, 13 chars, 3 pages).
1228///
1229/// The rule is deliberately tight so genuinely sparse *digital* documents are
1230/// not misrouted into OCR: only a document averaging at most one line per page
1231/// **and** totalling fewer than 32 characters is called vestigial.
1232pub fn text_layer_is_vestigial(pages: &[crate::pdfium_backend::PdfPage]) -> bool {
1233    let mut tally = TextLayerTally::default();
1234    pages.iter().for_each(|p| tally.add(p));
1235    tally.is_vestigial()
1236}
1237
1238/// [`text_layer_is_vestigial`]'s counts, accumulated a page at a time so a
1239/// streaming caller need not keep the pages around to decide.
1240#[derive(Debug, Default, Clone, Copy)]
1241pub struct TextLayerTally {
1242    pages: usize,
1243    lines: usize,
1244    chars: usize,
1245}
1246
1247impl TextLayerTally {
1248    pub fn add(&mut self, page: &crate::pdfium_backend::PdfPage) {
1249        self.pages += 1;
1250        self.lines += page.cells.len();
1251        self.chars += page
1252            .cells
1253            .iter()
1254            .map(|c| c.text.chars().count())
1255            .sum::<usize>();
1256    }
1257
1258    /// The verdict over the pages added so far.
1259    pub fn is_vestigial(&self) -> bool {
1260        self.lines == 0 || (self.lines <= self.pages && self.chars < 32)
1261    }
1262
1263    /// Whether the pages added so far already rule vestigial out for a
1264    /// document of `total_pages`, whatever the remaining pages hold — both
1265    /// counts only grow, so a caller may stop parsing pages it does not need.
1266    pub fn proves_text(&self, total_pages: usize) -> bool {
1267        self.lines > 0 && (self.lines > total_pages || self.chars >= 32)
1268    }
1269}
1270
1271/// Why the cross-reference repair did or did not fire, for the `text_layer`
1272/// diagnostic. A PDF that will not load is indistinguishable from a scan in
1273/// production (both convert to nothing), so the reason has to be askable.
1274pub fn xref_repair_status(bytes: &[u8]) -> String {
1275    if Document::load_mem(bytes).is_ok() {
1276        return "loads unaided; no repair needed".into();
1277    }
1278    match pad_short_xref_entries(bytes) {
1279        Ok(fixed) => match Document::load_mem(&fixed) {
1280            Ok(_) => "repaired: cross-reference entries padded to 20 bytes".into(),
1281            Err(e) => format!("padded the entries, but it still will not load: {e}"),
1282        },
1283        Err(why) => format!("repair declined — {why}"),
1284    }
1285}
1286
1287/// Load a PDF, repairing the one malformation that otherwise costs us the whole
1288/// document: **19-byte cross-reference entries**.
1289///
1290/// The spec fixes an xref entry at 20 bytes — `nnnnnnnnnn ggggg n` plus a
1291/// *two*-byte EOL. Some generators (an Austrian telecom's invoices, for one)
1292/// emit a bare LF instead, making each entry 19 bytes. lopdf rejects the file
1293/// outright (`invalid file trailer`) where pdfium reads it happily, so a
1294/// perfectly good text layer looked to the browser exactly like a scan and cost
1295/// ten seconds of OCR.
1296///
1297/// Padding is only attempted when it cannot move anything the xref points at:
1298/// a single `xref` section that begins after the last object. The repair then
1299/// has to prove itself — the padded bytes are used only if they load — so a
1300/// mis-repair degrades to today's behaviour rather than to silent garbage.
1301pub(crate) fn load_document(bytes: &[u8]) -> Option<Document> {
1302    open_document(bytes, None).ok()
1303}
1304
1305/// Why [`open_document`] could not hand back a readable document.
1306#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1307pub(crate) enum OpenError {
1308    /// lopdf cannot read the file even after the repairs.
1309    Unreadable,
1310    /// The file is encrypted and the password given (or none) does not open it.
1311    Password,
1312}
1313
1314/// Load `bytes` with the document's password. lopdf decrypts while reading —
1315/// with `password`, or the empty user password most "protected" PDFs carry
1316/// (what every viewer opens silently) — so every reader downstream sees plain
1317/// streams; a file the password does not open loads with its streams still
1318/// encrypted, which is reported as [`OpenError::Password`] rather than handed
1319/// on as a document whose every stream decodes to nothing.
1320pub(crate) fn open_document(bytes: &[u8], password: Option<&str>) -> Result<Document, OpenError> {
1321    let Some(doc) = load_document_raw(bytes, password) else {
1322        // lopdf refuses to load at all under a *wrong* password (a missing
1323        // one loads the file with its streams still encrypted); tell the two
1324        // apart by loading without it.
1325        if password.is_some() && load_document_raw(bytes, None).is_some_and(|d| d.is_encrypted()) {
1326            return Err(OpenError::Password);
1327        }
1328        return Err(OpenError::Unreadable);
1329    };
1330    if doc.is_encrypted() {
1331        return Err(OpenError::Password);
1332    }
1333    Ok(doc)
1334}
1335
1336fn load_options(password: Option<&str>) -> lopdf::LoadOptions {
1337    lopdf::LoadOptions {
1338        password: password.map(str::to_string),
1339        ..lopdf::LoadOptions::default()
1340    }
1341}
1342
1343fn load_document_raw(bytes: &[u8], password: Option<&str>) -> Option<Document> {
1344    // Try progressively more repair, and accept a candidate only once the pages
1345    // actually carry content — a document whose streams were dropped still
1346    // "loads", so loading alone is not evidence the repair helped. A
1347    // well-formed file returns on the first attempt and pays for nothing.
1348    let mut fallback = None;
1349    if let Some(doc) = best_effort_load(bytes, password, &mut fallback) {
1350        return Some(doc);
1351    }
1352    let xref_fixed = pad_short_xref_entries(bytes).ok();
1353    if let Some(fixed) = &xref_fixed {
1354        if let Some(doc) = best_effort_load(fixed, password, &mut fallback) {
1355            return Some(doc);
1356        }
1357    }
1358    // Both defects can coexist, and the second only becomes visible once the
1359    // first is repaired, so build on whatever the previous step produced.
1360    let lengths_fixed = fix_stream_lengths(xref_fixed.as_deref().unwrap_or(bytes));
1361    if let Some(doc) = best_effort_load(&lengths_fixed, password, &mut fallback) {
1362        return Some(doc);
1363    }
1364    fallback
1365}
1366
1367/// Load `data`, returning it only when its pages carry content; a document that
1368/// merely parses is remembered as the fallback for when nothing does better.
1369fn best_effort_load(
1370    data: &[u8],
1371    password: Option<&str>,
1372    fallback: &mut Option<Document>,
1373) -> Option<Document> {
1374    match Document::load_mem_with_options(data, load_options(password)) {
1375        Ok(doc) if has_page_content(&doc) => Some(doc),
1376        Ok(doc) => {
1377            fallback.get_or_insert(doc);
1378            None
1379        }
1380        Err(_) => None,
1381    }
1382}
1383
1384/// Does any page actually hand us a content stream? A document whose streams
1385/// were dropped still parses — it simply has nothing to read — so this is what
1386/// tells a successful repair from a pointless one.
1387fn has_page_content(doc: &Document) -> bool {
1388    doc.get_pages()
1389        .into_values()
1390        .take(4)
1391        .any(|pid| !doc.get_page_content(pid).is_empty())
1392}
1393
1394/// Correct `/Length` values that disagree with where `endstream` actually is.
1395///
1396/// The same generator that writes short xref entries also overstates its
1397/// content-stream lengths by a byte or two. lopdf trusts `/Length`, reads past
1398/// the data, fails to find `endstream` there and drops the stream — the object
1399/// comes back as a bare dictionary, so the page has no content at all and the
1400/// document looks like a scan. pdfium instead trusts `endstream`, which is what
1401/// this does.
1402///
1403/// The rewrite is length-preserving: the corrected number is written over the
1404/// old digits and padded with spaces, so every byte offset in the file — and
1405/// therefore the whole cross-reference table — stays valid.
1406fn fix_stream_lengths(bytes: &[u8]) -> Vec<u8> {
1407    let mut out = bytes.to_vec();
1408    let mut i = 0;
1409    while let Some(rel) = find(&out[i..], b"stream") {
1410        let kw = i + rel;
1411        i = kw + 6;
1412        // Skip `endstream` (the keyword we are measuring *to*).
1413        if kw >= 3 && &out[kw - 3..kw] == b"end" {
1414            continue;
1415        }
1416        // The stream data starts after the EOL that follows the keyword.
1417        let mut data = kw + 6;
1418        if out.get(data..data + 2) == Some(b"\r\n".as_slice()) {
1419            data += 2;
1420        } else if matches!(out.get(data), Some(b'\n' | b'\r')) {
1421            data += 1;
1422        }
1423        let Some(end) = find(&out[data..], b"endstream").map(|r| data + r) else {
1424            continue;
1425        };
1426        // `/Length <digits>` in the dictionary just before the keyword.
1427        let dict_start = out[..kw].iter().rposition(|&c| c == b'<').unwrap_or(0);
1428        let Some(lrel) = find(&out[dict_start..kw], b"/Length") else {
1429            continue;
1430        };
1431        let mut d = dict_start + lrel + 7;
1432        while matches!(out.get(d), Some(b' ')) {
1433            d += 1;
1434        }
1435        let digits = out[d..].iter().take_while(|c| c.is_ascii_digit()).count();
1436        if digits == 0 {
1437            continue;
1438        }
1439        let declared: usize = match std::str::from_utf8(&out[d..d + digits])
1440            .ok()
1441            .and_then(|s| s.parse().ok())
1442        {
1443            Some(v) => v,
1444            None => continue,
1445        };
1446        let actual = end - data;
1447        // Only shrink, and only when the new value fits the space the old one
1448        // occupied — growing the number would move every following byte.
1449        let replacement = actual.to_string();
1450        if actual == declared || replacement.len() > digits {
1451            continue;
1452        }
1453        out[d..d + digits].fill(b' ');
1454        out[d..d + replacement.len()].copy_from_slice(replacement.as_bytes());
1455    }
1456    out
1457}
1458
1459fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
1460    haystack.windows(needle.len()).position(|w| w == needle)
1461}
1462
1463/// Rewrite a classic cross-reference table's entries to the spec's 20 bytes,
1464/// or `None` when the file's shape makes that unsafe (see [`load_document`]).
1465pub(crate) fn pad_short_xref_entries(bytes: &[u8]) -> Result<Vec<u8>, &'static str> {
1466    // Exactly one xref section, and it must start after every object, so that
1467    // growing it shifts nothing the table's offsets refer to.
1468    let is_boundary = |i: usize| i == 0 || matches!(bytes[i - 1], b'\n' | b'\r');
1469    let mut starts = (0..bytes.len().saturating_sub(4))
1470        .filter(|&i| &bytes[i..i + 4] == b"xref" && is_boundary(i));
1471    let xref_at = starts
1472        .next()
1473        .ok_or("no classic `xref` section (an xref stream?)")?;
1474    if starts.next().is_some() {
1475        return Err("more than one xref section (incremental update)");
1476    }
1477    let last_obj = bytes
1478        .windows(3)
1479        .rposition(|w| w == b"obj")
1480        .ok_or("no objects found")?;
1481    if last_obj > xref_at {
1482        return Err("an object follows the xref — padding would move it");
1483    }
1484
1485    let mut out = bytes[..xref_at].to_vec();
1486    out.extend_from_slice(b"xref\n");
1487    let mut i = xref_at + 4;
1488    let skip_ws = |i: &mut usize| {
1489        while matches!(bytes.get(*i), Some(b'\r' | b'\n' | b' ')) {
1490            *i += 1;
1491        }
1492    };
1493    loop {
1494        skip_ws(&mut i);
1495        // Either the next subsection header ("first count") or the trailer.
1496        if bytes[i..].starts_with(b"trailer") {
1497            out.extend_from_slice(&bytes[i..]);
1498            return Ok(out);
1499        }
1500        let header_end = i + bytes[i..]
1501            .iter()
1502            .position(|c| matches!(c, b'\n' | b'\r'))
1503            .ok_or("subsection header runs off the end")?;
1504        let header = std::str::from_utf8(&bytes[i..header_end])
1505            .map_err(|_| "subsection header is not text")?
1506            .trim();
1507        let mut parts = header.split_whitespace();
1508        let count: usize = parts
1509            .nth(1)
1510            .and_then(|c| c.parse().ok())
1511            .ok_or("unparseable subsection header")?;
1512        if parts.next().is_some() || count == 0 {
1513            return Err("unexpected subsection header shape");
1514        }
1515        out.extend_from_slice(header.as_bytes());
1516        out.push(b'\n');
1517        i = header_end;
1518        for _ in 0..count {
1519            skip_ws(&mut i);
1520            // `nnnnnnnnnn ggggg n` — the 18 bytes before whatever EOL follows.
1521            let entry = bytes.get(i..i + 18).ok_or("xref entry runs off the end")?;
1522            let well_formed = entry[..10].iter().all(u8::is_ascii_digit)
1523                && entry[10] == b' '
1524                && entry[11..16].iter().all(u8::is_ascii_digit)
1525                && entry[16] == b' '
1526                && matches!(entry[17], b'n' | b'f');
1527            if !well_formed {
1528                return Err("xref entry is not `nnnnnnnnnn ggggg n`");
1529            }
1530            out.extend_from_slice(entry);
1531            out.extend_from_slice(b" \n"); // the spec's 2-byte EOL -> 20 bytes
1532            i += 18;
1533        }
1534    }
1535}
1536
1537/// Debug: raw glyph stream `(ch, ll, lr, lb, lt)` (native coords) for page
1538/// `index`, before the sanitizer. For comparing char cells to docling-parse.
1539pub fn debug_glyphs(bytes: &[u8], index: usize) -> Vec<(char, f32, f32, f32, f32)> {
1540    let Some(doc) = load_document(bytes) else {
1541        return Vec::new();
1542    };
1543    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1544    pages.sort_by_key(|(n, _)| *n);
1545    let Some((_, pid)) = pages.get(index) else {
1546        return Vec::new();
1547    };
1548    page_glyphs(&doc, *pid)
1549        .into_iter()
1550        .map(|g| (g.ch, g.ll, g.lr, g.lb, g.lt))
1551        .collect()
1552}
1553
1554/// Public entry: per-page (width, height, line cells) for a PDF, via the Rust
1555/// text parser + the docling-parse line sanitizer. Used by the pipeline and the
1556/// `textparse_dump` example.
1557pub fn pdf_textlines(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1558    let Some(doc) = load_document(bytes) else {
1559        return Vec::new();
1560    };
1561    let mut caches = DocCaches::default();
1562    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1563    pages.sort_by_key(|(n, _)| *n);
1564    pages
1565        .into_iter()
1566        .map(|(_, pid)| {
1567            let (w, h) = page_size(&doc, pid);
1568            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1569            let cells = crate::dp_lines::line_cells(&glyphs, h, true);
1570            (w, h, cells)
1571        })
1572        .collect()
1573}
1574
1575/// Debug/diagnostic entry: per-page (width, height, word cells) for a PDF, via
1576/// the Rust parser glyphs run through the docling-parse word grouping. Used to
1577/// compare parser word cells against docling-parse's `word_cells` oracle (roadmap
1578/// item 6).
1579pub fn pdf_words(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1580    let Some(doc) = load_document(bytes) else {
1581        return Vec::new();
1582    };
1583    let mut caches = DocCaches::default();
1584    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1585    pages.sort_by_key(|(n, _)| *n);
1586    pages
1587        .into_iter()
1588        .map(|(_, pid)| {
1589            let (w, h) = page_size(&doc, pid);
1590            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1591            let cells = crate::dp_lines::word_cells(&glyphs, h, true);
1592            (w, h, cells)
1593        })
1594        .collect()
1595}
1596
1597/// One page's text cells from the pure-Rust parser: prose line cells, per-word
1598/// cells, and code line cells — all from a single glyph parse. Replaces the
1599/// pdfium text path (roadmap item 6) when the parser drop is enabled.
1600#[derive(Default)]
1601pub struct PageParserCells {
1602    pub prose: Vec<crate::pdfium_backend::TextCell>,
1603    pub words: Vec<crate::pdfium_backend::TextCell>,
1604    pub code: Vec<crate::pdfium_backend::TextCell>,
1605    /// Drawn checkbox squares (#609), in the cells' frame.
1606    pub checkboxes: Vec<crate::checkbox::CheckBox>,
1607    /// Image placements (#999), in the cells' frame — see
1608    /// [`crate::pdfium_backend::PdfPage::bitmaps`].
1609    pub bitmaps: Vec<[f32; 4]>,
1610    /// The page's shapes for the reading-order separators (#1092).
1611    pub shapes: PageShapes,
1612}
1613
1614/// The parser text layer, driven one page at a time: the document is loaded
1615/// (and repaired, see [`load_document`]) once, the font/form caches persist
1616/// across pages, and each page's glyphs are parsed only when asked for.
1617///
1618/// The eager whole-document walk this replaces ran *before* the first page
1619/// was rendered, so on a long PDF it was a serial prefix the page-worker pool
1620/// sat idle through — 6.2 s on the 1913-page .NET reference, in front of a
1621/// pipeline that otherwise overlaps parsing with inference — and a `--pages`
1622/// window still paid for every page in the file. Pulling pages on demand
1623/// keeps the parse on the producer thread but interleaved with rendering,
1624/// and skips unselected pages entirely. Output per page is unchanged: same
1625/// glyph walk, same shared caches, same contraction.
1626pub struct PageTextParser {
1627    doc: Document,
1628    caches: DocCaches,
1629    /// Page object ids in document order (page 1 first).
1630    pages: Vec<lopdf::ObjectId>,
1631    /// How words split where no space glyph does (#921); from
1632    /// `DOCLING_RS_PDF_WORD_GAP` unless [`Self::set_word_gap`] says.
1633    word_gap: crate::dp_lines::WordGap,
1634    /// Whether [`Self::text_page`]'s page keeps typographic punctuation as
1635    /// drawn (#921); `DOCLING_RS_PDF_KEEP_TYPOGRAPHY` unless
1636    /// [`Self::set_keep_typography`] says.
1637    keep_typography: bool,
1638}
1639
1640impl PageTextParser {
1641    /// Load the document; `None` when lopdf cannot read it at all.
1642    pub fn open(bytes: &[u8]) -> Option<Self> {
1643        Self::open_with_password(bytes, None)
1644    }
1645
1646    /// [`open`](Self::open) with the document's password (an encrypted file
1647    /// the password does not open reads as `None` here; the pipeline reports
1648    /// it through [`crate::pdf_meta::PdfMeta::open_with_password`] first).
1649    pub fn open_with_password(bytes: &[u8], password: Option<&str>) -> Option<Self> {
1650        let doc = open_document(bytes, password).ok()?;
1651        let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1652        pages.sort_by_key(|(n, _)| *n);
1653        Some(Self {
1654            doc,
1655            caches: DocCaches::default(),
1656            pages: pages.into_iter().map(|(_, pid)| pid).collect(),
1657            word_gap: crate::dp_lines::WordGap::from_env(),
1658            keep_typography: crate::assemble::keep_typography_from_env(),
1659        })
1660    }
1661
1662    /// The word-gap rule the line and word cells are built with (#921);
1663    /// `None` keeps the environment's (`DOCLING_RS_PDF_WORD_GAP`, else
1664    /// `adaptive`).
1665    pub fn set_word_gap(&mut self, gap: Option<crate::dp_lines::WordGap>) {
1666        if let Some(g) = gap {
1667            self.word_gap = g;
1668        }
1669    }
1670
1671    /// Whether the pages [`Self::text_page`] builds keep typographic
1672    /// punctuation as drawn (#921); `None` keeps the environment's
1673    /// (`DOCLING_RS_PDF_KEEP_TYPOGRAPHY`, else folding).
1674    pub fn set_keep_typography(&mut self, keep: Option<bool>) {
1675        if let Some(k) = keep {
1676            self.keep_typography = k;
1677        }
1678    }
1679
1680    /// The glyph boxes and font styles of the 0-based page `index` — the
1681    /// heading-hierarchy stage's style signal (#302): every non-space glyph's
1682    /// box (font ascent + descent at its size — the font-size proxy pdfium's
1683    /// loose char box also gave) in top-left coordinates, with the weight
1684    /// class and slant its `/BaseFont` name declares. Empty for a page
1685    /// without a text layer (a scan), and the stage falls back to its other
1686    /// signals.
1687    pub(crate) fn glyph_styles(
1688        &mut self,
1689        index: usize,
1690    ) -> Vec<crate::heading_hierarchy::GlyphStyle> {
1691        let Some(&pid) = self.pages.get(index) else {
1692            return Vec::new();
1693        };
1694        let (_w, h) = page_size(&self.doc, pid);
1695        let glyphs = page_glyphs_cached(&self.doc, pid, &mut self.caches);
1696        // Font hash → style, over the fonts the walk just parsed (an inline,
1697        // uncached font dictionary reads as unstyled).
1698        let styles: HashMap<u64, crate::font_style::FontStyle> = self
1699            .caches
1700            .fonts
1701            .values()
1702            .map(|f| (f.hash, f.style))
1703            .collect();
1704        glyphs
1705            .iter()
1706            .filter(|g| !g.ch.is_whitespace() && g.ll.is_finite())
1707            .map(|g| {
1708                let st = styles.get(&g.font).copied().unwrap_or_default();
1709                crate::heading_hierarchy::GlyphStyle {
1710                    l: g.ll,
1711                    t: h - g.lt,
1712                    r: g.lr,
1713                    b: h - g.lb,
1714                    height: g.height(),
1715                    weight_cls: crate::font_style::weight_class(st.weight),
1716                    italic: st.italic,
1717                    styled: st.known,
1718                }
1719            })
1720            .collect()
1721    }
1722
1723    /// The characters of the 0-based page `index` as the script-position
1724    /// stage measures them (#1005, docling's `read_page_chars`): every
1725    /// visible, non-blank glyph with its font-metric box, baseline and drawn
1726    /// size in the unrotated page frame (y up), and beside it the same box in
1727    /// the display frame (`/Rotate` applied, top-left origin: the frame the
1728    /// items' boxes live in). Invisible text (`Tr` 3 / 7, an OCR layer over a
1729    /// scan) is left out, as docling leaves it: it follows the scanned lines,
1730    /// not a typeset baseline.
1731    pub(crate) fn script_chars(
1732        &mut self,
1733        index: usize,
1734    ) -> Vec<(crate::script_position::ScriptChar, [f32; 4])> {
1735        let Some(&pid) = self.pages.get(index) else {
1736            return Vec::new();
1737        };
1738        let (w, h) = page_size(&self.doc, pid);
1739        let rotation = page_rotation(&self.doc, pid);
1740        page_glyphs_cached(&self.doc, pid, &mut self.caches)
1741            .iter()
1742            .filter(|g| !g.invisible && !g.ch.is_whitespace() && g.ll.is_finite())
1743            .filter_map(|g| {
1744                let [_, y, size] = g.pen?;
1745                let c = crate::script_position::ScriptChar {
1746                    text: g.ch,
1747                    left: g.ll,
1748                    bottom: g.lb,
1749                    right: g.lr,
1750                    top: g.lt,
1751                    baseline: y,
1752                    size,
1753                    horizontal: g.quad.is_none(),
1754                };
1755                let (l, t, r, b) = crate::pdfium_backend::to_display_frame(
1756                    (g.ll, h - g.lt, g.lr, h - g.lb),
1757                    rotation,
1758                    w,
1759                    h,
1760                );
1761                Some((c, [l, t, r, b]))
1762            })
1763            .collect()
1764    }
1765
1766    /// Prose, word and code cells of the 0-based page `index` — empty for an
1767    /// index the parser's page tree doesn't have (a damaged file whose page
1768    /// tree disagrees with the object model's count).
1769    pub fn cells(&mut self, index: usize) -> PageParserCells {
1770        let Some(&pid) = self.pages.get(index) else {
1771            return PageParserCells::default();
1772        };
1773        let (_w, h) = page_size(&self.doc, pid);
1774        let (glyphs, inks, bitmaps, shapes) =
1775            page_glyphs_and_inks(&self.doc, pid, &mut self.caches);
1776        let (prose, words) =
1777            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1778        PageParserCells {
1779            prose,
1780            words,
1781            code: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1782            checkboxes: crate::checkbox::find(&inks, h),
1783            bitmaps,
1784            shapes,
1785        }
1786    }
1787
1788    /// Pages in the document's page tree.
1789    pub fn page_count(&self) -> usize {
1790        self.pages.len()
1791    }
1792
1793    /// The 0-based page `index` as the text-layer-only conversion sees it —
1794    /// one element of [`pdf_text_pages`], parsed on demand so a caller can
1795    /// drop each page once it is assembled instead of holding the whole
1796    /// document's cells. A blank page (no cells) for an index the page tree
1797    /// doesn't have.
1798    pub fn text_page(&mut self, index: usize) -> crate::pdfium_backend::PdfPage {
1799        let (w, h, (glyphs, inks, _, shapes)) = match self.pages.get(index) {
1800            Some(&pid) => {
1801                let (w, h) = page_size(&self.doc, pid);
1802                (w, h, page_glyphs_and_inks(&self.doc, pid, &mut self.caches))
1803            }
1804            None => (0.0, 0.0, Default::default()),
1805        };
1806        let (mut prose, mut words) =
1807            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1808        drop_overpainted_cells(&mut prose);
1809        drop_overpainted_cells(&mut words);
1810        crate::pdfium_backend::PdfPage {
1811            #[cfg(feature = "ocr-prep")]
1812            image_layout: None,
1813            width: w,
1814            height: h,
1815            // Cells are native PDF points; there is no rendered bitmap.
1816            scale: 1.0,
1817            cells: prose,
1818            code_cells: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1819            word_cells: words,
1820            checkboxes: crate::checkbox::find(&inks, h),
1821            bitmaps: Vec::new(),
1822            shapes,
1823            #[cfg(feature = "ocr-prep")]
1824            image: image::RgbImage::new(1, 1),
1825            links: Vec::new(),
1826            rotation: 0,
1827            keep_typography: self.keep_typography,
1828            #[cfg(feature = "render")]
1829            embedded_images: Vec::new(),
1830            #[cfg(feature = "render")]
1831            pending_image: None,
1832        }
1833    }
1834}
1835
1836/// Full parser text layer: prose + word + code cells per page, glyphs parsed once.
1837/// `prose`/`words` come from the docling-parse contraction ([`crate::dp_lines`]);
1838/// `code` splits only at the parser's own space glyphs (monospace keeps its
1839/// source spacing). The eager form of [`PageTextParser`].
1840pub fn pdf_all_cells(bytes: &[u8]) -> Vec<PageParserCells> {
1841    let Some(mut parser) = PageTextParser::open(bytes) else {
1842        return Vec::new();
1843    };
1844    (0..parser.pages.len()).map(|i| parser.cells(i)).collect()
1845}
1846
1847/// Whole pages for the text-layer-only conversion ([`crate::convert_text_layer`]):
1848/// the parser's prose/word/code cells plus page geometry, assembled into
1849/// [`PdfPage`]s with no rendered image and no link annotations. Everything here
1850/// is pure Rust (lopdf), so it compiles without the `ml` feature — including on
1851/// wasm32. A page the parser can't read (no text layer) comes back with empty
1852/// cells; there is no pdfium fallback on this path.
1853pub fn pdf_text_pages(bytes: &[u8]) -> Vec<crate::pdfium_backend::PdfPage> {
1854    let Some(mut parser) = PageTextParser::open(bytes) else {
1855        return Vec::new();
1856    };
1857    (0..parser.page_count())
1858        .map(|i| parser.text_page(i))
1859        .collect()
1860}
1861
1862/// Drop line cells that are *painted over each other* — glyphs used as artwork.
1863///
1864/// Some generators draw their logo with a symbol font: on the reporting
1865/// invoice, a `TeleLogo` Type1 paints the T-Mobile mark by stacking the glyphs
1866/// encoded as `"` and `==` on top of one another, and the flat text-layer
1867/// output opened with that garbage. Nothing in the font metadata gives it away
1868/// (the *text* fonts in the same file are also flagged symbolic, and the logo
1869/// font names its glyphs `quotedbl` &c.), but the geometry does: two cells with
1870/// different text where one lies inside the other on the same line is
1871/// physically impossible for prose — ink from two words never occupies the
1872/// same box. Both cells of such a pair are paint, not text.
1873///
1874/// Containment (not mere overlap) keeps this narrow: adjacent words touch but
1875/// never contain each other, and a same-text near-duplicate (double-draw faux
1876/// bold) is left alone for the sanitizer's usual handling. Applied on the
1877/// flat/browser path only — the ML pipeline's text layer is byte-pinned by the
1878/// PDF corpus, and there the layout model already sinks logo marks into
1879/// `picture` regions.
1880fn drop_overpainted_cells(cells: &mut Vec<crate::pdfium_backend::TextCell>) {
1881    let mut paint = vec![false; cells.len()];
1882    for i in 0..cells.len() {
1883        for j in 0..cells.len() {
1884            if i == j || cells[i].text == cells[j].text {
1885                continue;
1886            }
1887            let (a, b) = (&cells[i], &cells[j]);
1888            // Same line band: the vertical overlap covers most of the shorter.
1889            let vo = (a.b.min(b.b) - a.t.max(b.t)).max(0.0);
1890            if vo < 0.6 * (a.b - a.t).min(b.b - b.t) {
1891                continue;
1892            }
1893            // `a` horizontally inside `b` (with a small tolerance).
1894            let ho = (a.r.min(b.r) - a.l.max(b.l)).max(0.0);
1895            if ho >= 0.8 * (a.r - a.l) && (a.r - a.l) <= (b.r - b.l) {
1896                paint[i] = true;
1897                paint[j] = true;
1898            }
1899        }
1900    }
1901    let mut keep = paint.iter().map(|p| !p);
1902    cells.retain(|_| keep.next().unwrap());
1903}
1904
1905/// The text-state scalars inherited by a Form XObject when it is invoked via
1906/// `Do` (the PDF graphics state includes the text parameters, but not the text
1907/// matrices, which a form re-establishes inside its own `BT`/`ET`).
1908#[derive(Clone, Copy)]
1909struct TextState {
1910    tc: f64,
1911    tw: f64,
1912    th: f64,
1913    tl: f64,
1914    trise: f64,
1915    fsize: f64,
1916    /// The text rendering mode (`Tr`).
1917    tr: i64,
1918}
1919
1920impl TextState {
1921    const INIT: TextState = TextState {
1922        tc: 0.0,
1923        tw: 0.0,
1924        th: 1.0,
1925        tl: 0.0,
1926        trise: 0.0,
1927        fsize: 0.0,
1928        tr: 0,
1929    };
1930}
1931
1932/// pdfium's `CPDF_Page::GetPageRotation` — the inherited `/Rotate`, in
1933/// quarter turns, as degrees (`pdf_meta`'s rule).
1934fn page_rotation(doc: &Document, pid: lopdf::ObjectId) -> u16 {
1935    let mut id = pid;
1936    for _ in 0..32 {
1937        let Ok(dict) = doc.get_dictionary(id) else {
1938            return 0;
1939        };
1940        if let Some(obj) = dict.get(b"Rotate").ok().and_then(|o| deref(doc, o)) {
1941            let quarter = (num(obj).unwrap_or(0.0) as i64 / 90).rem_euclid(4);
1942            return (quarter * 90) as u16;
1943        }
1944        match dict.get(b"Parent").ok().and_then(|o| o.as_reference().ok()) {
1945            Some(parent) => id = parent,
1946            None => return 0,
1947        }
1948    }
1949    0
1950}
1951
1952/// The effective `/Resources` dictionary for a page (inline or via reference,
1953/// falling back to an inherited one from a `/Parent`).
1954fn page_res(doc: &Document, page_id: lopdf::ObjectId) -> Option<&Dictionary> {
1955    let (inline, ids) = doc.get_page_resources(page_id).ok()?;
1956    if let Some(d) = inline {
1957        return Some(d);
1958    }
1959    ids.into_iter().find_map(|id| doc.get_dictionary(id).ok())
1960}
1961
1962/// Build the code→[`Font`] map for a resources dictionary's `/Font` sub-dict,
1963/// reusing the per-document cache for fonts referenced indirectly (the common
1964/// case — the same font objects recur on every page).
1965fn fonts_from_res(
1966    doc: &Document,
1967    res: &Dictionary,
1968    caches: &mut DocCaches,
1969) -> HashMap<Vec<u8>, Arc<Font>> {
1970    let mut map = HashMap::new();
1971    let font_dict = res
1972        .get(b"Font")
1973        .ok()
1974        .and_then(|o| deref(doc, o))
1975        .and_then(|o| o.as_dict().ok());
1976    if let Some(fd) = font_dict {
1977        for (name, value) in fd.iter() {
1978            let font = match value {
1979                Object::Reference(id) => {
1980                    let key = (*id, name.clone());
1981                    if let Some(f) = caches.fonts.get(&key) {
1982                        Arc::clone(f)
1983                    } else if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
1984                        let f = Arc::new(parse_font(doc, name, fdict));
1985                        caches.fonts.insert(key, Arc::clone(&f));
1986                        f
1987                    } else {
1988                        continue;
1989                    }
1990                }
1991                _ => {
1992                    if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
1993                        Arc::new(parse_font(doc, name, fdict))
1994                    } else {
1995                        continue;
1996                    }
1997                }
1998            };
1999            map.insert(name.clone(), font);
2000        }
2001    }
2002    map
2003}
2004
2005/// Extract every glyph on a page as a native-coordinate [`Glyph`].
2006/// [`PageTextParser::glyph_styles`] for the given **1-based** pages of a
2007/// document, keyed by page number — a separate, on-demand pass (no
2008/// rendering), so the extraction pipeline stays byte-identical whether or not
2009/// the heading-hierarchy stage runs. Empty when lopdf cannot open the file.
2010pub(crate) fn glyph_styles(
2011    bytes: &[u8],
2012    pages: &[usize],
2013) -> HashMap<usize, Vec<crate::heading_hierarchy::GlyphStyle>> {
2014    let mut out = HashMap::new();
2015    let Some(mut parser) = PageTextParser::open(bytes) else {
2016        return out;
2017    };
2018    for &page_no in pages {
2019        if page_no == 0 {
2020            continue;
2021        }
2022        let styles = parser.glyph_styles(page_no - 1);
2023        if !styles.is_empty() {
2024            out.insert(page_no, styles);
2025        }
2026    }
2027    out
2028}
2029
2030pub(crate) fn page_glyphs(doc: &Document, page_id: lopdf::ObjectId) -> Vec<Glyph> {
2031    page_glyphs_cached(doc, page_id, &mut DocCaches::default())
2032}
2033
2034/// [`page_glyphs`] with an explicit per-document cache, so a multi-page walk
2035/// parses each font / decodes each form once instead of once per page.
2036fn page_glyphs_cached(
2037    doc: &Document,
2038    page_id: lopdf::ObjectId,
2039    caches: &mut DocCaches,
2040) -> Vec<Glyph> {
2041    page_glyphs_and_inks(doc, page_id, caches).0
2042}
2043
2044/// [`page_glyphs_cached`] plus the small painted path pieces the same walk
2045/// passed — what [`crate::checkbox::find`] looks for checkbox squares in
2046/// (#609) — the page's image placements (#999, `[l, t, r, b]` top-left
2047/// page points clipped to the page, the cells' frame), and its shapes for the
2048/// reading-order separators (#1092, [`PageShapes`]).
2049fn page_glyphs_and_inks(
2050    doc: &Document,
2051    page_id: lopdf::ObjectId,
2052    caches: &mut DocCaches,
2053) -> (
2054    Vec<Glyph>,
2055    Vec<crate::checkbox::Ink>,
2056    Vec<[f32; 4]>,
2057    PageShapes,
2058) {
2059    let mut out = Vec::new();
2060    let mut inks = Vec::new();
2061    // get_page_content returns the assembled content-stream bytes directly
2062    // (an empty Vec when the page has none).
2063    let content_bytes = doc.get_page_content(page_id);
2064    let Ok(content) = lopdf::content::Content::decode(&content_bytes) else {
2065        return (out, inks, Vec::new(), PageShapes::default());
2066    };
2067    let mut bitmaps = Vec::new();
2068    let mut page_shapes = PageShapes::default();
2069    if let Some(res) = page_res(doc, page_id) {
2070        // pdfium's page matrix: user space translated so the display box's
2071        // lower-left corner is the origin (see [`PageBox`]).
2072        let pb = page_box(doc, page_id);
2073        let base = Mat {
2074            e: -(pb.l as f64),
2075            f: -(pb.b as f64),
2076            ..Mat::ID
2077        };
2078        let mut shapes = Shapes::default();
2079        run_content(
2080            doc,
2081            res,
2082            &content,
2083            base,
2084            TextState::INIT,
2085            0,
2086            caches,
2087            &mut out,
2088            &mut inks,
2089            ShapeGs::INIT,
2090            &mut shapes,
2091        );
2092        out.retain(|g| on_page(g, pb.w, pb.h));
2093        drop_overprinted_spaces(&mut out);
2094        drop_shadow_runs(&mut out);
2095        let (w, h) = (f64::from(pb.w), f64::from(pb.h));
2096        inks.retain(|ink| match *ink {
2097            crate::checkbox::Ink::Bitmap { l, b, r, t } => {
2098                let (l, r, b, t) = (l.max(0.0), r.min(w), b.max(0.0), t.min(h));
2099                if r > l && t > b {
2100                    bitmaps.push([l as f32, (h - t) as f32, r as f32, (h - b) as f32]);
2101                }
2102                false
2103            }
2104            _ => true,
2105        });
2106        let top_left =
2107            |[l, b, r, t]: [f64; 4]| [l as f32, (h - t) as f32, r as f32, (h - b) as f32];
2108        page_shapes.lines = shapes.lines.into_iter().map(top_left).collect();
2109        if shapes.boxes.len() <= MAX_SHAPE_BOXES {
2110            page_shapes.boxes = connected_boxes(&shapes.boxes)
2111                .into_iter()
2112                .map(top_left)
2113                .collect();
2114        } else {
2115            docling_core::debug_log!(
2116                "textparse: {} painted paths on the page; their boxes are not merged",
2117                shapes.boxes.len()
2118            );
2119        }
2120    }
2121    (out, inks, bitmaps, page_shapes)
2122}
2123
2124/// A page's shapes for the reading-order separators (#1092), in the cells'
2125/// frame (`[l, t, r, b]`, top-left page points): docling-parse's
2126/// `get_shape_lines` and `get_connected_shape_bounding_boxes`.
2127#[derive(Default, Clone, Debug, PartialEq)]
2128pub struct PageShapes {
2129    pub lines: Vec<[f32; 4]>,
2130    pub boxes: Vec<[f32; 4]>,
2131}
2132
2133/// Drop a run of glyphs painted again over itself — a text shadow. A word
2134/// processor exports a heading with a shadow effect as the same run drawn
2135/// twice: the shadow, then the text over it, shifted by a fraction of a point
2136/// (0.5–0.8 pt up and left, black under red, in the manuals that showed it).
2137/// docling-parse keeps both copies and reads `Safety instructions Safety
2138/// instructions`; pdfium's text page, behind docling's pypdfium2 backend,
2139/// drops the second, and the page shows the heading once. A deliberate
2140/// divergence (`docs/MIGRATION.md` §4 item 25): a glyph that repeats an
2141/// earlier one — same character, font and size, its pen shifted from the
2142/// earlier pen by at most 0.15 em on each axis, and by less than the earlier
2143/// glyph's step to its own successor — starts a copy when the glyphs after it
2144/// repeat the earlier one's successors by the same offset and size, and the
2145/// whole stretch that keeps the offset is dropped. Two glyphs at least: a lone
2146/// mark stamped twice is left alone, and so is the per-glyph re-stamp the
2147/// sanitizer already folds (`dp_lines`, the overprint duplicate). Text that
2148/// really repeats sits an advance apart or more — a doubled letter (`ll`,
2149/// 0.22 em in Helvetica), a word set twice (`very very`), the same text on
2150/// the next line or in the next table cell — and is kept, and so is a run of
2151/// one character set tighter than a shadow's shift (dot leaders squeezed by
2152/// `Tc`): its glyph two on is farther than the next one. A line stamped two
2153/// or three times at a small offset to fake bold reads once the same way. The
2154/// original must start within the 1024 glyphs before its copy, so a shadowed
2155/// block longer than that is not caught; a shadowed heading is drawn right
2156/// after itself. A rotated run, whose pen positions are not compared here, is
2157/// left alone.
2158fn drop_shadow_runs(glyphs: &mut Vec<Glyph>) {
2159    // How far back a copy's original may start.
2160    const LOOKBACK: usize = 1024;
2161    // The largest shift of a copy, per axis, in ems of the glyph's size.
2162    const NEAR_EM: f32 = 0.15;
2163    // How much the shift and the size may vary along a copy, in ems.
2164    const SAME_EM: f32 = 0.01;
2165    let pen = |g: &Glyph| if g.quad.is_none() { g.pen } else { None };
2166    let n = glyphs.len();
2167    let mut keep = vec![true; n];
2168    // The glyphs a copy may start on, by character and font, in stream
2169    // order: kept (not part of a copy), upright, sized and not whitespace. A
2170    // copy repeats both, so a glyph walks only its own list back through the
2171    // window; scanning the whole window nearly doubled a `--text-layer-only`
2172    // read of pages without a shadow.
2173    let mut earlier: HashMap<(char, u64), Vec<usize>> = HashMap::new();
2174    let mut j = 0;
2175    while j < n {
2176        let b = &glyphs[j];
2177        let start = pen(b).filter(|p| p[2] > 0.0 && !b.ch.is_whitespace());
2178        let mut copy = 0;
2179        if let Some([xj, yj, sj]) = start {
2180            let lo = j.saturating_sub(LOOKBACK);
2181            let candidates = earlier.entry((b.ch, b.font)).or_default();
2182            for &i in candidates.iter().rev().take_while(|&&i| i >= lo) {
2183                let Some([xi, yi, si]) = pen(&glyphs[i]) else {
2184                    continue;
2185                };
2186                if (si - sj).abs() > SAME_EM * sj {
2187                    continue;
2188                }
2189                let (dx, dy) = (xj - xi, yj - yi);
2190                if dx.abs() > NEAR_EM * sj || dy.abs() > NEAR_EM * sj {
2191                    continue;
2192                }
2193                // A shadow is shifted: the characters one glyph expands to
2194                // (a code-like name read as `SM590000`, a decomposed
2195                // ligature) share the glyph's pen and are not a copy.
2196                if dx.abs() + dy.abs() <= SAME_EM * sj {
2197                    continue;
2198                }
2199                // And it lies over its original, nearer to it than the
2200                // original's next glyph is: one character set tighter than
2201                // half the largest shift (dot leaders squeezed by `Tc`) meets
2202                // its own successors at one offset two glyphs on.
2203                if let Some([xn, yn, _]) = glyphs.get(i + 1).and_then(pen) {
2204                    let step = (xn - xi).abs().max((yn - yi).abs());
2205                    if step > SAME_EM * sj && dx.abs().max(dy.abs()) >= step {
2206                        continue;
2207                    }
2208                }
2209                // The copy goes on while each next glyph repeats the
2210                // original's next one by the same shift; the original
2211                // ends before the copy starts.
2212                let mut m = 1;
2213                while j + m < n && i + m < j {
2214                    let (a, b) = (&glyphs[i + m], &glyphs[j + m]);
2215                    let (Some([xa, ya, sa]), Some([xb, yb, sb])) = (pen(a), pen(b)) else {
2216                        break;
2217                    };
2218                    if a.ch != b.ch
2219                        || a.font != b.font
2220                        || (sa - sb).abs() > SAME_EM * sj
2221                        || ((xb - xa) - dx).abs() > SAME_EM * sj
2222                        || ((yb - ya) - dy).abs() > SAME_EM * sj
2223                    {
2224                        break;
2225                    }
2226                    m += 1;
2227                }
2228                if m >= 2 {
2229                    copy = m;
2230                    break;
2231                }
2232            }
2233            if copy == 0 {
2234                candidates.push(j);
2235            }
2236        }
2237        if copy > 0 {
2238            keep[j..j + copy].iter_mut().for_each(|k| *k = false);
2239            j += copy;
2240        } else {
2241            j += 1;
2242        }
2243    }
2244    let mut it = keep.into_iter();
2245    glyphs.retain(|_| it.next().unwrap_or(true));
2246}
2247
2248/// How many marked-content spans may be open for one's `/ActualText` to be
2249/// read (see [`apply_actual_text`]); real documents nest a few.
2250const MAX_ACTUAL_TEXT_DEPTH: usize = 64;
2251
2252/// Put a marked-content span's `/ActualText` in place of the glyphs drawn
2253/// inside it, as docling-parse does since 7.9 (`apply_actual_text`,
2254/// docling-parse#300, on by default; PDF 32000-1 §14.9.4). A writer draws a
2255/// glyph cluster whose characters the font cannot spell this way: a base
2256/// letter and a zero-advance accent whose code collides with a Latin letter
2257/// (`Oreši` read as `OresMi`, docling#3783), or the Arabic marks of a
2258/// LibreOffice font subset whose `/ToUnicode` skips them, so the byte reads as
2259/// WinAnsi (`منعًا` as `منع8ا`). docling-parse's rules, cell for cell: a span
2260/// with no glyphs (over graphics) is left alone, and so is a text implausibly
2261/// long for it (more than 8 characters and 4 per cell: `/Alt` written into
2262/// the wrong key); an empty text removes the glyphs; as many characters as
2263/// cells replace them one by one, keeping each cell's geometry; any other
2264/// count puts the whole text on the span's first cell, over the union of the
2265/// span's boxes, its baseline running to the cell that reaches furthest along
2266/// it (an accent drawn after its base letter trails behind it). Nested spans
2267/// apply innermost first, so the outer text wins. docling-parse has a cell per
2268/// character code; a code whose `/ToUnicode` spells several characters is
2269/// several glyphs here, all on the code's pen, and counts as one cell.
2270///
2271/// Each `EMC` rewrites the glyphs under its span, so a crafted nest of spans
2272/// over the same glyphs would cost glyphs × depth: past
2273/// [`MAX_ACTUAL_TEXT_DEPTH`] open spans, a span still pairs with its `EMC` but
2274/// its `/ActualText` is not read.
2275fn apply_actual_text(out: &mut Vec<Glyph>, begin: usize, text: &str) {
2276    let begin = begin.min(out.len());
2277    let mut cells: Vec<std::ops::Range<usize>> = Vec::new();
2278    for i in begin..out.len() {
2279        match cells.last_mut() {
2280            Some(cell) if out[i].pen.is_some() && out[i].pen == out[cell.end - 1].pen => {
2281                cell.end = i + 1;
2282            }
2283            _ => cells.push(i..i + 1),
2284        }
2285    }
2286    if cells.is_empty() {
2287        return;
2288    }
2289    let chars: Vec<char> = text.chars().collect();
2290    if chars.len() > (4 * cells.len()).max(8) {
2291        return;
2292    }
2293    if chars.is_empty() {
2294        out.truncate(begin);
2295        return;
2296    }
2297    if chars.len() == cells.len() {
2298        // Each cell's first glyph takes its character, the rest of the cell
2299        // goes: one pass, writing behind the cells read.
2300        for (w, (cell, &ch)) in cells.iter().zip(&chars).enumerate() {
2301            out[begin + w] = Glyph {
2302                ch,
2303                ..out[cell.start]
2304            };
2305        }
2306        out.truncate(begin + cells.len());
2307        return;
2308    }
2309    // The leading edge (quad corners 0 and 3) stays the first cell's; the
2310    // trailing one (1 and 2) is the cell's that reaches furthest along the
2311    // baseline, projected on it so a rotated run keeps its direction.
2312    let quad_of = |g: &Glyph| {
2313        g.quad
2314            .unwrap_or([g.ll, g.lb, g.lr, g.lb, g.lr, g.lt, g.ll, g.lt])
2315    };
2316    let mut merged = Glyph { ..out[begin] };
2317    let mut quad = quad_of(&merged);
2318    let dir = (quad[2] - quad[0], quad[3] - quad[1]);
2319    let along = |q: &[f32; 8]| q[2] * dir.0 + q[3] * dir.1;
2320    let mut reach = along(&quad);
2321    for g in &out[begin + 1..] {
2322        merged.l = merged.l.min(g.l);
2323        merged.b = merged.b.min(g.b);
2324        merged.r = merged.r.max(g.r);
2325        merged.t = merged.t.max(g.t);
2326        merged.ll = merged.ll.min(g.ll);
2327        merged.lb = merged.lb.min(g.lb);
2328        merged.lr = merged.lr.max(g.lr);
2329        merged.lt = merged.lt.max(g.lt);
2330        let q = quad_of(g);
2331        if along(&q) > reach {
2332            reach = along(&q);
2333            quad[2..6].copy_from_slice(&q[2..6]);
2334        }
2335    }
2336    if merged.quad.is_some() {
2337        merged.quad = Some(quad);
2338    }
2339    out.truncate(begin);
2340    // The characters of one cell share its box, as a ligature's do.
2341    out.extend(chars.iter().map(|&ch| Glyph { ch, ..merged }));
2342}
2343
2344/// Drop a space glyph the next glyph is drawn over (#992). A typesetter
2345/// that sets a superscript footnote marker as `( 1 )` — space glyphs around
2346/// the raised digit — and then cancels each space's advance with a negative
2347/// `TJ` offset (Arbortext on EUR-Lex: the `1` starts 0.05 pt *before* its
2348/// space begins) renders `(1)`; the spaces exist in the content stream
2349/// only. docling-parse keeps them as characters and its line cell reads
2350/// `( 1 )`; pdftotext and pdfium, which break words by geometry alone,
2351/// read `(1)`, and so does the page. A deliberate divergence
2352/// (`docs/MIGRATION.md` §4 item 22): a space whose successor in stream
2353/// order sits on the same line and starts within the first 30 % of the
2354/// space's advance — the pen went back to (or before) where the space
2355/// began, so it left no gap — is not a character of the text. A space
2356/// followed by a glyph on another line (a trailing space before the
2357/// cursor jumps to the next line's left margin, far to the left of it) or
2358/// by one that starts past that mark (a kerned pair, a justified word gap
2359/// pulled in a little) is kept; the word-gap rules then see the geometry
2360/// as before.
2361fn drop_overprinted_spaces(glyphs: &mut Vec<Glyph>) {
2362    if glyphs.len() < 2 {
2363        return;
2364    }
2365    let overprinted = |sp: &Glyph, next: &Glyph| {
2366        if sp.ch != ' ' || next.ch == ' ' {
2367            return false;
2368        }
2369        // Upright glyphs only: a rotated run's loose boxes are the extents
2370        // of its quads, every glyph of a 90° line shares one left edge and
2371        // the advance runs along y, so the test below would read every
2372        // space of a landscape slide drawn through a 90° `cm` as
2373        // overprinted (`Nullammollis`, odf_presentation_02).
2374        if sp.quad.is_some() || next.quad.is_some() {
2375            return false;
2376        }
2377        let width = sp.lr - sp.ll;
2378        if !width.is_finite() || width <= 0.0 || !next.ll.is_finite() {
2379            return false;
2380        }
2381        // Same line: the loose boxes overlap vertically (a raised digit's box
2382        // still overlaps its neighbours' em boxes).
2383        let same_line = next.lb < sp.lt && next.lt > sp.lb;
2384        same_line && next.ll >= sp.ll - 0.1 * width && next.ll <= sp.ll + 0.3 * width
2385    };
2386    let mut keep = vec![true; glyphs.len()];
2387    for i in 0..glyphs.len() - 1 {
2388        if overprinted(&glyphs[i], &glyphs[i + 1]) {
2389            keep[i] = false;
2390        }
2391    }
2392    let mut it = keep.into_iter();
2393    glyphs.retain(|_| it.next().unwrap_or(true));
2394}
2395
2396/// Whether a glyph is on the page (#529): docling-parse keeps a character
2397/// only when its whole box lies inside the display box — the CropBox, or
2398/// the MediaBox without one — edges included, so a FrameMaker print slug
2399/// drawn beside the CropBox or a tiled page's neighbouring text beyond the
2400/// MediaBox never becomes a cell, and a line crossing the edge is cut at the
2401/// last glyph that fits (`Crossing the rig`). The box is docling-parse's
2402/// char box — the advance by the font's ascent/descent, the loose box here
2403/// (its axis-aligned extent for rotated text) — in the frame `page_glyphs`
2404/// already moved to the display box's corner, so the page is `[0, w] × [0,
2405/// h]`. A glyph without a finite box is kept, as before. (A standard-14 font
2406/// without a FontDescriptor gets the 750 / −250 default ascent / descent
2407/// here where docling-parse reads the AFM's — Helvetica's 718 / −207 — so
2408/// for those a glyph within ~0.04 em of the top or bottom edge can fall the
2409/// other way; horizontally the advance is the same.)
2410fn on_page(g: &Glyph, w: f32, h: f32) -> bool {
2411    // f32 noise only: docling-parse already drops a glyph 0.01 pt over.
2412    const EPS: f32 = 1e-3;
2413    let (l, b, r, t) = if [g.ll, g.lb, g.lr, g.lt].iter().all(|v| v.is_finite()) {
2414        (g.ll, g.lb, g.lr, g.lt)
2415    } else {
2416        (g.l, g.b, g.r, g.t)
2417    };
2418    if ![l, b, r, t].iter().all(|v| v.is_finite()) {
2419        return true;
2420    }
2421    l >= -EPS && b >= -EPS && r <= w + EPS && t <= h + EPS
2422}
2423
2424/// Run a content stream's operators, emitting glyphs into `out`. Recurses into
2425/// Form XObjects on `Do` (bulk body text in heavy PDFs lives inside a form, not
2426/// the page content stream). `res` is the resources dict in scope (the page's,
2427/// or the form's own); `base_ctm` is the CTM at the point of invocation.
2428#[allow(clippy::too_many_arguments)]
2429fn run_content(
2430    doc: &Document,
2431    res: &Dictionary,
2432    content: &lopdf::content::Content,
2433    base_ctm: Mat,
2434    init: TextState,
2435    depth: u32,
2436    caches: &mut DocCaches,
2437    out: &mut Vec<Glyph>,
2438    inks: &mut Vec<crate::checkbox::Ink>,
2439    shape_init: ShapeGs,
2440    shapes: &mut Shapes,
2441) {
2442    let fonts = fonts_from_res(doc, res, caches);
2443    let xobjects = res
2444        .get(b"XObject")
2445        .ok()
2446        .and_then(|o| deref(doc, o))
2447        .and_then(|o| o.as_dict().ok());
2448
2449    // Graphics + text state. `q`/`Q` save and restore the whole graphics state,
2450    // which includes the text parameters (Tc, Tw, Tz, TL, Tfs, Trise, font) —
2451    // *not* the text matrix (that is reset by BT). Saving only the CTM let a Tc
2452    // set inside a `q…Q` block leak out and drift every later glyph.
2453    #[allow(clippy::type_complexity)]
2454    let mut gstate_stack: Vec<(Mat, f64, f64, f64, f64, f64, f64, i64, Option<&Arc<Font>>)> =
2455        Vec::new();
2456    let mut ctm = base_ctm;
2457    let mut tm = Mat::ID;
2458    let mut tlm = Mat::ID;
2459    let mut font: Option<&Arc<Font>> = None;
2460    let mut fsize = init.fsize;
2461    let mut tc = init.tc; // char spacing
2462    let mut tw = init.tw; // word spacing
2463    let mut th = init.th; // horizontal scale (Tz/100)
2464    let mut tl = init.tl; // leading
2465    let mut trise = init.trise;
2466    let mut tr = init.tr; // rendering mode
2467
2468    let op_f = |operands: &[Object], i: usize| operands.get(i).and_then(num).unwrap_or(0.0);
2469    // The path under construction, already in page space (a path is built
2470    // under one CTM — `cm` is not allowed inside it), for the checkbox
2471    // squares (#609). Text extraction never reads it.
2472    let mut path: Vec<PathPiece> = Vec::new();
2473    let (mut cur, mut start) = ((0.0, 0.0), (0.0, 0.0));
2474    // Whether the fill colour is (near) white, saved/restored with `q`/`Q`:
2475    // a white-filled stroked box is a checkbox outline, a coloured one a
2476    // chart legend's swatch. The initial fill colour is black.
2477    let mut fill_light = false;
2478    let mut fill_stack: Vec<bool> = Vec::new();
2479    // Open marked-content sequences (`BMC`/`BDC` … `EMC`): where each one's
2480    // glyphs start in `out`, and its inline `/ActualText`, if any.
2481    let mut marked: Vec<(usize, Option<String>)> = Vec::new();
2482    // The reading-order shapes (#1092): docling-parse's view of the path
2483    // under construction, its graphics state and the pending clip.
2484    let mut shape_gs = shape_init;
2485    let mut shape_stack: Vec<ShapeGs> = Vec::new();
2486    let mut shape_path: Vec<ShapeSub> = Vec::new();
2487    let mut clip_pending = false;
2488    let ext_gstates = res
2489        .get(b"ExtGState")
2490        .ok()
2491        .and_then(|o| deref(doc, o))
2492        .and_then(|o| o.as_dict().ok());
2493    let colorspaces = res
2494        .get(b"ColorSpace")
2495        .ok()
2496        .and_then(|o| deref(doc, o))
2497        .and_then(|o| o.as_dict().ok());
2498
2499    for op in &content.operations {
2500        let operands = &op.operands;
2501        let shown = out.len();
2502        match op.operator.as_str() {
2503            "q" => {
2504                gstate_stack.push((ctm, tc, tw, th, tl, trise, fsize, tr, font));
2505                fill_stack.push(fill_light);
2506                shape_stack.push(shape_gs.clone());
2507            }
2508            "BMC" => marked.push((out.len(), None)),
2509            // Only an inline property list is read: a named one (`/OC /MC0`)
2510            // would need the `/Properties` resource, as in docling-parse.
2511            "BDC" => {
2512                let actual = operands
2513                    .get(1)
2514                    .filter(|_| marked.len() < MAX_ACTUAL_TEXT_DEPTH)
2515                    .and_then(|o| o.as_dict().ok())
2516                    .and_then(|d| d.get(b"ActualText").ok())
2517                    .and_then(|o| match o {
2518                        Object::String(bytes, _) => Some(crate::info::decode_text(bytes)),
2519                        _ => None,
2520                    });
2521                marked.push((out.len(), actual));
2522            }
2523            "EMC" => {
2524                if let Some((begin, Some(text))) = marked.pop() {
2525                    apply_actual_text(out, begin, &text);
2526                }
2527                // Shows no text: the invisible-mode stamp below has nothing
2528                // new to mark, and the span may have shortened `out`.
2529                continue;
2530            }
2531            "Q" => {
2532                fill_light = fill_stack.pop().unwrap_or(false);
2533                if let Some(gs) = shape_stack.pop() {
2534                    shape_gs = gs;
2535                }
2536                if let Some((c, a, b, h, l, r, fs, mode, f)) = gstate_stack.pop() {
2537                    ctm = c;
2538                    tc = a;
2539                    tw = b;
2540                    th = h;
2541                    tl = l;
2542                    trise = r;
2543                    fsize = fs;
2544                    tr = mode;
2545                    font = f;
2546                }
2547            }
2548            "cm" => {
2549                let m = Mat {
2550                    a: op_f(operands, 0),
2551                    b: op_f(operands, 1),
2552                    c: op_f(operands, 2),
2553                    d: op_f(operands, 3),
2554                    e: op_f(operands, 4),
2555                    f: op_f(operands, 5),
2556                };
2557                ctm = m.then(ctm);
2558            }
2559            "BT" => {
2560                tm = Mat::ID;
2561                tlm = Mat::ID;
2562            }
2563            "ET" => {}
2564            "Tf" => {
2565                if let Some(Object::Name(n)) = operands.first() {
2566                    font = fonts.get(n.as_slice());
2567                }
2568                fsize = op_f(operands, 1);
2569            }
2570            "Td" => {
2571                tlm = Mat {
2572                    a: 1.0,
2573                    b: 0.0,
2574                    c: 0.0,
2575                    d: 1.0,
2576                    e: op_f(operands, 0),
2577                    f: op_f(operands, 1),
2578                }
2579                .then(tlm);
2580                tm = tlm;
2581            }
2582            "TD" => {
2583                tl = -op_f(operands, 1);
2584                tlm = Mat {
2585                    a: 1.0,
2586                    b: 0.0,
2587                    c: 0.0,
2588                    d: 1.0,
2589                    e: op_f(operands, 0),
2590                    f: op_f(operands, 1),
2591                }
2592                .then(tlm);
2593                tm = tlm;
2594            }
2595            "Tm" => {
2596                tlm = Mat {
2597                    a: op_f(operands, 0),
2598                    b: op_f(operands, 1),
2599                    c: op_f(operands, 2),
2600                    d: op_f(operands, 3),
2601                    e: op_f(operands, 4),
2602                    f: op_f(operands, 5),
2603                };
2604                tm = tlm;
2605            }
2606            "T*" => {
2607                tlm = Mat {
2608                    a: 1.0,
2609                    b: 0.0,
2610                    c: 0.0,
2611                    d: 1.0,
2612                    e: 0.0,
2613                    f: -tl,
2614                }
2615                .then(tlm);
2616                tm = tlm;
2617            }
2618            "Tc" => tc = op_f(operands, 0),
2619            "Tw" => tw = op_f(operands, 0),
2620            "Tz" => th = op_f(operands, 0) / 100.0,
2621            "TL" => tl = op_f(operands, 0),
2622            "Ts" => trise = op_f(operands, 0),
2623            "Tr" => tr = op_f(operands, 0) as i64,
2624            "Tj" | "'" | "\"" => {
2625                if op.operator == "'" || op.operator == "\"" {
2626                    // move to next line first
2627                    tlm = Mat {
2628                        a: 1.0,
2629                        b: 0.0,
2630                        c: 0.0,
2631                        d: 1.0,
2632                        e: 0.0,
2633                        f: -tl,
2634                    }
2635                    .then(tlm);
2636                    tm = tlm;
2637                }
2638                if op.operator == "\"" {
2639                    // `aw ac string "` sets word- and char-spacing before
2640                    // showing the string (PDF 32000-1 §9.4.3), persisting after.
2641                    tw = op_f(operands, 0);
2642                    tc = op_f(operands, 1);
2643                }
2644                if let (Some(f), Some(Object::String(s, _))) = (font, operands.last()) {
2645                    show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out);
2646                }
2647            }
2648            "TJ" => {
2649                if let (Some(f), Some(Object::Array(arr))) = (font, operands.first()) {
2650                    for el in arr {
2651                        match el {
2652                            Object::String(s, _) => {
2653                                show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out)
2654                            }
2655                            other => {
2656                                if let Some(adj) = num(other) {
2657                                    // negative number moves text right (PDF: subtract)
2658                                    let tx = -adj / 1000.0 * fsize * th;
2659                                    tm = Mat {
2660                                        a: 1.0,
2661                                        b: 0.0,
2662                                        c: 0.0,
2663                                        d: 1.0,
2664                                        e: tx,
2665                                        f: 0.0,
2666                                    }
2667                                    .then(tm);
2668                                }
2669                            }
2670                        }
2671                    }
2672                }
2673            }
2674            "m" => {
2675                cur = ctm.apply(op_f(operands, 0), op_f(operands, 1));
2676                start = cur;
2677                shape_path.push(ShapeSub::at(cur));
2678            }
2679            "l" => {
2680                let p = ctm.apply(op_f(operands, 0), op_f(operands, 1));
2681                path.push(PathPiece::Line(cur, p));
2682                cur = p;
2683                if let Some(sub) = shape_path.last_mut() {
2684                    sub.lines.push((sub.cur, p));
2685                    sub.pts.push(p);
2686                    sub.cur = p;
2687                }
2688            }
2689            "c" | "v" | "y" => {
2690                let pts: Vec<(f64, f64)> = (0..operands.len() / 2)
2691                    .map(|k| ctm.apply(op_f(operands, 2 * k), op_f(operands, 2 * k + 1)))
2692                    .collect();
2693                if let Some(&end) = pts.last() {
2694                    path.push(PathPiece::Curve(
2695                        std::iter::once(cur).chain(pts.iter().copied()).collect(),
2696                    ));
2697                    cur = end;
2698                    if let Some(sub) = shape_path.last_mut() {
2699                        sub.pts.extend(pts.iter().copied());
2700                        sub.cur = end;
2701                    }
2702                }
2703            }
2704            "h" => {
2705                if cur != start {
2706                    path.push(PathPiece::Line(cur, start));
2707                }
2708                cur = start;
2709                // docling-parse closes the subpath and opens the next one at
2710                // its start, where a following `l` continues.
2711                if let Some(sub) = shape_path.last_mut() {
2712                    sub.closed = true;
2713                    sub.pts.push(sub.start);
2714                    let at = sub.start;
2715                    shape_path.push(ShapeSub::at(at));
2716                }
2717            }
2718            "re" => {
2719                let (x, y, w, h) = (
2720                    op_f(operands, 0),
2721                    op_f(operands, 1),
2722                    op_f(operands, 2),
2723                    op_f(operands, 3),
2724                );
2725                path.push(PathPiece::Rect([
2726                    ctm.apply(x, y),
2727                    ctm.apply(x + w, y),
2728                    ctm.apply(x + w, y + h),
2729                    ctm.apply(x, y + h),
2730                ]));
2731                cur = ctm.apply(x, y);
2732                start = cur;
2733                // `m` + three `l` + `h`, as docling-parse builds it.
2734                let c = [
2735                    ctm.apply(x, y),
2736                    ctm.apply(x + w, y),
2737                    ctm.apply(x + w, y + h),
2738                    ctm.apply(x, y + h),
2739                ];
2740                shape_path.push(ShapeSub {
2741                    start: c[0],
2742                    cur: c[3],
2743                    pts: vec![c[0], c[1], c[2], c[3], c[0]],
2744                    lines: vec![(c[0], c[1]), (c[1], c[2]), (c[2], c[3])],
2745                    closed: true,
2746                });
2747                shape_path.push(ShapeSub::at(c[0]));
2748            }
2749            "S" | "s" | "f" | "F" | "f*" | "B" | "B*" | "b" | "b*" => {
2750                let op = op.operator.as_str();
2751                if matches!(op, "s" | "b" | "b*") && cur != start {
2752                    path.push(PathPiece::Line(cur, start));
2753                }
2754                let stroke = matches!(op, "S" | "s" | "B" | "B*" | "b" | "b*");
2755                let fill = !matches!(op, "S" | "s");
2756                paint_path(&path, stroke, fill, fill_light, inks);
2757                path.clear();
2758                // `s`, `f`, `f*`, `b`, `b*` close the last subpath first.
2759                if !matches!(op, "S" | "B" | "B*") {
2760                    if let Some(sub) = shape_path.last_mut() {
2761                        sub.closed = true;
2762                    }
2763                }
2764                let scale = (ctm.a * ctm.d - ctm.b * ctm.c).abs().sqrt();
2765                record_shape(
2766                    &shape_path,
2767                    stroke,
2768                    fill,
2769                    &shape_gs,
2770                    if scale > 0.0 { scale } else { 1.0 },
2771                    shapes,
2772                );
2773                // A pending `W` takes effect after the painting operator.
2774                if std::mem::take(&mut clip_pending) {
2775                    capture_clip(&shape_path, &mut shape_gs.clips);
2776                }
2777                shape_path.clear();
2778            }
2779            "n" => {
2780                path.clear();
2781                if std::mem::take(&mut clip_pending) {
2782                    capture_clip(&shape_path, &mut shape_gs.clips);
2783                }
2784                shape_path.clear();
2785            }
2786            "W" | "W*" => clip_pending = true,
2787            "w" => shape_gs.line_width = op_f(operands, 0),
2788            "gs" => {
2789                let params = operands
2790                    .first()
2791                    .and_then(|o| o.as_name().ok())
2792                    .and_then(|n| ext_gstates?.get(n).ok())
2793                    .and_then(|o| deref(doc, o))
2794                    .and_then(|o| o.as_dict().ok());
2795                if let Some(params) = params {
2796                    let get = |k: &[u8]| params.get(k).ok().and_then(num);
2797                    if let Some(lw) = get(b"LW") {
2798                        shape_gs.line_width = lw;
2799                    }
2800                    if let Some(a) = get(b"CA") {
2801                        shape_gs.stroke_alpha = a;
2802                    }
2803                    if let Some(a) = get(b"ca") {
2804                        shape_gs.fill_alpha = a;
2805                    }
2806                }
2807            }
2808            // The fill colour, by its components: gray `g`, `rg`, CMYK `k`,
2809            // and `sc`/`scn` by operand count (a pattern name is not light).
2810            "g" | "rg" | "k" | "sc" | "scn" => {
2811                // docling-parse: a colour resolves a pattern fill; a pattern
2812                // name in `scn` makes it one.
2813                if matches!(op.operator.as_str(), "scn")
2814                    && matches!(operands.last(), Some(Object::Name(_)))
2815                {
2816                    shape_gs.fill_pattern = true;
2817                } else if !operands.is_empty() {
2818                    shape_gs.fill_pattern = false;
2819                }
2820                let v: Vec<f64> = operands
2821                    .iter()
2822                    .map(num)
2823                    .collect::<Option<_>>()
2824                    .unwrap_or_default();
2825                fill_light = match v.as_slice() {
2826                    [g] => *g >= 0.9,
2827                    [r, g, b] => r.min(*g).min(*b) >= 0.9,
2828                    [c, m, y, k] => c.max(*m).max(*y).max(*k) <= 0.1,
2829                    _ => false,
2830                };
2831            }
2832            "cs" => {
2833                fill_light = false;
2834                // A Pattern space carries no colour until `scn` names one.
2835                let pattern = match operands.first() {
2836                    Some(Object::Name(n)) if n.as_slice() == b"Pattern" => true,
2837                    Some(Object::Name(n)) => colorspaces
2838                        .and_then(|cs| cs.get(n).ok())
2839                        .and_then(|o| deref(doc, o))
2840                        .is_some_and(|o| match o {
2841                            Object::Name(f) => f.as_slice() == b"Pattern",
2842                            Object::Array(a) => a
2843                                .first()
2844                                .and_then(|f| f.as_name().ok())
2845                                .is_some_and(|f| f == b"Pattern"),
2846                            _ => false,
2847                        }),
2848                    _ => false,
2849                };
2850                if pattern {
2851                    shape_gs.fill_pattern = true;
2852                }
2853            }
2854            // An inline image: the unit square under the CTM (#999).
2855            "BI" => inks.push(image_ink(ctm)),
2856            "Do" => {
2857                // Invoke a Form XObject: bulk body text in many PDFs lives inside
2858                // a form, reached only here. An image XObject carries no text,
2859                // only its placement (#999).
2860                if depth >= 8 {
2861                    continue;
2862                }
2863                let Some(Object::Name(n)) = operands.first() else {
2864                    continue;
2865                };
2866                let obj = xobjects.and_then(|d| d.get(n.as_slice()).ok());
2867                let form_id = match obj {
2868                    Some(Object::Reference(id)) => Some(*id),
2869                    _ => None,
2870                };
2871                let stream = obj
2872                    .and_then(|o| deref(doc, o))
2873                    .and_then(|o| o.as_stream().ok());
2874                let Some(stream) = stream else { continue };
2875                let is_form = stream
2876                    .dict
2877                    .get(b"Subtype")
2878                    .ok()
2879                    .and_then(|o| o.as_name().ok())
2880                    == Some(b"Form".as_slice());
2881                if !is_form {
2882                    let is_image = stream
2883                        .dict
2884                        .get(b"Subtype")
2885                        .ok()
2886                        .and_then(|o| o.as_name().ok())
2887                        == Some(b"Image".as_slice());
2888                    if is_image {
2889                        inks.push(image_ink(ctm));
2890                    }
2891                    continue;
2892                }
2893                // Decode a reused form's content once per document
2894                // (headers/footers and bulk body text invoke the same form on
2895                // every page); see `DocCaches::forms_seen` for single-use ones.
2896                let cached = form_id.and_then(|id| caches.forms.get(&id).cloned());
2897                let form_content = match cached {
2898                    Some(c) => c,
2899                    None => {
2900                        let Ok(data) = stream.decompressed_content() else {
2901                            continue;
2902                        };
2903                        let Ok(c) = lopdf::content::Content::decode(&data) else {
2904                            continue;
2905                        };
2906                        let c = Arc::new(c);
2907                        if let Some(id) = form_id {
2908                            if !caches.forms_seen.insert(id) {
2909                                caches.forms.insert(id, Arc::clone(&c));
2910                            }
2911                        }
2912                        c
2913                    }
2914                };
2915                // The form's /Matrix maps form space into the CTM at invocation.
2916                let form_mat = match stream.dict.get(b"Matrix").ok() {
2917                    Some(Object::Array(a)) if a.len() == 6 => {
2918                        let v: Vec<f64> = a.iter().filter_map(num).collect();
2919                        if v.len() == 6 {
2920                            Mat {
2921                                a: v[0],
2922                                b: v[1],
2923                                c: v[2],
2924                                d: v[3],
2925                                e: v[4],
2926                                f: v[5],
2927                            }
2928                        } else {
2929                            Mat::ID
2930                        }
2931                    }
2932                    _ => Mat::ID,
2933                };
2934                // The form's own /Resources, falling back to the inherited ones.
2935                let form_res = stream
2936                    .dict
2937                    .get(b"Resources")
2938                    .ok()
2939                    .and_then(|o| deref(doc, o))
2940                    .and_then(|o| o.as_dict().ok())
2941                    .unwrap_or(res);
2942                let state = TextState {
2943                    tc,
2944                    tw,
2945                    th,
2946                    tl,
2947                    trise,
2948                    fsize,
2949                    tr,
2950                };
2951                run_content(
2952                    doc,
2953                    form_res,
2954                    &form_content,
2955                    form_mat.then(ctm),
2956                    state,
2957                    depth + 1,
2958                    caches,
2959                    out,
2960                    inks,
2961                    shape_gs.clone(),
2962                    shapes,
2963                );
2964            }
2965            _ => {}
2966        }
2967        // Invisible text (`Tr` 3, or 7: clip only) — an OCR layer over a scan
2968        // follows the scanned lines, not a typeset baseline (#1005).
2969        if matches!(tr, 3 | 7) {
2970            for g in &mut out[shown..] {
2971                g.invisible = true;
2972            }
2973        }
2974    }
2975}
2976
2977/// An image placement's box — the unit square under `ctm`, y-up page
2978/// points — as an [`Ink::Bitmap`](crate::checkbox::Ink::Bitmap).
2979fn image_ink(ctm: Mat) -> crate::checkbox::Ink {
2980    let pts = [(0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0)].map(|(x, y)| ctm.apply(x, y));
2981    let (mut l, mut b, mut r, mut t) = (
2982        f64::INFINITY,
2983        f64::INFINITY,
2984        f64::NEG_INFINITY,
2985        f64::NEG_INFINITY,
2986    );
2987    for (x, y) in pts {
2988        (l, b, r, t) = (l.min(x), b.min(y), r.max(x), t.max(y));
2989    }
2990    crate::checkbox::Ink::Bitmap { l, b, r, t }
2991}
2992
2993/// The page's vector shapes as docling-parse's page decoder reports them to
2994/// docling's reading order (#1092): `get_shape_lines` — every stroked,
2995/// axis-aligned straight segment, clipped — and the boxes
2996/// `get_connected_shape_bounding_boxes` merges — every visible painted path,
2997/// its stroke's half width included, clipped. `[l, b, r, t]`, page points,
2998/// bottom-left origin, in the frame `run_content` walks (the display box's
2999/// lower-left corner at the origin, before `/Rotate`).
3000#[derive(Default, Clone, Debug, PartialEq)]
3001pub(crate) struct Shapes {
3002    pub lines: Vec<[f64; 4]>,
3003    pub boxes: Vec<[f64; 4]>,
3004}
3005
3006/// More painted paths than this on a page and their boxes are not merged —
3007/// docling-parse's merge is quadratic in them, and a page that dense (a map,
3008/// a seat plan, text drawn as outlines) yields no trustworthy rule anyway;
3009/// the stroked lines still count.
3010const MAX_SHAPE_BOXES: usize = 10_000;
3011
3012/// One subpath of the shape path under construction, page space: what
3013/// docling-parse's `page_item<PAGE_SHAPE>` keeps of it.
3014struct ShapeSub {
3015    start: (f64, f64),
3016    cur: (f64, f64),
3017    /// Every point the path passes or is controlled by — a curve's control
3018    /// points included, as docling-parse's segment data has them.
3019    pts: Vec<(f64, f64)>,
3020    /// Its straight segments, in order.
3021    lines: Vec<((f64, f64), (f64, f64))>,
3022    closed: bool,
3023}
3024
3025impl ShapeSub {
3026    fn at(p: (f64, f64)) -> Self {
3027        ShapeSub {
3028            start: p,
3029            cur: p,
3030            pts: vec![p],
3031            lines: Vec::new(),
3032            closed: false,
3033        }
3034    }
3035
3036    /// `keep_shape`: two points at least, spanning more than a speck (the
3037    /// box diagonal, squared, over 1e-3).
3038    fn kept(&self) -> bool {
3039        let Some((l, b, r, t)) = bbox_of(&self.pts) else {
3040            return false;
3041        };
3042        self.pts.len() >= 2 && (r - l).powi(2) + (t - b).powi(2) >= 1e-3
3043    }
3044}
3045
3046fn bbox_of(pts: &[(f64, f64)]) -> Option<(f64, f64, f64, f64)> {
3047    let (&first, rest) = pts.split_first()?;
3048    Some(rest.iter().fold(
3049        (first.0, first.1, first.0, first.1),
3050        |(l, b, r, t), &(x, y)| (l.min(x), b.min(y), r.max(x), t.max(y)),
3051    ))
3052}
3053
3054/// The graphics state docling-parse's shapes read: line width (user space),
3055/// stroke and fill alpha (`CA` / `ca`), whether the fill colour is a pattern
3056/// not yet resolved to a paint, and the clip paths' boxes. Saved and
3057/// restored with `q` / `Q`.
3058#[derive(Clone)]
3059struct ShapeGs {
3060    line_width: f64,
3061    stroke_alpha: f64,
3062    fill_alpha: f64,
3063    fill_pattern: bool,
3064    clips: Vec<[f64; 4]>,
3065}
3066
3067impl ShapeGs {
3068    const INIT: ShapeGs = ShapeGs {
3069        line_width: 1.0,
3070        stroke_alpha: 1.0,
3071        fill_alpha: 1.0,
3072        fill_pattern: false,
3073        clips: Vec::new(),
3074    };
3075}
3076
3077/// docling-parse's `bbox_intersects`: the boxes share area (a degenerate box
3078/// lying inside the other counts).
3079fn boxes_intersect(a: [f64; 4], b: [f64; 4]) -> bool {
3080    a[0] < b[2] && a[2] > b[0] && a[1] < b[3] && a[3] > b[1]
3081}
3082
3083/// `clipped_bbox`: a box cut to every clip path's box in turn; `None` once
3084/// one of them misses it.
3085fn clipped(mut bbox: [f64; 4], clips: &[[f64; 4]]) -> Option<[f64; 4]> {
3086    for c in clips {
3087        if !boxes_intersect(bbox, *c) {
3088            return None;
3089        }
3090        bbox = [
3091            bbox[0].max(c[0]),
3092            bbox[1].max(c[1]),
3093            bbox[2].min(c[2]),
3094            bbox[3].min(c[3]),
3095        ];
3096    }
3097    Some(bbox)
3098}
3099
3100/// `get_shape_lines`' `maybe_add_line` + `clip_axis_aligned_segment`: a
3101/// straight segment that is horizontal or vertical within 1e-3 (not both: a
3102/// point), cut to every clip path's box along its axis and dropped when one
3103/// misses it.
3104fn shape_line(a: (f64, f64), z: (f64, f64), clips: &[[f64; 4]], out: &mut Vec<[f64; 4]>) {
3105    const TOL: f64 = 1e-3;
3106    let horizontal = (z.1 - a.1).abs() <= TOL;
3107    let vertical = (z.0 - a.0).abs() <= TOL;
3108    if horizontal == vertical {
3109        return;
3110    }
3111    let mut bbox = [a.0.min(z.0), a.1.min(z.1), a.0.max(z.0), a.1.max(z.1)];
3112    for c in clips {
3113        if horizontal {
3114            if a.1 < c[1] - TOL || a.1 > c[3] + TOL {
3115                return;
3116            }
3117            bbox[0] = bbox[0].max(c[0]);
3118            bbox[2] = bbox[2].min(c[2]);
3119            if bbox[0] > bbox[2] + TOL {
3120                return;
3121            }
3122        } else {
3123            if a.0 < c[0] - TOL || a.0 > c[2] + TOL {
3124                return;
3125            }
3126            bbox[1] = bbox[1].max(c[1]);
3127            bbox[3] = bbox[3].min(c[3]);
3128            if bbox[1] > bbox[3] + TOL {
3129                return;
3130            }
3131        }
3132    }
3133    out.push(bbox);
3134}
3135
3136/// `capture_pending_clip`: each subpath of the clipping path with a segment
3137/// (two points or more) bounds what later operators paint — appended, which
3138/// approximates the intersection the spec asks for, as docling-parse does.
3139fn capture_clip(subs: &[ShapeSub], clips: &mut Vec<[f64; 4]>) {
3140    for sub in subs.iter().filter(|s| s.pts.len() >= 2) {
3141        if let Some((l, b, r, t)) = bbox_of(&sub.pts) {
3142            clips.push([l, b, r, t]);
3143        }
3144    }
3145}
3146
3147/// docling-parse's `register_paths` for the reading-order geometry: a painted
3148/// path's kept subpaths become shape lines (when its stroke shows) and a
3149/// shape box (when anything of it shows). A fill whose colour is an
3150/// unresolved pattern paints nothing, so a fill-only path is no shape and a
3151/// fill-and-stroke one keeps its stroke. `scale` maps the user-space line
3152/// width to page space (the CTM's `sqrt|det|`; a hairline stays 0).
3153fn record_shape(
3154    subs: &[ShapeSub],
3155    stroke: bool,
3156    fill: bool,
3157    gs: &ShapeGs,
3158    scale: f64,
3159    shapes: &mut Shapes,
3160) {
3161    let kept: Vec<&ShapeSub> = subs.iter().filter(|s| s.kept()).collect();
3162    if kept.is_empty() {
3163        return;
3164    }
3165    let fill = fill && !gs.fill_pattern;
3166    if gs.fill_pattern && !stroke {
3167        return;
3168    }
3169    let strokes_visible = stroke && gs.stroke_alpha > 0.0;
3170    let visible = strokes_visible || (fill && gs.fill_alpha > 0.0);
3171    if strokes_visible {
3172        for sub in &kept {
3173            for &(a, z) in &sub.lines {
3174                shape_line(a, z, &gs.clips, &mut shapes.lines);
3175            }
3176            if sub.closed {
3177                shape_line(sub.cur, sub.start, &gs.clips, &mut shapes.lines);
3178            }
3179        }
3180    }
3181    if visible {
3182        let pts: Vec<(f64, f64)> = kept.iter().flat_map(|s| s.pts.iter().copied()).collect();
3183        if let Some((l, b, r, t)) = bbox_of(&pts) {
3184            let pad = if strokes_visible {
3185                (gs.line_width * scale).max(0.0) * 0.5
3186            } else {
3187                0.0
3188            };
3189            if let Some(bbox) = clipped([l - pad, b - pad, r + pad, t + pad], &gs.clips) {
3190                shapes.boxes.push(bbox);
3191            }
3192        }
3193    }
3194}
3195
3196/// `get_connected_shape_bounding_boxes`: each box in turn (unless already
3197/// taken) grows by every untaken box its growing union touches, until none
3198/// does — docling-parse's greedy pass, so a component finished early keeps a
3199/// box a later one would also have reached. Tolerance 0, as docling calls
3200/// it: two `re` that meet in the file meet here because [`real`] reads their
3201/// operands as the doubles docling-parse reads.
3202fn connected_boxes(boxes: &[[f64; 4]]) -> Vec<[f64; 4]> {
3203    let touches =
3204        |a: [f64; 4], b: [f64; 4]| a[0] <= b[2] && a[2] >= b[0] && a[1] <= b[3] && a[3] >= b[1];
3205    let mut taken = vec![false; boxes.len()];
3206    let mut out = Vec::new();
3207    for i in 0..boxes.len() {
3208        if taken[i] {
3209            continue;
3210        }
3211        taken[i] = true;
3212        let mut component = boxes[i];
3213        let mut changed = true;
3214        while changed {
3215            changed = false;
3216            for j in 0..boxes.len() {
3217                if !taken[j] && touches(component, boxes[j]) {
3218                    let b = boxes[j];
3219                    component = [
3220                        component[0].min(b[0]),
3221                        component[1].min(b[1]),
3222                        component[2].max(b[2]),
3223                        component[3].max(b[3]),
3224                    ];
3225                    taken[j] = true;
3226                    changed = true;
3227                }
3228            }
3229        }
3230        out.push(component);
3231    }
3232    out
3233}
3234
3235/// One piece of a path under construction, page space (see `run_content`).
3236enum PathPiece {
3237    Line((f64, f64), (f64, f64)),
3238    /// An `re`: its four corners, counter-clockwise from `(x, y)`.
3239    Rect([(f64, f64); 4]),
3240    /// A Bézier segment: its start and control/end points (their hull).
3241    Curve(Vec<(f64, f64)>),
3242}
3243
3244/// Record a painted path's pieces for the checkbox finder (#609): a stroked
3245/// line is a [`Seg`](crate::checkbox::Ink::Seg), a stroked rectangle that
3246/// stays axis-aligned a [`Rect`](crate::checkbox::Ink::Rect) (one filled
3247/// white too — still an outline), and a coloured fill or a curve only its
3248/// bounding box, a [`Blot`](crate::checkbox::Ink::Blot): a possible tick
3249/// mark, or a swatch. White fills draw nothing and are dropped. Only pieces small
3250/// enough to be a checkbox edge or a mark inside one are kept.
3251fn paint_path(
3252    path: &[PathPiece],
3253    stroke: bool,
3254    fill: bool,
3255    fill_light: bool,
3256    inks: &mut Vec<crate::checkbox::Ink>,
3257) {
3258    use crate::checkbox::Ink;
3259    let bbox = |pts: &mut dyn Iterator<Item = (f64, f64)>| {
3260        pts.fold(
3261            (
3262                f64::INFINITY,
3263                f64::INFINITY,
3264                f64::NEG_INFINITY,
3265                f64::NEG_INFINITY,
3266            ),
3267            |(l, b, r, t), (x, y)| (l.min(x), b.min(y), r.max(x), t.max(y)),
3268        )
3269    };
3270    let mut keep = |ink: Ink| {
3271        if ink.worth_keeping() {
3272            inks.push(ink);
3273        }
3274    };
3275    if fill && !fill_light {
3276        // A coloured fill is a blot — a tick mark, or (the size of a square)
3277        // a legend swatch, which `checkbox::find` then rules out. A white
3278        // fill is no ink on paper: it only leaves the stroke, if any, below.
3279        let (l, b, r, t) = bbox(&mut path.iter().flat_map(|p| match p {
3280            PathPiece::Line(a, z) => vec![*a, *z],
3281            PathPiece::Rect(c) => c.to_vec(),
3282            PathPiece::Curve(c) => c.clone(),
3283        }));
3284        if l.is_finite() {
3285            keep(Ink::Blot { l, b, r, t });
3286        }
3287        return;
3288    }
3289    if !stroke {
3290        return;
3291    }
3292    for piece in path {
3293        match piece {
3294            PathPiece::Line(a, z) => keep(Ink::Seg {
3295                x0: a.0,
3296                y0: a.1,
3297                x1: z.0,
3298                y1: z.1,
3299            }),
3300            PathPiece::Rect(c) => {
3301                let axis = ((c[0].1 - c[1].1).abs() < 1e-6 && (c[1].0 - c[2].0).abs() < 1e-6)
3302                    || ((c[0].0 - c[1].0).abs() < 1e-6 && (c[1].1 - c[2].1).abs() < 1e-6);
3303                if axis {
3304                    let (l, b, r, t) = bbox(&mut c.iter().copied());
3305                    keep(Ink::Rect { l, b, r, t });
3306                } else {
3307                    for k in 0..4 {
3308                        let (a, z) = (c[k], c[(k + 1) % 4]);
3309                        keep(Ink::Seg {
3310                            x0: a.0,
3311                            y0: a.1,
3312                            x1: z.0,
3313                            y1: z.1,
3314                        });
3315                    }
3316                }
3317            }
3318            PathPiece::Curve(c) => {
3319                let (l, b, r, t) = bbox(&mut c.iter().copied());
3320                keep(Ink::Blot { l, b, r, t });
3321            }
3322        }
3323    }
3324}
3325
3326#[allow(clippy::too_many_arguments)]
3327fn show_text(
3328    font: &Font,
3329    bytes: &[u8],
3330    fsize: f64,
3331    tc: f64,
3332    tw: f64,
3333    th: f64,
3334    trise: f64,
3335    tm: &mut Mat,
3336    ctm: Mat,
3337    out: &mut Vec<Glyph>,
3338) {
3339    for code in codes(font, bytes) {
3340        let (text, w, blank) = font.decode_code(code);
3341        let w0 = w / 1000.0; // advance in text-space (em) units
3342                             // The glyph→user transform: scale glyph space by font size, then Tm, CTM.
3343        let scale = Mat {
3344            a: fsize * th,
3345            b: 0.0,
3346            c: 0.0,
3347            d: fsize,
3348            e: 0.0,
3349            f: trise,
3350        };
3351        let trm = scale.then(*tm).then(ctm);
3352        // Box in glyph space (1000-unit em): x 0..w, y descent..ascent.
3353        // A blank unmapped glyph keeps its advance but no ink height
3354        // (docling-parse collapses its ascent and descent).
3355        let (desc, asc) = if blank {
3356            (0.0, 0.0)
3357        } else if let Some(&(lo, hi)) = font.ink_y.get(&code) {
3358            (lo / 1000.0, hi / 1000.0)
3359        } else {
3360            (font.descent / 1000.0, font.ascent / 1000.0)
3361        };
3362        let (x0, y0) = trm.apply(0.0, desc);
3363        let (x1, y1) = trm.apply(w0, desc);
3364        let (x2, y2) = trm.apply(w0, asc);
3365        let (x3, y3) = trm.apply(0.0, asc);
3366        // Upright = the baseline runs left-to-right along +x. Only then is the
3367        // loose box the rectangle spanned by the baseline's x and the em's y
3368        // (a synthetic oblique's skew is ignored, as it always was). A rotated
3369        // matrix — the `0 s -s 0 tx ty Tm` landscape pages are built with
3370        // (#528) — collapsed that rectangle to zero width, and every such run
3371        // vanished; those glyphs keep their real quad (docling-parse's char
3372        // rect) and its extent instead.
3373        let upright = trm.a > 0.0 && trm.b.abs() <= 1e-6 * trm.a;
3374        let (left, bot, right, top, quad) = if upright {
3375            (x0.min(x1), y0.min(y3), x0.max(x1), y0.max(y3), None)
3376        } else {
3377            let (xs, ys) = ([x0, x1, x2, x3], [y0, y1, y2, y3]);
3378            let fold = |v: [f64; 4], f: fn(f64, f64) -> f64| v.into_iter().reduce(f).unwrap();
3379            let q = [x0, y0, x1, y1, x2, y2, x3, y3].map(|v| v as f32);
3380            (
3381                fold(xs, f64::min),
3382                fold(ys, f64::min),
3383                fold(xs, f64::max),
3384                fold(ys, f64::max),
3385                Some(q),
3386            )
3387        };
3388        // The pen's origin (rise included) and the font size's extent from
3389        // it, as docling-parse maps them (`map_text_point(0, rise)` and
3390        // `(0, rise + font_size)`).
3391        let (ox, oy) = trm.apply(0.0, 0.0);
3392        let (sx, sy) = trm.apply(0.0, 1.0);
3393        let pen = Some([ox as f32, oy as f32, (sx - ox).hypot(sy - oy) as f32]);
3394        if let Some(s) = text {
3395            // A run may map one code to multiple chars (ligature/fraction); share box.
3396            for ch in s.chars() {
3397                // A NUL from the `/ToUnicode` is an unmapped glyph too
3398                // (docling#5070): U+FFFD.
3399                let ch = if ch == '\u{0}' { '\u{FFFD}' } else { ch };
3400                out.push(Glyph {
3401                    ch,
3402                    l: left as f32,
3403                    b: bot as f32,
3404                    r: right as f32,
3405                    t: top as f32,
3406                    ll: left as f32,
3407                    lb: bot as f32,
3408                    lr: right as f32,
3409                    lt: top as f32,
3410                    font: font.hash,
3411                    quad,
3412                    pen,
3413                    math_font: font.math,
3414                    invisible: false,
3415                });
3416            }
3417        }
3418        // Advance the text matrix. Word spacing applies to single-byte code 32.
3419        let is_space = !font.two_byte && code == 32;
3420        let tx = (w0 * fsize + tc + if is_space { tw } else { 0.0 }) * th;
3421        *tm = Mat {
3422            a: 1.0,
3423            b: 0.0,
3424            c: 0.0,
3425            d: 1.0,
3426            e: tx,
3427            f: 0.0,
3428        }
3429        .then(*tm);
3430    }
3431}
3432
3433/// Build a simple font's code→char table from its `/Encoding`: the base
3434/// encoding (WinAnsi / MacRoman) plus any `/Differences` overrides (glyph names
3435/// resolved through a small Adobe-glyph-name subset).
3436fn simple_encoding_table(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
3437    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o));
3438    let base_name = match enc {
3439        Some(Object::Name(n)) => n.clone(),
3440        Some(Object::Dictionary(d)) => d
3441            .get(b"BaseEncoding")
3442            .ok()
3443            .and_then(|o| o.as_name().ok())
3444            .map(|n| n.to_vec())
3445            .unwrap_or_default(),
3446        _ => Vec::new(),
3447    };
3448    let face = || {
3449        let font_name = sanitizer_font_name(doc, fdict, false);
3450        let base = fdict.get(b"BaseFont").ok().and_then(|o| o.as_name().ok());
3451        let names: Vec<&[u8]> = std::iter::once(font_name.as_slice()).chain(base).collect();
3452        crate::symbol_fonts::symbol_face(&names)
3453    };
3454    let mut m = if base_name == b"MacRomanEncoding" {
3455        macroman_table()
3456    } else if let Some(face) = base_name.is_empty().then(face).flatten() {
3457        // A font matched to Symbol or ZapfDingbats that names no encoding:
3458        // the face's own (docling-parse's base-font AFM, a "FontSpecific"
3459        // scheme), never the Latin tables — a non-embedded ZapfDingbats'
3460        // `l` is `●` (docling-parse#431, #1113). An embedded program's
3461        // builtin encoding still comes first ([`Font::builtin`]).
3462        crate::symbol_fonts::face_table(face)
3463    } else if base_name.is_empty() {
3464        // No PDF /Encoding at all: the font's *built-in* encoding applies. For
3465        // the standard TeX math fonts that is their fixed TeX layout — falling
3466        // back to StandardEncoding read CMSY's braces as `f`/`g`, `→` as `!`,
3467        // `∈` as `2` (2203's `{ahn,…}` author line). The font program (often
3468        // CFF, which this parser does not read) carries the same mapping;
3469        // docling-parse decodes it from there.
3470        tex_math_builtin(fdict).unwrap_or_else(winansi_table)
3471    } else if base_name == b"WinAnsiEncoding" {
3472        let mut m = winansi_table();
3473        // PDF 32000-1:2008 Annex D.2, note 3 to the Latin set: "In
3474        // WinAnsiEncoding, all unused codes greater than 40 [octal] map to the
3475        // bullet character" — pdfium and pdf.js read them so. ReportLab draws
3476        // its default list bullet as `\177` in a non-embedded WinAnsi
3477        // Helvetica, which read U+FFFD before (#1102). Only a font that names
3478        // WinAnsiEncoding gets them: the no-/Encoding fallback above stands
3479        // in for a built-in encoding, where these codes stay undefined.
3480        for b in [0x7Fu8, 0x81, 0x8D, 0x8F, 0x90, 0x9D] {
3481            m.insert(b, '\u{2022}');
3482        }
3483        m
3484    } else {
3485        winansi_table()
3486    };
3487    // Apply /Differences: `code /glyphname /glyphname ... code ...`.
3488    if let Some(Object::Dictionary(d)) = enc {
3489        if let Some(Object::Array(diffs)) = d.get(b"Differences").ok().and_then(|o| deref(doc, o)) {
3490            let mut code = 0u8;
3491            for el in diffs {
3492                match el {
3493                    Object::Integer(i) => code = *i as u8,
3494                    Object::Name(name) => {
3495                        if let Some(ch) = glyph_name_to_char(name) {
3496                            m.insert(code, ch);
3497                        }
3498                        code = code.wrapping_add(1);
3499                    }
3500                    _ => {}
3501                }
3502            }
3503        }
3504    }
3505    m
3506}
3507
3508/// The fixed built-in encodings of the standard TeX math fonts (TeXbook
3509/// Appendix F), keyed off the base font name: `CMSY*` (symbols; `CMBSY` is its
3510/// bold) and `CMMI*` (math italic). These fonts ship no PDF `/Encoding` and no
3511/// ToUnicode, and their program is usually CFF — without this table the codes
3512/// fell through to StandardEncoding and rendered as the wrong ASCII.
3513fn tex_math_builtin(fdict: &Dictionary) -> Option<HashMap<u8, char>> {
3514    const CMSY: [char; 128] = [
3515        '−', '·', '×', '∗', '÷', '⋄', '±', '∓', '⊕', '⊖', '⊗', '⊘', '⊙', '◯', '∘', '•', '≍', '≡',
3516        '⊆', '⊇', '≤', '≥', '≼', '≽', '∼', '≈', '⊂', '⊃', '≪', '≫', '≺', '≻', '←', '→', '↑', '↓',
3517        '↔', '↗', '↘', '≃', '⇐', '⇒', '⇑', '⇓', '⇔', '↖', '↙', '∝', '′', '∞', '∈', '∋', '△', '▽',
3518        '\u{338}', '↦', '∀', '∃', '¬', '∅', 'ℜ', 'ℑ', '⊤', '⊥', 'ℵ', 'A', 'B', 'C', 'D', 'E', 'F',
3519        'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
3520        'Y', 'Z', '∪', '∩', '⊎', '∧', '∨', '⊢', '⊣', '⌊', '⌋', '⌈', '⌉', '{', '}', '⟨', '⟩', '|',
3521        '∥', '↕', '⇕', '\\', '≀', '√', '∐', '∇', '∫', '⊔', '⊓', '⊑', '⊒', '§', '†', '‡', '¶', '♣',
3522        '♢', '♡', '♠',
3523    ];
3524    const CMMI: [char; 128] = [
3525        'Γ', 'Δ', 'Θ', 'Λ', 'Ξ', 'Π', 'Σ', 'Υ', 'Φ', 'Ψ', 'Ω', 'α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η',
3526        'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'π', 'ρ', 'σ', 'τ', 'υ', 'φ', 'χ', 'ψ', 'ω', 'ϵ', 'ϑ',
3527        'ϖ', 'ϱ', 'ς', 'ϕ', '↼', '↽', '⇀', '⇁', '↩', '↪', '▷', '◁', '0', '1', '2', '3', '4', '5',
3528        '6', '7', '8', '9', '.', ',', '<', '/', '>', '⋆', '∂', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
3529        'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y',
3530        'Z', '♭', '♮', '♯', '⌣', '⌢', 'ℓ', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
3531        'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'ı', 'ȷ', '℘',
3532        '\u{20d7}', '⁀',
3533    ];
3534    let name = base_font_name(fdict)?;
3535    let up = name.to_ascii_uppercase();
3536    let table: &[char; 128] = if up.starts_with(b"CMSY") || up.starts_with(b"CMBSY") {
3537        &CMSY
3538    } else if up.starts_with(b"CMMI") {
3539        &CMMI
3540    } else {
3541        return None;
3542    };
3543    Some(
3544        table
3545            .iter()
3546            .enumerate()
3547            .map(|(i, &c)| (i as u8, c))
3548            .collect(),
3549    )
3550}
3551
3552/// The Adobe Glyph List's names for Latin-1 Supplement and Latin Extended-A
3553/// (plus `Euro`, `florin`, the comma-accent pairs): what a Type1 font's
3554/// `/Differences` or builtin encoding calls an accented or stroked letter.
3555/// 2203.01017v2 names `Ł` as `/Lslash` in a NimbusRomNo9L subset with no
3556/// ToUnicode, and the resolver, which knew only the ASCII, ligature and symbol
3557/// names, dropped the glyph outright (`Ł ukasz Kaiser` read `ukasz Kaiser`).
3558/// Sorted by name for the binary search in [`glyph_name_to_char`].
3559const AGL_LATIN: &[(&str, char)] = &[
3560    ("AE", '\u{00C6}'),
3561    ("Aacute", '\u{00C1}'),
3562    ("Abreve", '\u{0102}'),
3563    ("Acircumflex", '\u{00C2}'),
3564    ("Adieresis", '\u{00C4}'),
3565    ("Agrave", '\u{00C0}'),
3566    ("Amacron", '\u{0100}'),
3567    ("Aogonek", '\u{0104}'),
3568    ("Aring", '\u{00C5}'),
3569    ("Atilde", '\u{00C3}'),
3570    ("Cacute", '\u{0106}'),
3571    ("Ccaron", '\u{010C}'),
3572    ("Ccedilla", '\u{00C7}'),
3573    ("Ccircumflex", '\u{0108}'),
3574    ("Cdotaccent", '\u{010A}'),
3575    ("Dcaron", '\u{010E}'),
3576    ("Dcroat", '\u{0110}'),
3577    ("Eacute", '\u{00C9}'),
3578    ("Ebreve", '\u{0114}'),
3579    ("Ecaron", '\u{011A}'),
3580    ("Ecircumflex", '\u{00CA}'),
3581    ("Edieresis", '\u{00CB}'),
3582    ("Edotaccent", '\u{0116}'),
3583    ("Egrave", '\u{00C8}'),
3584    ("Emacron", '\u{0112}'),
3585    ("Eng", '\u{014A}'),
3586    ("Eogonek", '\u{0118}'),
3587    ("Eth", '\u{00D0}'),
3588    ("Euro", '\u{20AC}'),
3589    ("Gbreve", '\u{011E}'),
3590    ("Gcircumflex", '\u{011C}'),
3591    ("Gcommaaccent", '\u{0122}'),
3592    ("Gdotaccent", '\u{0120}'),
3593    ("Hbar", '\u{0126}'),
3594    ("Hcircumflex", '\u{0124}'),
3595    ("IJ", '\u{0132}'),
3596    ("Iacute", '\u{00CD}'),
3597    ("Ibreve", '\u{012C}'),
3598    ("Icircumflex", '\u{00CE}'),
3599    ("Idieresis", '\u{00CF}'),
3600    ("Idotaccent", '\u{0130}'),
3601    ("Igrave", '\u{00CC}'),
3602    ("Imacron", '\u{012A}'),
3603    ("Iogonek", '\u{012E}'),
3604    ("Itilde", '\u{0128}'),
3605    ("Jcircumflex", '\u{0134}'),
3606    ("Kcommaaccent", '\u{0136}'),
3607    ("Lacute", '\u{0139}'),
3608    ("Lcaron", '\u{013D}'),
3609    ("Lcommaaccent", '\u{013B}'),
3610    ("Ldot", '\u{013F}'),
3611    ("Lslash", '\u{0141}'),
3612    ("Nacute", '\u{0143}'),
3613    ("Ncaron", '\u{0147}'),
3614    ("Ncommaaccent", '\u{0145}'),
3615    ("Ntilde", '\u{00D1}'),
3616    ("OE", '\u{0152}'),
3617    ("Oacute", '\u{00D3}'),
3618    ("Obreve", '\u{014E}'),
3619    ("Ocircumflex", '\u{00D4}'),
3620    ("Odieresis", '\u{00D6}'),
3621    ("Ograve", '\u{00D2}'),
3622    ("Ohungarumlaut", '\u{0150}'),
3623    ("Omacron", '\u{014C}'),
3624    ("Oslash", '\u{00D8}'),
3625    ("Otilde", '\u{00D5}'),
3626    ("Racute", '\u{0154}'),
3627    ("Rcaron", '\u{0158}'),
3628    ("Rcommaaccent", '\u{0156}'),
3629    ("Sacute", '\u{015A}'),
3630    ("Scaron", '\u{0160}'),
3631    ("Scedilla", '\u{015E}'),
3632    ("Scircumflex", '\u{015C}'),
3633    ("Scommaaccent", '\u{0218}'),
3634    ("Tbar", '\u{0166}'),
3635    ("Tcaron", '\u{0164}'),
3636    ("Tcommaaccent", '\u{0162}'),
3637    ("Thorn", '\u{00DE}'),
3638    ("Uacute", '\u{00DA}'),
3639    ("Ubreve", '\u{016C}'),
3640    ("Ucircumflex", '\u{00DB}'),
3641    ("Udieresis", '\u{00DC}'),
3642    ("Ugrave", '\u{00D9}'),
3643    ("Uhungarumlaut", '\u{0170}'),
3644    ("Umacron", '\u{016A}'),
3645    ("Uogonek", '\u{0172}'),
3646    ("Uring", '\u{016E}'),
3647    ("Utilde", '\u{0168}'),
3648    ("Wcircumflex", '\u{0174}'),
3649    ("Yacute", '\u{00DD}'),
3650    ("Ycircumflex", '\u{0176}'),
3651    ("Ydieresis", '\u{0178}'),
3652    ("Zacute", '\u{0179}'),
3653    ("Zcaron", '\u{017D}'),
3654    ("Zdotaccent", '\u{017B}'),
3655    ("aacute", '\u{00E1}'),
3656    ("abreve", '\u{0103}'),
3657    ("acircumflex", '\u{00E2}'),
3658    ("acute", '\u{00B4}'),
3659    ("adieresis", '\u{00E4}'),
3660    ("ae", '\u{00E6}'),
3661    ("agrave", '\u{00E0}'),
3662    ("amacron", '\u{0101}'),
3663    ("aogonek", '\u{0105}'),
3664    ("aring", '\u{00E5}'),
3665    ("atilde", '\u{00E3}'),
3666    ("brokenbar", '\u{00A6}'),
3667    ("cacute", '\u{0107}'),
3668    ("ccaron", '\u{010D}'),
3669    ("ccedilla", '\u{00E7}'),
3670    ("ccircumflex", '\u{0109}'),
3671    ("cdotaccent", '\u{010B}'),
3672    ("cedilla", '\u{00B8}'),
3673    ("cent", '\u{00A2}'),
3674    ("currency", '\u{00A4}'),
3675    ("dcaron", '\u{010F}'),
3676    ("dcroat", '\u{0111}'),
3677    ("dieresis", '\u{00A8}'),
3678    ("dotlessi", '\u{0131}'),
3679    ("eacute", '\u{00E9}'),
3680    ("ebreve", '\u{0115}'),
3681    ("ecaron", '\u{011B}'),
3682    ("ecircumflex", '\u{00EA}'),
3683    ("edieresis", '\u{00EB}'),
3684    ("edotaccent", '\u{0117}'),
3685    ("egrave", '\u{00E8}'),
3686    ("emacron", '\u{0113}'),
3687    ("eng", '\u{014B}'),
3688    ("eogonek", '\u{0119}'),
3689    ("eth", '\u{00F0}'),
3690    ("exclamdown", '\u{00A1}'),
3691    ("florin", '\u{0192}'),
3692    ("gbreve", '\u{011F}'),
3693    ("gcircumflex", '\u{011D}'),
3694    ("gcommaaccent", '\u{0123}'),
3695    ("gdotaccent", '\u{0121}'),
3696    ("germandbls", '\u{00DF}'),
3697    ("guillemotleft", '\u{00AB}'),
3698    ("guillemotright", '\u{00BB}'),
3699    ("hbar", '\u{0127}'),
3700    ("hcircumflex", '\u{0125}'),
3701    ("iacute", '\u{00ED}'),
3702    ("ibreve", '\u{012D}'),
3703    ("icircumflex", '\u{00EE}'),
3704    ("idieresis", '\u{00EF}'),
3705    ("igrave", '\u{00EC}'),
3706    ("ij", '\u{0133}'),
3707    ("imacron", '\u{012B}'),
3708    ("iogonek", '\u{012F}'),
3709    ("itilde", '\u{0129}'),
3710    ("jcircumflex", '\u{0135}'),
3711    ("kcommaaccent", '\u{0137}'),
3712    ("kgreenlandic", '\u{0138}'),
3713    ("lacute", '\u{013A}'),
3714    ("lcaron", '\u{013E}'),
3715    ("lcommaaccent", '\u{013C}'),
3716    ("ldot", '\u{0140}'),
3717    ("logicalnot", '\u{00AC}'),
3718    ("longs", '\u{017F}'),
3719    ("lslash", '\u{0142}'),
3720    ("macron", '\u{00AF}'),
3721    ("nacute", '\u{0144}'),
3722    ("napostrophe", '\u{0149}'),
3723    ("ncaron", '\u{0148}'),
3724    ("ncommaaccent", '\u{0146}'),
3725    ("ntilde", '\u{00F1}'),
3726    ("oacute", '\u{00F3}'),
3727    ("obreve", '\u{014F}'),
3728    ("ocircumflex", '\u{00F4}'),
3729    ("odieresis", '\u{00F6}'),
3730    ("oe", '\u{0153}'),
3731    ("ograve", '\u{00F2}'),
3732    ("ohungarumlaut", '\u{0151}'),
3733    ("omacron", '\u{014D}'),
3734    ("onehalf", '\u{00BD}'),
3735    ("onequarter", '\u{00BC}'),
3736    ("onesuperior", '\u{00B9}'),
3737    ("ordfeminine", '\u{00AA}'),
3738    ("ordmasculine", '\u{00BA}'),
3739    ("oslash", '\u{00F8}'),
3740    ("otilde", '\u{00F5}'),
3741    ("paragraph", '\u{00B6}'),
3742    ("questiondown", '\u{00BF}'),
3743    ("racute", '\u{0155}'),
3744    ("rcaron", '\u{0159}'),
3745    ("rcommaaccent", '\u{0157}'),
3746    ("sacute", '\u{015B}'),
3747    ("scaron", '\u{0161}'),
3748    ("scedilla", '\u{015F}'),
3749    ("scircumflex", '\u{015D}'),
3750    ("scommaaccent", '\u{0219}'),
3751    ("section", '\u{00A7}'),
3752    ("sfthyphen", '\u{00AD}'),
3753    ("sterling", '\u{00A3}'),
3754    ("tbar", '\u{0167}'),
3755    ("tcaron", '\u{0165}'),
3756    ("tcommaaccent", '\u{0163}'),
3757    ("thorn", '\u{00FE}'),
3758    ("threequarters", '\u{00BE}'),
3759    ("threesuperior", '\u{00B3}'),
3760    ("twosuperior", '\u{00B2}'),
3761    ("uacute", '\u{00FA}'),
3762    ("ubreve", '\u{016D}'),
3763    ("ucircumflex", '\u{00FB}'),
3764    ("udieresis", '\u{00FC}'),
3765    ("ugrave", '\u{00F9}'),
3766    ("uhungarumlaut", '\u{0171}'),
3767    ("umacron", '\u{016B}'),
3768    ("uogonek", '\u{0173}'),
3769    ("uring", '\u{016F}'),
3770    ("utilde", '\u{0169}'),
3771    ("wcircumflex", '\u{0175}'),
3772    ("yacute", '\u{00FD}'),
3773    ("ycircumflex", '\u{0177}'),
3774    ("ydieresis", '\u{00FF}'),
3775    ("yen", '\u{00A5}'),
3776    ("zacute", '\u{017A}'),
3777    ("zcaron", '\u{017E}'),
3778    ("zdotaccent", '\u{017C}'),
3779];
3780
3781/// docling-parse 7.25's reading text of a PostScript glyph name in a
3782/// program's builtin encoding (`builtin_font_encoding::glyph_name_to_utf8`,
3783/// #1113): `.notdef` and an index-style name (`g12`, `gid00012`, `cid12`,
3784/// `index12`) carry none; a stylistic `.suffix` is dropped; then the glyph
3785/// tables (the symbol fonts' names, [`glyph_name_to_char`]'s), a `uniXXXX`
3786/// sequence (a ligature's code points), a `uXXXX`–`uXXXXXX` name.
3787pub(crate) fn builtin_glyph_text(name: &str) -> Option<String> {
3788    if name.is_empty() || name == ".notdef" {
3789        return None;
3790    }
3791    let name = match name.find('.') {
3792        Some(dot) if dot > 0 => &name[..dot],
3793        _ => name,
3794    };
3795    let lower = name.to_ascii_lowercase();
3796    let index_name = ["gid", "glyph", "g", "cid", "index"].iter().any(|p| {
3797        lower
3798            .strip_prefix(p)
3799            .is_some_and(|d| !d.is_empty() && d.bytes().all(|b| b.is_ascii_digit()))
3800    });
3801    if index_name {
3802        return None;
3803    }
3804    if let Some(c) = crate::symbol_fonts::symbol_glyph_char(name) {
3805        return Some(c.to_string());
3806    }
3807    let hex_points = |hex: &str, width: usize| -> Option<String> {
3808        (!hex.is_empty()
3809            && hex.len().is_multiple_of(width)
3810            && hex.bytes().all(|b| b.is_ascii_hexdigit()))
3811        .then(|| {
3812            (0..hex.len())
3813                .step_by(width)
3814                .map(|i| {
3815                    u32::from_str_radix(&hex[i..i + width], 16)
3816                        .ok()
3817                        .and_then(char::from_u32)
3818                })
3819                .collect::<Option<String>>()
3820        })
3821        .flatten()
3822    };
3823    if let Some(hex) = name.strip_prefix("uni") {
3824        if let Some(t) = hex_points(hex, 4) {
3825            return Some(t);
3826        }
3827    }
3828    if let Some(hex) = name.strip_prefix('u') {
3829        if (4..=6).contains(&hex.len()) {
3830            if let Some(t) = hex_points(hex, hex.len()) {
3831                return Some(t);
3832            }
3833        }
3834    }
3835    glyph_name_to_char(name.as_bytes()).map(String::from)
3836}
3837
3838/// Resolve an Adobe glyph name to Unicode: `uniXXXX`, single ASCII letters, the
3839/// digit/punctuation names from the Adobe Glyph List, and common typographic
3840/// names. A `.suffix` (`one.taboldstyle`, `a.sc`) is stripped and the base name
3841/// retried — docling renders these as the base character.
3842pub(crate) fn glyph_name_to_char(name: &[u8]) -> Option<char> {
3843    let s = std::str::from_utf8(name).ok()?;
3844    // `uniXXXX`. A name that merely starts with `uni` (`union`) goes on to
3845    // the tables: it used to end here, unresolved, for want of four digits.
3846    if let Some(cp) = s
3847        .strip_prefix("uni")
3848        .and_then(|hex| hex.get(0..4))
3849        .and_then(|hex| u32::from_str_radix(hex, 16).ok())
3850    {
3851        return char::from_u32(cp);
3852    }
3853    // Single ASCII letter names (`A`, `m`) map to themselves.
3854    if s.len() == 1 {
3855        let b = s.as_bytes()[0];
3856        if b.is_ascii_alphabetic() {
3857            return Some(b as char);
3858        }
3859    }
3860    let resolved = match s {
3861        "space" => ' ',
3862        "exclam" => '!',
3863        "quotedbl" => '"',
3864        "numbersign" => '#',
3865        "dollar" => '$',
3866        "percent" => '%',
3867        "ampersand" => '&',
3868        "quotesingle" => '\'',
3869        "parenleft" => '(',
3870        "parenright" => ')',
3871        "asterisk" => '*',
3872        "plus" => '+',
3873        "comma" => ',',
3874        "hyphen" => '-',
3875        "period" => '.',
3876        "slash" => '/',
3877        "zero" => '0',
3878        "one" => '1',
3879        "two" => '2',
3880        "three" => '3',
3881        "four" => '4',
3882        "five" => '5',
3883        "six" => '6',
3884        "seven" => '7',
3885        "eight" => '8',
3886        "nine" => '9',
3887        "colon" => ':',
3888        "semicolon" => ';',
3889        "less" => '<',
3890        "equal" => '=',
3891        "greater" => '>',
3892        "question" => '?',
3893        "at" => '@',
3894        "bracketleft" => '[',
3895        "backslash" => '\\',
3896        "bracketright" => ']',
3897        "asciicircum" => '^',
3898        "underscore" => '_',
3899        "grave" => '`',
3900        "braceleft" => '{',
3901        "bar" => '|',
3902        "braceright" => '}',
3903        "asciitilde" => '~',
3904        "bullet" => '\u{2022}',
3905        "periodcentered" => '\u{00B7}',
3906        "endash" => '\u{2013}',
3907        "emdash" => '\u{2014}',
3908        "quoteright" => '\u{2019}',
3909        "quoteleft" => '\u{2018}',
3910        "quotedblleft" => '\u{201C}',
3911        "quotedblright" => '\u{201D}',
3912        "quotedblbase" => '\u{201E}',
3913        "quotesinglbase" => '\u{201A}',
3914        // Latin f-ligatures named in `/Differences` (e.g. 2305's body font). These
3915        // map to the presentation-form code points, which `decompose_ligatures`
3916        // then spells back out (`ff`→"ff") — without them the glyph decodes to
3917        // nothing and the sanitizer fills the gap with a space (`di erences`).
3918        "ff" => '\u{FB00}',
3919        "fi" => '\u{FB01}',
3920        "fl" => '\u{FB02}',
3921        "ffi" => '\u{FB03}',
3922        "ffl" => '\u{FB04}',
3923        "ft" => '\u{FB05}',
3924        "st" => '\u{FB06}',
3925        "degree" => '\u{00B0}',
3926        "trademark" => '\u{2122}',
3927        "registered" => '\u{00AE}',
3928        "copyright" => '\u{00A9}',
3929        "ellipsis" => '\u{2026}',
3930        "minus" => '\u{2212}',
3931        "fraction" => '\u{2044}',
3932        "nbspace" => '\u{00A0}',
3933        // Greek + math glyph names (standard Adobe Glyph List). Standard TeX math
3934        // fonts (CMMI/CMSY/…) name their glyphs this way in the embedded font
3935        // program's `/Encoding`; without these a `λ`/`≤` decodes to nothing and is
3936        // dropped from body text (`and λ set to 0.5` → `and set to 0.5`).
3937        "alpha" => '\u{03B1}',
3938        "beta" => '\u{03B2}',
3939        "gamma" => '\u{03B3}',
3940        "delta" => '\u{03B4}',
3941        "epsilon" | "epsilon1" => '\u{03B5}',
3942        "zeta" => '\u{03B6}',
3943        "eta" => '\u{03B7}',
3944        "theta" | "theta1" => '\u{03B8}',
3945        "iota" => '\u{03B9}',
3946        "kappa" => '\u{03BA}',
3947        "lambda" => '\u{03BB}',
3948        "mu" => '\u{03BC}',
3949        "nu" => '\u{03BD}',
3950        "xi" => '\u{03BE}',
3951        "omicron" => '\u{03BF}',
3952        "pi" | "pi1" => '\u{03C0}',
3953        "rho" | "rho1" => '\u{03C1}',
3954        "sigma" => '\u{03C3}',
3955        "sigma1" => '\u{03C2}',
3956        "tau" => '\u{03C4}',
3957        "upsilon" => '\u{03C5}',
3958        "phi" | "phi1" => '\u{03C6}',
3959        "chi" => '\u{03C7}',
3960        "psi" => '\u{03C8}',
3961        "omega" | "omega1" => '\u{03C9}',
3962        "Gamma" => '\u{0393}',
3963        "Delta" => '\u{0394}',
3964        "Theta" => '\u{0398}',
3965        "Lambda" => '\u{039B}',
3966        "Xi" => '\u{039E}',
3967        "Pi" => '\u{03A0}',
3968        "Sigma" => '\u{03A3}',
3969        "Upsilon" => '\u{03A5}',
3970        "Phi" => '\u{03A6}',
3971        "Psi" => '\u{03A8}',
3972        "Omega" => '\u{03A9}',
3973        "lessequal" => '\u{2264}',
3974        "greaterequal" => '\u{2265}',
3975        "notequal" => '\u{2260}',
3976        "approxequal" => '\u{2248}',
3977        "equivalence" => '\u{2261}',
3978        "element" => '\u{2208}',
3979        "plusminus" => '\u{00B1}',
3980        "multiply" => '\u{00D7}',
3981        "divide" => '\u{00F7}',
3982        "infinity" => '\u{221E}',
3983        "partialdiff" => '\u{2202}',
3984        "gradient" => '\u{2207}',
3985        "summation" => '\u{2211}',
3986        "product" => '\u{220F}',
3987        "integral" => '\u{222B}',
3988        "radical" => '\u{221A}',
3989        "proportional" => '\u{221D}',
3990        "arrowright" => '\u{2192}',
3991        "arrowleft" => '\u{2190}',
3992        "arrowup" => '\u{2191}',
3993        "arrowdown" => '\u{2193}',
3994        "arrowboth" => '\u{2194}',
3995        "arrowdblright" => '\u{21D2}',
3996        "logicaland" => '\u{2227}',
3997        "logicalor" => '\u{2228}',
3998        "intersection" => '\u{2229}',
3999        "union" => '\u{222A}',
4000        "similar" => '\u{223C}',
4001        "congruent" => '\u{2245}',
4002        "dotmath" => '\u{22C5}',
4003        "asteriskmath" => '\u{2217}',
4004        _ => {
4005            if let Ok(i) = AGL_LATIN.binary_search_by_key(&s, |(n, _)| n) {
4006                return Some(AGL_LATIN[i].1);
4007            }
4008            // The math names the arms above leave out (#1121): the TeX math
4009            // fonts' own (`parenleftbig`, `radicalbig`, `integraldisplay`,
4010            // `prime`), then the Symbol font's — Glyph List names such as
4011            // `universal`, `existential`, `arrowup`, which a Latin Modern
4012            // math font's `/Differences` uses. The arms above keep their
4013            // readings where docling-parse's lists differ (`Delta` is the
4014            // Greek letter here, the Glyph List's `∆` increment sign there).
4015            // Last, the Adobe Glyph List in full (the `glyph-list` feature):
4016            // every name none of the above reads — the `afii` names of a
4017            // Cyrillic, Hebrew or Arabic font, the rest of the symbols.
4018            if let Some(ch) = crate::glyph_names::lookup(s)
4019                .or_else(|| crate::symbol_fonts::symbol_glyph_char(s))
4020                .or_else(|| crate::adobe_glyph_list::lookup(s))
4021            {
4022                return Some(ch);
4023            }
4024            // Strip an AGL `.suffix` (oldstyle/small-cap variant) and retry.
4025            if let Some((base, _)) = s.split_once('.') {
4026                if !base.is_empty() {
4027                    return glyph_name_to_char(base.as_bytes());
4028                }
4029            }
4030            // An AGL ligature name joins its components with `_` (`f_f_i`,
4031            // `f_l` — what a re-encoded Libertine subset calls them); joined,
4032            // the presentation-form arms above resolve them.
4033            if s.contains('_') {
4034                let joined: String = s.chars().filter(|&c| c != '_').collect();
4035                if !joined.is_empty() && joined.len() < s.len() {
4036                    return glyph_name_to_char(joined.as_bytes());
4037                }
4038            }
4039            return None;
4040        }
4041    };
4042    Some(resolved)
4043}
4044
4045/// Minimal WinAnsiEncoding (Latin-1-ish) for simple fonts lacking ToUnicode.
4046fn winansi_table() -> HashMap<u8, char> {
4047    let mut m = HashMap::new();
4048    for b in 0x20u8..=0x7e {
4049        m.insert(b, b as char);
4050    }
4051    // High range: Windows-1252 printable points that differ from Latin-1.
4052    let extra: &[(u8, char)] = &[
4053        (0x91, '\u{2018}'),
4054        (0x92, '\u{2019}'),
4055        (0x93, '\u{201C}'),
4056        (0x94, '\u{201D}'),
4057        (0x95, '\u{2022}'),
4058        (0x96, '\u{2013}'),
4059        (0x97, '\u{2014}'),
4060        (0x85, '\u{2026}'),
4061        (0xA0, '\u{00A0}'),
4062        // The rest of Windows-1252's 0x80–0x9F row. Left out, the German
4063        // opening quote `„` (0x84) vanished from the text layer while its
4064        // closing `“` (0x93) stayed (#922); `€ ‚ ƒ † ‡ ˆ ‰ Š ‹ Œ Ž ˜ ™ š › œ ž Ÿ`
4065        // were lost the same way. 0x81/0x8D/0x8F/0x90/0x9D (and 0x7F) are
4066        // unused; a font naming WinAnsiEncoding reads them as bullets
4067        // (`simple_encoding_table`, #1102).
4068        (0x80, '\u{20AC}'),
4069        (0x82, '\u{201A}'),
4070        (0x83, '\u{0192}'),
4071        (0x84, '\u{201E}'),
4072        (0x86, '\u{2020}'),
4073        (0x87, '\u{2021}'),
4074        (0x88, '\u{02C6}'),
4075        (0x89, '\u{2030}'),
4076        (0x8A, '\u{0160}'),
4077        (0x8B, '\u{2039}'),
4078        (0x8C, '\u{0152}'),
4079        (0x8E, '\u{017D}'),
4080        (0x98, '\u{02DC}'),
4081        (0x99, '\u{2122}'),
4082        (0x9A, '\u{0161}'),
4083        (0x9B, '\u{203A}'),
4084        (0x9C, '\u{0153}'),
4085        (0x9E, '\u{017E}'),
4086        (0x9F, '\u{0178}'),
4087    ];
4088    for &(b, c) in extra {
4089        m.insert(b, c);
4090    }
4091    for b in 0xA1u8..=0xFF {
4092        m.entry(b).or_insert(b as char);
4093    }
4094    m
4095}
4096
4097/// MacRomanEncoding: ASCII plus the whole high range, 0x80–0xFF, as Mac OS
4098/// Roman maps it (#744) — the table pdfium, pdftotext and pdf.js decode with.
4099/// Two codes follow the PDF reference instead of Apple's later revisions:
4100/// 0xDB is `currency` (¤, not €), and 0xF0 (`apple`, a private-use glyph) is
4101/// left unmapped. 0xCA, the PDF table's second `space`, reads as a no-break
4102/// space.
4103fn macroman_table() -> HashMap<u8, char> {
4104    #[rustfmt::skip]
4105    const HIGH: [char; 128] = [
4106        'Ä', 'Å', 'Ç', 'É', 'Ñ', 'Ö', 'Ü', 'á',  // 0x80
4107        'à', 'â', 'ä', 'ã', 'å', 'ç', 'é', 'è',  // 0x88
4108        'ê', 'ë', 'í', 'ì', 'î', 'ï', 'ñ', 'ó',  // 0x90
4109        'ò', 'ô', 'ö', 'õ', 'ú', 'ù', 'û', 'ü',  // 0x98
4110        '†', '°', '¢', '£', '§', '•', '¶', 'ß',  // 0xA0
4111        '®', '©', '™', '´', '¨', '≠', 'Æ', 'Ø',  // 0xA8
4112        '∞', '±', '≤', '≥', '¥', 'µ', '∂', '∑',  // 0xB0
4113        '∏', 'π', '∫', 'ª', 'º', 'Ω', 'æ', 'ø',  // 0xB8
4114        '¿', '¡', '¬', '√', 'ƒ', '≈', '∆', '«',  // 0xC0
4115        '»', '…', '\u{00A0}', 'À', 'Ã', 'Õ', 'Œ', 'œ',  // 0xC8
4116        '–', '—', '“', '”', '‘', '’', '÷', '◊',  // 0xD0
4117        'ÿ', 'Ÿ', '⁄', '¤', '‹', '›', 'fi', 'fl',  // 0xD8
4118        '‡', '·', '‚', '„', '‰', 'Â', 'Ê', 'Á',  // 0xE0
4119        'Ë', 'È', 'Í', 'Î', 'Ï', 'Ì', 'Ó', 'Ô',  // 0xE8
4120        '\0', 'Ò', 'Ú', 'Û', 'Ù', 'ı', 'ˆ', '˜',  // 0xF0
4121        '¯', '˘', '˙', '˚', '¸', '˝', '˛', 'ˇ',  // 0xF8
4122    ];
4123    let mut m = HashMap::new();
4124    for b in 0x20u8..=0x7e {
4125        m.insert(b, b as char);
4126    }
4127    for (b, &c) in (0x80u8..=0xFF).zip(HIGH.iter()) {
4128        if c != '\0' {
4129            m.insert(b, c);
4130        }
4131    }
4132    m
4133}
4134
4135#[cfg(test)]
4136mod glyph_names {
4137    /// `/Lslash` and the other Latin-1 / Latin Extended-A glyph names a Type1
4138    /// `/Differences` uses resolve (2203.01017v2's `Łukasz`); the table the
4139    /// lookup bisects is sorted and the ligature / symbol arms still win.
4140    #[test]
4141    fn latin_glyph_names_resolve() {
4142        for w in super::AGL_LATIN.windows(2) {
4143            assert!(
4144                w[0].0 < w[1].0,
4145                "AGL_LATIN not sorted at {:?} / {:?}",
4146                w[0].0,
4147                w[1].0
4148            );
4149        }
4150        for (name, ch) in [
4151            ("Lslash", 'Ł'),
4152            ("lslash", 'ł'),
4153            ("Aacute", 'Á'),
4154            ("aacute", 'á'),
4155            ("Scaron", 'Š'),
4156            ("germandbls", 'ß'),
4157            ("oe", 'œ'),
4158            ("Ydieresis", 'Ÿ'),
4159            ("Euro", '€'),
4160            ("Lslash.sc", 'Ł'),
4161            ("fi", '\u{FB01}'),
4162            ("Omega", '\u{03A9}'),
4163        ] {
4164            assert_eq!(
4165                super::glyph_name_to_char(name.as_bytes()),
4166                Some(ch),
4167                "{name}"
4168            );
4169        }
4170        assert_eq!(super::glyph_name_to_char(b"g123"), None);
4171        // AGL ligature names: components joined with `_`.
4172        assert_eq!(super::glyph_name_to_char(b"f_f_i"), Some('\u{FB03}'));
4173        assert_eq!(super::glyph_name_to_char(b"f_l"), Some('\u{FB02}'));
4174        assert_eq!(super::glyph_name_to_char(b"a_b"), None);
4175    }
4176
4177    /// The stale-CMap case: ToUnicode maps a code to its own identity while
4178    /// `/Differences` names a ligature there — the name wins; a CMap that
4179    /// maps the code anywhere else, or a code with no entry, is untouched.
4180    #[test]
4181    fn a_stale_tounicode_yields_to_differences() {
4182        use lopdf::Object;
4183        use std::collections::HashMap;
4184        let mut tu: HashMap<u32, String> = HashMap::new();
4185        tu.insert(33, "!".into()); // identity — stale
4186        tu.insert(34, "x".into()); // the producer meant it
4187        let diffs = vec![
4188            Object::Integer(33),
4189            Object::Name(b"f_f_i".to_vec()),
4190            Object::Name(b"f_i".to_vec()),
4191            Object::Name(b"f_f".to_vec()),
4192            Object::Integer(65),
4193            Object::Name(b"A".to_vec()), // identity on both sides: nothing to do
4194        ];
4195        super::reconcile_stale_tounicode(&mut tu, &diffs);
4196        assert_eq!(tu.get(&33).map(String::as_str), Some("\u{FB03}"));
4197        assert_eq!(tu.get(&34).map(String::as_str), Some("x"));
4198        assert_eq!(tu.get(&35), None);
4199        assert_eq!(tu.get(&65), None);
4200    }
4201
4202    /// #1121: a ToUnicode value that says nothing — empty, a NUL, U+FFFD, a
4203    /// noncharacter — yields to the `/Differences` name (docling-parse's
4204    /// `has_to_unicode`). 1503.04203's embedded figure maps its re-encoded
4205    /// TeX glyphs to `<0000>`: `/minus` and `/Delta` at code 33. A value that
4206    /// says something stays, and so does one whose name does not resolve.
4207    #[test]
4208    fn a_tounicode_value_that_says_nothing_yields_to_differences() {
4209        use lopdf::Object;
4210        use std::collections::HashMap;
4211        let name = |n: &str| Object::Name(n.as_bytes().to_vec());
4212        let mut tu: HashMap<u32, String> = HashMap::new();
4213        tu.insert(33, "\u{0}".into());
4214        tu.insert(34, "\u{FFFD}".into());
4215        tu.insert(35, "\u{FFFF}".into());
4216        tu.insert(36, String::new());
4217        tu.insert(37, "\u{0}".into()); // the name is unknown: left alone
4218        tu.insert(38, "\u{0}\u{0}".into()); // not a single code point
4219        tu.insert(39, "−".into()); // says something
4220        let diffs = vec![
4221            Object::Integer(33),
4222            name("minus"),
4223            name("Delta"),
4224            name("radicalbig"),
4225            name("parenleftbig"),
4226            name("nosuchglyphname"),
4227            name("plus"),
4228            name("hyphen"),
4229        ];
4230        super::reconcile_stale_tounicode(&mut tu, &diffs);
4231        let got = |c: u32| tu.get(&c).map(String::as_str);
4232        assert_eq!(got(33), Some("−"));
4233        assert_eq!(got(34), Some("Δ"));
4234        assert_eq!(got(35), Some("√"));
4235        assert_eq!(got(36), Some("("));
4236        assert_eq!(got(37), Some("\u{0}"));
4237        assert_eq!(got(38), Some("\u{0}\u{0}"));
4238        assert_eq!(got(39), Some("−"));
4239    }
4240
4241    /// #1121: the TeX delimiter and operator names resolve through the glyph
4242    /// rules, the Symbol font's Glyph List names through its table; the
4243    /// names this resolver already knew keep their reading, and a name that
4244    /// only starts with `uni` is no `uniXXXX`.
4245    #[test]
4246    fn tex_glyph_names_resolve() {
4247        let c = |n: &str| super::glyph_name_to_char(n.as_bytes());
4248        assert_eq!(c("parenleftbig"), Some('('));
4249        assert_eq!(c("parenrightBigg"), Some(')'));
4250        assert_eq!(c("integraldisplay"), Some('∫'));
4251        assert_eq!(c("trianglerightsld"), Some('▶'));
4252        assert_eq!(c("prime"), Some('′'));
4253        assert_eq!(c("universal"), Some('∀'));
4254        assert_eq!(c("existential"), Some('∃'));
4255        assert_eq!(c("arrowup"), Some('↑'));
4256        assert_eq!(c("union"), Some('∪'));
4257        assert_eq!(c("uni2212"), Some('−'));
4258        assert_eq!(c("minus"), Some('−'));
4259        assert_eq!(c("Delta"), Some('Δ'), "the Greek letter, not the list's ∆");
4260        assert_eq!(c("angbracketleft"), Some('⟨'));
4261        assert_eq!(c("nosuchglyphname"), None);
4262    }
4263}
4264
4265#[cfg(test)]
4266mod page_box_frame {
4267    use super::*;
4268    use lopdf::dictionary;
4269
4270    /// One page, `boxes` spliced into the page dictionary verbatim, one text
4271    /// run at user-space `(x, y)`.
4272    fn pdf(boxes: &str, x: f32, y: f32) -> Vec<u8> {
4273        pdf_content(
4274            boxes,
4275            &format!("BT /F1 12 Tf {x} {y} Td (First printing) Tj ET\n"),
4276        )
4277    }
4278
4279    /// One page, `boxes` spliced into the page dictionary, `content` as its
4280    /// content stream (font `/F1` = Helvetica).
4281    fn pdf_content(boxes: &str, content: &str) -> Vec<u8> {
4282        let objs: Vec<String> = vec![
4283            "<</Type/Catalog/Pages 2 0 R>>".into(),
4284            format!("<</Type/Pages/Kids[3 0 R]/Count 1{boxes}>>"),
4285            "<</Type/Page/Parent 2 0 R/Contents 4 0 R/Resources<</Font<</F1 5 0 R>>>>>>".into(),
4286            format!("<</Length {}>>stream\n{content}endstream", content.len()),
4287            "<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".into(),
4288        ];
4289        let mut out = b"%PDF-1.4\n".to_vec();
4290        let mut offsets = Vec::new();
4291        for (i, body) in objs.iter().enumerate() {
4292            offsets.push(out.len());
4293            out.extend_from_slice(format!("{} 0 obj{body}endobj\n", i + 1).as_bytes());
4294        }
4295        let xref_at = out.len();
4296        out.extend_from_slice(
4297            format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes(),
4298        );
4299        for off in &offsets {
4300            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
4301        }
4302        out.extend_from_slice(
4303            format!(
4304                "trailer<</Size {}/Root 1 0 R>>\nstartxref\n{xref_at}\n%%EOF\n",
4305                objs.len() + 1
4306            )
4307            .as_bytes(),
4308        );
4309        out
4310    }
4311
4312    /// One page whose `/Contents` is an array of separate streams.
4313    fn pdf_content_streams(contents: &[&[u8]]) -> Vec<u8> {
4314        let mut doc = Document::with_version("1.5");
4315        let streams = contents
4316            .iter()
4317            .map(|content| {
4318                Object::Reference(
4319                    doc.add_object(lopdf::Stream::new(Dictionary::new(), content.to_vec())),
4320                )
4321            })
4322            .collect::<Vec<_>>();
4323        let font_id = doc.add_object(dictionary! {
4324            "Type" => "Font",
4325            "Subtype" => "Type1",
4326            "BaseFont" => "Helvetica",
4327        });
4328        let resources_id = doc.add_object(dictionary! {
4329            "Font" => dictionary! { "F1" => Object::Reference(font_id) },
4330        });
4331        let pages_id = doc.new_object_id();
4332        let page_id = doc.add_object(dictionary! {
4333            "Type" => "Page",
4334            "Parent" => Object::Reference(pages_id),
4335            "MediaBox" => vec![0.into(), 0.into(), 300.into(), 400.into()],
4336            "Contents" => streams,
4337            "Resources" => Object::Reference(resources_id),
4338        });
4339        doc.objects.insert(
4340            pages_id,
4341            Object::Dictionary(dictionary! {
4342                "Type" => "Pages",
4343                "Kids" => vec![Object::Reference(page_id)],
4344                "Count" => 1,
4345            }),
4346        );
4347        let catalog_id = doc.add_object(dictionary! {
4348            "Type" => "Catalog",
4349            "Pages" => Object::Reference(pages_id),
4350        });
4351        doc.trailer.set("Root", Object::Reference(catalog_id));
4352        let mut out = Vec::new();
4353        doc.save_to(&mut out).expect("save synthetic PDF");
4354        out
4355    }
4356
4357    fn only_page(bytes: &[u8]) -> (PageBox, Vec<Glyph>) {
4358        let doc = load_document(bytes).expect("loads");
4359        let pid = *doc.get_pages().values().next().expect("one page");
4360        (page_box(&doc, pid), page_glyphs(&doc, pid))
4361    }
4362
4363    /// The trimmed-book-page shape: MediaBox and CropBox both away from the
4364    /// origin (inherited from `/Pages`). The display box is the CropBox, and a
4365    /// glyph's coordinates count from its lower-left corner — the same numbers
4366    /// the same text gets on a page whose CropBox *is* `[0 0 w h]`.
4367    #[test]
4368    fn glyphs_count_from_the_cropbox_corner_like_pdfium() {
4369        let (pb, shifted) = only_page(&pdf(
4370            "/MediaBox[-56.505 -58.25 576.303 723.31]/CropBox[1.095 -0.65 518.703 665.71]",
4371            37.0 + 1.095,
4372            58.0 - 0.65,
4373        ));
4374        assert!((pb.l - 1.095).abs() < 1e-3 && (pb.b + 0.65).abs() < 1e-3);
4375        assert!(
4376            (pb.w - 517.608).abs() < 1e-3 && (pb.h - 666.36).abs() < 1e-3,
4377            "{pb:?}"
4378        );
4379        let (pb0, plain) = only_page(&pdf("/MediaBox[0 0 517.608 666.36]", 37.0, 58.0));
4380        assert!((pb0.w - pb.w).abs() < 1e-3 && (pb0.h - pb.h).abs() < 1e-3);
4381        assert_eq!(shifted.len(), plain.len());
4382        assert!(!plain.is_empty());
4383        for (a, b) in shifted.iter().zip(&plain) {
4384            assert!(
4385                (a.l - b.l).abs() < 1e-3 && (a.b - b.b).abs() < 1e-3,
4386                "{:?} vs {:?}",
4387                (a.l, a.b),
4388                (b.l, b.b)
4389            );
4390        }
4391        assert!((plain[0].l - 37.0).abs() < 1e-3, "{}", plain[0].l);
4392    }
4393
4394    fn text(glyphs: &[Glyph]) -> String {
4395        glyphs.iter().map(|g| g.ch).collect()
4396    }
4397
4398    /// #529: text drawn outside the display box — a print slug beside the
4399    /// CropBox, a tiled page's neighbour beyond the MediaBox — is dropped,
4400    /// glyph by glyph like docling-parse, instead of being clamped onto the
4401    /// page edge; a line crossing the edge keeps the glyphs that fit.
4402    #[test]
4403    fn glyphs_outside_the_display_box_are_dropped() {
4404        let (_, g) = only_page(&pdf_content(
4405            "/MediaBox[0 0 400 400]/CropBox[100 100 400 400]",
4406            "BT /F1 14 Tf 120 300 Td (Visible.) Tj ET\nBT /F1 8 Tf 5 40 Td (Slug) Tj ET\n",
4407        ));
4408        assert_eq!(text(&g), "Visible.");
4409        let (_, g) = only_page(&pdf_content(
4410            "/MediaBox[0 0 300 400]",
4411            "BT /F1 14 Tf 40 300 Td (On page) Tj ET\nBT /F1 14 Tf 400 250 Td (Beyond) Tj ET\n",
4412        ));
4413        assert_eq!(text(&g), "On page");
4414        // docling-parse: "Crossing the right edge" on a 300 pt page → "Crossing the rig".
4415        let (_, g) = only_page(&pdf_content(
4416            "/MediaBox[0 0 300 400]",
4417            "BT /F1 14 Tf 200 250 Td (Crossing the right edge) Tj ET\n",
4418        ));
4419        assert_eq!(text(&g), "Crossing the rig");
4420    }
4421
4422    #[test]
4423    fn leading_comment_content_stream_does_not_hide_later_text() {
4424        // The comment has its own EOL; lopdf inserts another while assembling
4425        // the streams, which is the boundary its 0.44 parser stopped at.
4426        let (_, glyphs) = only_page(&pdf_content_streams(&[
4427            b"% CANON_PFINF_TYPE2_TEXTOFF\n",
4428            b"BT /F1 14 Tf 40 300 Td (Second stream text) Tj ET",
4429        ]));
4430        assert_eq!(text(&glyphs), "Second stream text");
4431    }
4432
4433    /// The containment test is docling-parse's: the whole char box (advance
4434    /// × the font's ascent / descent) inside the display box, edges
4435    /// included — a Helvetica `W` at 50 pt (47.2 wide) ending exactly on the
4436    /// right edge stays, 0.01 pt further it goes; likewise on the left.
4437    #[test]
4438    fn a_glyph_on_the_edge_stays_one_over_it_goes() {
4439        let w_at = |x: f32, y: f32| {
4440            let (_, g) = only_page(&pdf_content(
4441                "/MediaBox[0 0 300 400]",
4442                &format!("BT /F1 50 Tf {x} {y} Td (W) Tj ET\n"),
4443            ));
4444            text(&g)
4445        };
4446        assert_eq!(w_at(252.8, 200.0), "W");
4447        assert_eq!(w_at(252.81, 200.0), "");
4448        assert_eq!(w_at(0.0, 200.0), "W");
4449        assert_eq!(w_at(-0.01, 200.0), "");
4450        // Vertically the box is the font's ascent / descent: a baseline far
4451        // enough up keeps the descender on the page, one at 0 does not.
4452        assert_eq!(w_at(100.0, 20.0), "W");
4453        assert_eq!(w_at(100.0, 0.0), "");
4454        assert_eq!(w_at(100.0, 380.0), "");
4455    }
4456
4457    /// pdfium's fallbacks: no MediaBox → Letter; a CropBox is clipped to the
4458    /// MediaBox, and one that misses it entirely is ignored.
4459    #[test]
4460    fn page_box_follows_pdfium_fallbacks() {
4461        let (pb, _) = only_page(&pdf("", 10.0, 10.0));
4462        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 612.0, 792.0));
4463        let (pb, _) = only_page(&pdf(
4464            "/MediaBox[0 0 500 700]/CropBox[-100 100 600 900]",
4465            10.0,
4466            10.0,
4467        ));
4468        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 100.0, 500.0, 600.0));
4469        let (pb, _) = only_page(&pdf(
4470            "/MediaBox[0 0 500 700]/CropBox[800 800 900 900]",
4471            10.0,
4472            10.0,
4473        ));
4474        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
4475        // Reversed corners normalize.
4476        let (pb, _) = only_page(&pdf("/MediaBox[500 700 0 0]", 10.0, 10.0));
4477        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
4478    }
4479}
4480
4481#[cfg(test)]
4482mod xref_repair {
4483    /// Build a tiny one-page PDF whose cross-reference entries are either the
4484    /// spec's 20 bytes (`two_byte_eol`) or the 19-byte form some generators
4485    /// emit — everything else about the two files is identical.
4486    fn pdf_with_xref(two_byte_eol: bool) -> Vec<u8> {
4487        let content = b"BT /F1 12 Tf 72 700 Td (Invoice 922769430725) Tj ET\n";
4488        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
4489        let objs: Vec<Vec<u8>> = vec![
4490            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
4491            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
4492            b"<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
4493               /Resources<</Font<</F1 5 0 R>>>>>>"
4494                .to_vec(),
4495            [stream.as_slice(), content.as_slice(), b"endstream"].concat(),
4496            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".to_vec(),
4497        ];
4498
4499        let mut out = b"%PDF-1.4\n".to_vec();
4500        let mut offsets = Vec::new();
4501        for (i, body) in objs.iter().enumerate() {
4502            offsets.push(out.len());
4503            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
4504            out.extend_from_slice(body);
4505            out.extend_from_slice(b"endobj\n");
4506        }
4507        let xref_at = out.len();
4508        let eol: &[u8] = if two_byte_eol { b" \n" } else { b"\n" };
4509        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
4510        out.extend_from_slice(b"0000000000 65535 f");
4511        out.extend_from_slice(eol);
4512        for off in &offsets {
4513            out.extend_from_slice(format!("{off:010} 00000 n").as_bytes());
4514            out.extend_from_slice(eol);
4515        }
4516        out.extend_from_slice(
4517            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
4518        );
4519        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
4520        out
4521    }
4522
4523    /// A 19-byte cross-reference entry (a bare LF where the spec wants a
4524    /// two-byte EOL) used to make lopdf reject the whole file. Lopdf 0.45
4525    /// accepts it directly, so exercise the fallback itself and prove it still
4526    /// produces the exact 20-byte entries lopdf 0.44 needed.
4527    #[test]
4528    fn short_xref_entries_are_padded_for_fallback() {
4529        let good = pdf_with_xref(true);
4530        let broken = pdf_with_xref(false);
4531        assert!(
4532            broken.len() < good.len(),
4533            "the broken file is the shorter one"
4534        );
4535        assert!(
4536            lopdf::Document::load_mem(&good).is_ok(),
4537            "the control file must load unaided"
4538        );
4539        assert!(
4540            lopdf::Document::load_mem(&broken).is_ok(),
4541            "lopdf 0.45 should accept 19-byte entries"
4542        );
4543
4544        let repaired = super::pad_short_xref_entries(&broken).expect("repair short entries");
4545        assert_eq!(repaired, good, "repair should recreate the valid xref");
4546        assert!(
4547            lopdf::Document::load_mem(&repaired).is_ok(),
4548            "the bytes produced for lopdf 0.44 must load"
4549        );
4550
4551        let cells = |b: &[u8]| -> Vec<String> {
4552            super::pdf_textlines(b)
4553                .into_iter()
4554                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
4555                .collect()
4556        };
4557        let from_good = cells(&good);
4558        assert!(
4559            from_good.iter().any(|t| t.contains("922769430725")),
4560            "control text: {from_good:?}"
4561        );
4562        assert_eq!(
4563            cells(&repaired),
4564            from_good,
4565            "repair must match the good parse"
4566        );
4567    }
4568
4569    /// The same generator overstates `/Length`. Lopdf 0.45 recovers at
4570    /// `endstream`, so exercise our fallback directly and prove it still writes
4571    /// the exact length lopdf 0.44 needed without moving any bytes.
4572    #[test]
4573    fn overstated_stream_length_is_fixed_for_fallback() {
4574        let good = pdf_with_xref(true);
4575        // Inflate the content stream's /Length by one, exactly as the invoice
4576        // that prompted this does.
4577        let broken = {
4578            let at = good
4579                .windows(8)
4580                .position(|w| w == b"/Length ")
4581                .expect("a /Length")
4582                + 8;
4583            let digits = good[at..].iter().take_while(|c| c.is_ascii_digit()).count();
4584            let n: usize = std::str::from_utf8(&good[at..at + digits])
4585                .unwrap()
4586                .parse()
4587                .unwrap();
4588            let inflated = (n + 1).to_string();
4589            assert_eq!(inflated.len(), digits, "keep the digit count");
4590            let mut b = good.clone();
4591            b[at..at + digits].copy_from_slice(inflated.as_bytes());
4592            b
4593        };
4594        assert_eq!(broken.len(), good.len(), "the defect must not move bytes");
4595        // Lopdf 0.45 keeps the stream despite the overstated length.
4596        let raw = lopdf::Document::load_mem(&broken).expect("still loads");
4597        assert!(
4598            raw.get_pages()
4599                .into_values()
4600                .any(|p| !raw.get_page_content(p).is_empty()),
4601            "lopdf 0.45 should recover content at endstream"
4602        );
4603
4604        let repaired = super::fix_stream_lengths(&broken);
4605        assert_eq!(repaired.len(), broken.len(), "repair must preserve offsets");
4606        assert_eq!(repaired, good, "repair should restore the true length");
4607        let repaired_doc = lopdf::Document::load_mem(&repaired).expect("repaired PDF loads");
4608        assert!(
4609            repaired_doc
4610                .get_pages()
4611                .into_values()
4612                .any(|p| !repaired_doc.get_page_content(p).is_empty()),
4613            "the bytes produced for lopdf 0.44 must retain page content"
4614        );
4615
4616        // Ours recovers the same text the well-formed file gives.
4617        let text = |b: &[u8]| -> Vec<String> {
4618            super::pdf_textlines(b)
4619                .into_iter()
4620                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
4621                .collect()
4622        };
4623        let expected = text(&good);
4624        assert!(!expected.is_empty(), "control must produce text");
4625        assert_eq!(text(&repaired), expected);
4626    }
4627
4628    /// The repair only fires where padding cannot move an object: it declines a
4629    /// file whose xref precedes an object (an incremental update), rather than
4630    /// shifting every offset the table records.
4631    #[test]
4632    fn repair_declines_when_padding_would_move_objects() {
4633        let mut incremental = pdf_with_xref(false);
4634        incremental.extend_from_slice(b"6 0 obj<</Type/Whatever>>endobj\n");
4635        let declined = super::pad_short_xref_entries(&incremental).unwrap_err();
4636        assert!(
4637            declined.contains("object follows the xref"),
4638            "reason: {declined}"
4639        );
4640    }
4641}
4642
4643/// #187: standard-14 fonts referenced without an embedded program (and thus
4644/// usually without `/Widths` or a `/FontDescriptor`) must decode with the
4645/// built-in Adobe Core 14 metrics instead of collapsing every cell to zero
4646/// width — the failure mode where a valid text layer was silently dropped
4647/// while pdfium read the same file fine.
4648#[cfg(test)]
4649mod base14_fonts {
4650    /// A one-page PDF whose single `Tj` uses `fontdict` (no embedded program).
4651    fn pdf_with_font(fontdict: &[u8], text: &[u8]) -> Vec<u8> {
4652        let content = [b"BT /F1 12 Tf 72 700 Td (".as_slice(), text, b") Tj ET\n"].concat();
4653        pdf_with_content(fontdict, &content)
4654    }
4655
4656    /// A one-page A4 PDF drawing `content` with `fontdict` as `/F1`.
4657    pub(super) fn pdf_with_content(fontdict: &[u8], content: &[u8]) -> Vec<u8> {
4658        pdf_with_fonts(&[fontdict], content)
4659    }
4660
4661    /// A one-page A4 PDF drawing `content` with `fontdicts[i]` bound as
4662    /// `/F{i+1}` (objects 5, 6, …).
4663    fn pdf_with_fonts(fontdicts: &[&[u8]], content: &[u8]) -> Vec<u8> {
4664        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
4665        let font_res: String = fontdicts
4666            .iter()
4667            .enumerate()
4668            .map(|(i, _)| format!("/F{} {} 0 R", i + 1, i + 5))
4669            .collect();
4670        let mut objs: Vec<Vec<u8>> = vec![
4671            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
4672            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
4673            format!(
4674                "<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
4675                 /Resources<</Font<<{font_res}>>>>>>"
4676            )
4677            .into_bytes(),
4678            [stream.as_slice(), content, b"endstream"].concat(),
4679        ];
4680        objs.extend(fontdicts.iter().map(|d| d.to_vec()));
4681        let mut out = b"%PDF-1.4\n".to_vec();
4682        let mut offsets = Vec::new();
4683        for (i, body) in objs.iter().enumerate() {
4684            offsets.push(out.len());
4685            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
4686            out.extend_from_slice(body);
4687            out.extend_from_slice(b"endobj\n");
4688        }
4689        let xref_at = out.len();
4690        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
4691        out.extend_from_slice(b"0000000000 65535 f \n");
4692        for off in &offsets {
4693            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
4694        }
4695        out.extend_from_slice(
4696            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
4697        );
4698        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
4699        out
4700    }
4701
4702    /// The parsed cells of the only page.
4703    pub(super) fn cells(pdf: &[u8]) -> Vec<crate::pdfium_backend::TextCell> {
4704        super::pdf_textlines(pdf)
4705            .into_iter()
4706            .flat_map(|(_, _, c)| c)
4707            .collect()
4708    }
4709
4710    /// Text drawn with a rotated text matrix reads as one line in its own
4711    /// reading order, boxed by the run's axis-aligned extent (#528). The
4712    /// landscape case is `0 s -s 0 tx ty Tm` on a `/Rotate 90` page; every
4713    /// such run used to collapse to zero width and vanish (90°/270°/tilted)
4714    /// or read backwards (180°). Expected boxes are docling-parse 7.22's line
4715    /// cells for the same runs (y-up, PDF points).
4716    #[test]
4717    fn rotated_text_matrix_reads_in_order() {
4718        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
4719        const TEXT: &str = "Upright text on a rotated page.";
4720        for (tm, [l, b, r, t]) in [
4721            ("0 14 -14 0 200 100", [189.9, 100.0, 202.9, 289.1]), // 90°
4722            ("-14 0 0 -14 350 300", [160.9, 289.9, 350.0, 302.9]), // 180°
4723            ("0 -14 14 0 200 500", [197.1, 310.9, 210.1, 500.0]), // 270°
4724            (
4725                "9.8995 9.8995 -9.8995 9.8995 100 100",
4726                [92.9, 98.0, 235.8, 240.8],
4727            ), // 45°
4728        ] {
4729            let content = format!("BT /F1 1 Tf {tm} Tm ({TEXT}) Tj ET\n");
4730            let pdf = pdf_with_content(HELV, content.as_bytes());
4731            let cs = cells(&pdf);
4732            assert_eq!(cs.len(), 1, "{tm}: {cs:?}");
4733            let c = &cs[0];
4734            assert_eq!(c.text, TEXT, "{tm}");
4735            // `cells` is top-left-origin on the 842 pt page. The descriptor-
4736            // less base-14 face gets the parser's 1-em box where docling-parse
4737            // reads Helvetica's AFM (718/−207) — the same ≤ 0.6 pt
4738            // across-the-baseline offset upright text has — so allow 1 pt.
4739            let got = [c.l, 842.0 - c.b, c.r, 842.0 - c.t];
4740            for (g, w) in got.iter().zip([l, b, r, t]) {
4741                assert!(
4742                    (g - w).abs() < 1.0,
4743                    "{tm}: box {got:?} vs docling-parse {:?}",
4744                    [l, b, r, t]
4745                );
4746            }
4747            let words: Vec<String> = super::pdf_words(&pdf)
4748                .into_iter()
4749                .flat_map(|(_, _, c)| c)
4750                .map(|c| c.text)
4751                .collect();
4752            assert_eq!(words, TEXT.split(' ').collect::<Vec<_>>(), "{tm}");
4753        }
4754    }
4755
4756    /// A rotated run past the page edge (a cover's spine title) is dropped
4757    /// by the on-page test (#529) — measured on the quad's extent, not the
4758    /// zero-width box rotated glyphs used to get — instead of clamping onto
4759    /// the edge as a zero-width cell. One inside the page is kept.
4760    #[test]
4761    fn rotated_text_off_the_page_is_dropped() {
4762        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
4763        let off = pdf_with_content(
4764            HELV,
4765            b"BT /F1 1 Tf 0 14 -14 0 -4 200 Tm (Spine title) Tj ET\n",
4766        );
4767        assert!(cells(&off).is_empty(), "{:?}", cells(&off));
4768        let on = pdf_with_content(
4769            HELV,
4770            b"BT /F1 1 Tf 0 14 -14 0 30 200 Tm (Margin note) Tj ET\n",
4771        );
4772        assert_eq!(cells(&on).len(), 1);
4773    }
4774
4775    /// Upright text keeps the plain loose rectangle — no quad, and the
4776    /// sanitizer path is the one every pinned PDF baseline was made with.
4777    #[test]
4778    fn upright_glyphs_carry_no_quad() {
4779        let only_page = |pdf: &[u8]| {
4780            let doc = super::load_document(pdf).expect("loads");
4781            let pid = *doc.get_pages().values().next().expect("one page");
4782            super::page_glyphs(&doc, pid)
4783        };
4784        let pdf = pdf_with_font(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", b"Plain");
4785        let glyphs = only_page(&pdf);
4786        assert!(!glyphs.is_empty());
4787        assert!(glyphs.iter().all(|g| g.quad.is_none() && g.lr > g.ll));
4788        // A 90° glyph's style height is its font size, not its advance.
4789        let pdf = pdf_with_content(
4790            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>",
4791            b"BT /F1 1 Tf 0 14 -14 0 200 100 Tm (W) Tj ET\n",
4792        );
4793        let glyphs = only_page(&pdf);
4794        let g = &glyphs[0];
4795        assert!(g.quad.is_some());
4796        // 14 pt × the face's 1-em box (no descriptor: ascent − descent = 1).
4797        assert!((g.height() - 14.0).abs() < 0.05, "{}", g.height());
4798    }
4799
4800    /// #609: ReportLab's chart axis sets each tick label with the *same* `Tm`
4801    /// inside its own `q … cm … Q`, so only the CTM stacks `3`…`8` one under
4802    /// another at one x. Each tick is its own line cell with its own box (as
4803    /// docling-parse reads it), not one `345678` cell boxed like the first.
4804    #[test]
4805    fn ticks_stacked_by_cm_are_separate_line_cells() {
4806        let mut content = String::new();
4807        for (k, tick) in "345678".chars().enumerate() {
4808            let y = 300.0 + 25.92 * k as f64;
4809            content.push_str(&format!(
4810                "q 1 0 0 1 125 {y} cm BT /F1 10 Tf 1 0 0 1 -5 -4 Tm ({tick}) Tj ET Q\n"
4811            ));
4812        }
4813        let pdf = pdf_with_content(
4814            b"<</Type/Font/Subtype/Type1/BaseFont/Times-Roman>>",
4815            content.as_bytes(),
4816        );
4817        let cells = cells(&pdf);
4818        let texts: Vec<&str> = cells.iter().map(|c| c.text.as_str()).collect();
4819        assert_eq!(texts, ["3", "4", "5", "6", "7", "8"]);
4820        for (k, c) in cells.iter().enumerate() {
4821            // Top-left y of a tick whose baseline is 4 pt under its cm origin.
4822            let base = 842.0 - (300.0 + 25.92 * k as f32 - 4.0);
4823            assert!((c.l - 120.0).abs() < 0.1, "{k}: l {}", c.l);
4824            assert!(
4825                c.t < base && base - c.t < 10.0,
4826                "{k}: t {} base {base}",
4827                c.t
4828            );
4829        }
4830    }
4831
4832    /// #609: the path walk finds drawn checkbox squares — ReportLab's four
4833    /// `m … l S` edges under a `cm`, and a stroked `re` with a tick inside
4834    /// (checked) — in the text cells' top-left frame, and a stroked box
4835    /// filled white; a filled bar, a big frame and a colour-filled legend
4836    /// swatch are not checkboxes.
4837    #[test]
4838    fn drawn_checkbox_squares_are_found() {
4839        let content = b"q 1 0 0 1 119.52 449.04 cm \
4840            n 0 12.96 m 12.96 12.96 l S n 0 0 m 12.96 0 l S \
4841            n 0 0 m 0 12.96 l S n 12.96 0 m 12.96 12.96 l S Q\n\
4842            200 400 10 10 re S 202 405 m 204.5 402 l 208.5 408.5 l S\n\
4843            300 400 12 12 re f 50 50 100 100 re S\n\
4844            q 0.2 0.4 0.8 rg 400 400 12 12 re B Q q 1 g 500 400 12 12 re B Q\n";
4845        let pdf = pdf_with_content(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", content);
4846        let mut parser = super::PageTextParser::open(&pdf).expect("parses");
4847        let mut boxes = parser.cells(0).checkboxes;
4848        boxes.sort_by(|a, b| a.l.total_cmp(&b.l));
4849        let got: Vec<([i32; 4], bool)> = boxes
4850            .iter()
4851            .map(|c| {
4852                let r = |v: f32| (v * 100.0).round() as i32;
4853                ([r(c.l), r(c.t), r(c.r), r(c.b)], c.checked)
4854            })
4855            .collect();
4856        // Page height 842: y-up 449.04..462.0 → top-left 380.0..392.96.
4857        assert_eq!(
4858            got,
4859            [
4860                ([11952, 38000, 13248, 39296], false),
4861                ([20000, 43200, 21000, 44200], true),
4862                ([50000, 43000, 51200, 44200], false),
4863            ]
4864        );
4865    }
4866
4867    /// Every standard-14 alias/style decodes with real (positive-width) boxes.
4868    #[test]
4869    fn standard14_faces_get_builtin_widths() {
4870        for fontdict in [
4871            // ReportLab's default: base-14 Helvetica, WinAnsi, nothing else.
4872            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>".as_slice(),
4873            // No /Encoding at all (StandardEncoding-ish default).
4874            b"<</Type/Font/Subtype/Type1/BaseFont/Times-BoldItalic>>",
4875            // Substitution aliases + a subset prefix.
4876            b"<</Type/Font/Subtype/TrueType/BaseFont/Arial,Bold>>",
4877            b"<</Type/Font/Subtype/Type1/BaseFont/ABCDEF+Courier-Oblique>>",
4878        ] {
4879            let pdf = pdf_with_font(fontdict, b"Words have width now");
4880            let cs = cells(&pdf);
4881            let text: String = cs
4882                .iter()
4883                .map(|c| c.text.as_str())
4884                .collect::<Vec<_>>()
4885                .join(" ");
4886            assert!(
4887                text.contains("Words have width now"),
4888                "{}: text lost: {text:?}",
4889                String::from_utf8_lossy(fontdict)
4890            );
4891            assert!(
4892                cs.iter().all(|c| c.r > c.l),
4893                "{}: zero-width cells: {cs:?}",
4894                String::from_utf8_lossy(fontdict)
4895            );
4896        }
4897    }
4898
4899    /// MacRomanEncoding without a ToUnicode CMap (#744) — Word for Mac's
4900    /// Quartz output: the whole high range decodes, Danish æ ø å, é, ü, ß
4901    /// and § included, through the plain encoding name and as the base of a
4902    /// `/Differences` dictionary alike.
4903    #[test]
4904    fn macroman_fonts_keep_accented_letters() {
4905        let sentence = b"Bl\x8cb\xbergr\xbfd p\x8c \xbfen \xd0 caf\x8e M\x9fller, Stra\xa7e \xa4 3";
4906        for (fontdict, want) in [
4907            (
4908                b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/MacRomanEncoding>>"
4909                    .as_slice(),
4910                "Blåbærgrød på øen – café Müller, Straße § 3",
4911            ),
4912            (
4913                b"<</Type/Font/Subtype/TrueType/BaseFont/ABCDEF+Arial\
4914                   /Encoding<</Type/Encoding/BaseEncoding/MacRomanEncoding/Differences[51/four]>>>>",
4915                "Blåbærgrød på øen – café Müller, Straße § 4",
4916            ),
4917        ] {
4918            let text: String = cells(&pdf_with_font(fontdict, sentence))
4919                .iter()
4920                .map(|c| c.text.as_str())
4921                .collect::<Vec<_>>()
4922                .join(" ");
4923            assert_eq!(text, want, "{}", String::from_utf8_lossy(fontdict));
4924        }
4925
4926        // The table itself: all of 0x80–0xFF but the Apple logo, ¤ at 0xDB.
4927        let table = super::macroman_table();
4928        assert_eq!(
4929            (0x80u8..=0xFF).filter(|b| table.contains_key(b)).count(),
4930            127
4931        );
4932        assert_eq!(table.get(&0xF0), None);
4933        assert_eq!(table[&0xDB], '¤');
4934        assert_eq!(
4935            [0x80, 0x8C, 0xAE, 0xAF, 0xBE, 0xBF, 0xFF].map(|b| table[&b]),
4936            ['Ä', 'å', 'Æ', 'Ø', 'æ', 'ø', 'ˇ']
4937        );
4938    }
4939
4940    /// WinAnsiEncoding's unused codes above octal 40 are bullets (PDF
4941    /// 32000-1 Annex D.2) — ReportLab's default list bullet is `\177` in a
4942    /// non-embedded WinAnsi Helvetica and read U+FFFD (#1102). A font
4943    /// without an `/Encoding` keeps them undefined.
4944    #[test]
4945    fn winansi_unused_codes_are_bullets() {
4946        let read = |fontdict: &[u8], text: &[u8]| -> String {
4947            cells(&pdf_with_font(fontdict, text))
4948                .iter()
4949                .map(|c| c.text.as_str())
4950                .collect::<Vec<_>>()
4951                .join(" ")
4952        };
4953        let winansi = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4954        assert_eq!(read(winansi, b"\\177 Cut costs"), "\u{2022} Cut costs");
4955        for code in [0x7Fu8, 0x81, 0x8D, 0x8F, 0x90, 0x9D] {
4956            assert_eq!(
4957                read(winansi, &[b'x', code, b'x']),
4958                "x\u{2022}x",
4959                "{code:#x}"
4960            );
4961        }
4962        let dict = b"<</Type/Font/Subtype/TrueType/BaseFont/Arial\
4963                     /Encoding<</Type/Encoding/BaseEncoding/WinAnsiEncoding>>>>";
4964        assert_eq!(read(dict, b"\\177 Cut costs"), "\u{2022} Cut costs");
4965        let bare = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
4966        assert!(!read(bare, b"x\\177x").contains('\u{2022}'));
4967    }
4968
4969    /// Unmapped glyphs read U+FFFD (#1003, docling#5070): a ToUnicode NUL —
4970    /// docling's `test_threaded_page_backend_replaces_nul_with_replacement_char`
4971    /// case, `mg.mL \x00 1` — and a bare glyph-index `/Differences` name,
4972    /// while an unknown code-like name keeps its text without the slash
4973    /// (docling-parse 7.24's `SM590000`).
4974    #[test]
4975    fn unmapped_glyphs_read_as_the_replacement_char() {
4976        let cmap = b"/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n\
4977            1 begincodespacerange <00> <FF> endcodespacerange\n\
4978            1 beginbfchar <7e> <0000> endbfchar\n\
4979            endcmap CMapName currentdict /CMap defineresource pop end end";
4980        let stream = [
4981            format!("<</Length {}>>stream\n", cmap.len()).as_bytes(),
4982            cmap.as_slice(),
4983            b"\nendstream",
4984        ]
4985        .concat();
4986        let pdf = pdf_with_fonts(
4987            &[
4988                b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/ToUnicode 6 0 R\
4989                   /Encoding<</Type/Encoding/Differences[1/g115 2/SM590000]>>>>",
4990                &stream,
4991            ],
4992            b"BT /F1 12 Tf 72 700 Td (mg.mL ~ 1 \x01 x \x02 y) Tj ET\n",
4993        );
4994        let text: String = cells(&pdf)
4995            .iter()
4996            .map(|c| c.text.as_str())
4997            .collect::<Vec<_>>()
4998            .join(" ");
4999        assert_eq!(text, "mg.mL \u{FFFD} 1 \u{FFFD} x SM590000 y");
5000    }
5001
5002    /// redp5110's subset font names every glyph by index (`/g115/g3 /g40…`):
5003    /// the painted ones read U+FFFD, the blank `/g3` program is a space, so
5004    /// the line keeps its words (docling-parse 7.24's line cell).
5005    #[cfg(feature = "render")]
5006    #[test]
5007    fn a_blank_glyph_program_is_a_space() {
5008        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
5009            .join("../../tests/data/pdf/sources/redp5110_sampled.pdf");
5010        let Ok(bytes) = std::fs::read(path) else {
5011            return;
5012        };
5013        let lines: Vec<String> = super::pdf_textlines(&bytes)
5014            .into_iter()
5015            .flat_map(|(_, _, c)| c)
5016            .map(|c| c.text)
5017            .collect();
5018        let want = "\u{FFFD} \u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD} \
5019                    \u{FFFD}\u{FFFD}\u{FFFD} ";
5020        assert!(
5021            lines.iter().any(|l| l.starts_with(want)),
5022            "no `{want}` line in {:?}",
5023            lines
5024                .iter()
5025                .filter(|l| l.contains('\u{FFFD}'))
5026                .collect::<Vec<_>>()
5027        );
5028    }
5029
5030    /// 2305.03393's figure-caption font (`SFRM0900`) declares `/Ascent 0
5031    /// /Descent 0`, which docling-parse 7.24 rejects: its glyphs are boxed by
5032    /// their outline, so the underscores of `"PMC2807444_006_00.png"` — ink
5033    /// under the baseline — are line cells of their own, as docling-parse's
5034    /// are (same boxes to 0.01 pt), not part of the file name.
5035    #[cfg(feature = "render")]
5036    #[test]
5037    fn an_invalid_metric_font_is_boxed_by_its_ink() {
5038        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
5039            .join("../../tests/data/pdf/sources/2305.03393v1.pdf");
5040        let Ok(bytes) = std::fs::read(path) else {
5041            return;
5042        };
5043        let pages = super::pdf_textlines(&bytes);
5044        let line = |text: &str| {
5045            pages[9]
5046                .2
5047                .iter()
5048                .find(|c| c.text == text)
5049                .unwrap_or_else(|| panic!("no `{text}` line cell"))
5050                .clone()
5051        };
5052        let name = line("\"PMC2807444");
5053        let under: Vec<_> = pages[9].2.iter().filter(|c| c.text == "_").collect();
5054        assert_eq!(under.len(), 2, "two `_` cells");
5055        for u in under {
5056            assert!(
5057                u.t > name.b,
5058                "`_` under the digits' ink: {:?} vs {:?}",
5059                u,
5060                name
5061            );
5062            assert!(
5063                (u.b - u.t - 0.403).abs() < 0.01,
5064                "an outline-tight box: {u:?}"
5065            );
5066        }
5067    }
5068
5069    /// docling-parse's descriptor checks (`init_ascent_and_descent`): metrics
5070    /// that do not straddle the baseline give way to a valid `/FontBBox` and
5071    /// mark the font for outline boxes; an implausible one is ignored for its
5072    /// `/FontBBox` edge; plain metrics are kept.
5073    #[test]
5074    fn descriptor_metrics_follow_docling_parse() {
5075        use lopdf::{dictionary, Object};
5076        let font = |asc: Option<i64>, desc: Option<i64>, bbox: bool| {
5077            let mut fd = dictionary! { "Type" => "FontDescriptor", "FontName" => "X" };
5078            if let Some(a) = asc {
5079                fd.set("Ascent", a);
5080            }
5081            if let Some(d) = desc {
5082                fd.set("Descent", d);
5083            }
5084            if bbox {
5085                let b: Vec<Object> = [-196, -322, 1502, 937].map(Object::from).to_vec();
5086                fd.set("FontBBox", b);
5087            }
5088            let f = dictionary! {
5089                "Type" => "Font", "Subtype" => "Type1", "BaseFont" => "X",
5090                "FontDescriptor" => fd,
5091            };
5092            super::font_ascent_descent(&lopdf::Document::with_version("1.5"), &f, false)
5093        };
5094        assert_eq!(font(Some(0), Some(0), true), (937.0, -322.0, true));
5095        assert_eq!(font(Some(700), Some(200), true), (937.0, -322.0, true));
5096        assert_eq!(font(Some(90000), Some(-250), true), (937.0, -250.0, true));
5097        assert_eq!(font(Some(718), Some(-207), true), (718.0, -207.0, false));
5098        assert_eq!(font(Some(0), Some(0), false), (750.0, -250.0, false));
5099        assert_eq!(font(None, None, true), (750.0, -250.0, false));
5100    }
5101
5102    /// redp5110 p. 14 draws its navigation arrows in `SymbolMT`, whose
5103    /// `/ToUnicode` maps them into the private-use symbol charset
5104    /// (`U+F0AE`): read through Symbol's encoding they are `→`, as
5105    /// docling-parse 7.25 reads them (#1113).
5106    #[test]
5107    fn symbol_private_use_values_read_as_the_glyphs() {
5108        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
5109            .join("../../tests/data/pdf/sources/redp5110_sampled.pdf");
5110        let Ok(bytes) = std::fs::read(path) else {
5111            return;
5112        };
5113        let lines: Vec<String> = super::pdf_textlines(&bytes)
5114            .into_iter()
5115            .flat_map(|(_, _, c)| c)
5116            .map(|c| c.text)
5117            .collect();
5118        assert!(
5119            lines
5120                .iter()
5121                .any(|l| l.ends_with("click Schemas → HR_SCHEMA→")),
5122            "{:?}",
5123            lines
5124                .iter()
5125                .filter(|l| l.contains("Schemas"))
5126                .collect::<Vec<_>>()
5127        );
5128        assert!(lines.iter().all(|l| !l.contains('\u{F0AE}')));
5129    }
5130
5131    /// 2203.01017's p. 4 table marks its datasets with an embedded Type 1
5132    /// `Dingbats` — symbolic, no `/Encoding`, no `/ToUnicode` — whose
5133    /// program encodes code `3` as `/a19` and `7` as `/a23`: `✓` and `✗`
5134    /// through the builtin encoding, as docling-parse 7.25 reads them
5135    /// (#1113), where the Latin tables read the digits.
5136    #[cfg(feature = "render")]
5137    #[test]
5138    fn a_symbolic_font_reads_through_its_builtin_encoding() {
5139        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
5140            .join("../../tests/data/pdf/sources/2203.01017v2.pdf");
5141        let Ok(bytes) = std::fs::read(path) else {
5142            return;
5143        };
5144        let pages = super::pdf_textlines(&bytes);
5145        let text: Vec<&str> = pages[3].2.iter().map(|c| c.text.as_str()).collect();
5146        let page = text.join("\n");
5147        assert_eq!(page.matches('✓').count(), 11, "{text:?}");
5148        assert_eq!(page.matches('✗').count(), 1, "{text:?}");
5149        assert!(text.contains(&"✓ ✗ 145k JPEG"), "{text:?}");
5150    }
5151
5152    /// docling-parse 7.25's `glyph_name_to_utf8`.
5153    #[test]
5154    fn builtin_glyph_names_read_as_docling_parse() {
5155        let t = |n: &str| super::builtin_glyph_text(n);
5156        assert_eq!(t("a19").as_deref(), Some("✓"));
5157        assert_eq!(t("arrowright").as_deref(), Some("→"));
5158        assert_eq!(t("A").as_deref(), Some("A"));
5159        assert_eq!(t("one.oldstyle").as_deref(), Some("1"), "suffix dropped");
5160        assert_eq!(t("uni00660069").as_deref(), Some("fi"), "a uni sequence");
5161        assert_eq!(t("u1F600").as_deref(), Some("😀"));
5162        assert_eq!(t("g12"), None, "an index name");
5163        assert_eq!(t("GID00012"), None);
5164        assert_eq!(t("cid7"), None);
5165        assert_eq!(t(".notdef"), None);
5166        assert_eq!(t(""), None);
5167        assert_eq!(t("nosuchglyphname"), None);
5168    }
5169
5170    /// The page's image placements reach the cells (#999): the scan under
5171    /// the #999 fixture's one native word is the whole page.
5172    #[test]
5173    fn image_placements_are_the_pages_bitmaps() {
5174        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
5175            .join("../docling/tests/fixtures/partial_text_layer/scan_with_native_word.pdf");
5176        let bytes = std::fs::read(path).expect("fixture");
5177        let mut parser = super::PageTextParser::open(&bytes).expect("parser");
5178        let page = parser.cells(0);
5179        assert_eq!(page.bitmaps, vec![[0.0, 0.0, 612.0, 792.0]]);
5180        assert_eq!(page.words.len(), 1);
5181        assert!(page.checkboxes.is_empty());
5182    }
5183
5184    /// A `bfrange` whose destination is a surrogate pair steps its last code
5185    /// unit (#1003: equations.pdf's `H𝛂`, `<1d59> <1d5a> <d835dec2>`); a
5186    /// single-unit base keeps counting in code points.
5187    #[test]
5188    fn surrogate_pair_bfrange_maps_each_code() {
5189        let map = super::parse_tounicode(
5190            b"1 beginbfrange <1d59> <1d5a> <d835dec2> endbfrange\n\
5191              1 beginbfrange <0010> <0011> <0041> endbfrange",
5192        );
5193        assert_eq!(map.get(&0x1d59).map(String::as_str), Some("\u{1D6C2}"));
5194        assert_eq!(map.get(&0x1d5a).map(String::as_str), Some("\u{1D6C3}"));
5195        assert_eq!(map.get(&0x11).map(String::as_str), Some("B"));
5196    }
5197
5198    /// Which unknown glyph names are text and which are bare indexes
5199    /// (docling-parse's `resolve_unknown_name`; Skia's hexadecimal `g%X`
5200    /// since 7.25, #1113).
5201    #[test]
5202    fn unknown_glyph_names_follow_docling_parse() {
5203        for name in [
5204            "g115", "gid00043", "Glyph7", "cid42", "index9", "g3.alt", "g15C", "gA1", "G1F",
5205        ] {
5206            assert_eq!(super::unknown_name_text(name), None, "{name}");
5207            assert!(
5208                super::is_gid_name(name.split('.').next().unwrap().as_bytes()),
5209                "{name}"
5210            );
5211        }
5212        assert!(!super::is_gid_name(b"glyphs"), "not hexadecimal");
5213        assert!(!super::is_gid_name(b"uni0041"));
5214        assert_eq!(
5215            super::unknown_name_text("SM590000").as_deref(),
5216            Some("SM590000")
5217        );
5218        assert_eq!(
5219            super::unknown_name_text("SM590000.sc").as_deref(),
5220            Some("SM590000")
5221        );
5222    }
5223
5224    /// An explicit `/Widths` array always wins over the built-in metrics, and a
5225    /// non-standard face without `/Widths` stays as before (no invented boxes).
5226    #[test]
5227    fn explicit_widths_win_and_unknown_faces_are_untouched() {
5228        // Helvetica with explicit 100/1000-em widths: the word's box must be
5229        // ~4×100 units at 12pt = 4.8pt wide — far narrower than the ~2.7×
5230        // wider built-in Helvetica advances would make it.
5231        let explicit = pdf_with_font(
5232            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/FirstChar 65\
5233               /Widths[100 100 100 100]/Encoding/WinAnsiEncoding>>",
5234            b"ABBA",
5235        );
5236        let builtin = pdf_with_font(
5237            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>",
5238            b"ABBA",
5239        );
5240        let w = |pdf: &[u8]| {
5241            let cs = cells(pdf);
5242            assert_eq!(cs.len(), 1, "one word cell: {cs:?}");
5243            cs[0].r - cs[0].l
5244        };
5245        let (we, wb) = (w(&explicit), w(&builtin));
5246        assert!(
5247            (we - 4.8).abs() < 0.1,
5248            "explicit widths must win: got {we}, want 4×100×12/1000"
5249        );
5250        assert!(
5251            wb > 2.0 * we,
5252            "built-in Helvetica is much wider: {wb} vs {we}"
5253        );
5254
5255        // An unknown face with no /Widths: still parses (text kept), but no
5256        // built-in table applies — the old zero-width behavior is preserved
5257        // rather than inventing Helvetica metrics for an arbitrary font.
5258        let unknown = pdf_with_font(
5259            b"<</Type/Font/Subtype/Type1/BaseFont/FancyCorp-Display>>",
5260            b"Mystery",
5261        );
5262        let cs = cells(&unknown);
5263        let text: String = cs.iter().map(|c| c.text.as_str()).collect();
5264        assert!(text.contains("Mystery"), "text still decodes: {cs:?}");
5265    }
5266
5267    /// #1121: a TeX math font names its glyphs at the low char codes —
5268    /// `/parenleftbig` at 0, `/parenrightbig` at 1 in Latin Modern's math
5269    /// extension font (1805.06723) — with no ToUnicode. The names are not
5270    /// Adobe Glyph List names; unresolved, the codes read U+FFFD
5271    /// (`rt�A(M)�`). They read as docling-parse reads them, and a font whose
5272    /// CMap maps the code to NUL reads its `/Differences` name.
5273    #[test]
5274    fn tex_math_glyphs_at_low_codes_decode() {
5275        let font = b"<</Type/Font/Subtype/Type1/BaseFont/ABCDEF+LMMathExtension10-Regular\
5276            /FirstChar 0/LastChar 90/Widths[458 458 0 0 0 0 0 0 0 0 0 0 0 0 0 0 597 597 \
5277            0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 \
5278            0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 472]\
5279            /Encoding<</Type/Encoding/Differences[0/parenleftbig/parenrightbig \
5280            16/parenleftBig/parenrightBig 90/integraldisplay]>>>>";
5281        let text = |bytes: &[u8]| -> String {
5282            cells(&pdf_with_font(font, bytes))
5283                .iter()
5284                .map(|c| c.text.as_str())
5285                .collect()
5286        };
5287        assert_eq!(text(b"\\000\\001"), "()");
5288        assert_eq!(text(b"\\020\\021Z"), "()∫");
5289        assert!(!text(b"\\000\\001\\020\\021Z").contains('\u{FFFD}'));
5290    }
5291
5292    /// The Adobe Glyph List in full (the `glyph-list` feature): a Cyrillic
5293    /// font re-encoded with `/Differences` and no ToUnicode names its glyphs
5294    /// `afii10017`, `afii10018`, …. Unresolved, each read the Latin letter of
5295    /// the base encoding at its code — plausible text in the wrong script.
5296    #[cfg(feature = "glyph-list")]
5297    #[test]
5298    fn glyph_list_names_decode_a_reencoded_cyrillic_font() {
5299        let font = b"<</Type/Font/Subtype/Type1/BaseFont/ABCDEF+SchoolBookC\
5300            /FirstChar 65/LastChar 68/Widths[700 650 650 600]\
5301            /Encoding<</Type/Encoding/BaseEncoding/WinAnsiEncoding\
5302            /Differences[65/afii10017/afii10018/afii10019/afii10020]>>>>";
5303        let text: String = cells(&pdf_with_font(font, b"ABCD"))
5304            .iter()
5305            .map(|c| c.text.as_str())
5306            .collect();
5307        assert_eq!(text, "АБВГ");
5308    }
5309
5310    /// A non-embedded ZapfDingbats or Symbol that names no encoding reads
5311    /// through the face's own (docling-parse's base-font AFM; its
5312    /// `annots_01`: `l` is `●`, #1113) and advances by its AFM widths — no
5313    /// Latin letters, no zero-width cells.
5314    #[test]
5315    fn non_embedded_symbol_faces_read_through_their_encoding() {
5316        let dingbats = pdf_with_font(
5317            b"<</Type/Font/Subtype/Type1/BaseFont/ZapfDingbats/Name/F3>>",
5318            b"l",
5319        );
5320        let cs = cells(&dingbats);
5321        assert_eq!(cs.len(), 1, "{cs:?}");
5322        assert_eq!(cs[0].text, "●");
5323        assert!(
5324            (cs[0].r - cs[0].l - 791.0 * 12.0 / 1000.0).abs() < 0.1,
5325            "{cs:?}"
5326        );
5327        let symbol = pdf_with_font(b"<</Type/Font/Subtype/Type1/BaseFont/Symbol>>", b"abg");
5328        let text: String = cells(&symbol).iter().map(|c| c.text.as_str()).collect();
5329        assert_eq!(text, "αβγ");
5330        // A declared encoding keeps its table (multi_page's `•`, a
5331        // deliberate divergence from docling-parse's FontSpecific reading).
5332        let declared = pdf_with_font(
5333            b"<</Type/Font/Subtype/Type1/BaseFont/Symbol/Encoding/WinAnsiEncoding>>",
5334            b"a",
5335        );
5336        let text: String = cells(&declared).iter().map(|c| c.text.as_str()).collect();
5337        assert_eq!(text, "a");
5338    }
5339
5340    /// #728: one face bound under two resource keys — `/F1` Helvetica in
5341    /// WinAnsi, `/F2` the same Helvetica re-encoded so code 0x81 is the `fi`
5342    /// glyph — with every fi-word drawn in `/F2` right after an `/F1` run.
5343    /// docling-parse names a font by `/FontDescriptor/FontName` → `/Name` →
5344    /// `/BaseFont`, never by the resource key, so both resources are the same
5345    /// font and the line reads `financial`. Keyed by resource key the `fi`
5346    /// ligature (font-neutral in `applicable_for_merge`) joined the `/F1`
5347    /// run and the font barrier then split it from `nancial`.
5348    #[test]
5349    fn same_face_under_two_resource_keys_is_one_font_for_the_sanitizer() {
5350        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
5351        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding<</Type/Encoding\
5352                   /BaseEncoding/WinAnsiEncoding/Differences[129/fi]>>>>";
5353        let content = b"BT 12 TL 72 720 Td \
5354            /F1 12 Tf (The ) Tj /F2 12 Tf (\\201nancial position) Tj \
5355            /F1 12 Tf ( of the company is shown in ) Tj /F2 12 Tf (\\201gure 3) Tj \
5356            /F1 12 Tf ( with the ) Tj /F2 12 Tf (\\201rst results.) Tj \
5357            T* T* /F2 12 Tf (Same words in /F2 only: \\201nancial, \\201gure, \\201rst.) Tj ET\n";
5358        let cs = cells(&pdf_with_fonts(&[f1, f2], content));
5359        let lines: Vec<&str> = cs.iter().map(|c| c.text.as_str()).collect();
5360        assert_eq!(
5361            lines,
5362            [
5363                "The financial position of the company is shown in figure 3 with the first results.",
5364                "Same words in /F2 only: financial, figure, first.",
5365            ],
5366            "{cs:?}"
5367        );
5368    }
5369
5370    /// Under `legacy` the font barrier itself is untouched: two *different*
5371    /// faces (regular and bold Helvetica, which docling-parse names apart) set
5372    /// flush against each other stay separate line cells, as
5373    /// `enforce_same_font` demands. docling-parse 7 dropped the barrier — its
5374    /// line runs follow geometry alone — so under the default `adaptive` the
5375    /// flush pair is one line and one word.
5376    #[test]
5377    fn different_faces_keep_the_font_barrier_only_under_legacy() {
5378        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
5379        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica-Bold/Encoding/WinAnsiEncoding>>";
5380        let content = b"BT 72 720 Td /F2 12 Tf (Total:) Tj /F1 12 Tf (42) Tj ET\n";
5381        let pdf = pdf_with_fonts(&[f1, f2], content);
5382        let lines = |gap| {
5383            let mut p = super::PageTextParser::open(&pdf).expect("opens");
5384            p.set_word_gap(Some(gap));
5385            p.cells(0)
5386                .prose
5387                .into_iter()
5388                .map(|c| c.text)
5389                .collect::<Vec<_>>()
5390        };
5391        assert_eq!(lines(crate::dp_lines::WordGap::Legacy), ["Total:", "42"]);
5392        assert_eq!(lines(crate::dp_lines::WordGap::Adaptive), ["Total:42"]);
5393    }
5394
5395    /// #992: a space glyph whose advance the next `TJ` offset takes back —
5396    /// EUR-Lex's superscript footnote marker `( 1 )`, set with spaces the
5397    /// digit is then drawn over — is not a character of the text: the page
5398    /// shows `(1)`, pdftotext and pdfium read `(1)`, and so does the line
5399    /// cell here (docling-parse keeps the spaces, `( 1 )`). A space the next
5400    /// glyph only kerns into (−100/1000 em, well past the space's start) is
5401    /// a real gap and stays, as does one before a jump to the next line.
5402    #[test]
5403    fn a_space_the_next_glyph_is_drawn_over_is_not_a_character() {
5404        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
5405        let lines = |content: &[u8]| {
5406            let pdf = pdf_with_content(f1, content);
5407            super::PageTextParser::open(&pdf)
5408                .expect("opens")
5409                .cells(0)
5410                .prose
5411                .into_iter()
5412                .map(|c| c.text)
5413                .collect::<Vec<_>>()
5414        };
5415        // Helvetica's space is 278/1000 em: an offset of 300 moves the pen
5416        // back past where the space began, 100 leaves two thirds of it.
5417        assert_eq!(
5418            lines(b"BT /F1 12 Tf 72 700 Td [(EG \\() ( ) 300 (1) ( ) 300 (\\) erf)] TJ ET\n"),
5419            ["EG (1) erf"]
5420        );
5421        assert_eq!(
5422            lines(b"BT /F1 12 Tf 72 700 Td [(EG \\() ( ) 100 (1) ( ) 100 (\\) erf)] TJ ET\n"),
5423            ["EG ( 1 ) erf"]
5424        );
5425        // A trailing space, then the next line starts at the left margin.
5426        assert_eq!(
5427            lines(b"BT /F1 12 Tf 72 700 Td (first line ) Tj 0 -14 Td (second) Tj ET\n"),
5428            ["first line", "second"]
5429        );
5430        // A line drawn rotated 90° (a landscape slide on a portrait page,
5431        // #528): its glyphs' loose boxes stack on one left edge, which is
5432        // not an overprint — the space stays.
5433        assert_eq!(
5434            lines(b"BT 0 1 -1 0 300 100 Tm /F1 12 Tf (Nullam mollis) Tj ET\n"),
5435            ["Nullam mollis"]
5436        );
5437    }
5438
5439    /// A text shadow: a word processor exports a heading with a shadow
5440    /// effect as the same run drawn twice — the shadow in black, then the
5441    /// text in colour over it, shifted up and left by a fraction of a point.
5442    /// pdfium's text page reads the heading once and so does the line cell
5443    /// here (docling-parse reads `Safety instructions Safety instructions`).
5444    /// #1092: docling-parse's view of a page's shapes — a stroked straight
5445    /// rule is a shape line and a shape box, a thin filled box only a box, a
5446    /// curve or a diagonal no line, a path outside the clip nothing, and a
5447    /// fill in an unresolved Pattern space nothing at all. In the cells'
5448    /// frame: top-left, on the 595 × 842 test page.
5449    #[test]
5450    fn the_page_shapes_follow_docling_parse() {
5451        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
5452        let shapes = |content: &[u8]| {
5453            let pdf = pdf_with_content(f1, content);
5454            super::PageTextParser::open(&pdf)
5455                .expect("opens")
5456                .cells(0)
5457                .shapes
5458        };
5459        let s = shapes(b"2 w 72 400 m 540 400 l S\n");
5460        assert_eq!(s.lines, vec![[72.0, 442.0, 540.0, 442.0]]);
5461        // The stroke's half width pads the box.
5462        assert_eq!(s.boxes, vec![[71.0, 441.0, 541.0, 443.0]]);
5463
5464        let s = shapes(b"72 300 468 1 re f\n");
5465        assert!(s.lines.is_empty(), "a fill strokes no line");
5466        assert_eq!(s.boxes, vec![[72.0, 541.0, 540.0, 542.0]]);
5467
5468        // A stroked rectangle: four edges, `h`'s closing one included.
5469        let s = shapes(b"100 100 200 50 re S\n");
5470        assert_eq!(s.lines.len(), 4);
5471
5472        let s = shapes(b"72 400 m 100 450 l 200 300 300 500 400 400 c S\n");
5473        assert!(s.lines.is_empty(), "neither a diagonal nor a curve");
5474        assert_eq!(s.boxes.len(), 1);
5475
5476        let s = shapes(b"q 0 0 300 842 re W n 72 400 m 540 400 l S 400 100 m 400 700 l S Q\n");
5477        assert_eq!(
5478            s.lines,
5479            vec![[72.0, 442.0, 300.0, 442.0]],
5480            "cut to the clip"
5481        );
5482        assert_eq!(s.boxes.len(), 1, "the vertical rule is clipped away");
5483
5484        let s = shapes(b"/Pattern cs /P0 scn 72 300 468 1 re f\n");
5485        assert!(s.lines.is_empty() && s.boxes.is_empty());
5486
5487        // Two touching boxes merge into one component.
5488        let s = shapes(b"72 300 100 1 re f 172 300 100 1 re f 400 600 10 10 re f\n");
5489        assert_eq!(
5490            s.boxes,
5491            vec![[72.0, 541.0, 272.0, 542.0], [400.0, 232.0, 410.0, 242.0]]
5492        );
5493
5494        // A table rule a word processor draws piece by piece, cell borders
5495        // and their corner squares: `90.144 + 1.44` is `91.584` in the file
5496        // and in docling-parse's doubles, not in lopdf's `f32` reals cast to
5497        // `f64`, where every such joint missed by a hair and the rule fell
5498        // into pieces: `real` reads the operands as the file wrote them. A
5499        // gap the producer left (`147.5 + 159.86` = 307.36, then 307.37)
5500        // still parts two components, as in docling-parse.
5501        let s = shapes(
5502            b"56.64 690.22 33.504 1.44 re f 90.144 690.22 1.44 1.44 re f \
5503              91.584 690.22 54.48 1.44 re f 146.06 690.22 1.44 1.44 re f \
5504              147.5 690.22 159.86 1.44 re f 307.37 690.22 1.44 1.44 re f \
5505              308.81 690.22 50.304 1.44 re f\n",
5506        );
5507        let spans: Vec<(f32, f32)> = s.boxes.iter().map(|b| (b[0], b[2])).collect();
5508        assert_eq!(spans, vec![(56.64, 307.36), (307.37, 359.114)]);
5509    }
5510
5511    /// A transparent stroke or fill is no shape (`CA` / `ca` 0).
5512    #[test]
5513    fn transparent_paths_are_no_shapes() {
5514        let rule = |closed| super::ShapeSub {
5515            start: (0.0, 0.0),
5516            cur: (100.0, 0.0),
5517            pts: vec![(0.0, 0.0), (100.0, 0.0)],
5518            lines: vec![((0.0, 0.0), (100.0, 0.0))],
5519            closed,
5520        };
5521        let gs = |stroke_alpha, fill_alpha| super::ShapeGs {
5522            stroke_alpha,
5523            fill_alpha,
5524            ..super::ShapeGs::INIT
5525        };
5526        let mut shapes = super::Shapes::default();
5527        super::record_shape(&[rule(false)], true, false, &gs(0.0, 1.0), 1.0, &mut shapes);
5528        assert_eq!(shapes, super::Shapes::default());
5529        super::record_shape(&[rule(true)], false, true, &gs(1.0, 0.0), 1.0, &mut shapes);
5530        assert_eq!(shapes, super::Shapes::default());
5531        super::record_shape(&[rule(false)], true, false, &gs(0.5, 0.0), 1.0, &mut shapes);
5532        assert_eq!(shapes.lines.len(), 1);
5533    }
5534
5535    /// Text that really repeats sits an advance apart or more and stays: a
5536    /// word set twice, a doubled letter, the same words on the next line.
5537    #[test]
5538    fn a_run_drawn_again_over_itself_is_read_once() {
5539        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica-Bold/Encoding/WinAnsiEncoding>>";
5540        let lines = |content: &[u8]| {
5541            let pdf = pdf_with_content(f1, content);
5542            super::PageTextParser::open(&pdf)
5543                .expect("opens")
5544                .cells(0)
5545                .prose
5546                .into_iter()
5547                .map(|c| c.text)
5548                .collect::<Vec<_>>()
5549        };
5550        // The shadow, then the text 0.5 pt up and left of it, at 18 pt.
5551        assert_eq!(
5552            lines(
5553                b"BT 0 g /F1 18 Tf 1 0 0 1 72 720 Tm (Safety instructions) Tj ET\n\
5554                  BT 0.8 0 0 rg /F1 18 Tf 1 0 0 1 71.5 720.5 Tm (Safety instructions) Tj ET\n"
5555            ),
5556            ["Safety instructions"]
5557        );
5558        // 0.8 pt at 14 pt, and the line under it untouched.
5559        assert_eq!(
5560            lines(
5561                b"BT /F1 14 Tf 1 0 0 1 72 656 Tm (Maintenance) Tj ET\n\
5562                  BT /F1 14 Tf 1 0 0 1 71.2 656.8 Tm (Maintenance) Tj ET\n\
5563                  BT /F1 11 Tf 1 0 0 1 72 632 Tm (Clean the filter) Tj ET\n"
5564            ),
5565            ["Maintenance", "Clean the filter"]
5566        );
5567        // A word set twice, a doubled letter, the same words a line apart.
5568        assert_eq!(
5569            lines(b"BT /F1 12 Tf 72 700 Td (very very small) Tj ET\n"),
5570            ["very very small"]
5571        );
5572        assert_eq!(
5573            lines(b"BT /F1 12 Tf 72 700 Td (Hello, all) Tj ET\n"),
5574            ["Hello, all"]
5575        );
5576        assert_eq!(
5577            lines(b"BT /F1 12 Tf 72 700 Td (Total) Tj 0 -14 Td (Total) Tj ET\n"),
5578            ["Total", "Total"]
5579        );
5580        // Dot leaders squeezed by `Tc` to 0.05, 0.07 and 0.1 em: two dots on,
5581        // a dot sits within a shadow's shift and the dots after it repeat by
5582        // the same offset, but the next dot is nearer still.
5583        for tc in ["-2.28", "-2.08", "-1.78"] {
5584            let content = format!(
5585                "BT /F1 10 Tf 72 700 Td (Contents) Tj {tc} Tc (..........) Tj 0 Tc ( 5) Tj ET\n"
5586            );
5587            assert_eq!(
5588                lines(content.as_bytes()),
5589                ["Contents.......... 5"],
5590                "Tc {tc}"
5591            );
5592        }
5593    }
5594
5595    /// docling-parse's `test_unit_actual_text.py` (docling-parse#300): a
5596    /// marked-content span's `/ActualText` replaces the glyphs drawn in it.
5597    #[test]
5598    fn actual_text_replaces_the_glyphs_of_its_span() {
5599        let helvetica = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
5600        // A non-embedded TrueType whose only code, 0x4D (`M` in WinAnsi), is a
5601        // zero-advance caron: drawn after `s` it makes `š`.
5602        let caron = b"<</Type/Font/Subtype/TrueType/BaseFont/SynthCaron/FirstChar 77/LastChar 77\
5603            /Widths[0]/Encoding/WinAnsiEncoding/FontDescriptor<</Type/FontDescriptor\
5604            /FontName/SynthCaron/Flags 32/FontBBox[-540 -200 0 800]/ItalicAngle 0/Ascent 800\
5605            /Descent -200/CapHeight 700/StemV 80>>>>";
5606        // Code 1 is the `fi` ligature: two glyphs on one pen, one cell.
5607        let ligature = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica\
5608            /Encoding<</Type/Encoding/BaseEncoding/WinAnsiEncoding/Differences[1/fi]>>>>";
5609        let text = |content: &str| {
5610            let pdf = pdf_with_fonts(&[helvetica, caron, ligature], content.as_bytes());
5611            super::PageTextParser::open(&pdf)
5612                .expect("opens")
5613                .cells(0)
5614                .prose
5615                .into_iter()
5616                .map(|c| c.text)
5617                .collect::<String>()
5618        };
5619        // Unmarked, the caron reads as its WinAnsi letter.
5620        assert_eq!(
5621            text("BT /F1 24 Tf 40 50 Td (Ore) Tj (s) Tj /F2 24 Tf <4D> Tj /F1 24 Tf (i) Tj ET"),
5622            "OresMi"
5623        );
5624        // Two cells, one character: the accent composes onto its base letter.
5625        assert_eq!(
5626            text(
5627                "BT /F1 24 Tf 40 50 Td (Ore) Tj /Span <</ActualText <FEFF0161>>> BDC \
5628                  (s) Tj /F2 24 Tf <4D> Tj EMC /F1 24 Tf (i) Tj ET"
5629            ),
5630            "Oreši"
5631        );
5632        // As many characters as cells: one for one.
5633        assert_eq!(
5634            text("BT /F1 24 Tf 40 50 Td /Span <</ActualText (ab)>> BDC (xy) Tj EMC (z) Tj ET"),
5635            "abz"
5636        );
5637        // A cell of two glyphs takes one character.
5638        assert_eq!(
5639            text("BT /F3 24 Tf 40 50 Td <01> Tj /F1 24 Tf (a) Tj ET"),
5640            "fia"
5641        );
5642        assert_eq!(
5643            text(
5644                "BT /F3 24 Tf 40 50 Td /Span <</ActualText (XY)>> BDC <01> Tj \
5645                  /F1 24 Tf (a) Tj EMC (z) Tj ET"
5646            ),
5647            "XYz"
5648        );
5649        // An empty text declares the glyphs not text (a visual line-break
5650        // hyphen); its advance may still read as a gap.
5651        let hyphen = text(
5652            "BT /F1 24 Tf 40 50 Td (exam) Tj /Span <</ActualText <FEFF>>> BDC (-) Tj EMC \
5653             (ple) Tj ET",
5654        );
5655        assert!(!hyphen.contains('-'), "{hyphen}");
5656        assert_eq!(hyphen.replace(' ', ""), "example");
5657        // `/Alt` written into the wrong key: implausibly long, ignored.
5658        assert_eq!(
5659            text(
5660                "BT /F1 24 Tf 40 50 Td /Span <</ActualText (A long description that belongs \
5661                  in Alt not here)>> BDC (ab) Tj EMC ET"
5662            ),
5663            "ab"
5664        );
5665        // Over graphics only: nothing to replace.
5666        assert_eq!(
5667            text(
5668                "BT /F1 24 Tf 40 50 Td (x) Tj ET /Span <</ActualText (ghost)>> BDC \
5669                  10 10 50 50 re f EMC"
5670            ),
5671            "x"
5672        );
5673        // A named property list is not resolved.
5674        assert_eq!(
5675            text("BT /F1 24 Tf 40 50 Td /OC /MC0 BDC (ok) Tj EMC ET"),
5676            "ok"
5677        );
5678        // Nested spans: the outer text describes the whole run.
5679        assert_eq!(
5680            text(
5681                "BT /F1 24 Tf 40 50 Td /Span <</ActualText (AB)>> BDC (a) Tj \
5682                  /Span <</ActualText (Z)>> BDC (b) Tj EMC EMC ET"
5683            ),
5684            "AB"
5685        );
5686        // A span nested past the depth limit keeps its glyphs.
5687        let nested = |depth: usize| {
5688            text(&format!(
5689                "BT /F1 24 Tf 40 50 Td {}/Span <</ActualText (z)>> BDC (a) Tj EMC{} ET",
5690                "/X BMC ".repeat(depth),
5691                " EMC".repeat(depth)
5692            ))
5693        };
5694        assert_eq!(nested(super::MAX_ACTUAL_TEXT_DEPTH - 1), "z");
5695        assert_eq!(nested(super::MAX_ACTUAL_TEXT_DEPTH), "a");
5696    }
5697
5698    /// One for one, each character keeps its cell's box, and a cell of
5699    /// several glyphs on one pen (a ligature's) becomes its first glyph.
5700    #[test]
5701    fn actual_text_one_for_one_keeps_each_cells_box() {
5702        let glyph = |ch, x: f32, pen: f32| super::Glyph {
5703            ch,
5704            l: x,
5705            b: 0.0,
5706            r: x + 5.0,
5707            t: 10.0,
5708            ll: x,
5709            lb: -2.0,
5710            lr: x + 5.0,
5711            lt: 10.0,
5712            font: 0,
5713            quad: None,
5714            pen: Some([pen, 0.0, 10.0]),
5715            math_font: false,
5716            invisible: false,
5717        };
5718        let mut out = vec![
5719            glyph('o', 0.0, 0.0),
5720            glyph('f', 10.0, 10.0),
5721            glyph('i', 15.0, 10.0),
5722            glyph('a', 20.0, 20.0),
5723        ];
5724        super::apply_actual_text(&mut out, 1, "XY");
5725        let got: Vec<(char, f32)> = out.iter().map(|g| (g.ch, g.l)).collect();
5726        assert_eq!(got, [('o', 0.0), ('X', 10.0), ('Y', 20.0)]);
5727    }
5728
5729    /// The identity follows docling-parse's precedence: a `/FontDescriptor/FontName`
5730    /// wins over `/BaseFont`, so two dictionaries that share a `/BaseFont` but
5731    /// describe different programs are two fonts, and two that share the
5732    /// descriptor name are one whatever their `/BaseFont` says.
5733    #[test]
5734    fn math_faces_are_named_like_docling_parse_names_them() {
5735        for name in [
5736            "ABCDEF+CMR7",
5737            "CMMI10",
5738            "MSBM10",
5739            "Cambria-Math",
5740            "Symbol",
5741            "MTExtra",
5742        ] {
5743            assert!(super::is_math_font_name(name.as_bytes()), "{name}");
5744        }
5745        for name in ["Helvetica", "ABCDEF+TimesNewRomanPSMT", "Arial-BoldMT"] {
5746            assert!(!super::is_math_font_name(name.as_bytes()), "{name}");
5747        }
5748    }
5749
5750    #[test]
5751    fn sanitizer_font_name_follows_docling_parse_precedence() {
5752        use lopdf::{dictionary, Document, Object};
5753        let mut doc = Document::with_version("1.4");
5754        let fd_a =
5755            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "AAAAAA+Face" });
5756        let fd_b =
5757            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "BBBBBB+Face" });
5758        let with = |fd: Option<lopdf::ObjectId>, base: &str| {
5759            let mut d = dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => base };
5760            if let Some(id) = fd {
5761                d.set("FontDescriptor", Object::Reference(id));
5762            }
5763            d
5764        };
5765        let name = |d: &lopdf::Dictionary| {
5766            String::from_utf8(super::sanitizer_font_name(&doc, d, false)).unwrap()
5767        };
5768        assert_eq!(name(&with(Some(fd_a), "Face")), "AAAAAA+Face");
5769        assert_eq!(name(&with(Some(fd_b), "Face")), "BBBBBB+Face");
5770        assert_eq!(name(&with(None, "ABCDEF+Face")), "ABCDEF+Face");
5771        assert_eq!(
5772            name(&dictionary! { "Type" => "Font", "Name" => "F7" }),
5773            "F7"
5774        );
5775        assert_eq!(name(&dictionary! { "Type" => "Font" }), "null");
5776    }
5777}
5778
5779#[cfg(test)]
5780mod overpainted {
5781    use crate::pdfium_backend::TextCell;
5782
5783    fn cell(text: &str, l: f32, t: f32, r: f32, b: f32) -> TextCell {
5784        TextCell {
5785            text: text.into(),
5786            l,
5787            t,
5788            r,
5789            b,
5790        }
5791    }
5792
5793    /// The reporting invoice's logo: a `"` painted inside a `==` on one band —
5794    /// artwork drawn with glyphs. Both cells go; the real text on the next
5795    /// band stays.
5796    #[test]
5797    fn stacked_logo_glyphs_are_dropped() {
5798        let mut cells = vec![
5799            cell("\"", 72.7, 21.5, 86.4, 31.5),
5800            cell("==", 59.4, 21.5, 99.6, 31.5),
5801            cell("Herr", 65.2, 151.3, 81.7, 161.3),
5802        ];
5803        super::drop_overpainted_cells(&mut cells);
5804        assert_eq!(cells.len(), 1, "cells: {cells:?}");
5805        assert_eq!(cells[0].text, "Herr");
5806    }
5807
5808    /// Adjacent words on a line touch but never contain each other — prose is
5809    /// untouched, and so is a same-text near-duplicate (double-drawn faux
5810    /// bold), which is not evidence of artwork.
5811    #[test]
5812    fn prose_and_double_draw_are_kept() {
5813        let mut cells = vec![
5814            cell("Telefon", 354.3, 133.2, 381.5, 143.2),
5815            cell("0676/2000", 387.3, 133.2, 428.7, 143.2),
5816            cell("Bold", 100.0, 50.0, 130.0, 60.0),
5817            cell("Bold", 100.3, 50.0, 130.3, 60.0),
5818        ];
5819        super::drop_overpainted_cells(&mut cells);
5820        assert_eq!(cells.len(), 4);
5821    }
5822}
5823
5824#[cfg(test)]
5825mod vestigial_layer {
5826    use crate::pdfium_backend::{PdfPage, TextCell};
5827
5828    fn page_with(texts: &[&str]) -> PdfPage {
5829        let cells = texts
5830            .iter()
5831            .enumerate()
5832            .map(|(i, t)| TextCell {
5833                text: t.to_string(),
5834                l: 10.0,
5835                t: 10.0 + 12.0 * i as f32,
5836                r: 90.0,
5837                b: 20.0 + 12.0 * i as f32,
5838            })
5839            .collect();
5840        PdfPage::from_cells(595.0, 842.0, 1.0, cells)
5841    }
5842
5843    /// The reported scanned form: three typed-in field values ("03", "05",
5844    /// "2025") over three image pages. That must read as *no usable layer*,
5845    /// so the browser routes the document to OCR instead of extracting
5846    /// thirteen characters and skipping the letter entirely.
5847    #[test]
5848    fn typed_in_form_fields_are_not_a_text_layer() {
5849        let pages = vec![
5850            page_with(&["03", "05", "2025"]),
5851            page_with(&[]),
5852            page_with(&[]),
5853        ];
5854        assert!(super::text_layer_is_vestigial(&pages));
5855        assert!(super::text_layer_is_vestigial(&[page_with(&[])]));
5856    }
5857
5858    /// A short but genuine digital document — one page, a few real lines —
5859    /// keeps the fast text path.
5860    #[test]
5861    fn sparse_but_real_documents_pass() {
5862        let one_pager = vec![page_with(&[
5863            "Confidential briefing",
5864            "Prepared for the board meeting",
5865            "Do not distribute",
5866        ])];
5867        assert!(!super::text_layer_is_vestigial(&one_pager));
5868    }
5869}
5870
5871/// #994: decoded Form XObject content is cached per document only from a
5872/// form's second sighting, so one-form-per-page documents do not hold every
5873/// page's decoded form until the document ends.
5874#[cfg(test)]
5875mod form_cache {
5876    use lopdf::content::{Content, Operation};
5877    use lopdf::{dictionary, Document, Object, Stream};
5878
5879    fn form(doc: &mut Document, text: &str) -> lopdf::ObjectId {
5880        let ops = vec![
5881            Operation::new("BT", vec![]),
5882            Operation::new("Tf", vec!["F1".into(), 12.into()]),
5883            Operation::new("Td", vec![40.into(), 500.into()]),
5884            Operation::new("Tj", vec![Object::string_literal(text)]),
5885            Operation::new("ET", vec![]),
5886        ];
5887        let data = Content { operations: ops }.encode().unwrap();
5888        doc.add_object(Stream::new(
5889            dictionary! { "Type" => "XObject", "Subtype" => "Form",
5890            "BBox" => vec![0.into(), 0.into(), 595.into(), 842.into()] },
5891            data,
5892        ))
5893    }
5894
5895    #[test]
5896    fn single_use_forms_are_not_retained_shared_ones_are() {
5897        let mut doc = Document::with_version("1.5");
5898        let pages_id = doc.new_object_id();
5899        let font = doc.add_object(
5900            dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => "Helvetica" },
5901        );
5902        let header = form(&mut doc, "Header");
5903        let mut kids = Vec::new();
5904        let mut own_forms = Vec::new();
5905        for p in 0..3 {
5906            let own = form(&mut doc, &format!("Body{p}"));
5907            own_forms.push(own);
5908            let content = doc.add_object(Stream::new(
5909                dictionary! {},
5910                b"q /Hd Do Q q /Fx Do Q".to_vec(),
5911            ));
5912            kids.push(Object::Reference(doc.add_object(dictionary! {
5913                "Type" => "Page", "Parent" => pages_id,
5914                "MediaBox" => vec![0.into(), 0.into(), 595.into(), 842.into()],
5915                "Contents" => content,
5916                "Resources" => dictionary! {
5917                    "Font" => dictionary! { "F1" => font },
5918                    "XObject" => dictionary! { "Hd" => header, "Fx" => own },
5919                },
5920            })));
5921        }
5922        doc.objects.insert(
5923            pages_id,
5924            Object::Dictionary(dictionary! { "Type" => "Pages", "Kids" => kids, "Count" => 3 }),
5925        );
5926        let catalog = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages_id });
5927        doc.trailer.set("Root", catalog);
5928
5929        let mut caches = super::DocCaches::default();
5930        let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
5931        pages.sort_by_key(|(n, _)| *n);
5932        for (i, (_, pid)) in pages.iter().enumerate() {
5933            let text: String = super::page_glyphs_cached(&doc, *pid, &mut caches)
5934                .iter()
5935                .map(|g| g.ch)
5936                .collect();
5937            // Both forms' text is still read, cached or not.
5938            assert!(text.contains("Header"), "page {i}: {text:?}");
5939            assert!(text.contains(&format!("Body{i}")), "page {i}: {text:?}");
5940        }
5941        // The shared header form is cached from its second page on; the
5942        // per-page forms, each used once, are not held.
5943        assert!(caches.forms.contains_key(&header));
5944        for own in &own_forms {
5945            assert!(!caches.forms.contains_key(own));
5946        }
5947        assert_eq!(caches.forms.len(), 1);
5948    }
5949}
5950
5951#[cfg(test)]
5952mod cid_collections {
5953    use super::base14_fonts::{cells, pdf_with_content};
5954
5955    fn type0(encoding: &str) -> Vec<u8> {
5956        format!(
5957            "<</Type/Font/Subtype/Type0/BaseFont/Test-Font-Light/Encoding/{encoding}\
5958             /DescendantFonts[<</Type/Font/Subtype/CIDFontType0/BaseFont/Test-Font-Light\
5959             /CIDSystemInfo<</Registry(Adobe)/Ordering(Japan1)/Supplement 7>>/DW 1000>>]>>"
5960        )
5961        .into_bytes()
5962    }
5963
5964    fn text(fontdict: &[u8], hex: &str) -> String {
5965        let content = format!("BT /F1 12 Tf 72 700 Td <{hex}> Tj ET\n");
5966        cells(&pdf_with_content(fontdict, content.as_bytes()))
5967            .iter()
5968            .map(|c| c.text.as_str())
5969            .collect()
5970    }
5971
5972    #[test]
5973    #[cfg(feature = "cid-collections")]
5974    fn identity_codes_read_through_the_collection() {
5975        assert_eq!(text(&type0("Identity-H"), "0CD40E8A07A0"), "日本語");
5976    }
5977
5978    #[test]
5979    fn unicode_keyed_cmaps_read_the_code() {
5980        assert_eq!(text(&type0("UniJIS-UCS2-H"), "65E5672C8A9E"), "日本語");
5981    }
5982
5983    #[test]
5984    #[cfg(feature = "cid-collections")]
5985    fn collection_ranges_keep_supplementary_characters() {
5986        assert_eq!(text(&type0("Identity-H"), "0CD44507450807A0"), "日𢭏𢭐語");
5987        let text = |o: &[u8], cid| super::collection_text(super::collection_table(o).unwrap(), cid);
5988        assert_eq!(text(b"Japan1", 0x2714).as_deref(), Some("🄐"));
5989        assert_eq!(text(b"CNS1", 0x3a32).as_deref(), Some("𡤃"));
5990    }
5991
5992    #[test]
5993    #[cfg(feature = "cid-collections")]
5994    fn collection_entries_are_plain_characters() {
5995        assert_eq!(text(&type0("Identity-H"), "046D"), "逢");
5996        let text = |o: &[u8], cid| super::collection_text(super::collection_table(o).unwrap(), cid);
5997        assert_eq!(text(b"Korea1", 8752), None);
5998        assert_eq!(text(b"KR", 11704), None);
5999        assert_eq!(text(b"Japan1", 0x1ffd).as_deref(), Some("sec"));
6000    }
6001
6002    #[test]
6003    fn tounicode_ranges_increment_the_utf16_string() {
6004        let map = super::parse_tounicode(
6005            b"3 beginbfrange\n\
6006              <01> <02> <d842dfb7>\n\
6007              <03> <04> <00660069>\n\
6008              <05> <06> <0061>\n\
6009              endbfrange",
6010        );
6011        assert_eq!(map[&1], "𠮷");
6012        assert_eq!(map[&2], "𠮸");
6013        assert_eq!(map[&3], "fi");
6014        assert_eq!(map[&4], "fj");
6015        assert_eq!(map[&5], "a");
6016        assert_eq!(map[&6], "b");
6017    }
6018
6019    #[test]
6020    fn tounicode_skips_lone_surrogates_overflows_and_huge_ranges() {
6021        let map = super::parse_tounicode(
6022            b"2 beginbfchar\n<01> <d842>\n<02> <0041>\nendbfchar\n\
6023              4 beginbfrange\n\
6024              <10> <11> [<dc00> <0042>]\n\
6025              <20> <22> <0041fffe>\n\
6026              <30> <31> <d842dfff>\n\
6027              <00010000> <ffffffff> <>\n\
6028              endbfrange",
6029        );
6030        assert!(!map.contains_key(&0x01));
6031        assert_eq!(map[&0x02], "A");
6032        assert!(!map.contains_key(&0x10));
6033        assert_eq!(map[&0x11], "B");
6034        assert_eq!(map[&0x20], "A\u{fffe}");
6035        assert_eq!(map[&0x21], "A\u{ffff}");
6036        assert!(!map.contains_key(&0x22));
6037        assert_eq!(map[&0x30], "\u{20bff}");
6038        assert!(!map.contains_key(&0x31));
6039        assert_eq!(map[&0x1ffff], "");
6040        assert_eq!(map.len(), 5 + 0x10000);
6041    }
6042
6043    #[test]
6044    fn utf16_pairs_are_one_glyph_and_one_advance() {
6045        let font = type0("UniJIS-UTF16-H");
6046        assert_eq!(text(&font, "65E5D842DFB78A9E"), "日𠮷語");
6047        let content = b"BT /F1 12 Tf 72 700 Td <65E5D842DFB78A9E> Tj ET\n";
6048        let pdf = pdf_with_content(&font, content);
6049        let doc = lopdf::Document::load_mem(&pdf).unwrap();
6050        let glyphs = super::page_glyphs(&doc, doc.get_pages()[&1]);
6051        assert_eq!(glyphs.len(), 3);
6052        assert!((glyphs[2].l - glyphs[0].l - 24.0).abs() < 0.001);
6053    }
6054
6055    #[test]
6056    fn utf16_codes_preserve_tounicode_keys_and_identity_cids() {
6057        use lopdf::{dictionary, Document};
6058        let doc = Document::with_version("1.4");
6059        let dict = dictionary! {"Subtype" => "Type0", "Encoding" => "UniJIS-UTF16-H"};
6060        let mut font = super::parse_font(&doc, b"F1", &dict);
6061        assert_eq!(
6062            super::codes(&font, &[0xd8, 0x42, 0xdf, 0xb7]),
6063            vec![0xd842dfb7]
6064        );
6065        font.to_unicode.insert(0xd842dfb7, "override".into());
6066        assert_eq!(font.decode_code(0xd842dfb7).0.as_deref(), Some("override"));
6067        assert_eq!(
6068            super::codes(&font, &[0xd8, 0x42, 0x65, 0xe5]),
6069            vec![0xd842, 0x65e5]
6070        );
6071        let dict = dictionary! {"Subtype" => "Type0", "Encoding" => "Identity-H"};
6072        let font = super::parse_font(&doc, b"F1", &dict);
6073        assert_eq!(
6074            super::codes(&font, &[0xd8, 0x42, 0xdf, 0xb7]),
6075            vec![0xd842, 0xdfb7]
6076        );
6077    }
6078}