Skip to main content

docling_pdf/
textparse.rs

1//! Pure-Rust PDF text extraction (replacing pdfium's glyph layer).
2//!
3//! pdfium reports *rendered* glyph boxes, which diverge from docling's
4//! `docling-parse` C++ parser at exactly the points that drive conformance:
5//! generated spaces get a zero-width box, combining diacritics get a real-width
6//! box, and ligature/fraction glyphs land at different x. This module instead
7//! reconstructs each glyph's box from the **font's own advance widths** and the
8//! PDF text/graphics matrices — the same information docling-parse uses — so a
9//! space is as wide as the font says and a combining mark has zero advance.
10//!
11//! The output is the same [`Glyph`] stream pdfium produces (native PDF
12//! coordinates, y-up), fed straight into the existing docling-parse line
13//! sanitizer ([`crate::dp_lines`]). Only the digital text layer is handled here;
14//! pages without one still fall back to OCR upstream.
15
16use std::collections::HashMap;
17use std::sync::Arc;
18
19use lopdf::{Dictionary, Document, Object};
20
21use crate::pdfium_backend::Glyph;
22
23/// Per-document caches for the content-stream interpreter. Fonts are indirect
24/// objects shared by many pages, but were fully re-parsed — ToUnicode CMap
25/// decompression + tokenization, embedded Type1 program scan, width tables —
26/// for **every page and every Form XObject invocation**; decoded form content
27/// streams were likewise re-inflated on every `Do`. Cached per document,
28/// keyed by the referenced object id (fonts also by resource name — the
29/// `/Font` entry a page binds the object to). Inline (non-reference) dicts
30/// are rare and stay uncached.
31#[derive(Default)]
32struct DocCaches {
33    fonts: HashMap<(lopdf::ObjectId, Vec<u8>), Arc<Font>>,
34    forms: HashMap<lopdf::ObjectId, Arc<lopdf::content::Content>>,
35}
36
37/// A 2×3 affine matrix `[a b c d e f]`: maps `(x,y)` → `(a·x+c·y+e, b·x+d·y+f)`.
38#[derive(Clone, Copy)]
39struct Mat {
40    a: f64,
41    b: f64,
42    c: f64,
43    d: f64,
44    e: f64,
45    f: f64,
46}
47
48impl Mat {
49    const ID: Mat = Mat {
50        a: 1.0,
51        b: 0.0,
52        c: 0.0,
53        d: 1.0,
54        e: 0.0,
55        f: 0.0,
56    };
57
58    /// `self ∘ m`: the matrix that applies `self` first, then `m`.
59    fn then(self, m: Mat) -> Mat {
60        Mat {
61            a: self.a * m.a + self.b * m.c,
62            b: self.a * m.b + self.b * m.d,
63            c: self.c * m.a + self.d * m.c,
64            d: self.c * m.b + self.d * m.d,
65            e: self.e * m.a + self.f * m.c + m.e,
66            f: self.e * m.b + self.f * m.d + m.f,
67        }
68    }
69
70    fn apply(self, x: f64, y: f64) -> (f64, f64) {
71        (
72            self.a * x + self.c * y + self.e,
73            self.b * x + self.d * y + self.f,
74        )
75    }
76}
77
78/// A parsed font: how to turn raw string bytes into (unicode, advance) pairs.
79struct Font {
80    /// 2-byte codes (Type0 / Identity-H) vs 1-byte (simple fonts).
81    two_byte: bool,
82    /// code → Unicode string (from ToUnicode; may be multi-char, e.g. ligatures).
83    to_unicode: HashMap<u32, String>,
84    /// code → glyph advance, in 1000-unit glyph space.
85    widths: HashMap<u32, f64>,
86    default_width: f64,
87    /// 1-byte fallback decoding when ToUnicode lacks a code (WinAnsi-ish).
88    simple_encoding: Option<HashMap<u8, char>>,
89    /// code → raw `/Differences` glyph name, for GID-style names (`g115`) that
90    /// have no Unicode mapping. docling-parse emits these verbatim as `/g115`
91    /// (see the redp5110 bulleted list); matching it keeps no text skipped.
92    fallback_names: HashMap<u8, String>,
93    /// code → char from the embedded Type1 font program's own `/Encoding` vector,
94    /// used only as a last resort for glyphs the base encoding leaves unmapped
95    /// (standard TeX math fonts: `λ`, `≤`, …).
96    program_encoding: HashMap<u8, char>,
97    ascent: f64,
98    descent: f64,
99    hash: u64,
100    /// Whether the font's docling-parse name marks a math face
101    /// ([`is_math_font_name`]) — for the word contraction's local
102    /// attachments (#921).
103    math: bool,
104    /// Weight and slant read from the `/BaseFont` name (the heading
105    /// hierarchy's style signal, #302).
106    style: crate::font_style::FontStyle,
107}
108
109impl Font {
110    fn decode_code(&self, code: u32) -> (Option<String>, f64) {
111        let w = self
112            .widths
113            .get(&code)
114            .copied()
115            .unwrap_or(self.default_width);
116        if let Some(s) = self.to_unicode.get(&code) {
117            return (Some(decompose_ligatures(s)), w);
118        }
119        if !self.two_byte {
120            // A GID-style `/Differences` name (no Unicode) overrides the base
121            // encoding, matching docling's verbatim `/g115` fallback.
122            if let Some(name) = self.fallback_names.get(&(code as u8)) {
123                return (Some(format!("/{name}")), w);
124            }
125            if let Some(enc) = &self.simple_encoding {
126                if let Some(&ch) = enc.get(&(code as u8)) {
127                    return (Some(decompose_ligatures(&ch.to_string())), w);
128                }
129            }
130            // Last resort: the embedded Type1 font program's own `/Encoding`
131            // vector (`dup N /glyphname put`). Standard TeX math fonts (CMMI, CMSY,
132            // …) ship no PDF `/Encoding` and no ToUnicode, so a glyph like `λ`
133            // (CMMI code 21 → `/lambda`) or `≤` (CMSY code 20 → `/lessequal`) has
134            // no other mapping and would otherwise be silently dropped. docling
135            // recovers these from the same font program. This only fills codes the
136            // base encoding left unmapped, so it never changes an existing decode.
137            if let Some(&ch) = self.program_encoding.get(&(code as u8)) {
138                return (Some(decompose_ligatures(&ch.to_string())), w);
139            }
140        }
141        (None, w)
142    }
143}
144
145/// Spell out Latin presentation-form ligatures (`fi`→`fi`, `ffi`→`ffi`, …) the way
146/// docling does, so `configuration`/`difficult` don't keep the ligature glyph.
147/// The chars share the ligature's box, so the line sanitizer recomposes them.
148fn decompose_ligatures(s: &str) -> String {
149    if !s.chars().any(|c| ('\u{FB00}'..='\u{FB06}').contains(&c)) {
150        return s.to_string();
151    }
152    s.chars()
153        .map(|c| {
154            match c {
155                '\u{FB00}' => "ff",
156                '\u{FB01}' => "fi",
157                '\u{FB02}' => "fl",
158                '\u{FB03}' => "ffi",
159                '\u{FB04}' => "ffl",
160                '\u{FB05}' => "ft",
161                '\u{FB06}' => "st",
162                _ => return c.to_string(),
163            }
164            .to_string()
165        })
166        .collect()
167}
168
169fn hash_name(name: &[u8]) -> u64 {
170    use std::hash::{Hash, Hasher};
171    let mut h = std::collections::hash_map::DefaultHasher::new();
172    name.hash(&mut h);
173    h.finish()
174}
175
176/// Resolve a possibly-indirect object to a dictionary.
177fn as_dict<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Dictionary> {
178    match obj {
179        Object::Dictionary(d) => Some(d),
180        Object::Reference(id) => doc.get_object(*id).ok().and_then(|o| o.as_dict().ok()),
181        _ => None,
182    }
183}
184
185fn deref<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Object> {
186    match obj {
187        Object::Reference(id) => doc.get_object(*id).ok(),
188        other => Some(other),
189    }
190}
191
192/// The font identity the line sanitizer's `enforce_same_font` compares —
193/// docling-parse's `page_font::init_font_name`: `/FontDescriptor/FontName`
194/// (the descendant CIDFont's for a Type0), else `/Name`, else `/BaseFont`
195/// (its `base_font`), else the literal `"null"`. The resource key (`/F1`,
196/// `/F2`) is deliberately **not** part of it: a producer that references one
197/// face twice — the same Helvetica under two keys, one re-encoded so a code
198/// maps to the `fi` glyph — draws one word across the two resources, and
199/// docling-parse reads both as the same font. Keyed by resource name (as
200/// this port was until #728) the `fi` ligature, which bridges fonts, glued
201/// itself to the preceding run while the font barrier then cut it off from
202/// its own word (`fi nancial`); docling's `financial` needs the two
203/// resources to compare equal.
204/// docling-parse's `has_math_positioning_semantics`, the font-name half: a
205/// TeX math or symbol face (`cmmi`, `cmsy`, `cmex`, `cmr`, `cmbx`, `msam`,
206/// `msbm`) or any name saying `math`, `symbol` or `mtextra`.
207fn is_math_font_name(name: &[u8]) -> bool {
208    let lower = name.to_ascii_lowercase();
209    [
210        "cmmi", "cmsy", "cmex", "cmr", "cmbx", "msam", "msbm", "math", "symbol", "mtextra",
211    ]
212    .iter()
213    .any(|t| lower.windows(t.len()).any(|w| w == t.as_bytes()))
214}
215
216fn sanitizer_font_name(doc: &Document, fdict: &Dictionary, two_byte: bool) -> Vec<u8> {
217    let descriptor_font_name = |owner: &Dictionary| {
218        owner
219            .get(b"FontDescriptor")
220            .ok()
221            .and_then(|o| as_dict(doc, o))
222            .and_then(|fd| fd.get(b"FontName").ok())
223            .and_then(|o| deref(doc, o))
224            .and_then(|o| o.as_name().ok().map(<[u8]>::to_vec))
225    };
226    if let Some(n) = descriptor_font_name(fdict) {
227        return n;
228    }
229    if two_byte {
230        let descendant = fdict
231            .get(b"DescendantFonts")
232            .ok()
233            .and_then(|o| deref(doc, o))
234            .and_then(|o| match o {
235                Object::Array(a) => a.first(),
236                _ => None,
237            })
238            .and_then(|o| as_dict(doc, o));
239        if let Some(n) = descendant.and_then(descriptor_font_name) {
240            return n;
241        }
242    }
243    if let Some(n) = fdict
244        .get(b"Name")
245        .ok()
246        .and_then(|o| deref(doc, o))
247        .and_then(|o| o.as_name().ok())
248    {
249        return n.to_vec();
250    }
251    // docling-parse's `base_font` keeps the subset prefix (`ABCDEF+Helvetica`):
252    // two subsets of one face are two fonts there, and so here.
253    if let Some(n) = fdict.get(b"BaseFont").ok().and_then(|o| o.as_name().ok()) {
254        return n.to_vec();
255    }
256    b"null".to_vec()
257}
258
259/// Parse one font dictionary into a [`Font`].
260fn parse_font(doc: &Document, name: &[u8], fdict: &Dictionary) -> Font {
261    let subtype: &[u8] = fdict
262        .get(b"Subtype")
263        .ok()
264        .and_then(|o| o.as_name().ok())
265        .unwrap_or(&[]);
266    let two_byte = subtype == b"Type0".as_slice();
267
268    let mut to_unicode = fdict
269        .get(b"ToUnicode")
270        .ok()
271        .and_then(|o| deref(doc, o))
272        .and_then(|o| o.as_stream().ok())
273        .and_then(|s| s.decompressed_content().ok())
274        .map(|data| parse_tounicode(&data))
275        .unwrap_or_default();
276    if !two_byte {
277        if let Some(diffs) = differences_array(doc, fdict) {
278            reconcile_stale_tounicode(&mut to_unicode, &diffs);
279        }
280    }
281
282    let (mut widths, mut default_width) = if two_byte {
283        cid_widths(doc, fdict)
284    } else {
285        simple_widths(doc, fdict)
286    };
287
288    let simple_encoding = if two_byte {
289        None
290    } else {
291        Some(simple_encoding_table(doc, fdict))
292    };
293
294    // A standard-14 font referenced without an embedded program usually ships
295    // no `/Widths` and no `/FontDescriptor` either (ReportLab's default, #187).
296    // Every advance then resolved to 0, the cells collapsed to zero width, and
297    // the page's whole text layer was silently dropped — while pdfium, with
298    // its built-in metrics, reads the same file fine. Fill the widths from the
299    // built-in Adobe Core 14 AFM tables via the font's own code→char decode
300    // (base encoding + `/Differences`), so an explicit `/Widths` always wins.
301    if !two_byte && widths.is_empty() && default_width == 0.0 {
302        if let Some(std14) = base_font_name(fdict).and_then(|n| crate::std14::widths_for(&n)) {
303            if let Some(enc) = &simple_encoding {
304                for (&code, &ch) in enc {
305                    if let Some(w) = std14.width(ch) {
306                        widths.insert(u32::from(code), w);
307                    }
308                }
309            }
310            // Codes the table misses still advance a typical width instead of
311            // stacking at x=0 (the failure mode this whole branch fixes).
312            default_width = 500.0;
313        }
314    }
315    let fallback_names = if two_byte {
316        HashMap::new()
317    } else {
318        differences_gid_names(doc, fdict)
319    };
320    let program_encoding = if two_byte {
321        HashMap::new()
322    } else {
323        type1_program_encoding(doc, fdict)
324    };
325
326    let (ascent, descent) = font_ascent_descent(doc, fdict, two_byte);
327    let sanitizer_name = sanitizer_font_name(doc, fdict, two_byte);
328
329    Font {
330        two_byte,
331        to_unicode,
332        widths,
333        default_width,
334        simple_encoding,
335        fallback_names,
336        program_encoding,
337        ascent,
338        descent,
339        hash: hash_name(&sanitizer_name),
340        math: is_math_font_name(&sanitizer_name),
341        style: crate::font_style::parse_font_style(&String::from_utf8_lossy(
342            &base_font_name(fdict).unwrap_or_else(|| name.to_vec()),
343        )),
344    }
345}
346
347/// A simple font's `/Encoding` `/Differences` array, dereferenced.
348fn differences_array(doc: &Document, fdict: &Dictionary) -> Option<Vec<Object>> {
349    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))?;
350    let diffs = enc.as_dict().ok()?.get(b"Differences").ok()?;
351    match deref(doc, diffs)? {
352        Object::Array(a) => Some(a.clone()),
353        _ => None,
354    }
355}
356
357/// A ToUnicode CMap copied from the *original* font while the glyphs were
358/// re-encoded: the encrypted `2206.01062_pg3` fixture (a Quartz re-save of
359/// pdfTeX output) keeps the T1-encoding CMap, keyed on the old codes, next
360/// to a `/Differences [33 /f_f_i /f_i /f_f /f_l]` that names the ligatures at
361/// new ones, so the CMap maps code 33 to `!` — its own identity — and
362/// `difficulties` read `di ! culties`, in docling's pdfium chain too. A
363/// ToUnicode entry that is the code's identity, for a code whose
364/// `/Differences` name resolves to something else, is that stale copy: the
365/// name wins. A CMap that disagrees any other way is left alone — ToUnicode
366/// is the producer's word on what the glyph means.
367fn reconcile_stale_tounicode(to_unicode: &mut HashMap<u32, String>, diffs: &[Object]) {
368    let mut code = 0u8;
369    for el in diffs {
370        match el {
371            Object::Integer(i) => code = *i as u8,
372            Object::Name(name) => {
373                if let Some(ch) = glyph_name_to_char(name) {
374                    let identity = (code as char).to_string();
375                    if ch.to_string() != identity
376                        && to_unicode.get(&u32::from(code)) == Some(&identity)
377                    {
378                        to_unicode.insert(u32::from(code), ch.to_string());
379                    }
380                }
381                code = code.wrapping_add(1);
382            }
383            _ => {}
384        }
385    }
386}
387
388/// Collect `/Differences` entries whose glyph name is a GID placeholder
389/// (`g115`, `cid42`, `glyph7`, `index9`) with no Unicode mapping. docling-parse
390/// emits such glyphs as the literal name `/g115`; mapping them here keeps the
391/// text from being silently dropped (subsetted fonts with no ToUnicode). The
392/// GID-name restriction keeps real Adobe glyph names on the normal path so this
393/// never invents garbage on the clean files.
394fn differences_gid_names(doc: &Document, fdict: &Dictionary) -> HashMap<u8, String> {
395    let mut map = HashMap::new();
396    let Some(Object::Dictionary(enc)) = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))
397    else {
398        return map;
399    };
400    let Some(Object::Array(diffs)) = enc.get(b"Differences").ok().and_then(|o| deref(doc, o))
401    else {
402        return map;
403    };
404    let mut code = 0u8;
405    for el in diffs {
406        match el {
407            Object::Integer(i) => code = *i as u8,
408            Object::Name(name) => {
409                if glyph_name_to_char(name).is_none() && is_gid_name(name) {
410                    map.insert(code, String::from_utf8_lossy(name).into_owned());
411                }
412                code = code.wrapping_add(1);
413            }
414            _ => {}
415        }
416    }
417    map
418}
419
420/// Parse the embedded Type1 font program's built-in `/Encoding` vector
421/// (`dup <code> /<glyphname> put` entries in the clear-text header before
422/// `eexec`) into `code → char`. This is how docling recovers glyphs from
423/// standard TeX math fonts (CMMI/CMSY/…) that carry no PDF `/Encoding` and no
424/// ToUnicode — e.g. CMMI's `dup 21 /lambda` or CMSY's `dup 20 /lessequal`.
425/// Only `FontFile` (Type1) is parsed; CFF (`FontFile3`) and TrueType
426/// (`FontFile2`) store their encoding in a binary table and are left alone.
427fn type1_program_encoding(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
428    let mut map = HashMap::new();
429    let Some(desc) = fdict
430        .get(b"FontDescriptor")
431        .ok()
432        .and_then(|o| deref(doc, o))
433        .and_then(|o| o.as_dict().ok())
434    else {
435        return map;
436    };
437    let Some(data) = desc
438        .get(b"FontFile")
439        .ok()
440        .and_then(|o| deref(doc, o))
441        .and_then(|o| o.as_stream().ok())
442        .and_then(|s| s.decompressed_content().ok())
443    else {
444        return map;
445    };
446    // The clear-text header (PostScript) ends at `eexec`; the rest is encrypted.
447    let head_end = data
448        .windows(5)
449        .position(|w| w == b"eexec")
450        .unwrap_or(data.len());
451    let head = String::from_utf8_lossy(&data[..head_end]);
452    // Scan for `dup <code> /<name> put` tokens.
453    let toks: Vec<&str> = head.split_whitespace().collect();
454    for w in toks.windows(4) {
455        if w[0] == "dup" && w[3] == "put" {
456            if let (Ok(code), Some(name)) = (w[1].parse::<u32>(), w[2].strip_prefix('/')) {
457                if code <= 255 {
458                    if let Some(ch) = glyph_name_to_char(name.as_bytes()) {
459                        map.insert(code as u8, ch);
460                    }
461                }
462            }
463        }
464    }
465    map
466}
467
468/// A glyph name that is a synthetic placeholder, not a real Adobe name:
469/// `g115`, `cid42`, `glyph7`, `index9`, `G12`, or a short-prefix code name like
470/// `SM590000` (IBM BookMaster). These carry no Unicode meaning, and docling-parse
471/// emits them verbatim (`/SM590000`). `afii####` / `uni####` are real Adobe names
472/// and excluded. The restriction keeps genuine glyph names on the Unicode path.
473fn is_gid_name(name: &[u8]) -> bool {
474    let Ok(s) = std::str::from_utf8(name) else {
475        return false;
476    };
477    if s.starts_with("afii") || s.starts_with("uni") {
478        return false;
479    }
480    for prefix in ["g", "G", "cid", "CID", "glyph", "index"] {
481        if let Some(rest) = s.strip_prefix(prefix) {
482            if !rest.is_empty() && rest.bytes().all(|b| b.is_ascii_digit()) {
483                return true;
484            }
485        }
486    }
487    // Short alpha prefix (≤3 letters) followed by a run of ≥3 digits — synthetic
488    // code names like `SM590000`, distinct from real Adobe names (whole words or
489    // letter+`.suffix` variants).
490    let alpha = s.bytes().take_while(|b| b.is_ascii_alphabetic()).count();
491    let digits = s.len() - alpha;
492    (1..=3).contains(&alpha)
493        && digits >= 3
494        && s.as_bytes()[alpha..].iter().all(|b| b.is_ascii_digit())
495}
496
497fn font_ascent_descent(doc: &Document, fdict: &Dictionary, two_byte: bool) -> (f64, f64) {
498    // For Type0, the descriptor lives on the descendant CIDFont.
499    let descr_owner = if two_byte {
500        fdict
501            .get(b"DescendantFonts")
502            .ok()
503            .and_then(|o| deref(doc, o))
504            .and_then(|o| match o {
505                Object::Array(a) => a.first(),
506                _ => None,
507            })
508            .and_then(|o| as_dict(doc, o))
509    } else {
510        Some(fdict)
511    };
512    let fd = descr_owner
513        .and_then(|d| d.get(b"FontDescriptor").ok())
514        .and_then(|o| as_dict(doc, o));
515    let asc = fd
516        .and_then(|d| d.get(b"Ascent").ok())
517        .and_then(|o| {
518            o.as_float()
519                .ok()
520                .or_else(|| o.as_i64().ok().map(|i| i as f32))
521        })
522        .unwrap_or(750.0) as f64;
523    let desc = fd
524        .and_then(|d| d.get(b"Descent").ok())
525        .and_then(|o| {
526            o.as_float()
527                .ok()
528                .or_else(|| o.as_i64().ok().map(|i| i as f32))
529        })
530        .unwrap_or(-250.0) as f64;
531    // Some subsetted fonts carry a degenerate FontDescriptor (`/Ascent 0
532    // /Descent 0`) — the real metrics live in the font program. That collapses
533    // the loose box to zero height, so the line cells get zero area and the
534    // layout's region/text assignment drops them (2305's References list lost
535    // every prose line, keeping only the URLs). Fall back to typical text metrics
536    // so the box has height.
537    if asc - desc <= 1.0 {
538        return (750.0, -250.0);
539    }
540    (asc, desc)
541}
542
543/// The `/BaseFont` name with any `ABCDEF+` subset prefix stripped (#187).
544fn base_font_name(fdict: &Dictionary) -> Option<Vec<u8>> {
545    let name = fdict.get(b"BaseFont").ok()?.as_name().ok()?;
546    let stripped = match name.iter().position(|&b| b == b'+') {
547        Some(i) if i == 6 => &name[i + 1..],
548        _ => name,
549    };
550    Some(stripped.to_vec())
551}
552
553/// Simple-font widths: `/FirstChar` + `/Widths` array, `/MissingWidth` default.
554fn simple_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
555    let mut map = HashMap::new();
556    let first = fdict
557        .get(b"FirstChar")
558        .ok()
559        .and_then(|o| o.as_i64().ok())
560        .unwrap_or(0) as u32;
561    if let Some(Object::Array(arr)) = fdict.get(b"Widths").ok().and_then(|o| deref(doc, o)) {
562        for (i, w) in arr.iter().enumerate() {
563            if let Some(w) = num(w) {
564                map.insert(first + i as u32, w);
565            }
566        }
567    }
568    let dw = fdict
569        .get(b"FontDescriptor")
570        .ok()
571        .and_then(|o| as_dict(doc, o))
572        .and_then(|d| d.get(b"MissingWidth").ok())
573        .and_then(num)
574        .unwrap_or(0.0);
575    (map, dw)
576}
577
578/// CIDFont widths: the `/W` array on the descendant font (`/DW` default = 1000).
579fn cid_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
580    let mut map = HashMap::new();
581    let Some(desc) = fdict
582        .get(b"DescendantFonts")
583        .ok()
584        .and_then(|o| deref(doc, o))
585        .and_then(|o| match o {
586            Object::Array(a) => a.first(),
587            _ => None,
588        })
589        .and_then(|o| as_dict(doc, o))
590    else {
591        return (map, 1000.0);
592    };
593    let dw = desc.get(b"DW").ok().and_then(num).unwrap_or(1000.0);
594    if let Some(Object::Array(w)) = desc.get(b"W").ok().and_then(|o| deref(doc, o)) {
595        let mut i = 0;
596        while i < w.len() {
597            let c = w.get(i).and_then(num);
598            match (c, w.get(i + 1)) {
599                // `c [w1 w2 ...]`: consecutive CIDs starting at c.
600                (Some(c), Some(Object::Array(list))) => {
601                    for (k, wv) in list.iter().enumerate() {
602                        if let Some(wv) = num(wv) {
603                            map.insert(c as u32 + k as u32, wv);
604                        }
605                    }
606                    i += 2;
607                }
608                // `c_first c_last w`: a run all of width w.
609                (Some(c1), Some(o2)) => {
610                    if let (Some(c2), Some(wv)) = (num(o2), w.get(i + 2).and_then(num)) {
611                        for cid in c1 as u32..=c2 as u32 {
612                            map.insert(cid, wv);
613                        }
614                    }
615                    i += 3;
616                }
617                _ => break,
618            }
619        }
620    }
621    (map, dw)
622}
623
624fn num(o: &Object) -> Option<f64> {
625    match o {
626        Object::Integer(i) => Some(*i as f64),
627        Object::Real(r) => Some(*r as f64),
628        _ => None,
629    }
630}
631
632/// Parse a ToUnicode CMap's `bfchar` / `bfrange` sections into code→string.
633pub(crate) fn parse_tounicode(data: &[u8]) -> HashMap<u32, String> {
634    let text = String::from_utf8_lossy(data);
635    let mut map = HashMap::new();
636    let hex = |s: &str| -> Option<Vec<u16>> {
637        let s = s.trim();
638        if !s.starts_with('<') || !s.ends_with('>') {
639            return None;
640        }
641        let h = &s[1..s.len() - 1];
642        let bytes: Vec<u8> = (0..h.len())
643            .step_by(2)
644            .filter_map(|i| u8::from_str_radix(h.get(i..i + 2)?, 16).ok())
645            .collect();
646        Some(
647            bytes
648                .chunks(2)
649                .map(|c| {
650                    if c.len() == 2 {
651                        u16::from_be_bytes([c[0], c[1]])
652                    } else {
653                        c[0] as u16
654                    }
655                })
656                .collect(),
657        )
658    };
659    let u16s_to_string = |u: &[u16]| String::from_utf16_lossy(u);
660    let code_of = |u: &[u16]| u.iter().fold(0u32, |acc, &x| (acc << 16) | x as u32);
661
662    // Tokenize by structure, not whitespace: CMap hex groups are often written
663    // back-to-back with no separators (`<21><21><0054>`), so scan for `<…>`
664    // groups, `[`/`]` brackets, and bareword keywords.
665    let tokens: Vec<String> = {
666        let bytes = text.as_bytes();
667        let mut toks = Vec::new();
668        let mut i = 0;
669        while i < bytes.len() {
670            let c = bytes[i];
671            if c.is_ascii_whitespace() {
672                i += 1;
673            } else if c == b'<' {
674                let start = i;
675                while i < bytes.len() && bytes[i] != b'>' {
676                    i += 1;
677                }
678                i += 1; // include '>'
679                toks.push(String::from_utf8_lossy(&bytes[start..i.min(bytes.len())]).into_owned());
680            } else if c == b'[' || c == b']' {
681                toks.push((c as char).to_string());
682                i += 1;
683            } else {
684                let start = i;
685                while i < bytes.len()
686                    && !bytes[i].is_ascii_whitespace()
687                    && bytes[i] != b'<'
688                    && bytes[i] != b'['
689                    && bytes[i] != b']'
690                {
691                    i += 1;
692                }
693                toks.push(String::from_utf8_lossy(&bytes[start..i]).into_owned());
694            }
695        }
696        toks
697    };
698    let tokens: Vec<&str> = tokens.iter().map(|s| s.as_str()).collect();
699    let mut i = 0;
700    while i < tokens.len() {
701        match tokens[i] {
702            "beginbfchar" => {
703                i += 1;
704                while i + 1 < tokens.len() && tokens[i] != "endbfchar" {
705                    if let (Some(src), Some(dst)) = (hex(tokens[i]), hex(tokens[i + 1])) {
706                        map.insert(code_of(&src), u16s_to_string(&dst));
707                    }
708                    i += 2;
709                }
710            }
711            "beginbfrange" => {
712                i += 1;
713                while i + 2 < tokens.len() && tokens[i] != "endbfrange" {
714                    let (Some(lo), Some(hi)) = (hex(tokens[i]), hex(tokens[i + 1])) else {
715                        i += 1;
716                        continue;
717                    };
718                    let lo = code_of(&lo);
719                    let hi = code_of(&hi);
720                    if tokens[i + 2] == "[" {
721                        // `<lo> <hi> [ <d0> <d1> ... ]`: one dst per code in the range.
722                        let mut j = i + 3;
723                        let mut code = lo;
724                        while j < tokens.len() && tokens[j] != "]" {
725                            if let Some(dst) = hex(tokens[j]) {
726                                map.insert(code, u16s_to_string(&dst));
727                            }
728                            code += 1;
729                            j += 1;
730                        }
731                        i = j + 1;
732                    } else if let Some(dst) = hex(tokens[i + 2]) {
733                        // `<lo> <hi> <dst>`: consecutive Unicode from a base.
734                        let base = code_of(&dst);
735                        for (k, code) in (lo..=hi).enumerate() {
736                            if let Some(ch) = char::from_u32(base + k as u32) {
737                                map.insert(code, ch.to_string());
738                            }
739                        }
740                        i += 3;
741                    } else {
742                        i += 1;
743                    }
744                }
745            }
746            _ => i += 1,
747        }
748    }
749    map
750}
751
752/// Decode a PDF string literal in a Tj/TJ operand into raw code units.
753fn codes(font: &Font, bytes: &[u8]) -> Vec<u32> {
754    if font.two_byte {
755        bytes
756            .chunks(2)
757            .map(|c| {
758                if c.len() == 2 {
759                    ((c[0] as u32) << 8) | c[1] as u32
760                } else {
761                    c[0] as u32
762                }
763            })
764            .collect()
765    } else {
766        bytes.iter().map(|&b| b as u32).collect()
767    }
768}
769
770/// A page's display box, in PDF user space: the `/CropBox` clipped to the
771/// `/MediaBox`, both inherited through the page tree and normalized — a
772/// missing or empty MediaBox is US Letter, an empty CropBox is the MediaBox
773/// (pdfium's `CPDF_Page::UpdateDimensions`). pdfium reports the page size from
774/// this box, renders exactly it, and translates every content coordinate so
775/// its lower-left corner is the origin (`m_PageMatrix`); docling's backends
776/// inherit that frame, so text cells, `prov` boxes and destinations all count
777/// from the CropBox corner, not the MediaBox one. The parser used to flip
778/// glyphs with the MediaBox *height* and no translation at all, so a page
779/// whose boxes do not start at (0, 0) — a trimmed book page with
780/// `MediaBox [-56 -58 576 723]` / `CropBox [1 -0.6 519 666]`, or a LaTeX
781/// figure cropped to `[156 147 637 391]` — had its text displaced against the
782/// rendered bitmap by the box offset, the bottom lines pushed past the page
783/// edge and clamped to `t = b`.
784#[derive(Debug, Clone, Copy, PartialEq)]
785pub(crate) struct PageBox {
786    /// Left edge, user space.
787    pub l: f32,
788    /// Bottom edge, user space.
789    pub b: f32,
790    pub w: f32,
791    pub h: f32,
792}
793
794impl PageBox {
795    /// Top edge, user space — the y that becomes `0` in the y-down frame.
796    pub fn top(&self) -> f32 {
797        self.b + self.h
798    }
799}
800
801/// A page-tree rect attribute (`/MediaBox`, `/CropBox`), inherited from the
802/// nearest ancestor that sets it, as normalized `(l, b, r, t)`.
803fn inherited_rect(
804    doc: &Document,
805    page_id: lopdf::ObjectId,
806    key: &[u8],
807) -> Option<(f32, f32, f32, f32)> {
808    let mut id = page_id;
809    for _ in 0..32 {
810        let dict = doc.get_object(id).ok()?.as_dict().ok()?;
811        if let Some(Object::Array(a)) = dict.get(key).ok().and_then(|o| deref(doc, o)) {
812            let v: Vec<f32> = a.iter().filter_map(|o| num(o).map(|x| x as f32)).collect();
813            if v.len() == 4 && v.iter().all(|x| x.is_finite()) {
814                return Some((
815                    v[0].min(v[2]),
816                    v[1].min(v[3]),
817                    v[0].max(v[2]),
818                    v[1].max(v[3]),
819                ));
820            }
821            return None;
822        }
823        id = dict.get(b"Parent").ok()?.as_reference().ok()?;
824    }
825    None
826}
827
828pub(crate) fn page_box(doc: &Document, page_id: lopdf::ObjectId) -> PageBox {
829    let nonempty = |r: &(f32, f32, f32, f32)| r.2 > r.0 && r.3 > r.1;
830    let media = inherited_rect(doc, page_id, b"MediaBox")
831        .filter(nonempty)
832        .unwrap_or((0.0, 0.0, 612.0, 792.0));
833    let crop = inherited_rect(doc, page_id, b"CropBox")
834        .map(|c| {
835            (
836                c.0.max(media.0),
837                c.1.max(media.1),
838                c.2.min(media.2),
839                c.3.min(media.3),
840            )
841        })
842        .filter(nonempty)
843        .unwrap_or(media);
844    PageBox {
845        l: crop.0,
846        b: crop.1,
847        w: crop.2 - crop.0,
848        h: crop.3 - crop.1,
849    }
850}
851
852/// Page size (width, height) in PDF points — the display box's, like pdfium's
853/// `FPDF_GetPageWidthF/HeightF`.
854fn page_size(doc: &Document, page_id: lopdf::ObjectId) -> (f32, f32) {
855    let pb = page_box(doc, page_id);
856    (pb.w, pb.h)
857}
858
859/// Localize where a page's text is lost, for the `text_layer` diagnostic.
860/// Extraction can come up empty at three different points — no content stream
861/// reached the parser, the stream did not decode into operators, or it ran but
862/// produced no glyphs (fonts/encodings) — and from the outside all three look
863/// the same. Report them per page.
864pub fn content_diagnosis(bytes: &[u8]) -> String {
865    let Some(doc) = load_document(bytes) else {
866        return "document does not load".into();
867    };
868    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
869    pages.sort_by_key(|(n, _)| *n);
870    let mut out = String::new();
871    let mut caches = DocCaches::default();
872    for (n, pid) in pages.into_iter().take(4) {
873        let content_bytes = doc.get_page_content(pid);
874        let ops = lopdf::content::Content::decode(&content_bytes)
875            .map(|c| c.operations.len())
876            .ok();
877        let res = page_res(&doc, pid);
878        let fonts = res.map(|r| fonts_from_res(&doc, r, &mut caches).len());
879        let glyphs = page_glyphs_cached(&doc, pid, &mut caches).len();
880        out.push_str(&format!(
881            "\n   page {n}: content {} B, ops {}, resources {}, fonts {}, glyphs {}",
882            content_bytes.len(),
883            ops.map_or("UNDECODABLE".to_string(), |n| n.to_string()),
884            if res.is_some() { "ok" } else { "MISSING" },
885            fonts.map_or("-".to_string(), |n| n.to_string()),
886            glyphs,
887        ));
888    }
889    out
890}
891
892/// Is this "text layer" a vestige rather than the document's text?
893///
894/// Scanned forms often carry a handful of typed-in strings — a date filled
895/// into three form fields, say — on top of pages that are otherwise images.
896/// Treating that as a real text layer is the worst of both worlds: the text
897/// path proudly extracts thirteen characters, and no OCR ever runs on the
898/// letter the pages actually show. The reported form did exactly this (3
899/// lines, 13 chars, 3 pages).
900///
901/// The rule is deliberately tight so genuinely sparse *digital* documents are
902/// not misrouted into OCR: only a document averaging at most one line per page
903/// **and** totalling fewer than 32 characters is called vestigial.
904pub fn text_layer_is_vestigial(pages: &[crate::pdfium_backend::PdfPage]) -> bool {
905    let mut tally = TextLayerTally::default();
906    pages.iter().for_each(|p| tally.add(p));
907    tally.is_vestigial()
908}
909
910/// [`text_layer_is_vestigial`]'s counts, accumulated a page at a time so a
911/// streaming caller need not keep the pages around to decide.
912#[derive(Debug, Default, Clone, Copy)]
913pub struct TextLayerTally {
914    pages: usize,
915    lines: usize,
916    chars: usize,
917}
918
919impl TextLayerTally {
920    pub fn add(&mut self, page: &crate::pdfium_backend::PdfPage) {
921        self.pages += 1;
922        self.lines += page.cells.len();
923        self.chars += page
924            .cells
925            .iter()
926            .map(|c| c.text.chars().count())
927            .sum::<usize>();
928    }
929
930    /// The verdict over the pages added so far.
931    pub fn is_vestigial(&self) -> bool {
932        self.lines == 0 || (self.lines <= self.pages && self.chars < 32)
933    }
934
935    /// Whether the pages added so far already rule vestigial out for a
936    /// document of `total_pages`, whatever the remaining pages hold — both
937    /// counts only grow, so a caller may stop parsing pages it does not need.
938    pub fn proves_text(&self, total_pages: usize) -> bool {
939        self.lines > 0 && (self.lines > total_pages || self.chars >= 32)
940    }
941}
942
943/// Why the cross-reference repair did or did not fire, for the `text_layer`
944/// diagnostic. A PDF that will not load is indistinguishable from a scan in
945/// production (both convert to nothing), so the reason has to be askable.
946pub fn xref_repair_status(bytes: &[u8]) -> String {
947    if Document::load_mem(bytes).is_ok() {
948        return "loads unaided; no repair needed".into();
949    }
950    match pad_short_xref_entries(bytes) {
951        Ok(fixed) => match Document::load_mem(&fixed) {
952            Ok(_) => "repaired: cross-reference entries padded to 20 bytes".into(),
953            Err(e) => format!("padded the entries, but it still will not load: {e}"),
954        },
955        Err(why) => format!("repair declined — {why}"),
956    }
957}
958
959/// Load a PDF, repairing the one malformation that otherwise costs us the whole
960/// document: **19-byte cross-reference entries**.
961///
962/// The spec fixes an xref entry at 20 bytes — `nnnnnnnnnn ggggg n` plus a
963/// *two*-byte EOL. Some generators (an Austrian telecom's invoices, for one)
964/// emit a bare LF instead, making each entry 19 bytes. lopdf rejects the file
965/// outright (`invalid file trailer`) where pdfium reads it happily, so a
966/// perfectly good text layer looked to the browser exactly like a scan and cost
967/// ten seconds of OCR.
968///
969/// Padding is only attempted when it cannot move anything the xref points at:
970/// a single `xref` section that begins after the last object. The repair then
971/// has to prove itself — the padded bytes are used only if they load — so a
972/// mis-repair degrades to today's behaviour rather than to silent garbage.
973pub(crate) fn load_document(bytes: &[u8]) -> Option<Document> {
974    open_document(bytes, None).ok()
975}
976
977/// Why [`open_document`] could not hand back a readable document.
978#[derive(Debug, Clone, Copy, PartialEq, Eq)]
979pub(crate) enum OpenError {
980    /// lopdf cannot read the file even after the repairs.
981    Unreadable,
982    /// The file is encrypted and the password given (or none) does not open it.
983    Password,
984}
985
986/// Load `bytes` with the document's password. lopdf decrypts while reading —
987/// with `password`, or the empty user password most "protected" PDFs carry
988/// (what every viewer opens silently) — so every reader downstream sees plain
989/// streams; a file the password does not open loads with its streams still
990/// encrypted, which is reported as [`OpenError::Password`] rather than handed
991/// on as a document whose every stream decodes to nothing.
992pub(crate) fn open_document(bytes: &[u8], password: Option<&str>) -> Result<Document, OpenError> {
993    let Some(doc) = load_document_raw(bytes, password) else {
994        // lopdf refuses to load at all under a *wrong* password (a missing
995        // one loads the file with its streams still encrypted); tell the two
996        // apart by loading without it.
997        if password.is_some() && load_document_raw(bytes, None).is_some_and(|d| d.is_encrypted()) {
998            return Err(OpenError::Password);
999        }
1000        return Err(OpenError::Unreadable);
1001    };
1002    if doc.is_encrypted() {
1003        return Err(OpenError::Password);
1004    }
1005    Ok(doc)
1006}
1007
1008fn load_options(password: Option<&str>) -> lopdf::LoadOptions {
1009    lopdf::LoadOptions {
1010        password: password.map(str::to_string),
1011        ..lopdf::LoadOptions::default()
1012    }
1013}
1014
1015fn load_document_raw(bytes: &[u8], password: Option<&str>) -> Option<Document> {
1016    // Try progressively more repair, and accept a candidate only once the pages
1017    // actually carry content — a document whose streams were dropped still
1018    // "loads", so loading alone is not evidence the repair helped. A
1019    // well-formed file returns on the first attempt and pays for nothing.
1020    let mut fallback = None;
1021    if let Some(doc) = best_effort_load(bytes, password, &mut fallback) {
1022        return Some(doc);
1023    }
1024    let xref_fixed = pad_short_xref_entries(bytes).ok();
1025    if let Some(fixed) = &xref_fixed {
1026        if let Some(doc) = best_effort_load(fixed, password, &mut fallback) {
1027            return Some(doc);
1028        }
1029    }
1030    // Both defects can coexist, and the second only becomes visible once the
1031    // first is repaired, so build on whatever the previous step produced.
1032    let lengths_fixed = fix_stream_lengths(xref_fixed.as_deref().unwrap_or(bytes));
1033    if let Some(doc) = best_effort_load(&lengths_fixed, password, &mut fallback) {
1034        return Some(doc);
1035    }
1036    fallback
1037}
1038
1039/// Load `data`, returning it only when its pages carry content; a document that
1040/// merely parses is remembered as the fallback for when nothing does better.
1041fn best_effort_load(
1042    data: &[u8],
1043    password: Option<&str>,
1044    fallback: &mut Option<Document>,
1045) -> Option<Document> {
1046    match Document::load_mem_with_options(data, load_options(password)) {
1047        Ok(doc) if has_page_content(&doc) => Some(doc),
1048        Ok(doc) => {
1049            fallback.get_or_insert(doc);
1050            None
1051        }
1052        Err(_) => None,
1053    }
1054}
1055
1056/// Does any page actually hand us a content stream? A document whose streams
1057/// were dropped still parses — it simply has nothing to read — so this is what
1058/// tells a successful repair from a pointless one.
1059fn has_page_content(doc: &Document) -> bool {
1060    doc.get_pages()
1061        .into_values()
1062        .take(4)
1063        .any(|pid| !doc.get_page_content(pid).is_empty())
1064}
1065
1066/// Correct `/Length` values that disagree with where `endstream` actually is.
1067///
1068/// The same generator that writes short xref entries also overstates its
1069/// content-stream lengths by a byte or two. lopdf trusts `/Length`, reads past
1070/// the data, fails to find `endstream` there and drops the stream — the object
1071/// comes back as a bare dictionary, so the page has no content at all and the
1072/// document looks like a scan. pdfium instead trusts `endstream`, which is what
1073/// this does.
1074///
1075/// The rewrite is length-preserving: the corrected number is written over the
1076/// old digits and padded with spaces, so every byte offset in the file — and
1077/// therefore the whole cross-reference table — stays valid.
1078fn fix_stream_lengths(bytes: &[u8]) -> Vec<u8> {
1079    let mut out = bytes.to_vec();
1080    let mut i = 0;
1081    while let Some(rel) = find(&out[i..], b"stream") {
1082        let kw = i + rel;
1083        i = kw + 6;
1084        // Skip `endstream` (the keyword we are measuring *to*).
1085        if kw >= 3 && &out[kw - 3..kw] == b"end" {
1086            continue;
1087        }
1088        // The stream data starts after the EOL that follows the keyword.
1089        let mut data = kw + 6;
1090        if out.get(data..data + 2) == Some(b"\r\n".as_slice()) {
1091            data += 2;
1092        } else if matches!(out.get(data), Some(b'\n' | b'\r')) {
1093            data += 1;
1094        }
1095        let Some(end) = find(&out[data..], b"endstream").map(|r| data + r) else {
1096            continue;
1097        };
1098        // `/Length <digits>` in the dictionary just before the keyword.
1099        let dict_start = out[..kw].iter().rposition(|&c| c == b'<').unwrap_or(0);
1100        let Some(lrel) = find(&out[dict_start..kw], b"/Length") else {
1101            continue;
1102        };
1103        let mut d = dict_start + lrel + 7;
1104        while matches!(out.get(d), Some(b' ')) {
1105            d += 1;
1106        }
1107        let digits = out[d..].iter().take_while(|c| c.is_ascii_digit()).count();
1108        if digits == 0 {
1109            continue;
1110        }
1111        let declared: usize = match std::str::from_utf8(&out[d..d + digits])
1112            .ok()
1113            .and_then(|s| s.parse().ok())
1114        {
1115            Some(v) => v,
1116            None => continue,
1117        };
1118        let actual = end - data;
1119        // Only shrink, and only when the new value fits the space the old one
1120        // occupied — growing the number would move every following byte.
1121        let replacement = actual.to_string();
1122        if actual == declared || replacement.len() > digits {
1123            continue;
1124        }
1125        out[d..d + digits].fill(b' ');
1126        out[d..d + replacement.len()].copy_from_slice(replacement.as_bytes());
1127    }
1128    out
1129}
1130
1131fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
1132    haystack.windows(needle.len()).position(|w| w == needle)
1133}
1134
1135/// Rewrite a classic cross-reference table's entries to the spec's 20 bytes,
1136/// or `None` when the file's shape makes that unsafe (see [`load_document`]).
1137pub(crate) fn pad_short_xref_entries(bytes: &[u8]) -> Result<Vec<u8>, &'static str> {
1138    // Exactly one xref section, and it must start after every object, so that
1139    // growing it shifts nothing the table's offsets refer to.
1140    let is_boundary = |i: usize| i == 0 || matches!(bytes[i - 1], b'\n' | b'\r');
1141    let mut starts = (0..bytes.len().saturating_sub(4))
1142        .filter(|&i| &bytes[i..i + 4] == b"xref" && is_boundary(i));
1143    let xref_at = starts
1144        .next()
1145        .ok_or("no classic `xref` section (an xref stream?)")?;
1146    if starts.next().is_some() {
1147        return Err("more than one xref section (incremental update)");
1148    }
1149    let last_obj = bytes
1150        .windows(3)
1151        .rposition(|w| w == b"obj")
1152        .ok_or("no objects found")?;
1153    if last_obj > xref_at {
1154        return Err("an object follows the xref — padding would move it");
1155    }
1156
1157    let mut out = bytes[..xref_at].to_vec();
1158    out.extend_from_slice(b"xref\n");
1159    let mut i = xref_at + 4;
1160    let skip_ws = |i: &mut usize| {
1161        while matches!(bytes.get(*i), Some(b'\r' | b'\n' | b' ')) {
1162            *i += 1;
1163        }
1164    };
1165    loop {
1166        skip_ws(&mut i);
1167        // Either the next subsection header ("first count") or the trailer.
1168        if bytes[i..].starts_with(b"trailer") {
1169            out.extend_from_slice(&bytes[i..]);
1170            return Ok(out);
1171        }
1172        let header_end = i + bytes[i..]
1173            .iter()
1174            .position(|c| matches!(c, b'\n' | b'\r'))
1175            .ok_or("subsection header runs off the end")?;
1176        let header = std::str::from_utf8(&bytes[i..header_end])
1177            .map_err(|_| "subsection header is not text")?
1178            .trim();
1179        let mut parts = header.split_whitespace();
1180        let count: usize = parts
1181            .nth(1)
1182            .and_then(|c| c.parse().ok())
1183            .ok_or("unparseable subsection header")?;
1184        if parts.next().is_some() || count == 0 {
1185            return Err("unexpected subsection header shape");
1186        }
1187        out.extend_from_slice(header.as_bytes());
1188        out.push(b'\n');
1189        i = header_end;
1190        for _ in 0..count {
1191            skip_ws(&mut i);
1192            // `nnnnnnnnnn ggggg n` — the 18 bytes before whatever EOL follows.
1193            let entry = bytes.get(i..i + 18).ok_or("xref entry runs off the end")?;
1194            let well_formed = entry[..10].iter().all(u8::is_ascii_digit)
1195                && entry[10] == b' '
1196                && entry[11..16].iter().all(u8::is_ascii_digit)
1197                && entry[16] == b' '
1198                && matches!(entry[17], b'n' | b'f');
1199            if !well_formed {
1200                return Err("xref entry is not `nnnnnnnnnn ggggg n`");
1201            }
1202            out.extend_from_slice(entry);
1203            out.extend_from_slice(b" \n"); // the spec's 2-byte EOL -> 20 bytes
1204            i += 18;
1205        }
1206    }
1207}
1208
1209/// Debug: raw glyph stream `(ch, ll, lr, lb, lt)` (native coords) for page
1210/// `index`, before the sanitizer. For comparing char cells to docling-parse.
1211pub fn debug_glyphs(bytes: &[u8], index: usize) -> Vec<(char, f32, f32, f32, f32)> {
1212    let Some(doc) = load_document(bytes) else {
1213        return Vec::new();
1214    };
1215    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1216    pages.sort_by_key(|(n, _)| *n);
1217    let Some((_, pid)) = pages.get(index) else {
1218        return Vec::new();
1219    };
1220    page_glyphs(&doc, *pid)
1221        .into_iter()
1222        .map(|g| (g.ch, g.ll, g.lr, g.lb, g.lt))
1223        .collect()
1224}
1225
1226/// Public entry: per-page (width, height, line cells) for a PDF, via the Rust
1227/// text parser + the docling-parse line sanitizer. Used by the pipeline and the
1228/// `textparse_dump` example.
1229pub fn pdf_textlines(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1230    let Some(doc) = load_document(bytes) else {
1231        return Vec::new();
1232    };
1233    let mut caches = DocCaches::default();
1234    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1235    pages.sort_by_key(|(n, _)| *n);
1236    pages
1237        .into_iter()
1238        .map(|(_, pid)| {
1239            let (w, h) = page_size(&doc, pid);
1240            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1241            let cells = crate::dp_lines::line_cells(&glyphs, h, true);
1242            (w, h, cells)
1243        })
1244        .collect()
1245}
1246
1247/// Debug/diagnostic entry: per-page (width, height, word cells) for a PDF, via
1248/// the Rust parser glyphs run through the docling-parse word grouping. Used to
1249/// compare parser word cells against docling-parse's `word_cells` oracle (roadmap
1250/// item 6).
1251pub fn pdf_words(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1252    let Some(doc) = load_document(bytes) else {
1253        return Vec::new();
1254    };
1255    let mut caches = DocCaches::default();
1256    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1257    pages.sort_by_key(|(n, _)| *n);
1258    pages
1259        .into_iter()
1260        .map(|(_, pid)| {
1261            let (w, h) = page_size(&doc, pid);
1262            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1263            let cells = crate::dp_lines::word_cells(&glyphs, h, true);
1264            (w, h, cells)
1265        })
1266        .collect()
1267}
1268
1269/// One page's text cells from the pure-Rust parser: prose line cells, per-word
1270/// cells, and code line cells — all from a single glyph parse. Replaces the
1271/// pdfium text path (roadmap item 6) when the parser drop is enabled.
1272#[derive(Default)]
1273pub struct PageParserCells {
1274    pub prose: Vec<crate::pdfium_backend::TextCell>,
1275    pub words: Vec<crate::pdfium_backend::TextCell>,
1276    pub code: Vec<crate::pdfium_backend::TextCell>,
1277    /// Drawn checkbox squares (#609), in the cells' frame.
1278    pub checkboxes: Vec<crate::checkbox::CheckBox>,
1279}
1280
1281/// The parser text layer, driven one page at a time: the document is loaded
1282/// (and repaired, see [`load_document`]) once, the font/form caches persist
1283/// across pages, and each page's glyphs are parsed only when asked for.
1284///
1285/// The eager whole-document walk this replaces ran *before* the first page
1286/// was rendered, so on a long PDF it was a serial prefix the page-worker pool
1287/// sat idle through — 6.2 s on the 1913-page .NET reference, in front of a
1288/// pipeline that otherwise overlaps parsing with inference — and a `--pages`
1289/// window still paid for every page in the file. Pulling pages on demand
1290/// keeps the parse on the producer thread but interleaved with rendering,
1291/// and skips unselected pages entirely. Output per page is unchanged: same
1292/// glyph walk, same shared caches, same contraction.
1293pub struct PageTextParser {
1294    doc: Document,
1295    caches: DocCaches,
1296    /// Page object ids in document order (page 1 first).
1297    pages: Vec<lopdf::ObjectId>,
1298    /// How words split where no space glyph does (#921); from
1299    /// `DOCLING_RS_PDF_WORD_GAP` unless [`Self::set_word_gap`] says.
1300    word_gap: crate::dp_lines::WordGap,
1301    /// Whether [`Self::text_page`]'s page keeps typographic punctuation as
1302    /// drawn (#921); `DOCLING_RS_PDF_KEEP_TYPOGRAPHY` unless
1303    /// [`Self::set_keep_typography`] says.
1304    keep_typography: bool,
1305}
1306
1307impl PageTextParser {
1308    /// Load the document; `None` when lopdf cannot read it at all.
1309    pub fn open(bytes: &[u8]) -> Option<Self> {
1310        Self::open_with_password(bytes, None)
1311    }
1312
1313    /// [`open`](Self::open) with the document's password (an encrypted file
1314    /// the password does not open reads as `None` here; the pipeline reports
1315    /// it through [`crate::pdf_meta::PdfMeta::open_with_password`] first).
1316    pub fn open_with_password(bytes: &[u8], password: Option<&str>) -> Option<Self> {
1317        let doc = open_document(bytes, password).ok()?;
1318        let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1319        pages.sort_by_key(|(n, _)| *n);
1320        Some(Self {
1321            doc,
1322            caches: DocCaches::default(),
1323            pages: pages.into_iter().map(|(_, pid)| pid).collect(),
1324            word_gap: crate::dp_lines::WordGap::from_env(),
1325            keep_typography: crate::assemble::keep_typography_from_env(),
1326        })
1327    }
1328
1329    /// The word-gap rule the line and word cells are built with (#921);
1330    /// `None` keeps the environment's (`DOCLING_RS_PDF_WORD_GAP`, else
1331    /// `adaptive`).
1332    pub fn set_word_gap(&mut self, gap: Option<crate::dp_lines::WordGap>) {
1333        if let Some(g) = gap {
1334            self.word_gap = g;
1335        }
1336    }
1337
1338    /// Whether the pages [`Self::text_page`] builds keep typographic
1339    /// punctuation as drawn (#921); `None` keeps the environment's
1340    /// (`DOCLING_RS_PDF_KEEP_TYPOGRAPHY`, else folding).
1341    pub fn set_keep_typography(&mut self, keep: Option<bool>) {
1342        if let Some(k) = keep {
1343            self.keep_typography = k;
1344        }
1345    }
1346
1347    /// The glyph boxes and font styles of the 0-based page `index` — the
1348    /// heading-hierarchy stage's style signal (#302): every non-space glyph's
1349    /// box (font ascent + descent at its size — the font-size proxy pdfium's
1350    /// loose char box also gave) in top-left coordinates, with the weight
1351    /// class and slant its `/BaseFont` name declares. Empty for a page
1352    /// without a text layer (a scan), and the stage falls back to its other
1353    /// signals.
1354    pub(crate) fn glyph_styles(
1355        &mut self,
1356        index: usize,
1357    ) -> Vec<crate::heading_hierarchy::GlyphStyle> {
1358        let Some(&pid) = self.pages.get(index) else {
1359            return Vec::new();
1360        };
1361        let (_w, h) = page_size(&self.doc, pid);
1362        let glyphs = page_glyphs_cached(&self.doc, pid, &mut self.caches);
1363        // Font hash → style, over the fonts the walk just parsed (an inline,
1364        // uncached font dictionary reads as unstyled).
1365        let styles: HashMap<u64, crate::font_style::FontStyle> = self
1366            .caches
1367            .fonts
1368            .values()
1369            .map(|f| (f.hash, f.style))
1370            .collect();
1371        glyphs
1372            .iter()
1373            .filter(|g| !g.ch.is_whitespace() && g.ll.is_finite())
1374            .map(|g| {
1375                let st = styles.get(&g.font).copied().unwrap_or_default();
1376                crate::heading_hierarchy::GlyphStyle {
1377                    l: g.ll,
1378                    t: h - g.lt,
1379                    r: g.lr,
1380                    b: h - g.lb,
1381                    height: g.height(),
1382                    weight_cls: crate::font_style::weight_class(st.weight),
1383                    italic: st.italic,
1384                    styled: st.known,
1385                }
1386            })
1387            .collect()
1388    }
1389
1390    /// Prose, word and code cells of the 0-based page `index` — empty for an
1391    /// index the parser's page tree doesn't have (a damaged file whose page
1392    /// tree disagrees with the object model's count).
1393    pub fn cells(&mut self, index: usize) -> PageParserCells {
1394        let Some(&pid) = self.pages.get(index) else {
1395            return PageParserCells::default();
1396        };
1397        let (_w, h) = page_size(&self.doc, pid);
1398        let (glyphs, inks) = page_glyphs_and_inks(&self.doc, pid, &mut self.caches);
1399        let (prose, words) =
1400            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1401        PageParserCells {
1402            prose,
1403            words,
1404            code: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1405            checkboxes: crate::checkbox::find(&inks, h),
1406        }
1407    }
1408
1409    /// Pages in the document's page tree.
1410    pub fn page_count(&self) -> usize {
1411        self.pages.len()
1412    }
1413
1414    /// The 0-based page `index` as the text-layer-only conversion sees it —
1415    /// one element of [`pdf_text_pages`], parsed on demand so a caller can
1416    /// drop each page once it is assembled instead of holding the whole
1417    /// document's cells. A blank page (no cells) for an index the page tree
1418    /// doesn't have.
1419    pub fn text_page(&mut self, index: usize) -> crate::pdfium_backend::PdfPage {
1420        let (w, h, (glyphs, inks)) = match self.pages.get(index) {
1421            Some(&pid) => {
1422                let (w, h) = page_size(&self.doc, pid);
1423                (w, h, page_glyphs_and_inks(&self.doc, pid, &mut self.caches))
1424            }
1425            None => (0.0, 0.0, Default::default()),
1426        };
1427        let (mut prose, mut words) =
1428            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1429        drop_overpainted_cells(&mut prose);
1430        drop_overpainted_cells(&mut words);
1431        crate::pdfium_backend::PdfPage {
1432            #[cfg(feature = "ocr-prep")]
1433            image_layout: None,
1434            width: w,
1435            height: h,
1436            // Cells are native PDF points; there is no rendered bitmap.
1437            scale: 1.0,
1438            cells: prose,
1439            code_cells: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1440            word_cells: words,
1441            checkboxes: crate::checkbox::find(&inks, h),
1442            #[cfg(feature = "ocr-prep")]
1443            image: image::RgbImage::new(1, 1),
1444            links: Vec::new(),
1445            rotation: 0,
1446            keep_typography: self.keep_typography,
1447            #[cfg(feature = "render")]
1448            pending_image: None,
1449        }
1450    }
1451}
1452
1453/// Full parser text layer: prose + word + code cells per page, glyphs parsed once.
1454/// `prose`/`words` come from the docling-parse contraction ([`crate::dp_lines`]);
1455/// `code` splits only at the parser's own space glyphs (monospace keeps its
1456/// source spacing). The eager form of [`PageTextParser`].
1457pub fn pdf_all_cells(bytes: &[u8]) -> Vec<PageParserCells> {
1458    let Some(mut parser) = PageTextParser::open(bytes) else {
1459        return Vec::new();
1460    };
1461    (0..parser.pages.len()).map(|i| parser.cells(i)).collect()
1462}
1463
1464/// Whole pages for the text-layer-only conversion ([`crate::convert_text_layer`]):
1465/// the parser's prose/word/code cells plus page geometry, assembled into
1466/// [`PdfPage`]s with no rendered image and no link annotations. Everything here
1467/// is pure Rust (lopdf), so it compiles without the `ml` feature — including on
1468/// wasm32. A page the parser can't read (no text layer) comes back with empty
1469/// cells; there is no pdfium fallback on this path.
1470pub fn pdf_text_pages(bytes: &[u8]) -> Vec<crate::pdfium_backend::PdfPage> {
1471    let Some(mut parser) = PageTextParser::open(bytes) else {
1472        return Vec::new();
1473    };
1474    (0..parser.page_count())
1475        .map(|i| parser.text_page(i))
1476        .collect()
1477}
1478
1479/// Drop line cells that are *painted over each other* — glyphs used as artwork.
1480///
1481/// Some generators draw their logo with a symbol font: on the reporting
1482/// invoice, a `TeleLogo` Type1 paints the T-Mobile mark by stacking the glyphs
1483/// encoded as `"` and `==` on top of one another, and the flat text-layer
1484/// output opened with that garbage. Nothing in the font metadata gives it away
1485/// (the *text* fonts in the same file are also flagged symbolic, and the logo
1486/// font names its glyphs `quotedbl` &c.), but the geometry does: two cells with
1487/// different text where one lies inside the other on the same line is
1488/// physically impossible for prose — ink from two words never occupies the
1489/// same box. Both cells of such a pair are paint, not text.
1490///
1491/// Containment (not mere overlap) keeps this narrow: adjacent words touch but
1492/// never contain each other, and a same-text near-duplicate (double-draw faux
1493/// bold) is left alone for the sanitizer's usual handling. Applied on the
1494/// flat/browser path only — the ML pipeline's text layer is byte-pinned by the
1495/// PDF corpus, and there the layout model already sinks logo marks into
1496/// `picture` regions.
1497fn drop_overpainted_cells(cells: &mut Vec<crate::pdfium_backend::TextCell>) {
1498    let mut paint = vec![false; cells.len()];
1499    for i in 0..cells.len() {
1500        for j in 0..cells.len() {
1501            if i == j || cells[i].text == cells[j].text {
1502                continue;
1503            }
1504            let (a, b) = (&cells[i], &cells[j]);
1505            // Same line band: the vertical overlap covers most of the shorter.
1506            let vo = (a.b.min(b.b) - a.t.max(b.t)).max(0.0);
1507            if vo < 0.6 * (a.b - a.t).min(b.b - b.t) {
1508                continue;
1509            }
1510            // `a` horizontally inside `b` (with a small tolerance).
1511            let ho = (a.r.min(b.r) - a.l.max(b.l)).max(0.0);
1512            if ho >= 0.8 * (a.r - a.l) && (a.r - a.l) <= (b.r - b.l) {
1513                paint[i] = true;
1514                paint[j] = true;
1515            }
1516        }
1517    }
1518    let mut keep = paint.iter().map(|p| !p);
1519    cells.retain(|_| keep.next().unwrap());
1520}
1521
1522/// The text-state scalars inherited by a Form XObject when it is invoked via
1523/// `Do` (the PDF graphics state includes the text parameters, but not the text
1524/// matrices, which a form re-establishes inside its own `BT`/`ET`).
1525#[derive(Clone, Copy)]
1526struct TextState {
1527    tc: f64,
1528    tw: f64,
1529    th: f64,
1530    tl: f64,
1531    trise: f64,
1532    fsize: f64,
1533}
1534
1535impl TextState {
1536    const INIT: TextState = TextState {
1537        tc: 0.0,
1538        tw: 0.0,
1539        th: 1.0,
1540        tl: 0.0,
1541        trise: 0.0,
1542        fsize: 0.0,
1543    };
1544}
1545
1546/// The effective `/Resources` dictionary for a page (inline or via reference,
1547/// falling back to an inherited one from a `/Parent`).
1548fn page_res(doc: &Document, page_id: lopdf::ObjectId) -> Option<&Dictionary> {
1549    let (inline, ids) = doc.get_page_resources(page_id).ok()?;
1550    if let Some(d) = inline {
1551        return Some(d);
1552    }
1553    ids.into_iter().find_map(|id| doc.get_dictionary(id).ok())
1554}
1555
1556/// Build the code→[`Font`] map for a resources dictionary's `/Font` sub-dict,
1557/// reusing the per-document cache for fonts referenced indirectly (the common
1558/// case — the same font objects recur on every page).
1559fn fonts_from_res(
1560    doc: &Document,
1561    res: &Dictionary,
1562    caches: &mut DocCaches,
1563) -> HashMap<Vec<u8>, Arc<Font>> {
1564    let mut map = HashMap::new();
1565    let font_dict = res
1566        .get(b"Font")
1567        .ok()
1568        .and_then(|o| deref(doc, o))
1569        .and_then(|o| o.as_dict().ok());
1570    if let Some(fd) = font_dict {
1571        for (name, value) in fd.iter() {
1572            let font = match value {
1573                Object::Reference(id) => {
1574                    let key = (*id, name.clone());
1575                    if let Some(f) = caches.fonts.get(&key) {
1576                        Arc::clone(f)
1577                    } else if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
1578                        let f = Arc::new(parse_font(doc, name, fdict));
1579                        caches.fonts.insert(key, Arc::clone(&f));
1580                        f
1581                    } else {
1582                        continue;
1583                    }
1584                }
1585                _ => {
1586                    if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
1587                        Arc::new(parse_font(doc, name, fdict))
1588                    } else {
1589                        continue;
1590                    }
1591                }
1592            };
1593            map.insert(name.clone(), font);
1594        }
1595    }
1596    map
1597}
1598
1599/// Extract every glyph on a page as a native-coordinate [`Glyph`].
1600/// [`PageTextParser::glyph_styles`] for the given **1-based** pages of a
1601/// document, keyed by page number — a separate, on-demand pass (no
1602/// rendering), so the extraction pipeline stays byte-identical whether or not
1603/// the heading-hierarchy stage runs. Empty when lopdf cannot open the file.
1604pub(crate) fn glyph_styles(
1605    bytes: &[u8],
1606    pages: &[usize],
1607) -> HashMap<usize, Vec<crate::heading_hierarchy::GlyphStyle>> {
1608    let mut out = HashMap::new();
1609    let Some(mut parser) = PageTextParser::open(bytes) else {
1610        return out;
1611    };
1612    for &page_no in pages {
1613        if page_no == 0 {
1614            continue;
1615        }
1616        let styles = parser.glyph_styles(page_no - 1);
1617        if !styles.is_empty() {
1618            out.insert(page_no, styles);
1619        }
1620    }
1621    out
1622}
1623
1624pub(crate) fn page_glyphs(doc: &Document, page_id: lopdf::ObjectId) -> Vec<Glyph> {
1625    page_glyphs_cached(doc, page_id, &mut DocCaches::default())
1626}
1627
1628/// [`page_glyphs`] with an explicit per-document cache, so a multi-page walk
1629/// parses each font / decodes each form once instead of once per page.
1630fn page_glyphs_cached(
1631    doc: &Document,
1632    page_id: lopdf::ObjectId,
1633    caches: &mut DocCaches,
1634) -> Vec<Glyph> {
1635    page_glyphs_and_inks(doc, page_id, caches).0
1636}
1637
1638/// [`page_glyphs_cached`] plus the small painted path pieces the same walk
1639/// passed — what [`crate::checkbox::find`] looks for checkbox squares in
1640/// (#609) — in the glyphs' frame.
1641fn page_glyphs_and_inks(
1642    doc: &Document,
1643    page_id: lopdf::ObjectId,
1644    caches: &mut DocCaches,
1645) -> (Vec<Glyph>, Vec<crate::checkbox::Ink>) {
1646    let mut out = Vec::new();
1647    let mut inks = Vec::new();
1648    // get_page_content returns the assembled content-stream bytes directly
1649    // (an empty Vec when the page has none).
1650    let content_bytes = doc.get_page_content(page_id);
1651    let Ok(content) = lopdf::content::Content::decode(&content_bytes) else {
1652        return (out, inks);
1653    };
1654    if let Some(res) = page_res(doc, page_id) {
1655        // pdfium's page matrix: user space translated so the display box's
1656        // lower-left corner is the origin (see [`PageBox`]).
1657        let pb = page_box(doc, page_id);
1658        let base = Mat {
1659            e: -(pb.l as f64),
1660            f: -(pb.b as f64),
1661            ..Mat::ID
1662        };
1663        run_content(
1664            doc,
1665            res,
1666            &content,
1667            base,
1668            TextState::INIT,
1669            0,
1670            caches,
1671            &mut out,
1672            &mut inks,
1673        );
1674        out.retain(|g| on_page(g, pb.w, pb.h));
1675    }
1676    (out, inks)
1677}
1678
1679/// Whether a glyph is on the page (#529): docling-parse keeps a character
1680/// only when its whole box lies inside the display box — the CropBox, or
1681/// the MediaBox without one — edges included, so a FrameMaker print slug
1682/// drawn beside the CropBox or a tiled page's neighbouring text beyond the
1683/// MediaBox never becomes a cell, and a line crossing the edge is cut at the
1684/// last glyph that fits (`Crossing the rig`). The box is docling-parse's
1685/// char box — the advance by the font's ascent/descent, the loose box here
1686/// (its axis-aligned extent for rotated text) — in the frame `page_glyphs`
1687/// already moved to the display box's corner, so the page is `[0, w] × [0,
1688/// h]`. A glyph without a finite box is kept, as before. (A standard-14 font
1689/// without a FontDescriptor gets the 750 / −250 default ascent / descent
1690/// here where docling-parse reads the AFM's — Helvetica's 718 / −207 — so
1691/// for those a glyph within ~0.04 em of the top or bottom edge can fall the
1692/// other way; horizontally the advance is the same.)
1693fn on_page(g: &Glyph, w: f32, h: f32) -> bool {
1694    // f32 noise only: docling-parse already drops a glyph 0.01 pt over.
1695    const EPS: f32 = 1e-3;
1696    let (l, b, r, t) = if [g.ll, g.lb, g.lr, g.lt].iter().all(|v| v.is_finite()) {
1697        (g.ll, g.lb, g.lr, g.lt)
1698    } else {
1699        (g.l, g.b, g.r, g.t)
1700    };
1701    if ![l, b, r, t].iter().all(|v| v.is_finite()) {
1702        return true;
1703    }
1704    l >= -EPS && b >= -EPS && r <= w + EPS && t <= h + EPS
1705}
1706
1707/// Run a content stream's operators, emitting glyphs into `out`. Recurses into
1708/// Form XObjects on `Do` (bulk body text in heavy PDFs lives inside a form, not
1709/// the page content stream). `res` is the resources dict in scope (the page's,
1710/// or the form's own); `base_ctm` is the CTM at the point of invocation.
1711#[allow(clippy::too_many_arguments)]
1712fn run_content(
1713    doc: &Document,
1714    res: &Dictionary,
1715    content: &lopdf::content::Content,
1716    base_ctm: Mat,
1717    init: TextState,
1718    depth: u32,
1719    caches: &mut DocCaches,
1720    out: &mut Vec<Glyph>,
1721    inks: &mut Vec<crate::checkbox::Ink>,
1722) {
1723    let fonts = fonts_from_res(doc, res, caches);
1724    let xobjects = res
1725        .get(b"XObject")
1726        .ok()
1727        .and_then(|o| deref(doc, o))
1728        .and_then(|o| o.as_dict().ok());
1729
1730    // Graphics + text state. `q`/`Q` save and restore the whole graphics state,
1731    // which includes the text parameters (Tc, Tw, Tz, TL, Tfs, Trise, font) —
1732    // *not* the text matrix (that is reset by BT). Saving only the CTM let a Tc
1733    // set inside a `q…Q` block leak out and drift every later glyph.
1734    #[allow(clippy::type_complexity)]
1735    let mut gstate_stack: Vec<(Mat, f64, f64, f64, f64, f64, f64, Option<&Arc<Font>>)> = Vec::new();
1736    let mut ctm = base_ctm;
1737    let mut tm = Mat::ID;
1738    let mut tlm = Mat::ID;
1739    let mut font: Option<&Arc<Font>> = None;
1740    let mut fsize = init.fsize;
1741    let mut tc = init.tc; // char spacing
1742    let mut tw = init.tw; // word spacing
1743    let mut th = init.th; // horizontal scale (Tz/100)
1744    let mut tl = init.tl; // leading
1745    let mut trise = init.trise;
1746
1747    let op_f = |operands: &[Object], i: usize| operands.get(i).and_then(num).unwrap_or(0.0);
1748    // The path under construction, already in page space (a path is built
1749    // under one CTM — `cm` is not allowed inside it), for the checkbox
1750    // squares (#609). Text extraction never reads it.
1751    let mut path: Vec<PathPiece> = Vec::new();
1752    let (mut cur, mut start) = ((0.0, 0.0), (0.0, 0.0));
1753    // Whether the fill colour is (near) white, saved/restored with `q`/`Q`:
1754    // a white-filled stroked box is a checkbox outline, a coloured one a
1755    // chart legend's swatch. The initial fill colour is black.
1756    let mut fill_light = false;
1757    let mut fill_stack: Vec<bool> = Vec::new();
1758
1759    for op in &content.operations {
1760        let operands = &op.operands;
1761        match op.operator.as_str() {
1762            "q" => {
1763                gstate_stack.push((ctm, tc, tw, th, tl, trise, fsize, font));
1764                fill_stack.push(fill_light);
1765            }
1766            "Q" => {
1767                fill_light = fill_stack.pop().unwrap_or(false);
1768                if let Some((c, a, b, h, l, r, fs, f)) = gstate_stack.pop() {
1769                    ctm = c;
1770                    tc = a;
1771                    tw = b;
1772                    th = h;
1773                    tl = l;
1774                    trise = r;
1775                    fsize = fs;
1776                    font = f;
1777                }
1778            }
1779            "cm" => {
1780                let m = Mat {
1781                    a: op_f(operands, 0),
1782                    b: op_f(operands, 1),
1783                    c: op_f(operands, 2),
1784                    d: op_f(operands, 3),
1785                    e: op_f(operands, 4),
1786                    f: op_f(operands, 5),
1787                };
1788                ctm = m.then(ctm);
1789            }
1790            "BT" => {
1791                tm = Mat::ID;
1792                tlm = Mat::ID;
1793            }
1794            "ET" => {}
1795            "Tf" => {
1796                if let Some(Object::Name(n)) = operands.first() {
1797                    font = fonts.get(n.as_slice());
1798                }
1799                fsize = op_f(operands, 1);
1800            }
1801            "Td" => {
1802                tlm = Mat {
1803                    a: 1.0,
1804                    b: 0.0,
1805                    c: 0.0,
1806                    d: 1.0,
1807                    e: op_f(operands, 0),
1808                    f: op_f(operands, 1),
1809                }
1810                .then(tlm);
1811                tm = tlm;
1812            }
1813            "TD" => {
1814                tl = -op_f(operands, 1);
1815                tlm = Mat {
1816                    a: 1.0,
1817                    b: 0.0,
1818                    c: 0.0,
1819                    d: 1.0,
1820                    e: op_f(operands, 0),
1821                    f: op_f(operands, 1),
1822                }
1823                .then(tlm);
1824                tm = tlm;
1825            }
1826            "Tm" => {
1827                tlm = Mat {
1828                    a: op_f(operands, 0),
1829                    b: op_f(operands, 1),
1830                    c: op_f(operands, 2),
1831                    d: op_f(operands, 3),
1832                    e: op_f(operands, 4),
1833                    f: op_f(operands, 5),
1834                };
1835                tm = tlm;
1836            }
1837            "T*" => {
1838                tlm = Mat {
1839                    a: 1.0,
1840                    b: 0.0,
1841                    c: 0.0,
1842                    d: 1.0,
1843                    e: 0.0,
1844                    f: -tl,
1845                }
1846                .then(tlm);
1847                tm = tlm;
1848            }
1849            "Tc" => tc = op_f(operands, 0),
1850            "Tw" => tw = op_f(operands, 0),
1851            "Tz" => th = op_f(operands, 0) / 100.0,
1852            "TL" => tl = op_f(operands, 0),
1853            "Ts" => trise = op_f(operands, 0),
1854            "Tj" | "'" | "\"" => {
1855                if op.operator == "'" || op.operator == "\"" {
1856                    // move to next line first
1857                    tlm = Mat {
1858                        a: 1.0,
1859                        b: 0.0,
1860                        c: 0.0,
1861                        d: 1.0,
1862                        e: 0.0,
1863                        f: -tl,
1864                    }
1865                    .then(tlm);
1866                    tm = tlm;
1867                }
1868                if op.operator == "\"" {
1869                    // `aw ac string "` sets word- and char-spacing before
1870                    // showing the string (PDF 32000-1 §9.4.3), persisting after.
1871                    tw = op_f(operands, 0);
1872                    tc = op_f(operands, 1);
1873                }
1874                if let (Some(f), Some(Object::String(s, _))) = (font, operands.last()) {
1875                    show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out);
1876                }
1877            }
1878            "TJ" => {
1879                if let (Some(f), Some(Object::Array(arr))) = (font, operands.first()) {
1880                    for el in arr {
1881                        match el {
1882                            Object::String(s, _) => {
1883                                show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out)
1884                            }
1885                            other => {
1886                                if let Some(adj) = num(other) {
1887                                    // negative number moves text right (PDF: subtract)
1888                                    let tx = -adj / 1000.0 * fsize * th;
1889                                    tm = Mat {
1890                                        a: 1.0,
1891                                        b: 0.0,
1892                                        c: 0.0,
1893                                        d: 1.0,
1894                                        e: tx,
1895                                        f: 0.0,
1896                                    }
1897                                    .then(tm);
1898                                }
1899                            }
1900                        }
1901                    }
1902                }
1903            }
1904            "m" => {
1905                cur = ctm.apply(op_f(operands, 0), op_f(operands, 1));
1906                start = cur;
1907            }
1908            "l" => {
1909                let p = ctm.apply(op_f(operands, 0), op_f(operands, 1));
1910                path.push(PathPiece::Line(cur, p));
1911                cur = p;
1912            }
1913            "c" | "v" | "y" => {
1914                let pts: Vec<(f64, f64)> = (0..operands.len() / 2)
1915                    .map(|k| ctm.apply(op_f(operands, 2 * k), op_f(operands, 2 * k + 1)))
1916                    .collect();
1917                if let Some(&end) = pts.last() {
1918                    path.push(PathPiece::Curve(
1919                        std::iter::once(cur).chain(pts.iter().copied()).collect(),
1920                    ));
1921                    cur = end;
1922                }
1923            }
1924            "h" => {
1925                if cur != start {
1926                    path.push(PathPiece::Line(cur, start));
1927                }
1928                cur = start;
1929            }
1930            "re" => {
1931                let (x, y, w, h) = (
1932                    op_f(operands, 0),
1933                    op_f(operands, 1),
1934                    op_f(operands, 2),
1935                    op_f(operands, 3),
1936                );
1937                path.push(PathPiece::Rect([
1938                    ctm.apply(x, y),
1939                    ctm.apply(x + w, y),
1940                    ctm.apply(x + w, y + h),
1941                    ctm.apply(x, y + h),
1942                ]));
1943                cur = ctm.apply(x, y);
1944                start = cur;
1945            }
1946            "S" | "s" | "f" | "F" | "f*" | "B" | "B*" | "b" | "b*" => {
1947                let op = op.operator.as_str();
1948                if matches!(op, "s" | "b" | "b*") && cur != start {
1949                    path.push(PathPiece::Line(cur, start));
1950                }
1951                let stroke = matches!(op, "S" | "s" | "B" | "B*" | "b" | "b*");
1952                let fill = !matches!(op, "S" | "s");
1953                paint_path(&path, stroke, fill, fill_light, inks);
1954                path.clear();
1955            }
1956            "n" => path.clear(),
1957            // The fill colour, by its components: gray `g`, `rg`, CMYK `k`,
1958            // and `sc`/`scn` by operand count (a pattern name is not light).
1959            "g" | "rg" | "k" | "sc" | "scn" => {
1960                let v: Vec<f64> = operands
1961                    .iter()
1962                    .map(num)
1963                    .collect::<Option<_>>()
1964                    .unwrap_or_default();
1965                fill_light = match v.as_slice() {
1966                    [g] => *g >= 0.9,
1967                    [r, g, b] => r.min(*g).min(*b) >= 0.9,
1968                    [c, m, y, k] => c.max(*m).max(*y).max(*k) <= 0.1,
1969                    _ => false,
1970                };
1971            }
1972            "cs" => fill_light = false,
1973            "Do" => {
1974                // Invoke a Form XObject: bulk body text in many PDFs lives inside
1975                // a form, reached only here. Image XObjects are skipped (no text).
1976                if depth >= 8 {
1977                    continue;
1978                }
1979                let Some(Object::Name(n)) = operands.first() else {
1980                    continue;
1981                };
1982                let obj = xobjects.and_then(|d| d.get(n.as_slice()).ok());
1983                let form_id = match obj {
1984                    Some(Object::Reference(id)) => Some(*id),
1985                    _ => None,
1986                };
1987                let stream = obj
1988                    .and_then(|o| deref(doc, o))
1989                    .and_then(|o| o.as_stream().ok());
1990                let Some(stream) = stream else { continue };
1991                let is_form = stream
1992                    .dict
1993                    .get(b"Subtype")
1994                    .ok()
1995                    .and_then(|o| o.as_name().ok())
1996                    == Some(b"Form".as_slice());
1997                if !is_form {
1998                    continue;
1999                }
2000                // Decode the form's content once per document (headers/footers
2001                // and bulk body text invoke the same form on every page).
2002                let cached = form_id.and_then(|id| caches.forms.get(&id).cloned());
2003                let form_content = match cached {
2004                    Some(c) => c,
2005                    None => {
2006                        let Ok(data) = stream.decompressed_content() else {
2007                            continue;
2008                        };
2009                        let Ok(c) = lopdf::content::Content::decode(&data) else {
2010                            continue;
2011                        };
2012                        let c = Arc::new(c);
2013                        if let Some(id) = form_id {
2014                            caches.forms.insert(id, Arc::clone(&c));
2015                        }
2016                        c
2017                    }
2018                };
2019                // The form's /Matrix maps form space into the CTM at invocation.
2020                let form_mat = match stream.dict.get(b"Matrix").ok() {
2021                    Some(Object::Array(a)) if a.len() == 6 => {
2022                        let v: Vec<f64> = a.iter().filter_map(num).collect();
2023                        if v.len() == 6 {
2024                            Mat {
2025                                a: v[0],
2026                                b: v[1],
2027                                c: v[2],
2028                                d: v[3],
2029                                e: v[4],
2030                                f: v[5],
2031                            }
2032                        } else {
2033                            Mat::ID
2034                        }
2035                    }
2036                    _ => Mat::ID,
2037                };
2038                // The form's own /Resources, falling back to the inherited ones.
2039                let form_res = stream
2040                    .dict
2041                    .get(b"Resources")
2042                    .ok()
2043                    .and_then(|o| deref(doc, o))
2044                    .and_then(|o| o.as_dict().ok())
2045                    .unwrap_or(res);
2046                let state = TextState {
2047                    tc,
2048                    tw,
2049                    th,
2050                    tl,
2051                    trise,
2052                    fsize,
2053                };
2054                run_content(
2055                    doc,
2056                    form_res,
2057                    &form_content,
2058                    form_mat.then(ctm),
2059                    state,
2060                    depth + 1,
2061                    caches,
2062                    out,
2063                    inks,
2064                );
2065            }
2066            _ => {}
2067        }
2068    }
2069}
2070
2071/// One piece of a path under construction, page space (see `run_content`).
2072enum PathPiece {
2073    Line((f64, f64), (f64, f64)),
2074    /// An `re`: its four corners, counter-clockwise from `(x, y)`.
2075    Rect([(f64, f64); 4]),
2076    /// A Bézier segment: its start and control/end points (their hull).
2077    Curve(Vec<(f64, f64)>),
2078}
2079
2080/// Record a painted path's pieces for the checkbox finder (#609): a stroked
2081/// line is a [`Seg`](crate::checkbox::Ink::Seg), a stroked rectangle that
2082/// stays axis-aligned a [`Rect`](crate::checkbox::Ink::Rect) (one filled
2083/// white too — still an outline), and a coloured fill or a curve only its
2084/// bounding box, a [`Blot`](crate::checkbox::Ink::Blot): a possible tick
2085/// mark, or a swatch. White fills draw nothing and are dropped. Only pieces small
2086/// enough to be a checkbox edge or a mark inside one are kept.
2087fn paint_path(
2088    path: &[PathPiece],
2089    stroke: bool,
2090    fill: bool,
2091    fill_light: bool,
2092    inks: &mut Vec<crate::checkbox::Ink>,
2093) {
2094    use crate::checkbox::Ink;
2095    let bbox = |pts: &mut dyn Iterator<Item = (f64, f64)>| {
2096        pts.fold(
2097            (
2098                f64::INFINITY,
2099                f64::INFINITY,
2100                f64::NEG_INFINITY,
2101                f64::NEG_INFINITY,
2102            ),
2103            |(l, b, r, t), (x, y)| (l.min(x), b.min(y), r.max(x), t.max(y)),
2104        )
2105    };
2106    let mut keep = |ink: Ink| {
2107        if ink.worth_keeping() {
2108            inks.push(ink);
2109        }
2110    };
2111    if fill && !fill_light {
2112        // A coloured fill is a blot — a tick mark, or (the size of a square)
2113        // a legend swatch, which `checkbox::find` then rules out. A white
2114        // fill is no ink on paper: it only leaves the stroke, if any, below.
2115        let (l, b, r, t) = bbox(&mut path.iter().flat_map(|p| match p {
2116            PathPiece::Line(a, z) => vec![*a, *z],
2117            PathPiece::Rect(c) => c.to_vec(),
2118            PathPiece::Curve(c) => c.clone(),
2119        }));
2120        if l.is_finite() {
2121            keep(Ink::Blot { l, b, r, t });
2122        }
2123        return;
2124    }
2125    if !stroke {
2126        return;
2127    }
2128    for piece in path {
2129        match piece {
2130            PathPiece::Line(a, z) => keep(Ink::Seg {
2131                x0: a.0,
2132                y0: a.1,
2133                x1: z.0,
2134                y1: z.1,
2135            }),
2136            PathPiece::Rect(c) => {
2137                let axis = ((c[0].1 - c[1].1).abs() < 1e-6 && (c[1].0 - c[2].0).abs() < 1e-6)
2138                    || ((c[0].0 - c[1].0).abs() < 1e-6 && (c[1].1 - c[2].1).abs() < 1e-6);
2139                if axis {
2140                    let (l, b, r, t) = bbox(&mut c.iter().copied());
2141                    keep(Ink::Rect { l, b, r, t });
2142                } else {
2143                    for k in 0..4 {
2144                        let (a, z) = (c[k], c[(k + 1) % 4]);
2145                        keep(Ink::Seg {
2146                            x0: a.0,
2147                            y0: a.1,
2148                            x1: z.0,
2149                            y1: z.1,
2150                        });
2151                    }
2152                }
2153            }
2154            PathPiece::Curve(c) => {
2155                let (l, b, r, t) = bbox(&mut c.iter().copied());
2156                keep(Ink::Blot { l, b, r, t });
2157            }
2158        }
2159    }
2160}
2161
2162#[allow(clippy::too_many_arguments)]
2163fn show_text(
2164    font: &Font,
2165    bytes: &[u8],
2166    fsize: f64,
2167    tc: f64,
2168    tw: f64,
2169    th: f64,
2170    trise: f64,
2171    tm: &mut Mat,
2172    ctm: Mat,
2173    out: &mut Vec<Glyph>,
2174) {
2175    for code in codes(font, bytes) {
2176        let (text, w) = font.decode_code(code);
2177        let w0 = w / 1000.0; // advance in text-space (em) units
2178                             // The glyph→user transform: scale glyph space by font size, then Tm, CTM.
2179        let scale = Mat {
2180            a: fsize * th,
2181            b: 0.0,
2182            c: 0.0,
2183            d: fsize,
2184            e: 0.0,
2185            f: trise,
2186        };
2187        let trm = scale.then(*tm).then(ctm);
2188        // Box in glyph space (1000-unit em): x 0..w, y descent..ascent.
2189        let (desc, asc) = (font.descent / 1000.0, font.ascent / 1000.0);
2190        let (x0, y0) = trm.apply(0.0, desc);
2191        let (x1, y1) = trm.apply(w0, desc);
2192        let (x2, y2) = trm.apply(w0, asc);
2193        let (x3, y3) = trm.apply(0.0, asc);
2194        // Upright = the baseline runs left-to-right along +x. Only then is the
2195        // loose box the rectangle spanned by the baseline's x and the em's y
2196        // (a synthetic oblique's skew is ignored, as it always was). A rotated
2197        // matrix — the `0 s -s 0 tx ty Tm` landscape pages are built with
2198        // (#528) — collapsed that rectangle to zero width, and every such run
2199        // vanished; those glyphs keep their real quad (docling-parse's char
2200        // rect) and its extent instead.
2201        let upright = trm.a > 0.0 && trm.b.abs() <= 1e-6 * trm.a;
2202        let (left, bot, right, top, quad) = if upright {
2203            (x0.min(x1), y0.min(y3), x0.max(x1), y0.max(y3), None)
2204        } else {
2205            let (xs, ys) = ([x0, x1, x2, x3], [y0, y1, y2, y3]);
2206            let fold = |v: [f64; 4], f: fn(f64, f64) -> f64| v.into_iter().reduce(f).unwrap();
2207            let q = [x0, y0, x1, y1, x2, y2, x3, y3].map(|v| v as f32);
2208            (
2209                fold(xs, f64::min),
2210                fold(ys, f64::min),
2211                fold(xs, f64::max),
2212                fold(ys, f64::max),
2213                Some(q),
2214            )
2215        };
2216        // The pen's origin (rise included) and the font size's extent from
2217        // it, as docling-parse maps them (`map_text_point(0, rise)` and
2218        // `(0, rise + font_size)`).
2219        let (ox, oy) = trm.apply(0.0, 0.0);
2220        let (sx, sy) = trm.apply(0.0, 1.0);
2221        let pen = Some([ox as f32, oy as f32, (sx - ox).hypot(sy - oy) as f32]);
2222        if let Some(s) = text {
2223            // A run may map one code to multiple chars (ligature/fraction); share box.
2224            for ch in s.chars() {
2225                if ch != '\u{0}' {
2226                    out.push(Glyph {
2227                        ch,
2228                        l: left as f32,
2229                        b: bot as f32,
2230                        r: right as f32,
2231                        t: top as f32,
2232                        ll: left as f32,
2233                        lb: bot as f32,
2234                        lr: right as f32,
2235                        lt: top as f32,
2236                        font: font.hash,
2237                        quad,
2238                        pen,
2239                        math_font: font.math,
2240                    });
2241                }
2242            }
2243        }
2244        // Advance the text matrix. Word spacing applies to single-byte code 32.
2245        let is_space = !font.two_byte && code == 32;
2246        let tx = (w0 * fsize + tc + if is_space { tw } else { 0.0 }) * th;
2247        *tm = Mat {
2248            a: 1.0,
2249            b: 0.0,
2250            c: 0.0,
2251            d: 1.0,
2252            e: tx,
2253            f: 0.0,
2254        }
2255        .then(*tm);
2256    }
2257}
2258
2259/// Build a simple font's code→char table from its `/Encoding`: the base
2260/// encoding (WinAnsi / MacRoman) plus any `/Differences` overrides (glyph names
2261/// resolved through a small Adobe-glyph-name subset).
2262fn simple_encoding_table(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
2263    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o));
2264    let base_name = match enc {
2265        Some(Object::Name(n)) => n.clone(),
2266        Some(Object::Dictionary(d)) => d
2267            .get(b"BaseEncoding")
2268            .ok()
2269            .and_then(|o| o.as_name().ok())
2270            .map(|n| n.to_vec())
2271            .unwrap_or_default(),
2272        _ => Vec::new(),
2273    };
2274    let mut m = if base_name == b"MacRomanEncoding" {
2275        macroman_table()
2276    } else if base_name.is_empty() {
2277        // No PDF /Encoding at all: the font's *built-in* encoding applies. For
2278        // the standard TeX math fonts that is their fixed TeX layout — falling
2279        // back to StandardEncoding read CMSY's braces as `f`/`g`, `→` as `!`,
2280        // `∈` as `2` (2203's `{ahn,…}` author line). The font program (often
2281        // CFF, which this parser does not read) carries the same mapping;
2282        // docling-parse decodes it from there.
2283        tex_math_builtin(fdict).unwrap_or_else(winansi_table)
2284    } else {
2285        winansi_table()
2286    };
2287    // Apply /Differences: `code /glyphname /glyphname ... code ...`.
2288    if let Some(Object::Dictionary(d)) = enc {
2289        if let Some(Object::Array(diffs)) = d.get(b"Differences").ok().and_then(|o| deref(doc, o)) {
2290            let mut code = 0u8;
2291            for el in diffs {
2292                match el {
2293                    Object::Integer(i) => code = *i as u8,
2294                    Object::Name(name) => {
2295                        if let Some(ch) = glyph_name_to_char(name) {
2296                            m.insert(code, ch);
2297                        }
2298                        code = code.wrapping_add(1);
2299                    }
2300                    _ => {}
2301                }
2302            }
2303        }
2304    }
2305    m
2306}
2307
2308/// The fixed built-in encodings of the standard TeX math fonts (TeXbook
2309/// Appendix F), keyed off the base font name: `CMSY*` (symbols; `CMBSY` is its
2310/// bold) and `CMMI*` (math italic). These fonts ship no PDF `/Encoding` and no
2311/// ToUnicode, and their program is usually CFF — without this table the codes
2312/// fell through to StandardEncoding and rendered as the wrong ASCII.
2313fn tex_math_builtin(fdict: &Dictionary) -> Option<HashMap<u8, char>> {
2314    const CMSY: [char; 128] = [
2315        '−', '·', '×', '∗', '÷', '⋄', '±', '∓', '⊕', '⊖', '⊗', '⊘', '⊙', '◯', '∘', '•', '≍', '≡',
2316        '⊆', '⊇', '≤', '≥', '≼', '≽', '∼', '≈', '⊂', '⊃', '≪', '≫', '≺', '≻', '←', '→', '↑', '↓',
2317        '↔', '↗', '↘', '≃', '⇐', '⇒', '⇑', '⇓', '⇔', '↖', '↙', '∝', '′', '∞', '∈', '∋', '△', '▽',
2318        '\u{338}', '↦', '∀', '∃', '¬', '∅', 'ℜ', 'ℑ', '⊤', '⊥', 'ℵ', 'A', 'B', 'C', 'D', 'E', 'F',
2319        'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
2320        'Y', 'Z', '∪', '∩', '⊎', '∧', '∨', '⊢', '⊣', '⌊', '⌋', '⌈', '⌉', '{', '}', '⟨', '⟩', '|',
2321        '∥', '↕', '⇕', '\\', '≀', '√', '∐', '∇', '∫', '⊔', '⊓', '⊑', '⊒', '§', '†', '‡', '¶', '♣',
2322        '♢', '♡', '♠',
2323    ];
2324    const CMMI: [char; 128] = [
2325        'Γ', 'Δ', 'Θ', 'Λ', 'Ξ', 'Π', 'Σ', 'Υ', 'Φ', 'Ψ', 'Ω', 'α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η',
2326        'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'π', 'ρ', 'σ', 'τ', 'υ', 'φ', 'χ', 'ψ', 'ω', 'ϵ', 'ϑ',
2327        'ϖ', 'ϱ', 'ς', 'ϕ', '↼', '↽', '⇀', '⇁', '↩', '↪', '▷', '◁', '0', '1', '2', '3', '4', '5',
2328        '6', '7', '8', '9', '.', ',', '<', '/', '>', '⋆', '∂', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
2329        'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y',
2330        'Z', '♭', '♮', '♯', '⌣', '⌢', 'ℓ', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
2331        'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'ı', 'ȷ', '℘',
2332        '\u{20d7}', '⁀',
2333    ];
2334    let name = base_font_name(fdict)?;
2335    let up = name.to_ascii_uppercase();
2336    let table: &[char; 128] = if up.starts_with(b"CMSY") || up.starts_with(b"CMBSY") {
2337        &CMSY
2338    } else if up.starts_with(b"CMMI") {
2339        &CMMI
2340    } else {
2341        return None;
2342    };
2343    Some(
2344        table
2345            .iter()
2346            .enumerate()
2347            .map(|(i, &c)| (i as u8, c))
2348            .collect(),
2349    )
2350}
2351
2352/// The Adobe Glyph List's names for Latin-1 Supplement and Latin Extended-A
2353/// (plus `Euro`, `florin`, the comma-accent pairs): what a Type1 font's
2354/// `/Differences` or builtin encoding calls an accented or stroked letter.
2355/// 2203.01017v2 names `Ł` as `/Lslash` in a NimbusRomNo9L subset with no
2356/// ToUnicode, and the resolver, which knew only the ASCII, ligature and symbol
2357/// names, dropped the glyph outright (`Ł ukasz Kaiser` read `ukasz Kaiser`).
2358/// Sorted by name for the binary search in [`glyph_name_to_char`].
2359const AGL_LATIN: &[(&str, char)] = &[
2360    ("AE", '\u{00C6}'),
2361    ("Aacute", '\u{00C1}'),
2362    ("Abreve", '\u{0102}'),
2363    ("Acircumflex", '\u{00C2}'),
2364    ("Adieresis", '\u{00C4}'),
2365    ("Agrave", '\u{00C0}'),
2366    ("Amacron", '\u{0100}'),
2367    ("Aogonek", '\u{0104}'),
2368    ("Aring", '\u{00C5}'),
2369    ("Atilde", '\u{00C3}'),
2370    ("Cacute", '\u{0106}'),
2371    ("Ccaron", '\u{010C}'),
2372    ("Ccedilla", '\u{00C7}'),
2373    ("Ccircumflex", '\u{0108}'),
2374    ("Cdotaccent", '\u{010A}'),
2375    ("Dcaron", '\u{010E}'),
2376    ("Dcroat", '\u{0110}'),
2377    ("Eacute", '\u{00C9}'),
2378    ("Ebreve", '\u{0114}'),
2379    ("Ecaron", '\u{011A}'),
2380    ("Ecircumflex", '\u{00CA}'),
2381    ("Edieresis", '\u{00CB}'),
2382    ("Edotaccent", '\u{0116}'),
2383    ("Egrave", '\u{00C8}'),
2384    ("Emacron", '\u{0112}'),
2385    ("Eng", '\u{014A}'),
2386    ("Eogonek", '\u{0118}'),
2387    ("Eth", '\u{00D0}'),
2388    ("Euro", '\u{20AC}'),
2389    ("Gbreve", '\u{011E}'),
2390    ("Gcircumflex", '\u{011C}'),
2391    ("Gcommaaccent", '\u{0122}'),
2392    ("Gdotaccent", '\u{0120}'),
2393    ("Hbar", '\u{0126}'),
2394    ("Hcircumflex", '\u{0124}'),
2395    ("IJ", '\u{0132}'),
2396    ("Iacute", '\u{00CD}'),
2397    ("Ibreve", '\u{012C}'),
2398    ("Icircumflex", '\u{00CE}'),
2399    ("Idieresis", '\u{00CF}'),
2400    ("Idotaccent", '\u{0130}'),
2401    ("Igrave", '\u{00CC}'),
2402    ("Imacron", '\u{012A}'),
2403    ("Iogonek", '\u{012E}'),
2404    ("Itilde", '\u{0128}'),
2405    ("Jcircumflex", '\u{0134}'),
2406    ("Kcommaaccent", '\u{0136}'),
2407    ("Lacute", '\u{0139}'),
2408    ("Lcaron", '\u{013D}'),
2409    ("Lcommaaccent", '\u{013B}'),
2410    ("Ldot", '\u{013F}'),
2411    ("Lslash", '\u{0141}'),
2412    ("Nacute", '\u{0143}'),
2413    ("Ncaron", '\u{0147}'),
2414    ("Ncommaaccent", '\u{0145}'),
2415    ("Ntilde", '\u{00D1}'),
2416    ("OE", '\u{0152}'),
2417    ("Oacute", '\u{00D3}'),
2418    ("Obreve", '\u{014E}'),
2419    ("Ocircumflex", '\u{00D4}'),
2420    ("Odieresis", '\u{00D6}'),
2421    ("Ograve", '\u{00D2}'),
2422    ("Ohungarumlaut", '\u{0150}'),
2423    ("Omacron", '\u{014C}'),
2424    ("Oslash", '\u{00D8}'),
2425    ("Otilde", '\u{00D5}'),
2426    ("Racute", '\u{0154}'),
2427    ("Rcaron", '\u{0158}'),
2428    ("Rcommaaccent", '\u{0156}'),
2429    ("Sacute", '\u{015A}'),
2430    ("Scaron", '\u{0160}'),
2431    ("Scedilla", '\u{015E}'),
2432    ("Scircumflex", '\u{015C}'),
2433    ("Scommaaccent", '\u{0218}'),
2434    ("Tbar", '\u{0166}'),
2435    ("Tcaron", '\u{0164}'),
2436    ("Tcommaaccent", '\u{0162}'),
2437    ("Thorn", '\u{00DE}'),
2438    ("Uacute", '\u{00DA}'),
2439    ("Ubreve", '\u{016C}'),
2440    ("Ucircumflex", '\u{00DB}'),
2441    ("Udieresis", '\u{00DC}'),
2442    ("Ugrave", '\u{00D9}'),
2443    ("Uhungarumlaut", '\u{0170}'),
2444    ("Umacron", '\u{016A}'),
2445    ("Uogonek", '\u{0172}'),
2446    ("Uring", '\u{016E}'),
2447    ("Utilde", '\u{0168}'),
2448    ("Wcircumflex", '\u{0174}'),
2449    ("Yacute", '\u{00DD}'),
2450    ("Ycircumflex", '\u{0176}'),
2451    ("Ydieresis", '\u{0178}'),
2452    ("Zacute", '\u{0179}'),
2453    ("Zcaron", '\u{017D}'),
2454    ("Zdotaccent", '\u{017B}'),
2455    ("aacute", '\u{00E1}'),
2456    ("abreve", '\u{0103}'),
2457    ("acircumflex", '\u{00E2}'),
2458    ("acute", '\u{00B4}'),
2459    ("adieresis", '\u{00E4}'),
2460    ("ae", '\u{00E6}'),
2461    ("agrave", '\u{00E0}'),
2462    ("amacron", '\u{0101}'),
2463    ("aogonek", '\u{0105}'),
2464    ("aring", '\u{00E5}'),
2465    ("atilde", '\u{00E3}'),
2466    ("brokenbar", '\u{00A6}'),
2467    ("cacute", '\u{0107}'),
2468    ("ccaron", '\u{010D}'),
2469    ("ccedilla", '\u{00E7}'),
2470    ("ccircumflex", '\u{0109}'),
2471    ("cdotaccent", '\u{010B}'),
2472    ("cedilla", '\u{00B8}'),
2473    ("cent", '\u{00A2}'),
2474    ("currency", '\u{00A4}'),
2475    ("dcaron", '\u{010F}'),
2476    ("dcroat", '\u{0111}'),
2477    ("dieresis", '\u{00A8}'),
2478    ("dotlessi", '\u{0131}'),
2479    ("eacute", '\u{00E9}'),
2480    ("ebreve", '\u{0115}'),
2481    ("ecaron", '\u{011B}'),
2482    ("ecircumflex", '\u{00EA}'),
2483    ("edieresis", '\u{00EB}'),
2484    ("edotaccent", '\u{0117}'),
2485    ("egrave", '\u{00E8}'),
2486    ("emacron", '\u{0113}'),
2487    ("eng", '\u{014B}'),
2488    ("eogonek", '\u{0119}'),
2489    ("eth", '\u{00F0}'),
2490    ("exclamdown", '\u{00A1}'),
2491    ("florin", '\u{0192}'),
2492    ("gbreve", '\u{011F}'),
2493    ("gcircumflex", '\u{011D}'),
2494    ("gcommaaccent", '\u{0123}'),
2495    ("gdotaccent", '\u{0121}'),
2496    ("germandbls", '\u{00DF}'),
2497    ("guillemotleft", '\u{00AB}'),
2498    ("guillemotright", '\u{00BB}'),
2499    ("hbar", '\u{0127}'),
2500    ("hcircumflex", '\u{0125}'),
2501    ("iacute", '\u{00ED}'),
2502    ("ibreve", '\u{012D}'),
2503    ("icircumflex", '\u{00EE}'),
2504    ("idieresis", '\u{00EF}'),
2505    ("igrave", '\u{00EC}'),
2506    ("ij", '\u{0133}'),
2507    ("imacron", '\u{012B}'),
2508    ("iogonek", '\u{012F}'),
2509    ("itilde", '\u{0129}'),
2510    ("jcircumflex", '\u{0135}'),
2511    ("kcommaaccent", '\u{0137}'),
2512    ("kgreenlandic", '\u{0138}'),
2513    ("lacute", '\u{013A}'),
2514    ("lcaron", '\u{013E}'),
2515    ("lcommaaccent", '\u{013C}'),
2516    ("ldot", '\u{0140}'),
2517    ("logicalnot", '\u{00AC}'),
2518    ("longs", '\u{017F}'),
2519    ("lslash", '\u{0142}'),
2520    ("macron", '\u{00AF}'),
2521    ("nacute", '\u{0144}'),
2522    ("napostrophe", '\u{0149}'),
2523    ("ncaron", '\u{0148}'),
2524    ("ncommaaccent", '\u{0146}'),
2525    ("ntilde", '\u{00F1}'),
2526    ("oacute", '\u{00F3}'),
2527    ("obreve", '\u{014F}'),
2528    ("ocircumflex", '\u{00F4}'),
2529    ("odieresis", '\u{00F6}'),
2530    ("oe", '\u{0153}'),
2531    ("ograve", '\u{00F2}'),
2532    ("ohungarumlaut", '\u{0151}'),
2533    ("omacron", '\u{014D}'),
2534    ("onehalf", '\u{00BD}'),
2535    ("onequarter", '\u{00BC}'),
2536    ("onesuperior", '\u{00B9}'),
2537    ("ordfeminine", '\u{00AA}'),
2538    ("ordmasculine", '\u{00BA}'),
2539    ("oslash", '\u{00F8}'),
2540    ("otilde", '\u{00F5}'),
2541    ("paragraph", '\u{00B6}'),
2542    ("questiondown", '\u{00BF}'),
2543    ("racute", '\u{0155}'),
2544    ("rcaron", '\u{0159}'),
2545    ("rcommaaccent", '\u{0157}'),
2546    ("sacute", '\u{015B}'),
2547    ("scaron", '\u{0161}'),
2548    ("scedilla", '\u{015F}'),
2549    ("scircumflex", '\u{015D}'),
2550    ("scommaaccent", '\u{0219}'),
2551    ("section", '\u{00A7}'),
2552    ("sfthyphen", '\u{00AD}'),
2553    ("sterling", '\u{00A3}'),
2554    ("tbar", '\u{0167}'),
2555    ("tcaron", '\u{0165}'),
2556    ("tcommaaccent", '\u{0163}'),
2557    ("thorn", '\u{00FE}'),
2558    ("threequarters", '\u{00BE}'),
2559    ("threesuperior", '\u{00B3}'),
2560    ("twosuperior", '\u{00B2}'),
2561    ("uacute", '\u{00FA}'),
2562    ("ubreve", '\u{016D}'),
2563    ("ucircumflex", '\u{00FB}'),
2564    ("udieresis", '\u{00FC}'),
2565    ("ugrave", '\u{00F9}'),
2566    ("uhungarumlaut", '\u{0171}'),
2567    ("umacron", '\u{016B}'),
2568    ("uogonek", '\u{0173}'),
2569    ("uring", '\u{016F}'),
2570    ("utilde", '\u{0169}'),
2571    ("wcircumflex", '\u{0175}'),
2572    ("yacute", '\u{00FD}'),
2573    ("ycircumflex", '\u{0177}'),
2574    ("ydieresis", '\u{00FF}'),
2575    ("yen", '\u{00A5}'),
2576    ("zacute", '\u{017A}'),
2577    ("zcaron", '\u{017E}'),
2578    ("zdotaccent", '\u{017C}'),
2579];
2580
2581/// Resolve an Adobe glyph name to Unicode: `uniXXXX`, single ASCII letters, the
2582/// digit/punctuation names from the Adobe Glyph List, and common typographic
2583/// names. A `.suffix` (`one.taboldstyle`, `a.sc`) is stripped and the base name
2584/// retried — docling renders these as the base character.
2585pub(crate) fn glyph_name_to_char(name: &[u8]) -> Option<char> {
2586    let s = std::str::from_utf8(name).ok()?;
2587    if let Some(hex) = s.strip_prefix("uni") {
2588        if let Ok(cp) = u32::from_str_radix(hex.get(0..4)?, 16) {
2589            return char::from_u32(cp);
2590        }
2591    }
2592    // Single ASCII letter names (`A`, `m`) map to themselves.
2593    if s.len() == 1 {
2594        let b = s.as_bytes()[0];
2595        if b.is_ascii_alphabetic() {
2596            return Some(b as char);
2597        }
2598    }
2599    let resolved = match s {
2600        "space" => ' ',
2601        "exclam" => '!',
2602        "quotedbl" => '"',
2603        "numbersign" => '#',
2604        "dollar" => '$',
2605        "percent" => '%',
2606        "ampersand" => '&',
2607        "quotesingle" => '\'',
2608        "parenleft" => '(',
2609        "parenright" => ')',
2610        "asterisk" => '*',
2611        "plus" => '+',
2612        "comma" => ',',
2613        "hyphen" => '-',
2614        "period" => '.',
2615        "slash" => '/',
2616        "zero" => '0',
2617        "one" => '1',
2618        "two" => '2',
2619        "three" => '3',
2620        "four" => '4',
2621        "five" => '5',
2622        "six" => '6',
2623        "seven" => '7',
2624        "eight" => '8',
2625        "nine" => '9',
2626        "colon" => ':',
2627        "semicolon" => ';',
2628        "less" => '<',
2629        "equal" => '=',
2630        "greater" => '>',
2631        "question" => '?',
2632        "at" => '@',
2633        "bracketleft" => '[',
2634        "backslash" => '\\',
2635        "bracketright" => ']',
2636        "asciicircum" => '^',
2637        "underscore" => '_',
2638        "grave" => '`',
2639        "braceleft" => '{',
2640        "bar" => '|',
2641        "braceright" => '}',
2642        "asciitilde" => '~',
2643        "bullet" => '\u{2022}',
2644        "periodcentered" => '\u{00B7}',
2645        "endash" => '\u{2013}',
2646        "emdash" => '\u{2014}',
2647        "quoteright" => '\u{2019}',
2648        "quoteleft" => '\u{2018}',
2649        "quotedblleft" => '\u{201C}',
2650        "quotedblright" => '\u{201D}',
2651        "quotedblbase" => '\u{201E}',
2652        "quotesinglbase" => '\u{201A}',
2653        // Latin f-ligatures named in `/Differences` (e.g. 2305's body font). These
2654        // map to the presentation-form code points, which `decompose_ligatures`
2655        // then spells back out (`ff`→"ff") — without them the glyph decodes to
2656        // nothing and the sanitizer fills the gap with a space (`di erences`).
2657        "ff" => '\u{FB00}',
2658        "fi" => '\u{FB01}',
2659        "fl" => '\u{FB02}',
2660        "ffi" => '\u{FB03}',
2661        "ffl" => '\u{FB04}',
2662        "ft" => '\u{FB05}',
2663        "st" => '\u{FB06}',
2664        "degree" => '\u{00B0}',
2665        "trademark" => '\u{2122}',
2666        "registered" => '\u{00AE}',
2667        "copyright" => '\u{00A9}',
2668        "ellipsis" => '\u{2026}',
2669        "minus" => '\u{2212}',
2670        "fraction" => '\u{2044}',
2671        "nbspace" => '\u{00A0}',
2672        // Greek + math glyph names (standard Adobe Glyph List). Standard TeX math
2673        // fonts (CMMI/CMSY/…) name their glyphs this way in the embedded font
2674        // program's `/Encoding`; without these a `λ`/`≤` decodes to nothing and is
2675        // dropped from body text (`and λ set to 0.5` → `and set to 0.5`).
2676        "alpha" => '\u{03B1}',
2677        "beta" => '\u{03B2}',
2678        "gamma" => '\u{03B3}',
2679        "delta" => '\u{03B4}',
2680        "epsilon" | "epsilon1" => '\u{03B5}',
2681        "zeta" => '\u{03B6}',
2682        "eta" => '\u{03B7}',
2683        "theta" | "theta1" => '\u{03B8}',
2684        "iota" => '\u{03B9}',
2685        "kappa" => '\u{03BA}',
2686        "lambda" => '\u{03BB}',
2687        "mu" => '\u{03BC}',
2688        "nu" => '\u{03BD}',
2689        "xi" => '\u{03BE}',
2690        "omicron" => '\u{03BF}',
2691        "pi" | "pi1" => '\u{03C0}',
2692        "rho" | "rho1" => '\u{03C1}',
2693        "sigma" => '\u{03C3}',
2694        "sigma1" => '\u{03C2}',
2695        "tau" => '\u{03C4}',
2696        "upsilon" => '\u{03C5}',
2697        "phi" | "phi1" => '\u{03C6}',
2698        "chi" => '\u{03C7}',
2699        "psi" => '\u{03C8}',
2700        "omega" | "omega1" => '\u{03C9}',
2701        "Gamma" => '\u{0393}',
2702        "Delta" => '\u{0394}',
2703        "Theta" => '\u{0398}',
2704        "Lambda" => '\u{039B}',
2705        "Xi" => '\u{039E}',
2706        "Pi" => '\u{03A0}',
2707        "Sigma" => '\u{03A3}',
2708        "Upsilon" => '\u{03A5}',
2709        "Phi" => '\u{03A6}',
2710        "Psi" => '\u{03A8}',
2711        "Omega" => '\u{03A9}',
2712        "lessequal" => '\u{2264}',
2713        "greaterequal" => '\u{2265}',
2714        "notequal" => '\u{2260}',
2715        "approxequal" => '\u{2248}',
2716        "equivalence" => '\u{2261}',
2717        "element" => '\u{2208}',
2718        "plusminus" => '\u{00B1}',
2719        "multiply" => '\u{00D7}',
2720        "divide" => '\u{00F7}',
2721        "infinity" => '\u{221E}',
2722        "partialdiff" => '\u{2202}',
2723        "gradient" => '\u{2207}',
2724        "summation" => '\u{2211}',
2725        "product" => '\u{220F}',
2726        "integral" => '\u{222B}',
2727        "radical" => '\u{221A}',
2728        "proportional" => '\u{221D}',
2729        "arrowright" => '\u{2192}',
2730        "arrowleft" => '\u{2190}',
2731        "arrowup" => '\u{2191}',
2732        "arrowdown" => '\u{2193}',
2733        "arrowboth" => '\u{2194}',
2734        "arrowdblright" => '\u{21D2}',
2735        "logicaland" => '\u{2227}',
2736        "logicalor" => '\u{2228}',
2737        "intersection" => '\u{2229}',
2738        "union" => '\u{222A}',
2739        "similar" => '\u{223C}',
2740        "congruent" => '\u{2245}',
2741        "dotmath" => '\u{22C5}',
2742        "asteriskmath" => '\u{2217}',
2743        _ => {
2744            if let Ok(i) = AGL_LATIN.binary_search_by_key(&s, |(n, _)| n) {
2745                return Some(AGL_LATIN[i].1);
2746            }
2747            // Strip an AGL `.suffix` (oldstyle/small-cap variant) and retry.
2748            if let Some((base, _)) = s.split_once('.') {
2749                if !base.is_empty() {
2750                    return glyph_name_to_char(base.as_bytes());
2751                }
2752            }
2753            // An AGL ligature name joins its components with `_` (`f_f_i`,
2754            // `f_l` — what a re-encoded Libertine subset calls them); joined,
2755            // the presentation-form arms above resolve them.
2756            if s.contains('_') {
2757                let joined: String = s.chars().filter(|&c| c != '_').collect();
2758                if !joined.is_empty() && joined.len() < s.len() {
2759                    return glyph_name_to_char(joined.as_bytes());
2760                }
2761            }
2762            return None;
2763        }
2764    };
2765    Some(resolved)
2766}
2767
2768/// Minimal WinAnsiEncoding (Latin-1-ish) for simple fonts lacking ToUnicode.
2769fn winansi_table() -> HashMap<u8, char> {
2770    let mut m = HashMap::new();
2771    for b in 0x20u8..=0x7e {
2772        m.insert(b, b as char);
2773    }
2774    // High range: Windows-1252 printable points that differ from Latin-1.
2775    let extra: &[(u8, char)] = &[
2776        (0x91, '\u{2018}'),
2777        (0x92, '\u{2019}'),
2778        (0x93, '\u{201C}'),
2779        (0x94, '\u{201D}'),
2780        (0x95, '\u{2022}'),
2781        (0x96, '\u{2013}'),
2782        (0x97, '\u{2014}'),
2783        (0x85, '\u{2026}'),
2784        (0xA0, '\u{00A0}'),
2785        // The rest of Windows-1252's 0x80–0x9F row. Left out, the German
2786        // opening quote `„` (0x84) vanished from the text layer while its
2787        // closing `“` (0x93) stayed (#922); `€ ‚ ƒ † ‡ ˆ ‰ Š ‹ Œ Ž ˜ ™ š › œ ž Ÿ`
2788        // were lost the same way. 0x81/0x8D/0x8F/0x90/0x9D stay unmapped
2789        // (PDF's WinAnsi leaves them undefined).
2790        (0x80, '\u{20AC}'),
2791        (0x82, '\u{201A}'),
2792        (0x83, '\u{0192}'),
2793        (0x84, '\u{201E}'),
2794        (0x86, '\u{2020}'),
2795        (0x87, '\u{2021}'),
2796        (0x88, '\u{02C6}'),
2797        (0x89, '\u{2030}'),
2798        (0x8A, '\u{0160}'),
2799        (0x8B, '\u{2039}'),
2800        (0x8C, '\u{0152}'),
2801        (0x8E, '\u{017D}'),
2802        (0x98, '\u{02DC}'),
2803        (0x99, '\u{2122}'),
2804        (0x9A, '\u{0161}'),
2805        (0x9B, '\u{203A}'),
2806        (0x9C, '\u{0153}'),
2807        (0x9E, '\u{017E}'),
2808        (0x9F, '\u{0178}'),
2809    ];
2810    for &(b, c) in extra {
2811        m.insert(b, c);
2812    }
2813    for b in 0xA1u8..=0xFF {
2814        m.entry(b).or_insert(b as char);
2815    }
2816    m
2817}
2818
2819/// MacRomanEncoding: ASCII plus the whole high range, 0x80–0xFF, as Mac OS
2820/// Roman maps it (#744) — the table pdfium, pdftotext and pdf.js decode with.
2821/// Two codes follow the PDF reference instead of Apple's later revisions:
2822/// 0xDB is `currency` (¤, not €), and 0xF0 (`apple`, a private-use glyph) is
2823/// left unmapped. 0xCA, the PDF table's second `space`, reads as a no-break
2824/// space.
2825fn macroman_table() -> HashMap<u8, char> {
2826    #[rustfmt::skip]
2827    const HIGH: [char; 128] = [
2828        'Ä', 'Å', 'Ç', 'É', 'Ñ', 'Ö', 'Ü', 'á',  // 0x80
2829        'à', 'â', 'ä', 'ã', 'å', 'ç', 'é', 'è',  // 0x88
2830        'ê', 'ë', 'í', 'ì', 'î', 'ï', 'ñ', 'ó',  // 0x90
2831        'ò', 'ô', 'ö', 'õ', 'ú', 'ù', 'û', 'ü',  // 0x98
2832        '†', '°', '¢', '£', '§', '•', '¶', 'ß',  // 0xA0
2833        '®', '©', '™', '´', '¨', '≠', 'Æ', 'Ø',  // 0xA8
2834        '∞', '±', '≤', '≥', '¥', 'µ', '∂', '∑',  // 0xB0
2835        '∏', 'π', '∫', 'ª', 'º', 'Ω', 'æ', 'ø',  // 0xB8
2836        '¿', '¡', '¬', '√', 'ƒ', '≈', '∆', '«',  // 0xC0
2837        '»', '…', '\u{00A0}', 'À', 'Ã', 'Õ', 'Œ', 'œ',  // 0xC8
2838        '–', '—', '“', '”', '‘', '’', '÷', '◊',  // 0xD0
2839        'ÿ', 'Ÿ', '⁄', '¤', '‹', '›', 'fi', 'fl',  // 0xD8
2840        '‡', '·', '‚', '„', '‰', 'Â', 'Ê', 'Á',  // 0xE0
2841        'Ë', 'È', 'Í', 'Î', 'Ï', 'Ì', 'Ó', 'Ô',  // 0xE8
2842        '\0', 'Ò', 'Ú', 'Û', 'Ù', 'ı', 'ˆ', '˜',  // 0xF0
2843        '¯', '˘', '˙', '˚', '¸', '˝', '˛', 'ˇ',  // 0xF8
2844    ];
2845    let mut m = HashMap::new();
2846    for b in 0x20u8..=0x7e {
2847        m.insert(b, b as char);
2848    }
2849    for (b, &c) in (0x80u8..=0xFF).zip(HIGH.iter()) {
2850        if c != '\0' {
2851            m.insert(b, c);
2852        }
2853    }
2854    m
2855}
2856
2857#[cfg(test)]
2858mod glyph_names {
2859    /// `/Lslash` and the other Latin-1 / Latin Extended-A glyph names a Type1
2860    /// `/Differences` uses resolve (2203.01017v2's `Łukasz`); the table the
2861    /// lookup bisects is sorted and the ligature / symbol arms still win.
2862    #[test]
2863    fn latin_glyph_names_resolve() {
2864        for w in super::AGL_LATIN.windows(2) {
2865            assert!(
2866                w[0].0 < w[1].0,
2867                "AGL_LATIN not sorted at {:?} / {:?}",
2868                w[0].0,
2869                w[1].0
2870            );
2871        }
2872        for (name, ch) in [
2873            ("Lslash", 'Ł'),
2874            ("lslash", 'ł'),
2875            ("Aacute", 'Á'),
2876            ("aacute", 'á'),
2877            ("Scaron", 'Š'),
2878            ("germandbls", 'ß'),
2879            ("oe", 'œ'),
2880            ("Ydieresis", 'Ÿ'),
2881            ("Euro", '€'),
2882            ("Lslash.sc", 'Ł'),
2883            ("fi", '\u{FB01}'),
2884            ("Omega", '\u{03A9}'),
2885        ] {
2886            assert_eq!(
2887                super::glyph_name_to_char(name.as_bytes()),
2888                Some(ch),
2889                "{name}"
2890            );
2891        }
2892        assert_eq!(super::glyph_name_to_char(b"g123"), None);
2893        // AGL ligature names: components joined with `_`.
2894        assert_eq!(super::glyph_name_to_char(b"f_f_i"), Some('\u{FB03}'));
2895        assert_eq!(super::glyph_name_to_char(b"f_l"), Some('\u{FB02}'));
2896        assert_eq!(super::glyph_name_to_char(b"a_b"), None);
2897    }
2898
2899    /// The stale-CMap case: ToUnicode maps a code to its own identity while
2900    /// `/Differences` names a ligature there — the name wins; a CMap that
2901    /// maps the code anywhere else, or a code with no entry, is untouched.
2902    #[test]
2903    fn a_stale_tounicode_yields_to_differences() {
2904        use lopdf::Object;
2905        use std::collections::HashMap;
2906        let mut tu: HashMap<u32, String> = HashMap::new();
2907        tu.insert(33, "!".into()); // identity — stale
2908        tu.insert(34, "x".into()); // the producer meant it
2909        let diffs = vec![
2910            Object::Integer(33),
2911            Object::Name(b"f_f_i".to_vec()),
2912            Object::Name(b"f_i".to_vec()),
2913            Object::Name(b"f_f".to_vec()),
2914            Object::Integer(65),
2915            Object::Name(b"A".to_vec()), // identity on both sides: nothing to do
2916        ];
2917        super::reconcile_stale_tounicode(&mut tu, &diffs);
2918        assert_eq!(tu.get(&33).map(String::as_str), Some("\u{FB03}"));
2919        assert_eq!(tu.get(&34).map(String::as_str), Some("x"));
2920        assert_eq!(tu.get(&35), None);
2921        assert_eq!(tu.get(&65), None);
2922    }
2923}
2924
2925#[cfg(test)]
2926mod page_box_frame {
2927    use super::*;
2928    use lopdf::dictionary;
2929
2930    /// One page, `boxes` spliced into the page dictionary verbatim, one text
2931    /// run at user-space `(x, y)`.
2932    fn pdf(boxes: &str, x: f32, y: f32) -> Vec<u8> {
2933        pdf_content(
2934            boxes,
2935            &format!("BT /F1 12 Tf {x} {y} Td (First printing) Tj ET\n"),
2936        )
2937    }
2938
2939    /// One page, `boxes` spliced into the page dictionary, `content` as its
2940    /// content stream (font `/F1` = Helvetica).
2941    fn pdf_content(boxes: &str, content: &str) -> Vec<u8> {
2942        let objs: Vec<String> = vec![
2943            "<</Type/Catalog/Pages 2 0 R>>".into(),
2944            format!("<</Type/Pages/Kids[3 0 R]/Count 1{boxes}>>"),
2945            "<</Type/Page/Parent 2 0 R/Contents 4 0 R/Resources<</Font<</F1 5 0 R>>>>>>".into(),
2946            format!("<</Length {}>>stream\n{content}endstream", content.len()),
2947            "<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".into(),
2948        ];
2949        let mut out = b"%PDF-1.4\n".to_vec();
2950        let mut offsets = Vec::new();
2951        for (i, body) in objs.iter().enumerate() {
2952            offsets.push(out.len());
2953            out.extend_from_slice(format!("{} 0 obj{body}endobj\n", i + 1).as_bytes());
2954        }
2955        let xref_at = out.len();
2956        out.extend_from_slice(
2957            format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes(),
2958        );
2959        for off in &offsets {
2960            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
2961        }
2962        out.extend_from_slice(
2963            format!(
2964                "trailer<</Size {}/Root 1 0 R>>\nstartxref\n{xref_at}\n%%EOF\n",
2965                objs.len() + 1
2966            )
2967            .as_bytes(),
2968        );
2969        out
2970    }
2971
2972    /// One page whose `/Contents` is an array of separate streams.
2973    fn pdf_content_streams(contents: &[&[u8]]) -> Vec<u8> {
2974        let mut doc = Document::with_version("1.5");
2975        let streams = contents
2976            .iter()
2977            .map(|content| {
2978                Object::Reference(
2979                    doc.add_object(lopdf::Stream::new(Dictionary::new(), content.to_vec())),
2980                )
2981            })
2982            .collect::<Vec<_>>();
2983        let font_id = doc.add_object(dictionary! {
2984            "Type" => "Font",
2985            "Subtype" => "Type1",
2986            "BaseFont" => "Helvetica",
2987        });
2988        let resources_id = doc.add_object(dictionary! {
2989            "Font" => dictionary! { "F1" => Object::Reference(font_id) },
2990        });
2991        let pages_id = doc.new_object_id();
2992        let page_id = doc.add_object(dictionary! {
2993            "Type" => "Page",
2994            "Parent" => Object::Reference(pages_id),
2995            "MediaBox" => vec![0.into(), 0.into(), 300.into(), 400.into()],
2996            "Contents" => streams,
2997            "Resources" => Object::Reference(resources_id),
2998        });
2999        doc.objects.insert(
3000            pages_id,
3001            Object::Dictionary(dictionary! {
3002                "Type" => "Pages",
3003                "Kids" => vec![Object::Reference(page_id)],
3004                "Count" => 1,
3005            }),
3006        );
3007        let catalog_id = doc.add_object(dictionary! {
3008            "Type" => "Catalog",
3009            "Pages" => Object::Reference(pages_id),
3010        });
3011        doc.trailer.set("Root", Object::Reference(catalog_id));
3012        let mut out = Vec::new();
3013        doc.save_to(&mut out).expect("save synthetic PDF");
3014        out
3015    }
3016
3017    fn only_page(bytes: &[u8]) -> (PageBox, Vec<Glyph>) {
3018        let doc = load_document(bytes).expect("loads");
3019        let pid = *doc.get_pages().values().next().expect("one page");
3020        (page_box(&doc, pid), page_glyphs(&doc, pid))
3021    }
3022
3023    /// The trimmed-book-page shape: MediaBox and CropBox both away from the
3024    /// origin (inherited from `/Pages`). The display box is the CropBox, and a
3025    /// glyph's coordinates count from its lower-left corner — the same numbers
3026    /// the same text gets on a page whose CropBox *is* `[0 0 w h]`.
3027    #[test]
3028    fn glyphs_count_from_the_cropbox_corner_like_pdfium() {
3029        let (pb, shifted) = only_page(&pdf(
3030            "/MediaBox[-56.505 -58.25 576.303 723.31]/CropBox[1.095 -0.65 518.703 665.71]",
3031            37.0 + 1.095,
3032            58.0 - 0.65,
3033        ));
3034        assert!((pb.l - 1.095).abs() < 1e-3 && (pb.b + 0.65).abs() < 1e-3);
3035        assert!(
3036            (pb.w - 517.608).abs() < 1e-3 && (pb.h - 666.36).abs() < 1e-3,
3037            "{pb:?}"
3038        );
3039        let (pb0, plain) = only_page(&pdf("/MediaBox[0 0 517.608 666.36]", 37.0, 58.0));
3040        assert!((pb0.w - pb.w).abs() < 1e-3 && (pb0.h - pb.h).abs() < 1e-3);
3041        assert_eq!(shifted.len(), plain.len());
3042        assert!(!plain.is_empty());
3043        for (a, b) in shifted.iter().zip(&plain) {
3044            assert!(
3045                (a.l - b.l).abs() < 1e-3 && (a.b - b.b).abs() < 1e-3,
3046                "{:?} vs {:?}",
3047                (a.l, a.b),
3048                (b.l, b.b)
3049            );
3050        }
3051        assert!((plain[0].l - 37.0).abs() < 1e-3, "{}", plain[0].l);
3052    }
3053
3054    fn text(glyphs: &[Glyph]) -> String {
3055        glyphs.iter().map(|g| g.ch).collect()
3056    }
3057
3058    /// #529: text drawn outside the display box — a print slug beside the
3059    /// CropBox, a tiled page's neighbour beyond the MediaBox — is dropped,
3060    /// glyph by glyph like docling-parse, instead of being clamped onto the
3061    /// page edge; a line crossing the edge keeps the glyphs that fit.
3062    #[test]
3063    fn glyphs_outside_the_display_box_are_dropped() {
3064        let (_, g) = only_page(&pdf_content(
3065            "/MediaBox[0 0 400 400]/CropBox[100 100 400 400]",
3066            "BT /F1 14 Tf 120 300 Td (Visible.) Tj ET\nBT /F1 8 Tf 5 40 Td (Slug) Tj ET\n",
3067        ));
3068        assert_eq!(text(&g), "Visible.");
3069        let (_, g) = only_page(&pdf_content(
3070            "/MediaBox[0 0 300 400]",
3071            "BT /F1 14 Tf 40 300 Td (On page) Tj ET\nBT /F1 14 Tf 400 250 Td (Beyond) Tj ET\n",
3072        ));
3073        assert_eq!(text(&g), "On page");
3074        // docling-parse: "Crossing the right edge" on a 300 pt page → "Crossing the rig".
3075        let (_, g) = only_page(&pdf_content(
3076            "/MediaBox[0 0 300 400]",
3077            "BT /F1 14 Tf 200 250 Td (Crossing the right edge) Tj ET\n",
3078        ));
3079        assert_eq!(text(&g), "Crossing the rig");
3080    }
3081
3082    #[test]
3083    fn leading_comment_content_stream_does_not_hide_later_text() {
3084        // The comment has its own EOL; lopdf inserts another while assembling
3085        // the streams, which is the boundary its 0.44 parser stopped at.
3086        let (_, glyphs) = only_page(&pdf_content_streams(&[
3087            b"% CANON_PFINF_TYPE2_TEXTOFF\n",
3088            b"BT /F1 14 Tf 40 300 Td (Second stream text) Tj ET",
3089        ]));
3090        assert_eq!(text(&glyphs), "Second stream text");
3091    }
3092
3093    /// The containment test is docling-parse's: the whole char box (advance
3094    /// × the font's ascent / descent) inside the display box, edges
3095    /// included — a Helvetica `W` at 50 pt (47.2 wide) ending exactly on the
3096    /// right edge stays, 0.01 pt further it goes; likewise on the left.
3097    #[test]
3098    fn a_glyph_on_the_edge_stays_one_over_it_goes() {
3099        let w_at = |x: f32, y: f32| {
3100            let (_, g) = only_page(&pdf_content(
3101                "/MediaBox[0 0 300 400]",
3102                &format!("BT /F1 50 Tf {x} {y} Td (W) Tj ET\n"),
3103            ));
3104            text(&g)
3105        };
3106        assert_eq!(w_at(252.8, 200.0), "W");
3107        assert_eq!(w_at(252.81, 200.0), "");
3108        assert_eq!(w_at(0.0, 200.0), "W");
3109        assert_eq!(w_at(-0.01, 200.0), "");
3110        // Vertically the box is the font's ascent / descent: a baseline far
3111        // enough up keeps the descender on the page, one at 0 does not.
3112        assert_eq!(w_at(100.0, 20.0), "W");
3113        assert_eq!(w_at(100.0, 0.0), "");
3114        assert_eq!(w_at(100.0, 380.0), "");
3115    }
3116
3117    /// pdfium's fallbacks: no MediaBox → Letter; a CropBox is clipped to the
3118    /// MediaBox, and one that misses it entirely is ignored.
3119    #[test]
3120    fn page_box_follows_pdfium_fallbacks() {
3121        let (pb, _) = only_page(&pdf("", 10.0, 10.0));
3122        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 612.0, 792.0));
3123        let (pb, _) = only_page(&pdf(
3124            "/MediaBox[0 0 500 700]/CropBox[-100 100 600 900]",
3125            10.0,
3126            10.0,
3127        ));
3128        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 100.0, 500.0, 600.0));
3129        let (pb, _) = only_page(&pdf(
3130            "/MediaBox[0 0 500 700]/CropBox[800 800 900 900]",
3131            10.0,
3132            10.0,
3133        ));
3134        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
3135        // Reversed corners normalize.
3136        let (pb, _) = only_page(&pdf("/MediaBox[500 700 0 0]", 10.0, 10.0));
3137        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
3138    }
3139}
3140
3141#[cfg(test)]
3142mod xref_repair {
3143    /// Build a tiny one-page PDF whose cross-reference entries are either the
3144    /// spec's 20 bytes (`two_byte_eol`) or the 19-byte form some generators
3145    /// emit — everything else about the two files is identical.
3146    fn pdf_with_xref(two_byte_eol: bool) -> Vec<u8> {
3147        let content = b"BT /F1 12 Tf 72 700 Td (Invoice 922769430725) Tj ET\n";
3148        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
3149        let objs: Vec<Vec<u8>> = vec![
3150            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
3151            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
3152            b"<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
3153               /Resources<</Font<</F1 5 0 R>>>>>>"
3154                .to_vec(),
3155            [stream.as_slice(), content.as_slice(), b"endstream"].concat(),
3156            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".to_vec(),
3157        ];
3158
3159        let mut out = b"%PDF-1.4\n".to_vec();
3160        let mut offsets = Vec::new();
3161        for (i, body) in objs.iter().enumerate() {
3162            offsets.push(out.len());
3163            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
3164            out.extend_from_slice(body);
3165            out.extend_from_slice(b"endobj\n");
3166        }
3167        let xref_at = out.len();
3168        let eol: &[u8] = if two_byte_eol { b" \n" } else { b"\n" };
3169        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
3170        out.extend_from_slice(b"0000000000 65535 f");
3171        out.extend_from_slice(eol);
3172        for off in &offsets {
3173            out.extend_from_slice(format!("{off:010} 00000 n").as_bytes());
3174            out.extend_from_slice(eol);
3175        }
3176        out.extend_from_slice(
3177            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
3178        );
3179        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
3180        out
3181    }
3182
3183    /// A 19-byte cross-reference entry (a bare LF where the spec wants a
3184    /// two-byte EOL) used to make lopdf reject the whole file. Lopdf 0.45
3185    /// accepts it directly, so exercise the fallback itself and prove it still
3186    /// produces the exact 20-byte entries lopdf 0.44 needed.
3187    #[test]
3188    fn short_xref_entries_are_padded_for_fallback() {
3189        let good = pdf_with_xref(true);
3190        let broken = pdf_with_xref(false);
3191        assert!(
3192            broken.len() < good.len(),
3193            "the broken file is the shorter one"
3194        );
3195        assert!(
3196            lopdf::Document::load_mem(&good).is_ok(),
3197            "the control file must load unaided"
3198        );
3199        assert!(
3200            lopdf::Document::load_mem(&broken).is_ok(),
3201            "lopdf 0.45 should accept 19-byte entries"
3202        );
3203
3204        let repaired = super::pad_short_xref_entries(&broken).expect("repair short entries");
3205        assert_eq!(repaired, good, "repair should recreate the valid xref");
3206        assert!(
3207            lopdf::Document::load_mem(&repaired).is_ok(),
3208            "the bytes produced for lopdf 0.44 must load"
3209        );
3210
3211        let cells = |b: &[u8]| -> Vec<String> {
3212            super::pdf_textlines(b)
3213                .into_iter()
3214                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
3215                .collect()
3216        };
3217        let from_good = cells(&good);
3218        assert!(
3219            from_good.iter().any(|t| t.contains("922769430725")),
3220            "control text: {from_good:?}"
3221        );
3222        assert_eq!(
3223            cells(&repaired),
3224            from_good,
3225            "repair must match the good parse"
3226        );
3227    }
3228
3229    /// The same generator overstates `/Length`. Lopdf 0.45 recovers at
3230    /// `endstream`, so exercise our fallback directly and prove it still writes
3231    /// the exact length lopdf 0.44 needed without moving any bytes.
3232    #[test]
3233    fn overstated_stream_length_is_fixed_for_fallback() {
3234        let good = pdf_with_xref(true);
3235        // Inflate the content stream's /Length by one, exactly as the invoice
3236        // that prompted this does.
3237        let broken = {
3238            let at = good
3239                .windows(8)
3240                .position(|w| w == b"/Length ")
3241                .expect("a /Length")
3242                + 8;
3243            let digits = good[at..].iter().take_while(|c| c.is_ascii_digit()).count();
3244            let n: usize = std::str::from_utf8(&good[at..at + digits])
3245                .unwrap()
3246                .parse()
3247                .unwrap();
3248            let inflated = (n + 1).to_string();
3249            assert_eq!(inflated.len(), digits, "keep the digit count");
3250            let mut b = good.clone();
3251            b[at..at + digits].copy_from_slice(inflated.as_bytes());
3252            b
3253        };
3254        assert_eq!(broken.len(), good.len(), "the defect must not move bytes");
3255        // Lopdf 0.45 keeps the stream despite the overstated length.
3256        let raw = lopdf::Document::load_mem(&broken).expect("still loads");
3257        assert!(
3258            raw.get_pages()
3259                .into_values()
3260                .any(|p| !raw.get_page_content(p).is_empty()),
3261            "lopdf 0.45 should recover content at endstream"
3262        );
3263
3264        let repaired = super::fix_stream_lengths(&broken);
3265        assert_eq!(repaired.len(), broken.len(), "repair must preserve offsets");
3266        assert_eq!(repaired, good, "repair should restore the true length");
3267        let repaired_doc = lopdf::Document::load_mem(&repaired).expect("repaired PDF loads");
3268        assert!(
3269            repaired_doc
3270                .get_pages()
3271                .into_values()
3272                .any(|p| !repaired_doc.get_page_content(p).is_empty()),
3273            "the bytes produced for lopdf 0.44 must retain page content"
3274        );
3275
3276        // Ours recovers the same text the well-formed file gives.
3277        let text = |b: &[u8]| -> Vec<String> {
3278            super::pdf_textlines(b)
3279                .into_iter()
3280                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
3281                .collect()
3282        };
3283        let expected = text(&good);
3284        assert!(!expected.is_empty(), "control must produce text");
3285        assert_eq!(text(&repaired), expected);
3286    }
3287
3288    /// The repair only fires where padding cannot move an object: it declines a
3289    /// file whose xref precedes an object (an incremental update), rather than
3290    /// shifting every offset the table records.
3291    #[test]
3292    fn repair_declines_when_padding_would_move_objects() {
3293        let mut incremental = pdf_with_xref(false);
3294        incremental.extend_from_slice(b"6 0 obj<</Type/Whatever>>endobj\n");
3295        let declined = super::pad_short_xref_entries(&incremental).unwrap_err();
3296        assert!(
3297            declined.contains("object follows the xref"),
3298            "reason: {declined}"
3299        );
3300    }
3301}
3302
3303/// #187: standard-14 fonts referenced without an embedded program (and thus
3304/// usually without `/Widths` or a `/FontDescriptor`) must decode with the
3305/// built-in Adobe Core 14 metrics instead of collapsing every cell to zero
3306/// width — the failure mode where a valid text layer was silently dropped
3307/// while pdfium read the same file fine.
3308#[cfg(test)]
3309mod base14_fonts {
3310    /// A one-page PDF whose single `Tj` uses `fontdict` (no embedded program).
3311    fn pdf_with_font(fontdict: &[u8], text: &[u8]) -> Vec<u8> {
3312        let content = [b"BT /F1 12 Tf 72 700 Td (".as_slice(), text, b") Tj ET\n"].concat();
3313        pdf_with_content(fontdict, &content)
3314    }
3315
3316    /// A one-page A4 PDF drawing `content` with `fontdict` as `/F1`.
3317    fn pdf_with_content(fontdict: &[u8], content: &[u8]) -> Vec<u8> {
3318        pdf_with_fonts(&[fontdict], content)
3319    }
3320
3321    /// A one-page A4 PDF drawing `content` with `fontdicts[i]` bound as
3322    /// `/F{i+1}` (objects 5, 6, …).
3323    fn pdf_with_fonts(fontdicts: &[&[u8]], content: &[u8]) -> Vec<u8> {
3324        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
3325        let font_res: String = fontdicts
3326            .iter()
3327            .enumerate()
3328            .map(|(i, _)| format!("/F{} {} 0 R", i + 1, i + 5))
3329            .collect();
3330        let mut objs: Vec<Vec<u8>> = vec![
3331            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
3332            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
3333            format!(
3334                "<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
3335                 /Resources<</Font<<{font_res}>>>>>>"
3336            )
3337            .into_bytes(),
3338            [stream.as_slice(), content, b"endstream"].concat(),
3339        ];
3340        objs.extend(fontdicts.iter().map(|d| d.to_vec()));
3341        let mut out = b"%PDF-1.4\n".to_vec();
3342        let mut offsets = Vec::new();
3343        for (i, body) in objs.iter().enumerate() {
3344            offsets.push(out.len());
3345            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
3346            out.extend_from_slice(body);
3347            out.extend_from_slice(b"endobj\n");
3348        }
3349        let xref_at = out.len();
3350        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
3351        out.extend_from_slice(b"0000000000 65535 f \n");
3352        for off in &offsets {
3353            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
3354        }
3355        out.extend_from_slice(
3356            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
3357        );
3358        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
3359        out
3360    }
3361
3362    /// The parsed cells of the only page.
3363    fn cells(pdf: &[u8]) -> Vec<crate::pdfium_backend::TextCell> {
3364        super::pdf_textlines(pdf)
3365            .into_iter()
3366            .flat_map(|(_, _, c)| c)
3367            .collect()
3368    }
3369
3370    /// Text drawn with a rotated text matrix reads as one line in its own
3371    /// reading order, boxed by the run's axis-aligned extent (#528). The
3372    /// landscape case is `0 s -s 0 tx ty Tm` on a `/Rotate 90` page; every
3373    /// such run used to collapse to zero width and vanish (90°/270°/tilted)
3374    /// or read backwards (180°). Expected boxes are docling-parse 7.22's line
3375    /// cells for the same runs (y-up, PDF points).
3376    #[test]
3377    fn rotated_text_matrix_reads_in_order() {
3378        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
3379        const TEXT: &str = "Upright text on a rotated page.";
3380        for (tm, [l, b, r, t]) in [
3381            ("0 14 -14 0 200 100", [189.9, 100.0, 202.9, 289.1]), // 90°
3382            ("-14 0 0 -14 350 300", [160.9, 289.9, 350.0, 302.9]), // 180°
3383            ("0 -14 14 0 200 500", [197.1, 310.9, 210.1, 500.0]), // 270°
3384            (
3385                "9.8995 9.8995 -9.8995 9.8995 100 100",
3386                [92.9, 98.0, 235.8, 240.8],
3387            ), // 45°
3388        ] {
3389            let content = format!("BT /F1 1 Tf {tm} Tm ({TEXT}) Tj ET\n");
3390            let pdf = pdf_with_content(HELV, content.as_bytes());
3391            let cs = cells(&pdf);
3392            assert_eq!(cs.len(), 1, "{tm}: {cs:?}");
3393            let c = &cs[0];
3394            assert_eq!(c.text, TEXT, "{tm}");
3395            // `cells` is top-left-origin on the 842 pt page. The descriptor-
3396            // less base-14 face gets the parser's 1-em box where docling-parse
3397            // reads Helvetica's AFM (718/−207) — the same ≤ 0.6 pt
3398            // across-the-baseline offset upright text has — so allow 1 pt.
3399            let got = [c.l, 842.0 - c.b, c.r, 842.0 - c.t];
3400            for (g, w) in got.iter().zip([l, b, r, t]) {
3401                assert!(
3402                    (g - w).abs() < 1.0,
3403                    "{tm}: box {got:?} vs docling-parse {:?}",
3404                    [l, b, r, t]
3405                );
3406            }
3407            let words: Vec<String> = super::pdf_words(&pdf)
3408                .into_iter()
3409                .flat_map(|(_, _, c)| c)
3410                .map(|c| c.text)
3411                .collect();
3412            assert_eq!(words, TEXT.split(' ').collect::<Vec<_>>(), "{tm}");
3413        }
3414    }
3415
3416    /// A rotated run past the page edge (a cover's spine title) is dropped
3417    /// by the on-page test (#529) — measured on the quad's extent, not the
3418    /// zero-width box rotated glyphs used to get — instead of clamping onto
3419    /// the edge as a zero-width cell. One inside the page is kept.
3420    #[test]
3421    fn rotated_text_off_the_page_is_dropped() {
3422        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
3423        let off = pdf_with_content(
3424            HELV,
3425            b"BT /F1 1 Tf 0 14 -14 0 -4 200 Tm (Spine title) Tj ET\n",
3426        );
3427        assert!(cells(&off).is_empty(), "{:?}", cells(&off));
3428        let on = pdf_with_content(
3429            HELV,
3430            b"BT /F1 1 Tf 0 14 -14 0 30 200 Tm (Margin note) Tj ET\n",
3431        );
3432        assert_eq!(cells(&on).len(), 1);
3433    }
3434
3435    /// Upright text keeps the plain loose rectangle — no quad, and the
3436    /// sanitizer path is the one every pinned PDF baseline was made with.
3437    #[test]
3438    fn upright_glyphs_carry_no_quad() {
3439        let only_page = |pdf: &[u8]| {
3440            let doc = super::load_document(pdf).expect("loads");
3441            let pid = *doc.get_pages().values().next().expect("one page");
3442            super::page_glyphs(&doc, pid)
3443        };
3444        let pdf = pdf_with_font(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", b"Plain");
3445        let glyphs = only_page(&pdf);
3446        assert!(!glyphs.is_empty());
3447        assert!(glyphs.iter().all(|g| g.quad.is_none() && g.lr > g.ll));
3448        // A 90° glyph's style height is its font size, not its advance.
3449        let pdf = pdf_with_content(
3450            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>",
3451            b"BT /F1 1 Tf 0 14 -14 0 200 100 Tm (W) Tj ET\n",
3452        );
3453        let glyphs = only_page(&pdf);
3454        let g = &glyphs[0];
3455        assert!(g.quad.is_some());
3456        // 14 pt × the face's 1-em box (no descriptor: ascent − descent = 1).
3457        assert!((g.height() - 14.0).abs() < 0.05, "{}", g.height());
3458    }
3459
3460    /// #609: ReportLab's chart axis sets each tick label with the *same* `Tm`
3461    /// inside its own `q … cm … Q`, so only the CTM stacks `3`…`8` one under
3462    /// another at one x. Each tick is its own line cell with its own box (as
3463    /// docling-parse reads it), not one `345678` cell boxed like the first.
3464    #[test]
3465    fn ticks_stacked_by_cm_are_separate_line_cells() {
3466        let mut content = String::new();
3467        for (k, tick) in "345678".chars().enumerate() {
3468            let y = 300.0 + 25.92 * k as f64;
3469            content.push_str(&format!(
3470                "q 1 0 0 1 125 {y} cm BT /F1 10 Tf 1 0 0 1 -5 -4 Tm ({tick}) Tj ET Q\n"
3471            ));
3472        }
3473        let pdf = pdf_with_content(
3474            b"<</Type/Font/Subtype/Type1/BaseFont/Times-Roman>>",
3475            content.as_bytes(),
3476        );
3477        let cells = cells(&pdf);
3478        let texts: Vec<&str> = cells.iter().map(|c| c.text.as_str()).collect();
3479        assert_eq!(texts, ["3", "4", "5", "6", "7", "8"]);
3480        for (k, c) in cells.iter().enumerate() {
3481            // Top-left y of a tick whose baseline is 4 pt under its cm origin.
3482            let base = 842.0 - (300.0 + 25.92 * k as f32 - 4.0);
3483            assert!((c.l - 120.0).abs() < 0.1, "{k}: l {}", c.l);
3484            assert!(
3485                c.t < base && base - c.t < 10.0,
3486                "{k}: t {} base {base}",
3487                c.t
3488            );
3489        }
3490    }
3491
3492    /// #609: the path walk finds drawn checkbox squares — ReportLab's four
3493    /// `m … l S` edges under a `cm`, and a stroked `re` with a tick inside
3494    /// (checked) — in the text cells' top-left frame, and a stroked box
3495    /// filled white; a filled bar, a big frame and a colour-filled legend
3496    /// swatch are not checkboxes.
3497    #[test]
3498    fn drawn_checkbox_squares_are_found() {
3499        let content = b"q 1 0 0 1 119.52 449.04 cm \
3500            n 0 12.96 m 12.96 12.96 l S n 0 0 m 12.96 0 l S \
3501            n 0 0 m 0 12.96 l S n 12.96 0 m 12.96 12.96 l S Q\n\
3502            200 400 10 10 re S 202 405 m 204.5 402 l 208.5 408.5 l S\n\
3503            300 400 12 12 re f 50 50 100 100 re S\n\
3504            q 0.2 0.4 0.8 rg 400 400 12 12 re B Q q 1 g 500 400 12 12 re B Q\n";
3505        let pdf = pdf_with_content(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", content);
3506        let mut parser = super::PageTextParser::open(&pdf).expect("parses");
3507        let mut boxes = parser.cells(0).checkboxes;
3508        boxes.sort_by(|a, b| a.l.total_cmp(&b.l));
3509        let got: Vec<([i32; 4], bool)> = boxes
3510            .iter()
3511            .map(|c| {
3512                let r = |v: f32| (v * 100.0).round() as i32;
3513                ([r(c.l), r(c.t), r(c.r), r(c.b)], c.checked)
3514            })
3515            .collect();
3516        // Page height 842: y-up 449.04..462.0 → top-left 380.0..392.96.
3517        assert_eq!(
3518            got,
3519            [
3520                ([11952, 38000, 13248, 39296], false),
3521                ([20000, 43200, 21000, 44200], true),
3522                ([50000, 43000, 51200, 44200], false),
3523            ]
3524        );
3525    }
3526
3527    /// Every standard-14 alias/style decodes with real (positive-width) boxes.
3528    #[test]
3529    fn standard14_faces_get_builtin_widths() {
3530        for fontdict in [
3531            // ReportLab's default: base-14 Helvetica, WinAnsi, nothing else.
3532            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>".as_slice(),
3533            // No /Encoding at all (StandardEncoding-ish default).
3534            b"<</Type/Font/Subtype/Type1/BaseFont/Times-BoldItalic>>",
3535            // Substitution aliases + a subset prefix.
3536            b"<</Type/Font/Subtype/TrueType/BaseFont/Arial,Bold>>",
3537            b"<</Type/Font/Subtype/Type1/BaseFont/ABCDEF+Courier-Oblique>>",
3538        ] {
3539            let pdf = pdf_with_font(fontdict, b"Words have width now");
3540            let cs = cells(&pdf);
3541            let text: String = cs
3542                .iter()
3543                .map(|c| c.text.as_str())
3544                .collect::<Vec<_>>()
3545                .join(" ");
3546            assert!(
3547                text.contains("Words have width now"),
3548                "{}: text lost: {text:?}",
3549                String::from_utf8_lossy(fontdict)
3550            );
3551            assert!(
3552                cs.iter().all(|c| c.r > c.l),
3553                "{}: zero-width cells: {cs:?}",
3554                String::from_utf8_lossy(fontdict)
3555            );
3556        }
3557    }
3558
3559    /// MacRomanEncoding without a ToUnicode CMap (#744) — Word for Mac's
3560    /// Quartz output: the whole high range decodes, Danish æ ø å, é, ü, ß
3561    /// and § included, through the plain encoding name and as the base of a
3562    /// `/Differences` dictionary alike.
3563    #[test]
3564    fn macroman_fonts_keep_accented_letters() {
3565        let sentence = b"Bl\x8cb\xbergr\xbfd p\x8c \xbfen \xd0 caf\x8e M\x9fller, Stra\xa7e \xa4 3";
3566        for (fontdict, want) in [
3567            (
3568                b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/MacRomanEncoding>>"
3569                    .as_slice(),
3570                "Blåbærgrød på øen – café Müller, Straße § 3",
3571            ),
3572            (
3573                b"<</Type/Font/Subtype/TrueType/BaseFont/ABCDEF+Arial\
3574                   /Encoding<</Type/Encoding/BaseEncoding/MacRomanEncoding/Differences[51/four]>>>>",
3575                "Blåbærgrød på øen – café Müller, Straße § 4",
3576            ),
3577        ] {
3578            let text: String = cells(&pdf_with_font(fontdict, sentence))
3579                .iter()
3580                .map(|c| c.text.as_str())
3581                .collect::<Vec<_>>()
3582                .join(" ");
3583            assert_eq!(text, want, "{}", String::from_utf8_lossy(fontdict));
3584        }
3585
3586        // The table itself: all of 0x80–0xFF but the Apple logo, ¤ at 0xDB.
3587        let table = super::macroman_table();
3588        assert_eq!(
3589            (0x80u8..=0xFF).filter(|b| table.contains_key(b)).count(),
3590            127
3591        );
3592        assert_eq!(table.get(&0xF0), None);
3593        assert_eq!(table[&0xDB], '¤');
3594        assert_eq!(
3595            [0x80, 0x8C, 0xAE, 0xAF, 0xBE, 0xBF, 0xFF].map(|b| table[&b]),
3596            ['Ä', 'å', 'Æ', 'Ø', 'æ', 'ø', 'ˇ']
3597        );
3598    }
3599
3600    /// An explicit `/Widths` array always wins over the built-in metrics, and a
3601    /// non-standard face without `/Widths` stays as before (no invented boxes).
3602    #[test]
3603    fn explicit_widths_win_and_unknown_faces_are_untouched() {
3604        // Helvetica with explicit 100/1000-em widths: the word's box must be
3605        // ~4×100 units at 12pt = 4.8pt wide — far narrower than the ~2.7×
3606        // wider built-in Helvetica advances would make it.
3607        let explicit = pdf_with_font(
3608            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/FirstChar 65\
3609               /Widths[100 100 100 100]/Encoding/WinAnsiEncoding>>",
3610            b"ABBA",
3611        );
3612        let builtin = pdf_with_font(
3613            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>",
3614            b"ABBA",
3615        );
3616        let w = |pdf: &[u8]| {
3617            let cs = cells(pdf);
3618            assert_eq!(cs.len(), 1, "one word cell: {cs:?}");
3619            cs[0].r - cs[0].l
3620        };
3621        let (we, wb) = (w(&explicit), w(&builtin));
3622        assert!(
3623            (we - 4.8).abs() < 0.1,
3624            "explicit widths must win: got {we}, want 4×100×12/1000"
3625        );
3626        assert!(
3627            wb > 2.0 * we,
3628            "built-in Helvetica is much wider: {wb} vs {we}"
3629        );
3630
3631        // An unknown face with no /Widths: still parses (text kept), but no
3632        // built-in table applies — the old zero-width behavior is preserved
3633        // rather than inventing Helvetica metrics for an arbitrary font.
3634        let unknown = pdf_with_font(
3635            b"<</Type/Font/Subtype/Type1/BaseFont/FancyCorp-Display>>",
3636            b"Mystery",
3637        );
3638        let cs = cells(&unknown);
3639        let text: String = cs.iter().map(|c| c.text.as_str()).collect();
3640        assert!(text.contains("Mystery"), "text still decodes: {cs:?}");
3641    }
3642    /// #728: one face bound under two resource keys — `/F1` Helvetica in
3643    /// WinAnsi, `/F2` the same Helvetica re-encoded so code 0x81 is the `fi`
3644    /// glyph — with every fi-word drawn in `/F2` right after an `/F1` run.
3645    /// docling-parse names a font by `/FontDescriptor/FontName` → `/Name` →
3646    /// `/BaseFont`, never by the resource key, so both resources are the same
3647    /// font and the line reads `financial`. Keyed by resource key the `fi`
3648    /// ligature (font-neutral in `applicable_for_merge`) joined the `/F1`
3649    /// run and the font barrier then split it from `nancial`.
3650    #[test]
3651    fn same_face_under_two_resource_keys_is_one_font_for_the_sanitizer() {
3652        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
3653        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding<</Type/Encoding\
3654                   /BaseEncoding/WinAnsiEncoding/Differences[129/fi]>>>>";
3655        let content = b"BT 12 TL 72 720 Td \
3656            /F1 12 Tf (The ) Tj /F2 12 Tf (\\201nancial position) Tj \
3657            /F1 12 Tf ( of the company is shown in ) Tj /F2 12 Tf (\\201gure 3) Tj \
3658            /F1 12 Tf ( with the ) Tj /F2 12 Tf (\\201rst results.) Tj \
3659            T* T* /F2 12 Tf (Same words in /F2 only: \\201nancial, \\201gure, \\201rst.) Tj ET\n";
3660        let cs = cells(&pdf_with_fonts(&[f1, f2], content));
3661        let lines: Vec<&str> = cs.iter().map(|c| c.text.as_str()).collect();
3662        assert_eq!(
3663            lines,
3664            [
3665                "The financial position of the company is shown in figure 3 with the first results.",
3666                "Same words in /F2 only: financial, figure, first.",
3667            ],
3668            "{cs:?}"
3669        );
3670    }
3671
3672    /// Under `legacy` the font barrier itself is untouched: two *different*
3673    /// faces (regular and bold Helvetica, which docling-parse names apart) set
3674    /// flush against each other stay separate line cells, as
3675    /// `enforce_same_font` demands. docling-parse 7 dropped the barrier — its
3676    /// line runs follow geometry alone — so under the default `adaptive` the
3677    /// flush pair is one line and one word.
3678    #[test]
3679    fn different_faces_keep_the_font_barrier_only_under_legacy() {
3680        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
3681        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica-Bold/Encoding/WinAnsiEncoding>>";
3682        let content = b"BT 72 720 Td /F2 12 Tf (Total:) Tj /F1 12 Tf (42) Tj ET\n";
3683        let pdf = pdf_with_fonts(&[f1, f2], content);
3684        let lines = |gap| {
3685            let mut p = super::PageTextParser::open(&pdf).expect("opens");
3686            p.set_word_gap(Some(gap));
3687            p.cells(0)
3688                .prose
3689                .into_iter()
3690                .map(|c| c.text)
3691                .collect::<Vec<_>>()
3692        };
3693        assert_eq!(lines(crate::dp_lines::WordGap::Legacy), ["Total:", "42"]);
3694        assert_eq!(lines(crate::dp_lines::WordGap::Adaptive), ["Total:42"]);
3695    }
3696
3697    /// The identity follows docling-parse's precedence: a `/FontDescriptor/FontName`
3698    /// wins over `/BaseFont`, so two dictionaries that share a `/BaseFont` but
3699    /// describe different programs are two fonts, and two that share the
3700    /// descriptor name are one whatever their `/BaseFont` says.
3701    #[test]
3702    fn math_faces_are_named_like_docling_parse_names_them() {
3703        for name in [
3704            "ABCDEF+CMR7",
3705            "CMMI10",
3706            "MSBM10",
3707            "Cambria-Math",
3708            "Symbol",
3709            "MTExtra",
3710        ] {
3711            assert!(super::is_math_font_name(name.as_bytes()), "{name}");
3712        }
3713        for name in ["Helvetica", "ABCDEF+TimesNewRomanPSMT", "Arial-BoldMT"] {
3714            assert!(!super::is_math_font_name(name.as_bytes()), "{name}");
3715        }
3716    }
3717
3718    #[test]
3719    fn sanitizer_font_name_follows_docling_parse_precedence() {
3720        use lopdf::{dictionary, Document, Object};
3721        let mut doc = Document::with_version("1.4");
3722        let fd_a =
3723            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "AAAAAA+Face" });
3724        let fd_b =
3725            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "BBBBBB+Face" });
3726        let with = |fd: Option<lopdf::ObjectId>, base: &str| {
3727            let mut d = dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => base };
3728            if let Some(id) = fd {
3729                d.set("FontDescriptor", Object::Reference(id));
3730            }
3731            d
3732        };
3733        let name = |d: &lopdf::Dictionary| {
3734            String::from_utf8(super::sanitizer_font_name(&doc, d, false)).unwrap()
3735        };
3736        assert_eq!(name(&with(Some(fd_a), "Face")), "AAAAAA+Face");
3737        assert_eq!(name(&with(Some(fd_b), "Face")), "BBBBBB+Face");
3738        assert_eq!(name(&with(None, "ABCDEF+Face")), "ABCDEF+Face");
3739        assert_eq!(
3740            name(&dictionary! { "Type" => "Font", "Name" => "F7" }),
3741            "F7"
3742        );
3743        assert_eq!(name(&dictionary! { "Type" => "Font" }), "null");
3744    }
3745}
3746
3747#[cfg(test)]
3748mod overpainted {
3749    use crate::pdfium_backend::TextCell;
3750
3751    fn cell(text: &str, l: f32, t: f32, r: f32, b: f32) -> TextCell {
3752        TextCell {
3753            text: text.into(),
3754            l,
3755            t,
3756            r,
3757            b,
3758        }
3759    }
3760
3761    /// The reporting invoice's logo: a `"` painted inside a `==` on one band —
3762    /// artwork drawn with glyphs. Both cells go; the real text on the next
3763    /// band stays.
3764    #[test]
3765    fn stacked_logo_glyphs_are_dropped() {
3766        let mut cells = vec![
3767            cell("\"", 72.7, 21.5, 86.4, 31.5),
3768            cell("==", 59.4, 21.5, 99.6, 31.5),
3769            cell("Herr", 65.2, 151.3, 81.7, 161.3),
3770        ];
3771        super::drop_overpainted_cells(&mut cells);
3772        assert_eq!(cells.len(), 1, "cells: {cells:?}");
3773        assert_eq!(cells[0].text, "Herr");
3774    }
3775
3776    /// Adjacent words on a line touch but never contain each other — prose is
3777    /// untouched, and so is a same-text near-duplicate (double-drawn faux
3778    /// bold), which is not evidence of artwork.
3779    #[test]
3780    fn prose_and_double_draw_are_kept() {
3781        let mut cells = vec![
3782            cell("Telefon", 354.3, 133.2, 381.5, 143.2),
3783            cell("0676/2000", 387.3, 133.2, 428.7, 143.2),
3784            cell("Bold", 100.0, 50.0, 130.0, 60.0),
3785            cell("Bold", 100.3, 50.0, 130.3, 60.0),
3786        ];
3787        super::drop_overpainted_cells(&mut cells);
3788        assert_eq!(cells.len(), 4);
3789    }
3790}
3791
3792#[cfg(test)]
3793mod vestigial_layer {
3794    use crate::pdfium_backend::{PdfPage, TextCell};
3795
3796    fn page_with(texts: &[&str]) -> PdfPage {
3797        let cells = texts
3798            .iter()
3799            .enumerate()
3800            .map(|(i, t)| TextCell {
3801                text: t.to_string(),
3802                l: 10.0,
3803                t: 10.0 + 12.0 * i as f32,
3804                r: 90.0,
3805                b: 20.0 + 12.0 * i as f32,
3806            })
3807            .collect();
3808        PdfPage::from_cells(595.0, 842.0, 1.0, cells)
3809    }
3810
3811    /// The reported scanned form: three typed-in field values ("03", "05",
3812    /// "2025") over three image pages. That must read as *no usable layer*,
3813    /// so the browser routes the document to OCR instead of extracting
3814    /// thirteen characters and skipping the letter entirely.
3815    #[test]
3816    fn typed_in_form_fields_are_not_a_text_layer() {
3817        let pages = vec![
3818            page_with(&["03", "05", "2025"]),
3819            page_with(&[]),
3820            page_with(&[]),
3821        ];
3822        assert!(super::text_layer_is_vestigial(&pages));
3823        assert!(super::text_layer_is_vestigial(&[page_with(&[])]));
3824    }
3825
3826    /// A short but genuine digital document — one page, a few real lines —
3827    /// keeps the fast text path.
3828    #[test]
3829    fn sparse_but_real_documents_pass() {
3830        let one_pager = vec![page_with(&[
3831            "Confidential briefing",
3832            "Prepared for the board meeting",
3833            "Do not distribute",
3834        ])];
3835        assert!(!super::text_layer_is_vestigial(&one_pager));
3836    }
3837}