Skip to main content

docling_pdf/
textparse.rs

1//! Pure-Rust PDF text extraction (replacing pdfium's glyph layer).
2//!
3//! pdfium reports *rendered* glyph boxes, which diverge from docling's
4//! `docling-parse` C++ parser at exactly the points that drive conformance:
5//! generated spaces get a zero-width box, combining diacritics get a real-width
6//! box, and ligature/fraction glyphs land at different x. This module instead
7//! reconstructs each glyph's box from the **font's own advance widths** and the
8//! PDF text/graphics matrices — the same information docling-parse uses — so a
9//! space is as wide as the font says and a combining mark has zero advance.
10//!
11//! The output is the same [`Glyph`] stream pdfium produces (native PDF
12//! coordinates, y-up), fed straight into the existing docling-parse line
13//! sanitizer ([`crate::dp_lines`]). Only the digital text layer is handled here;
14//! pages without one still fall back to OCR upstream.
15
16use std::collections::{HashMap, HashSet};
17use std::sync::Arc;
18
19use lopdf::{Dictionary, Document, Object};
20
21use crate::pdfium_backend::Glyph;
22use crate::tounicode::parse_tounicode;
23
24/// Per-document caches for the content-stream interpreter. Fonts are indirect
25/// objects shared by many pages, but were fully re-parsed — ToUnicode CMap
26/// decompression + tokenization, embedded Type1 program scan, width tables —
27/// for **every page and every Form XObject invocation**; decoded form content
28/// streams were likewise re-inflated on every `Do`. Cached per document,
29/// keyed by the referenced object id (fonts also by resource name — the
30/// `/Font` entry a page binds the object to). Inline (non-reference) dicts
31/// are rare and stay uncached.
32#[derive(Default)]
33struct DocCaches {
34    fonts: HashMap<(lopdf::ObjectId, Vec<u8>), Arc<Font>>,
35    forms: HashMap<lopdf::ObjectId, Arc<lopdf::content::Content>>,
36    /// Forms decoded once and not (yet) cached (#994). A form enters
37    /// [`Self::forms`] on its *second* sighting: a shared header/footer form
38    /// is decoded twice and then hits the cache on every later page, while
39    /// a generator's one-form-per-page body (each page's chart in its own
40    /// form, used once) is dropped with its page instead of every decoded
41    /// `Content` — many times its stream's size — living until the document
42    /// ends. Caching on first use made peak RSS grow ~45 MB per page on such
43    /// files (7 GiB at 60 pages of the issue's repro, flat ~0.6 GiB now).
44    /// Only ids are kept here, so the set itself stays tiny.
45    forms_seen: HashSet<lopdf::ObjectId>,
46}
47
48/// A 2×3 affine matrix `[a b c d e f]`: maps `(x,y)` → `(a·x+c·y+e, b·x+d·y+f)`.
49#[derive(Clone, Copy)]
50struct Mat {
51    a: f64,
52    b: f64,
53    c: f64,
54    d: f64,
55    e: f64,
56    f: f64,
57}
58
59impl Mat {
60    const ID: Mat = Mat {
61        a: 1.0,
62        b: 0.0,
63        c: 0.0,
64        d: 1.0,
65        e: 0.0,
66        f: 0.0,
67    };
68
69    /// `self ∘ m`: the matrix that applies `self` first, then `m`.
70    fn then(self, m: Mat) -> Mat {
71        Mat {
72            a: self.a * m.a + self.b * m.c,
73            b: self.a * m.b + self.b * m.d,
74            c: self.c * m.a + self.d * m.c,
75            d: self.c * m.b + self.d * m.d,
76            e: self.e * m.a + self.f * m.c + m.e,
77            f: self.e * m.b + self.f * m.d + m.f,
78        }
79    }
80
81    fn apply(self, x: f64, y: f64) -> (f64, f64) {
82        (
83            self.a * x + self.c * y + self.e,
84            self.b * x + self.d * y + self.f,
85        )
86    }
87}
88
89/// A parsed font: how to turn raw string bytes into (unicode, advance) pairs.
90struct Font {
91    /// 2-byte codes (Type0 / Identity-H) vs 1-byte (simple fonts).
92    two_byte: bool,
93    /// code → Unicode string (from ToUnicode; may be multi-char, e.g. ligatures).
94    to_unicode: HashMap<u32, String>,
95    /// For codes ToUnicode lacks (ISO 32000-1, 9.10.2).
96    cid_fallback: CidFallback,
97    /// code → glyph advance, in 1000-unit glyph space.
98    widths: HashMap<u32, f64>,
99    default_width: f64,
100    /// 1-byte fallback decoding when ToUnicode lacks a code (WinAnsi-ish).
101    simple_encoding: Option<HashMap<u8, char>>,
102    /// code → raw `/Differences` glyph name, for placeholder names (`g115`,
103    /// `SM590000`) that have no Unicode mapping; they override the base
104    /// encoding (see [`unknown_name_text`]).
105    fallback_names: HashMap<u8, String>,
106    /// The unmapped codes whose embedded glyph paints nothing — semantic
107    /// spaces (docling-parse 7.24's blank glyph programs: the redp5110
108    /// list's `/g3`). Every other unmapped code reads U+FFFD.
109    blank_codes: HashSet<u32>,
110    /// code → char from the embedded Type1 font program's own `/Encoding` vector,
111    /// used only as a last resort for glyphs the base encoding leaves unmapped
112    /// (standard TeX math fonts: `λ`, `≤`, …).
113    program_encoding: HashMap<u8, char>,
114    ascent: f64,
115    descent: f64,
116    hash: u64,
117    /// Whether the font's docling-parse name marks a math face
118    /// ([`is_math_font_name`]) — for the word contraction's local
119    /// attachments (#921).
120    math: bool,
121    /// Weight and slant read from the `/BaseFont` name (the heading
122    /// hierarchy's style signal, #302).
123    style: crate::font_style::FontStyle,
124}
125
126impl Font {
127    /// The text of `code` and its advance; the flag marks a blank unmapped
128    /// glyph, whose box has no height.
129    ///
130    /// A code no source maps to Unicode is visibly painted but carries no
131    /// text: docling-parse 7.24 emits U+FFFD for it, and docling turns a NUL
132    /// into U+FFFD too (docling#5070), so the loss stays visible and counted
133    /// by the text-quality check instead of silently dropping the glyph
134    /// (`mL⁻¹` reading `mL1`). A glyph whose embedded program paints nothing
135    /// is a space — a word boundary — instead.
136    fn decode_code(&self, code: u32) -> (Option<String>, f64, bool) {
137        let w = self
138            .widths
139            .get(&code)
140            .copied()
141            .unwrap_or(self.default_width);
142        if let Some(s) = self.to_unicode.get(&code) {
143            return (Some(decompose_ligatures(s)), w, false);
144        }
145        let unmapped = |font: &Font| {
146            let blank = font.blank_codes.contains(&code);
147            (
148                Some(if blank { " " } else { "\u{FFFD}" }.to_string()),
149                w,
150                blank,
151            )
152        };
153        if !self.two_byte {
154            // A placeholder `/Differences` name (no Unicode) overrides the
155            // base encoding. docling-parse 7.24 reads a bare glyph index
156            // (`g115`, `gid00043`) as unmapped and any other unknown name as
157            // its text (`SM590000`, the redp5110 bullets).
158            if let Some(name) = self.fallback_names.get(&(code as u8)) {
159                return match unknown_name_text(name) {
160                    Some(text) => (Some(text), w, false),
161                    None => unmapped(self),
162                };
163            }
164            if let Some(enc) = &self.simple_encoding {
165                if let Some(&ch) = enc.get(&(code as u8)) {
166                    return (Some(decompose_ligatures(&ch.to_string())), w, false);
167                }
168            }
169            // Last resort: the embedded Type1 font program's own `/Encoding`
170            // vector (`dup N /glyphname put`). Standard TeX math fonts (CMMI, CMSY,
171            // …) ship no PDF `/Encoding` and no ToUnicode, so a glyph like `λ`
172            // (CMMI code 21 → `/lambda`) or `≤` (CMSY code 20 → `/lessequal`) has
173            // no other mapping and would otherwise be silently dropped. docling
174            // recovers these from the same font program. This only fills codes the
175            // base encoding left unmapped, so it never changes an existing decode.
176            if let Some(&ch) = self.program_encoding.get(&(code as u8)) {
177                return (Some(decompose_ligatures(&ch.to_string())), w, false);
178            }
179        } else {
180            match self.cid_fallback {
181                CidFallback::None => {}
182                CidFallback::UnicodeCodes { .. } => {
183                    if let Some(ch) = unicode_cmap_char(code).filter(|c| !c.is_control()) {
184                        return (Some(ch.to_string()), w, false);
185                    }
186                }
187                CidFallback::Collection(table) => {
188                    if let Some(s) = collection_text(table, code) {
189                        return (Some(decompose_ligatures(&s)), w, false);
190                    }
191                }
192            }
193        }
194        unmapped(self)
195    }
196}
197
198/// Spell out Latin presentation-form ligatures (`fi`→`fi`, `ffi`→`ffi`, …) the way
199/// docling does, so `configuration`/`difficult` don't keep the ligature glyph.
200/// The chars share the ligature's box, so the line sanitizer recomposes them.
201fn decompose_ligatures(s: &str) -> String {
202    if !s.chars().any(|c| ('\u{FB00}'..='\u{FB06}').contains(&c)) {
203        return s.to_string();
204    }
205    s.chars()
206        .map(|c| {
207            match c {
208                '\u{FB00}' => "ff",
209                '\u{FB01}' => "fi",
210                '\u{FB02}' => "fl",
211                '\u{FB03}' => "ffi",
212                '\u{FB04}' => "ffl",
213                // U+FB05 is LATIN SMALL LIGATURE LONG S T (`ſt`): `st`, as
214                // docling's ligature map and Unicode's decomposition (#1001).
215                '\u{FB05}' => "st",
216                '\u{FB06}' => "st",
217                _ => return c.to_string(),
218            }
219            .to_string()
220        })
221        .collect()
222}
223
224fn hash_name(name: &[u8]) -> u64 {
225    use std::hash::{Hash, Hasher};
226    let mut h = std::collections::hash_map::DefaultHasher::new();
227    name.hash(&mut h);
228    h.finish()
229}
230
231/// Resolve a possibly-indirect object to a dictionary.
232fn as_dict<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Dictionary> {
233    match obj {
234        Object::Dictionary(d) => Some(d),
235        Object::Reference(id) => doc.get_object(*id).ok().and_then(|o| o.as_dict().ok()),
236        _ => None,
237    }
238}
239
240fn deref<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Object> {
241    match obj {
242        Object::Reference(id) => doc.get_object(*id).ok(),
243        other => Some(other),
244    }
245}
246
247/// The font identity the line sanitizer's `enforce_same_font` compares —
248/// docling-parse's `page_font::init_font_name`: `/FontDescriptor/FontName`
249/// (the descendant CIDFont's for a Type0), else `/Name`, else `/BaseFont`
250/// (its `base_font`), else the literal `"null"`. The resource key (`/F1`,
251/// `/F2`) is deliberately **not** part of it: a producer that references one
252/// face twice — the same Helvetica under two keys, one re-encoded so a code
253/// maps to the `fi` glyph — draws one word across the two resources, and
254/// docling-parse reads both as the same font. Keyed by resource name (as
255/// this port was until #728) the `fi` ligature, which bridges fonts, glued
256/// itself to the preceding run while the font barrier then cut it off from
257/// its own word (`fi nancial`); docling's `financial` needs the two
258/// resources to compare equal.
259/// docling-parse's `has_math_positioning_semantics`, the font-name half: a
260/// TeX math or symbol face (`cmmi`, `cmsy`, `cmex`, `cmr`, `cmbx`, `msam`,
261/// `msbm`) or any name saying `math`, `symbol` or `mtextra`.
262fn is_math_font_name(name: &[u8]) -> bool {
263    let lower = name.to_ascii_lowercase();
264    [
265        "cmmi", "cmsy", "cmex", "cmr", "cmbx", "msam", "msbm", "math", "symbol", "mtextra",
266    ]
267    .iter()
268    .any(|t| lower.windows(t.len()).any(|w| w == t.as_bytes()))
269}
270
271fn sanitizer_font_name(doc: &Document, fdict: &Dictionary, two_byte: bool) -> Vec<u8> {
272    let descriptor_font_name = |owner: &Dictionary| {
273        owner
274            .get(b"FontDescriptor")
275            .ok()
276            .and_then(|o| as_dict(doc, o))
277            .and_then(|fd| fd.get(b"FontName").ok())
278            .and_then(|o| deref(doc, o))
279            .and_then(|o| o.as_name().ok().map(<[u8]>::to_vec))
280    };
281    if let Some(n) = descriptor_font_name(fdict) {
282        return n;
283    }
284    if two_byte {
285        let descendant = fdict
286            .get(b"DescendantFonts")
287            .ok()
288            .and_then(|o| deref(doc, o))
289            .and_then(|o| match o {
290                Object::Array(a) => a.first(),
291                _ => None,
292            })
293            .and_then(|o| as_dict(doc, o));
294        if let Some(n) = descendant.and_then(descriptor_font_name) {
295            return n;
296        }
297    }
298    if let Some(n) = fdict
299        .get(b"Name")
300        .ok()
301        .and_then(|o| deref(doc, o))
302        .and_then(|o| o.as_name().ok())
303    {
304        return n.to_vec();
305    }
306    // docling-parse's `base_font` keeps the subset prefix (`ABCDEF+Helvetica`):
307    // two subsets of one face are two fonts there, and so here.
308    if let Some(n) = fdict.get(b"BaseFont").ok().and_then(|o| o.as_name().ok()) {
309        return n.to_vec();
310    }
311    b"null".to_vec()
312}
313
314/// Parse one font dictionary into a [`Font`].
315fn parse_font(doc: &Document, name: &[u8], fdict: &Dictionary) -> Font {
316    let subtype: &[u8] = fdict
317        .get(b"Subtype")
318        .ok()
319        .and_then(|o| o.as_name().ok())
320        .unwrap_or(&[]);
321    let two_byte = subtype == b"Type0".as_slice();
322
323    let mut to_unicode = fdict
324        .get(b"ToUnicode")
325        .ok()
326        .and_then(|o| deref(doc, o))
327        .and_then(|o| o.as_stream().ok())
328        .and_then(|s| s.decompressed_content().ok())
329        .map(|data| parse_tounicode(&data))
330        .unwrap_or_default();
331    if !two_byte {
332        if let Some(diffs) = differences_array(doc, fdict) {
333            reconcile_stale_tounicode(&mut to_unicode, &diffs);
334        }
335    }
336
337    let cid_fallback = if two_byte {
338        cid_fallback(doc, fdict)
339    } else {
340        CidFallback::None
341    };
342
343    let (mut widths, mut default_width) = if two_byte {
344        cid_widths(doc, fdict)
345    } else {
346        simple_widths(doc, fdict)
347    };
348
349    let simple_encoding = if two_byte {
350        None
351    } else {
352        Some(simple_encoding_table(doc, fdict))
353    };
354
355    // A standard-14 font referenced without an embedded program usually ships
356    // no `/Widths` and no `/FontDescriptor` either (ReportLab's default, #187).
357    // Every advance then resolved to 0, the cells collapsed to zero width, and
358    // the page's whole text layer was silently dropped — while pdfium, with
359    // its built-in metrics, reads the same file fine. Fill the widths from the
360    // built-in Adobe Core 14 AFM tables via the font's own code→char decode
361    // (base encoding + `/Differences`), so an explicit `/Widths` always wins.
362    if !two_byte && widths.is_empty() && default_width == 0.0 {
363        if let Some(std14) = base_font_name(fdict).and_then(|n| crate::std14::widths_for(&n)) {
364            if let Some(enc) = &simple_encoding {
365                for (&code, &ch) in enc {
366                    if let Some(w) = std14.width(ch) {
367                        widths.insert(u32::from(code), w);
368                    }
369                }
370            }
371            // Codes the table misses still advance a typical width instead of
372            // stacking at x=0 (the failure mode this whole branch fixes).
373            default_width = 500.0;
374        }
375    }
376    let fallback_names = if two_byte {
377        HashMap::new()
378    } else {
379        differences_gid_names(doc, fdict)
380    };
381    let program_encoding = if two_byte {
382        HashMap::new()
383    } else {
384        type1_program_encoding(doc, fdict)
385    };
386
387    let (ascent, descent) = font_ascent_descent(doc, fdict, two_byte);
388    let sanitizer_name = sanitizer_font_name(doc, fdict, two_byte);
389    let mut font = Font {
390        two_byte,
391        to_unicode,
392        cid_fallback,
393        widths,
394        default_width,
395        simple_encoding,
396        fallback_names,
397        blank_codes: HashSet::new(),
398        program_encoding,
399        ascent,
400        descent,
401        hash: hash_name(&sanitizer_name),
402        math: is_math_font_name(&sanitizer_name),
403        style: crate::font_style::parse_font_style(&String::from_utf8_lossy(
404            &base_font_name(fdict).unwrap_or_else(|| name.to_vec()),
405        )),
406    };
407    font.blank_codes = blank_codes(doc, fdict, &font);
408    font
409}
410
411/// The codes `font` leaves unmapped whose embedded glyph paints nothing
412/// ([`Font::blank_codes`]). The program is loaded only when the font has
413/// unmapped codes among those it declares widths for; without the
414/// renderer (the wasm text-layer build) there is no program to ask, and
415/// every unmapped glyph reads U+FFFD.
416#[cfg(feature = "render")]
417fn blank_codes(doc: &Document, fdict: &Dictionary, font: &Font) -> HashSet<u32> {
418    // A CID font can declare tens of thousands of widths: past this many
419    // unmapped codes nothing is classified (the glyphs read U+FFFD).
420    const MAX_CANDIDATES: usize = 4096;
421    let unmapped: Vec<u32> = font
422        .widths
423        .keys()
424        .copied()
425        .filter(|&c| font.decode_code(c).0.as_deref() == Some("\u{FFFD}"))
426        .take(MAX_CANDIDATES + 1)
427        .collect();
428    if unmapped.is_empty() || unmapped.len() > MAX_CANDIDATES {
429        return HashSet::new();
430    }
431    let loaded = crate::render::font::LoadedFont::load(doc, fdict);
432    unmapped
433        .into_iter()
434        .filter(|&c| loaded.glyph_has_ink(c) == Some(false))
435        .collect()
436}
437
438#[cfg(not(feature = "render"))]
439fn blank_codes(_doc: &Document, _fdict: &Dictionary, _font: &Font) -> HashSet<u32> {
440    HashSet::new()
441}
442
443#[derive(Clone, Copy)]
444enum CidFallback {
445    None,
446    UnicodeCodes { utf16: bool },
447    Collection(&'static [u8]),
448}
449
450/// A Unicode CMap code retains its original bytes for ToUnicode lookups;
451/// a four-byte UTF-16 code is a packed surrogate pair, not a scalar value.
452pub(crate) fn unicode_cmap_char(code: u32) -> Option<char> {
453    if code <= 0xffff {
454        char::from_u32(code)
455    } else {
456        let units = [(code >> 16) as u16, code as u16];
457        let mut chars = char::decode_utf16(units);
458        let ch = chars.next()?.ok()?;
459        chars.next().is_none().then_some(ch)
460    }
461}
462
463#[cfg(not(feature = "cid-collections"))]
464fn collection_table(_ordering: &[u8]) -> Option<&'static [u8]> {
465    None
466}
467
468#[cfg(feature = "cid-collections")]
469fn collection_table(ordering: &[u8]) -> Option<&'static [u8]> {
470    macro_rules! table {
471        ($file:literal) => {
472            include_bytes!(concat!(env!("OUT_DIR"), "/", $file))
473        };
474    }
475    Some(match ordering {
476        b"Japan1" => table!("Adobe-Japan1-UCS2"),
477        b"GB1" => table!("Adobe-GB1-UCS2"),
478        b"CNS1" => table!("Adobe-CNS1-UCS2"),
479        b"Korea1" => table!("Adobe-Korea1-UCS2"),
480        b"KR" => table!("Adobe-KR-UCS2"),
481        _ => return None,
482    })
483}
484
485/// The text of `cid` in a table build.rs wrote: the CID count, then three
486/// bytes per CID (0 for none, a code point, or 0x110000 + the offset of a
487/// length-prefixed UTF-8 string after the entries).
488pub(crate) fn collection_text(table: &[u8], cid: u32) -> Option<String> {
489    let n = u32::from_le_bytes(table.get(..4)?.try_into().ok()?) as usize;
490    let at = 4 + 3 * usize::try_from(cid).ok().filter(|&c| c < n)?;
491    let &[a, b, c] = table.get(at..at + 3)? else {
492        return None;
493    };
494    match u32::from_be_bytes([0, a, b, c]) {
495        0 => None,
496        v @ ..0x110000 => char::from_u32(v).map(String::from),
497        v => {
498            let at = 4 + 3 * n + (v - 0x110000) as usize;
499            let len = *table.get(at)? as usize;
500            std::str::from_utf8(table.get(at + 1..at + 1 + len)?)
501                .ok()
502                .map(str::to_owned)
503        }
504    }
505}
506
507fn cid_fallback(doc: &Document, fdict: &Dictionary) -> CidFallback {
508    let encoding = fdict
509        .get(b"Encoding")
510        .ok()
511        .and_then(|o| deref(doc, o))
512        .and_then(|o| o.as_name().ok())
513        .unwrap_or(&[]);
514    if encoding.starts_with(b"Uni")
515        && (encoding.windows(4).any(|w| w == b"UCS2") || encoding.windows(5).any(|w| w == b"UTF16"))
516    {
517        return CidFallback::UnicodeCodes {
518            utf16: encoding.windows(5).any(|w| w == b"UTF16"),
519        };
520    }
521    if encoding != b"Identity-H" && encoding != b"Identity-V" {
522        // Other CMaps need code → CID tables that are not bundled.
523        return CidFallback::None;
524    }
525    cid_collection(doc, fdict).map_or(CidFallback::None, CidFallback::Collection)
526}
527
528/// The CID → Unicode table of the character collection a Type0 font uses.
529pub(crate) fn cid_collection(doc: &Document, fdict: &Dictionary) -> Option<&'static [u8]> {
530    let Object::Array(descendants) = deref(doc, fdict.get(b"DescendantFonts").ok()?)? else {
531        return None;
532    };
533    let descendant = as_dict(doc, descendants.first()?)?;
534    let info = as_dict(doc, descendant.get(b"CIDSystemInfo").ok()?)?;
535    let registry = deref(doc, info.get(b"Registry").ok()?)?.as_str().ok()?;
536    let ordering = deref(doc, info.get(b"Ordering").ok()?)?.as_str().ok()?;
537    (registry == b"Adobe").then(|| collection_table(ordering))?
538}
539
540/// A simple font's `/Encoding` `/Differences` array, dereferenced.
541fn differences_array(doc: &Document, fdict: &Dictionary) -> Option<Vec<Object>> {
542    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))?;
543    let diffs = enc.as_dict().ok()?.get(b"Differences").ok()?;
544    match deref(doc, diffs)? {
545        Object::Array(a) => Some(a.clone()),
546        _ => None,
547    }
548}
549
550/// A ToUnicode CMap copied from the *original* font while the glyphs were
551/// re-encoded: the encrypted `2206.01062_pg3` fixture (a Quartz re-save of
552/// pdfTeX output) keeps the T1-encoding CMap, keyed on the old codes, next
553/// to a `/Differences [33 /f_f_i /f_i /f_f /f_l]` that names the ligatures at
554/// new ones, so the CMap maps code 33 to `!` — its own identity — and
555/// `difficulties` read `di ! culties`, in docling's pdfium chain too. A
556/// ToUnicode entry that is the code's identity, for a code whose
557/// `/Differences` name resolves to something else, is that stale copy: the
558/// name wins. A CMap that disagrees any other way is left alone — ToUnicode
559/// is the producer's word on what the glyph means.
560fn reconcile_stale_tounicode(to_unicode: &mut HashMap<u32, String>, diffs: &[Object]) {
561    let mut code = 0u8;
562    for el in diffs {
563        match el {
564            Object::Integer(i) => code = *i as u8,
565            Object::Name(name) => {
566                if let Some(ch) = glyph_name_to_char(name) {
567                    let identity = (code as char).to_string();
568                    if ch.to_string() != identity
569                        && to_unicode.get(&u32::from(code)) == Some(&identity)
570                    {
571                        to_unicode.insert(u32::from(code), ch.to_string());
572                    }
573                }
574                code = code.wrapping_add(1);
575            }
576            _ => {}
577        }
578    }
579}
580
581/// Collect `/Differences` entries whose glyph name is a GID placeholder
582/// (`g115`, `cid42`, `glyph7`, `index9`, `SM590000`) with no Unicode mapping
583/// (subsetted fonts with no ToUnicode); [`unknown_name_text`] decides what
584/// each reads as. The placeholder-name restriction keeps real Adobe glyph
585/// names on the normal path so this never invents garbage on the clean files.
586fn differences_gid_names(doc: &Document, fdict: &Dictionary) -> HashMap<u8, String> {
587    let mut map = HashMap::new();
588    let Some(Object::Dictionary(enc)) = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))
589    else {
590        return map;
591    };
592    let Some(Object::Array(diffs)) = enc.get(b"Differences").ok().and_then(|o| deref(doc, o))
593    else {
594        return map;
595    };
596    let mut code = 0u8;
597    for el in diffs {
598        match el {
599            Object::Integer(i) => code = *i as u8,
600            Object::Name(name) => {
601                if glyph_name_to_char(name).is_none() && is_gid_name(name) {
602                    map.insert(code, String::from_utf8_lossy(name).into_owned());
603                }
604                code = code.wrapping_add(1);
605            }
606            _ => {}
607        }
608    }
609    map
610}
611
612/// Parse the embedded Type1 font program's built-in `/Encoding` vector
613/// (`dup <code> /<glyphname> put` entries in the clear-text header before
614/// `eexec`) into `code → char`. This is how docling recovers glyphs from
615/// standard TeX math fonts (CMMI/CMSY/…) that carry no PDF `/Encoding` and no
616/// ToUnicode — e.g. CMMI's `dup 21 /lambda` or CMSY's `dup 20 /lessequal`.
617/// Only `FontFile` (Type1) is parsed; CFF (`FontFile3`) and TrueType
618/// (`FontFile2`) store their encoding in a binary table and are left alone.
619fn type1_program_encoding(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
620    let mut map = HashMap::new();
621    let Some(desc) = fdict
622        .get(b"FontDescriptor")
623        .ok()
624        .and_then(|o| deref(doc, o))
625        .and_then(|o| o.as_dict().ok())
626    else {
627        return map;
628    };
629    let Some(data) = desc
630        .get(b"FontFile")
631        .ok()
632        .and_then(|o| deref(doc, o))
633        .and_then(|o| o.as_stream().ok())
634        .and_then(|s| s.decompressed_content().ok())
635    else {
636        return map;
637    };
638    // The clear-text header (PostScript) ends at `eexec`; the rest is encrypted.
639    let head_end = data
640        .windows(5)
641        .position(|w| w == b"eexec")
642        .unwrap_or(data.len());
643    let head = String::from_utf8_lossy(&data[..head_end]);
644    // Scan for `dup <code> /<name> put` tokens.
645    let toks: Vec<&str> = head.split_whitespace().collect();
646    for w in toks.windows(4) {
647        if w[0] == "dup" && w[3] == "put" {
648            if let (Ok(code), Some(name)) = (w[1].parse::<u32>(), w[2].strip_prefix('/')) {
649                if code <= 255 {
650                    if let Some(ch) = glyph_name_to_char(name.as_bytes()) {
651                        map.insert(code as u8, ch);
652                    }
653                }
654            }
655        }
656    }
657    map
658}
659
660/// The reading text docling-parse 7.24 keeps for a `/Differences` glyph name
661/// no table resolves (`resolve_unknown_name`): the name without any
662/// `.suffix` — `None` when that is a bare glyph index
663/// (`(gid|glyph|g|cid|index)\d+`, any case), which names a glyph of the
664/// font program but carries no text (docling-parse#238), so it is unmapped.
665fn unknown_name_text(name: &str) -> Option<String> {
666    let stem = name.rsplit_once('.').map_or(name, |(s, _)| s);
667    let lower = stem.to_ascii_lowercase();
668    let index = ["gid", "glyph", "g", "cid", "index"].iter().any(|p| {
669        lower
670            .strip_prefix(p)
671            .is_some_and(|d| !d.is_empty() && d.bytes().all(|b| b.is_ascii_digit()))
672    });
673    (!index).then(|| stem.to_string())
674}
675
676/// A glyph name that is a synthetic placeholder, not a real Adobe name:
677/// `g115`, `cid42`, `glyph7`, `index9`, `G12`, or a short-prefix code name like
678/// `SM590000` (IBM BookMaster). These carry no Unicode meaning (docling-parse
679/// keeps the code-like names as text, `SM590000`, and reads the glyph indexes
680/// as unmapped, see [`unknown_name_text`]). `afii####` / `uni####` are real Adobe names
681/// and excluded. The restriction keeps genuine glyph names on the Unicode path.
682fn is_gid_name(name: &[u8]) -> bool {
683    let Ok(s) = std::str::from_utf8(name) else {
684        return false;
685    };
686    if s.starts_with("afii") || s.starts_with("uni") {
687        return false;
688    }
689    for prefix in ["g", "G", "cid", "CID", "glyph", "index"] {
690        if let Some(rest) = s.strip_prefix(prefix) {
691            if !rest.is_empty() && rest.bytes().all(|b| b.is_ascii_digit()) {
692                return true;
693            }
694        }
695    }
696    // Short alpha prefix (≤3 letters) followed by a run of ≥3 digits — synthetic
697    // code names like `SM590000`, distinct from real Adobe names (whole words or
698    // letter+`.suffix` variants).
699    let alpha = s.bytes().take_while(|b| b.is_ascii_alphabetic()).count();
700    let digits = s.len() - alpha;
701    (1..=3).contains(&alpha)
702        && digits >= 3
703        && s.as_bytes()[alpha..].iter().all(|b| b.is_ascii_digit())
704}
705
706fn font_ascent_descent(doc: &Document, fdict: &Dictionary, two_byte: bool) -> (f64, f64) {
707    // For Type0, the descriptor lives on the descendant CIDFont.
708    let descr_owner = if two_byte {
709        fdict
710            .get(b"DescendantFonts")
711            .ok()
712            .and_then(|o| deref(doc, o))
713            .and_then(|o| match o {
714                Object::Array(a) => a.first(),
715                _ => None,
716            })
717            .and_then(|o| as_dict(doc, o))
718    } else {
719        Some(fdict)
720    };
721    let fd = descr_owner
722        .and_then(|d| d.get(b"FontDescriptor").ok())
723        .and_then(|o| as_dict(doc, o));
724    let asc = fd
725        .and_then(|d| d.get(b"Ascent").ok())
726        .and_then(|o| {
727            o.as_float()
728                .ok()
729                .or_else(|| o.as_i64().ok().map(|i| i as f32))
730        })
731        .unwrap_or(750.0) as f64;
732    let desc = fd
733        .and_then(|d| d.get(b"Descent").ok())
734        .and_then(|o| {
735            o.as_float()
736                .ok()
737                .or_else(|| o.as_i64().ok().map(|i| i as f32))
738        })
739        .unwrap_or(-250.0) as f64;
740    // Some subsetted fonts carry a degenerate FontDescriptor (`/Ascent 0
741    // /Descent 0`) — the real metrics live in the font program. That collapses
742    // the loose box to zero height, so the line cells get zero area and the
743    // layout's region/text assignment drops them (2305's References list lost
744    // every prose line, keeping only the URLs). Fall back to typical text metrics
745    // so the box has height.
746    if asc - desc <= 1.0 {
747        return (750.0, -250.0);
748    }
749    (asc, desc)
750}
751
752/// The `/BaseFont` name with any `ABCDEF+` subset prefix stripped (#187).
753fn base_font_name(fdict: &Dictionary) -> Option<Vec<u8>> {
754    let name = fdict.get(b"BaseFont").ok()?.as_name().ok()?;
755    let stripped = match name.iter().position(|&b| b == b'+') {
756        Some(i) if i == 6 => &name[i + 1..],
757        _ => name,
758    };
759    Some(stripped.to_vec())
760}
761
762/// Simple-font widths: `/FirstChar` + `/Widths` array, `/MissingWidth` default.
763fn simple_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
764    let mut map = HashMap::new();
765    let first = fdict
766        .get(b"FirstChar")
767        .ok()
768        .and_then(|o| o.as_i64().ok())
769        .unwrap_or(0) as u32;
770    if let Some(Object::Array(arr)) = fdict.get(b"Widths").ok().and_then(|o| deref(doc, o)) {
771        for (i, w) in arr.iter().enumerate() {
772            if let Some(w) = num(w) {
773                map.insert(first + i as u32, w);
774            }
775        }
776    }
777    let dw = fdict
778        .get(b"FontDescriptor")
779        .ok()
780        .and_then(|o| as_dict(doc, o))
781        .and_then(|d| d.get(b"MissingWidth").ok())
782        .and_then(num)
783        .unwrap_or(0.0);
784    (map, dw)
785}
786
787/// CIDFont widths: the `/W` array on the descendant font (`/DW` default = 1000).
788fn cid_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
789    let mut map = HashMap::new();
790    let Some(desc) = fdict
791        .get(b"DescendantFonts")
792        .ok()
793        .and_then(|o| deref(doc, o))
794        .and_then(|o| match o {
795            Object::Array(a) => a.first(),
796            _ => None,
797        })
798        .and_then(|o| as_dict(doc, o))
799    else {
800        return (map, 1000.0);
801    };
802    let dw = desc.get(b"DW").ok().and_then(num).unwrap_or(1000.0);
803    if let Some(Object::Array(w)) = desc.get(b"W").ok().and_then(|o| deref(doc, o)) {
804        let mut i = 0;
805        while i < w.len() {
806            let c = w.get(i).and_then(num);
807            match (c, w.get(i + 1)) {
808                // `c [w1 w2 ...]`: consecutive CIDs starting at c.
809                (Some(c), Some(Object::Array(list))) => {
810                    for (k, wv) in list.iter().enumerate() {
811                        if let Some(wv) = num(wv) {
812                            map.insert(c as u32 + k as u32, wv);
813                        }
814                    }
815                    i += 2;
816                }
817                // `c_first c_last w`: a run all of width w.
818                (Some(c1), Some(o2)) => {
819                    if let (Some(c2), Some(wv)) = (num(o2), w.get(i + 2).and_then(num)) {
820                        for cid in c1 as u32..=c2 as u32 {
821                            map.insert(cid, wv);
822                        }
823                    }
824                    i += 3;
825                }
826                _ => break,
827            }
828        }
829    }
830    (map, dw)
831}
832
833fn num(o: &Object) -> Option<f64> {
834    match o {
835        Object::Integer(i) => Some(*i as f64),
836        Object::Real(r) => Some(*r as f64),
837        _ => None,
838    }
839}
840
841/// Decode a PDF string literal into codes, retaining packed UTF-16 pairs
842/// as the original four-byte keys used by ToUnicode.
843fn codes(font: &Font, bytes: &[u8]) -> Vec<u32> {
844    if font.two_byte {
845        let mut units = bytes
846            .chunks(2)
847            .map(|c| {
848                if c.len() == 2 {
849                    ((c[0] as u32) << 8) | c[1] as u32
850                } else {
851                    c[0] as u32
852                }
853            })
854            .peekable();
855        let utf16 = matches!(font.cid_fallback, CidFallback::UnicodeCodes { utf16: true });
856        let mut out = Vec::with_capacity(bytes.len() / 2 + 1);
857        while let Some(mut code) = units.next() {
858            if utf16 && (0xd800..=0xdbff).contains(&code) {
859                if let Some(low) = units.next_if(|u| (0xdc00..=0xdfff).contains(u)) {
860                    code = (code << 16) | low;
861                }
862            }
863            out.push(code);
864        }
865        out
866    } else {
867        bytes.iter().map(|&b| b as u32).collect()
868    }
869}
870
871/// A page's display box, in PDF user space: the `/CropBox` clipped to the
872/// `/MediaBox`, both inherited through the page tree and normalized — a
873/// missing or empty MediaBox is US Letter, an empty CropBox is the MediaBox
874/// (pdfium's `CPDF_Page::UpdateDimensions`). pdfium reports the page size from
875/// this box, renders exactly it, and translates every content coordinate so
876/// its lower-left corner is the origin (`m_PageMatrix`); docling's backends
877/// inherit that frame, so text cells, `prov` boxes and destinations all count
878/// from the CropBox corner, not the MediaBox one. The parser used to flip
879/// glyphs with the MediaBox *height* and no translation at all, so a page
880/// whose boxes do not start at (0, 0) — a trimmed book page with
881/// `MediaBox [-56 -58 576 723]` / `CropBox [1 -0.6 519 666]`, or a LaTeX
882/// figure cropped to `[156 147 637 391]` — had its text displaced against the
883/// rendered bitmap by the box offset, the bottom lines pushed past the page
884/// edge and clamped to `t = b`.
885#[derive(Debug, Clone, Copy, PartialEq)]
886pub(crate) struct PageBox {
887    /// Left edge, user space.
888    pub l: f32,
889    /// Bottom edge, user space.
890    pub b: f32,
891    pub w: f32,
892    pub h: f32,
893}
894
895impl PageBox {
896    /// Top edge, user space — the y that becomes `0` in the y-down frame.
897    pub fn top(&self) -> f32 {
898        self.b + self.h
899    }
900}
901
902/// A page-tree rect attribute (`/MediaBox`, `/CropBox`), inherited from the
903/// nearest ancestor that sets it, as normalized `(l, b, r, t)`.
904fn inherited_rect(
905    doc: &Document,
906    page_id: lopdf::ObjectId,
907    key: &[u8],
908) -> Option<(f32, f32, f32, f32)> {
909    let mut id = page_id;
910    for _ in 0..32 {
911        let dict = doc.get_object(id).ok()?.as_dict().ok()?;
912        if let Some(Object::Array(a)) = dict.get(key).ok().and_then(|o| deref(doc, o)) {
913            let v: Vec<f32> = a.iter().filter_map(|o| num(o).map(|x| x as f32)).collect();
914            if v.len() == 4 && v.iter().all(|x| x.is_finite()) {
915                return Some((
916                    v[0].min(v[2]),
917                    v[1].min(v[3]),
918                    v[0].max(v[2]),
919                    v[1].max(v[3]),
920                ));
921            }
922            return None;
923        }
924        id = dict.get(b"Parent").ok()?.as_reference().ok()?;
925    }
926    None
927}
928
929pub(crate) fn page_box(doc: &Document, page_id: lopdf::ObjectId) -> PageBox {
930    let nonempty = |r: &(f32, f32, f32, f32)| r.2 > r.0 && r.3 > r.1;
931    let media = inherited_rect(doc, page_id, b"MediaBox")
932        .filter(nonempty)
933        .unwrap_or((0.0, 0.0, 612.0, 792.0));
934    let crop = inherited_rect(doc, page_id, b"CropBox")
935        .map(|c| {
936            (
937                c.0.max(media.0),
938                c.1.max(media.1),
939                c.2.min(media.2),
940                c.3.min(media.3),
941            )
942        })
943        .filter(nonempty)
944        .unwrap_or(media);
945    PageBox {
946        l: crop.0,
947        b: crop.1,
948        w: crop.2 - crop.0,
949        h: crop.3 - crop.1,
950    }
951}
952
953/// Page size (width, height) in PDF points — the display box's, like pdfium's
954/// `FPDF_GetPageWidthF/HeightF`.
955fn page_size(doc: &Document, page_id: lopdf::ObjectId) -> (f32, f32) {
956    let pb = page_box(doc, page_id);
957    (pb.w, pb.h)
958}
959
960/// Localize where a page's text is lost, for the `text_layer` diagnostic.
961/// Extraction can come up empty at three different points — no content stream
962/// reached the parser, the stream did not decode into operators, or it ran but
963/// produced no glyphs (fonts/encodings) — and from the outside all three look
964/// the same. Report them per page.
965pub fn content_diagnosis(bytes: &[u8]) -> String {
966    let Some(doc) = load_document(bytes) else {
967        return "document does not load".into();
968    };
969    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
970    pages.sort_by_key(|(n, _)| *n);
971    let mut out = String::new();
972    let mut caches = DocCaches::default();
973    for (n, pid) in pages.into_iter().take(4) {
974        let content_bytes = doc.get_page_content(pid);
975        let ops = lopdf::content::Content::decode(&content_bytes)
976            .map(|c| c.operations.len())
977            .ok();
978        let res = page_res(&doc, pid);
979        let fonts = res.map(|r| fonts_from_res(&doc, r, &mut caches).len());
980        let glyphs = page_glyphs_cached(&doc, pid, &mut caches).len();
981        out.push_str(&format!(
982            "\n   page {n}: content {} B, ops {}, resources {}, fonts {}, glyphs {}",
983            content_bytes.len(),
984            ops.map_or("UNDECODABLE".to_string(), |n| n.to_string()),
985            if res.is_some() { "ok" } else { "MISSING" },
986            fonts.map_or("-".to_string(), |n| n.to_string()),
987            glyphs,
988        ));
989    }
990    out
991}
992
993/// Is this "text layer" a vestige rather than the document's text?
994///
995/// Scanned forms often carry a handful of typed-in strings — a date filled
996/// into three form fields, say — on top of pages that are otherwise images.
997/// Treating that as a real text layer is the worst of both worlds: the text
998/// path proudly extracts thirteen characters, and no OCR ever runs on the
999/// letter the pages actually show. The reported form did exactly this (3
1000/// lines, 13 chars, 3 pages).
1001///
1002/// The rule is deliberately tight so genuinely sparse *digital* documents are
1003/// not misrouted into OCR: only a document averaging at most one line per page
1004/// **and** totalling fewer than 32 characters is called vestigial.
1005pub fn text_layer_is_vestigial(pages: &[crate::pdfium_backend::PdfPage]) -> bool {
1006    let mut tally = TextLayerTally::default();
1007    pages.iter().for_each(|p| tally.add(p));
1008    tally.is_vestigial()
1009}
1010
1011/// [`text_layer_is_vestigial`]'s counts, accumulated a page at a time so a
1012/// streaming caller need not keep the pages around to decide.
1013#[derive(Debug, Default, Clone, Copy)]
1014pub struct TextLayerTally {
1015    pages: usize,
1016    lines: usize,
1017    chars: usize,
1018}
1019
1020impl TextLayerTally {
1021    pub fn add(&mut self, page: &crate::pdfium_backend::PdfPage) {
1022        self.pages += 1;
1023        self.lines += page.cells.len();
1024        self.chars += page
1025            .cells
1026            .iter()
1027            .map(|c| c.text.chars().count())
1028            .sum::<usize>();
1029    }
1030
1031    /// The verdict over the pages added so far.
1032    pub fn is_vestigial(&self) -> bool {
1033        self.lines == 0 || (self.lines <= self.pages && self.chars < 32)
1034    }
1035
1036    /// Whether the pages added so far already rule vestigial out for a
1037    /// document of `total_pages`, whatever the remaining pages hold — both
1038    /// counts only grow, so a caller may stop parsing pages it does not need.
1039    pub fn proves_text(&self, total_pages: usize) -> bool {
1040        self.lines > 0 && (self.lines > total_pages || self.chars >= 32)
1041    }
1042}
1043
1044/// Why the cross-reference repair did or did not fire, for the `text_layer`
1045/// diagnostic. A PDF that will not load is indistinguishable from a scan in
1046/// production (both convert to nothing), so the reason has to be askable.
1047pub fn xref_repair_status(bytes: &[u8]) -> String {
1048    if Document::load_mem(bytes).is_ok() {
1049        return "loads unaided; no repair needed".into();
1050    }
1051    match pad_short_xref_entries(bytes) {
1052        Ok(fixed) => match Document::load_mem(&fixed) {
1053            Ok(_) => "repaired: cross-reference entries padded to 20 bytes".into(),
1054            Err(e) => format!("padded the entries, but it still will not load: {e}"),
1055        },
1056        Err(why) => format!("repair declined — {why}"),
1057    }
1058}
1059
1060/// Load a PDF, repairing the one malformation that otherwise costs us the whole
1061/// document: **19-byte cross-reference entries**.
1062///
1063/// The spec fixes an xref entry at 20 bytes — `nnnnnnnnnn ggggg n` plus a
1064/// *two*-byte EOL. Some generators (an Austrian telecom's invoices, for one)
1065/// emit a bare LF instead, making each entry 19 bytes. lopdf rejects the file
1066/// outright (`invalid file trailer`) where pdfium reads it happily, so a
1067/// perfectly good text layer looked to the browser exactly like a scan and cost
1068/// ten seconds of OCR.
1069///
1070/// Padding is only attempted when it cannot move anything the xref points at:
1071/// a single `xref` section that begins after the last object. The repair then
1072/// has to prove itself — the padded bytes are used only if they load — so a
1073/// mis-repair degrades to today's behaviour rather than to silent garbage.
1074pub(crate) fn load_document(bytes: &[u8]) -> Option<Document> {
1075    open_document(bytes, None).ok()
1076}
1077
1078/// Why [`open_document`] could not hand back a readable document.
1079#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1080pub(crate) enum OpenError {
1081    /// lopdf cannot read the file even after the repairs.
1082    Unreadable,
1083    /// The file is encrypted and the password given (or none) does not open it.
1084    Password,
1085}
1086
1087/// Load `bytes` with the document's password. lopdf decrypts while reading —
1088/// with `password`, or the empty user password most "protected" PDFs carry
1089/// (what every viewer opens silently) — so every reader downstream sees plain
1090/// streams; a file the password does not open loads with its streams still
1091/// encrypted, which is reported as [`OpenError::Password`] rather than handed
1092/// on as a document whose every stream decodes to nothing.
1093pub(crate) fn open_document(bytes: &[u8], password: Option<&str>) -> Result<Document, OpenError> {
1094    let Some(doc) = load_document_raw(bytes, password) else {
1095        // lopdf refuses to load at all under a *wrong* password (a missing
1096        // one loads the file with its streams still encrypted); tell the two
1097        // apart by loading without it.
1098        if password.is_some() && load_document_raw(bytes, None).is_some_and(|d| d.is_encrypted()) {
1099            return Err(OpenError::Password);
1100        }
1101        return Err(OpenError::Unreadable);
1102    };
1103    if doc.is_encrypted() {
1104        return Err(OpenError::Password);
1105    }
1106    Ok(doc)
1107}
1108
1109fn load_options(password: Option<&str>) -> lopdf::LoadOptions {
1110    lopdf::LoadOptions {
1111        password: password.map(str::to_string),
1112        ..lopdf::LoadOptions::default()
1113    }
1114}
1115
1116fn load_document_raw(bytes: &[u8], password: Option<&str>) -> Option<Document> {
1117    // Try progressively more repair, and accept a candidate only once the pages
1118    // actually carry content — a document whose streams were dropped still
1119    // "loads", so loading alone is not evidence the repair helped. A
1120    // well-formed file returns on the first attempt and pays for nothing.
1121    let mut fallback = None;
1122    if let Some(doc) = best_effort_load(bytes, password, &mut fallback) {
1123        return Some(doc);
1124    }
1125    let xref_fixed = pad_short_xref_entries(bytes).ok();
1126    if let Some(fixed) = &xref_fixed {
1127        if let Some(doc) = best_effort_load(fixed, password, &mut fallback) {
1128            return Some(doc);
1129        }
1130    }
1131    // Both defects can coexist, and the second only becomes visible once the
1132    // first is repaired, so build on whatever the previous step produced.
1133    let lengths_fixed = fix_stream_lengths(xref_fixed.as_deref().unwrap_or(bytes));
1134    if let Some(doc) = best_effort_load(&lengths_fixed, password, &mut fallback) {
1135        return Some(doc);
1136    }
1137    fallback
1138}
1139
1140/// Load `data`, returning it only when its pages carry content; a document that
1141/// merely parses is remembered as the fallback for when nothing does better.
1142fn best_effort_load(
1143    data: &[u8],
1144    password: Option<&str>,
1145    fallback: &mut Option<Document>,
1146) -> Option<Document> {
1147    match Document::load_mem_with_options(data, load_options(password)) {
1148        Ok(doc) if has_page_content(&doc) => Some(doc),
1149        Ok(doc) => {
1150            fallback.get_or_insert(doc);
1151            None
1152        }
1153        Err(_) => None,
1154    }
1155}
1156
1157/// Does any page actually hand us a content stream? A document whose streams
1158/// were dropped still parses — it simply has nothing to read — so this is what
1159/// tells a successful repair from a pointless one.
1160fn has_page_content(doc: &Document) -> bool {
1161    doc.get_pages()
1162        .into_values()
1163        .take(4)
1164        .any(|pid| !doc.get_page_content(pid).is_empty())
1165}
1166
1167/// Correct `/Length` values that disagree with where `endstream` actually is.
1168///
1169/// The same generator that writes short xref entries also overstates its
1170/// content-stream lengths by a byte or two. lopdf trusts `/Length`, reads past
1171/// the data, fails to find `endstream` there and drops the stream — the object
1172/// comes back as a bare dictionary, so the page has no content at all and the
1173/// document looks like a scan. pdfium instead trusts `endstream`, which is what
1174/// this does.
1175///
1176/// The rewrite is length-preserving: the corrected number is written over the
1177/// old digits and padded with spaces, so every byte offset in the file — and
1178/// therefore the whole cross-reference table — stays valid.
1179fn fix_stream_lengths(bytes: &[u8]) -> Vec<u8> {
1180    let mut out = bytes.to_vec();
1181    let mut i = 0;
1182    while let Some(rel) = find(&out[i..], b"stream") {
1183        let kw = i + rel;
1184        i = kw + 6;
1185        // Skip `endstream` (the keyword we are measuring *to*).
1186        if kw >= 3 && &out[kw - 3..kw] == b"end" {
1187            continue;
1188        }
1189        // The stream data starts after the EOL that follows the keyword.
1190        let mut data = kw + 6;
1191        if out.get(data..data + 2) == Some(b"\r\n".as_slice()) {
1192            data += 2;
1193        } else if matches!(out.get(data), Some(b'\n' | b'\r')) {
1194            data += 1;
1195        }
1196        let Some(end) = find(&out[data..], b"endstream").map(|r| data + r) else {
1197            continue;
1198        };
1199        // `/Length <digits>` in the dictionary just before the keyword.
1200        let dict_start = out[..kw].iter().rposition(|&c| c == b'<').unwrap_or(0);
1201        let Some(lrel) = find(&out[dict_start..kw], b"/Length") else {
1202            continue;
1203        };
1204        let mut d = dict_start + lrel + 7;
1205        while matches!(out.get(d), Some(b' ')) {
1206            d += 1;
1207        }
1208        let digits = out[d..].iter().take_while(|c| c.is_ascii_digit()).count();
1209        if digits == 0 {
1210            continue;
1211        }
1212        let declared: usize = match std::str::from_utf8(&out[d..d + digits])
1213            .ok()
1214            .and_then(|s| s.parse().ok())
1215        {
1216            Some(v) => v,
1217            None => continue,
1218        };
1219        let actual = end - data;
1220        // Only shrink, and only when the new value fits the space the old one
1221        // occupied — growing the number would move every following byte.
1222        let replacement = actual.to_string();
1223        if actual == declared || replacement.len() > digits {
1224            continue;
1225        }
1226        out[d..d + digits].fill(b' ');
1227        out[d..d + replacement.len()].copy_from_slice(replacement.as_bytes());
1228    }
1229    out
1230}
1231
1232fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
1233    haystack.windows(needle.len()).position(|w| w == needle)
1234}
1235
1236/// Rewrite a classic cross-reference table's entries to the spec's 20 bytes,
1237/// or `None` when the file's shape makes that unsafe (see [`load_document`]).
1238pub(crate) fn pad_short_xref_entries(bytes: &[u8]) -> Result<Vec<u8>, &'static str> {
1239    // Exactly one xref section, and it must start after every object, so that
1240    // growing it shifts nothing the table's offsets refer to.
1241    let is_boundary = |i: usize| i == 0 || matches!(bytes[i - 1], b'\n' | b'\r');
1242    let mut starts = (0..bytes.len().saturating_sub(4))
1243        .filter(|&i| &bytes[i..i + 4] == b"xref" && is_boundary(i));
1244    let xref_at = starts
1245        .next()
1246        .ok_or("no classic `xref` section (an xref stream?)")?;
1247    if starts.next().is_some() {
1248        return Err("more than one xref section (incremental update)");
1249    }
1250    let last_obj = bytes
1251        .windows(3)
1252        .rposition(|w| w == b"obj")
1253        .ok_or("no objects found")?;
1254    if last_obj > xref_at {
1255        return Err("an object follows the xref — padding would move it");
1256    }
1257
1258    let mut out = bytes[..xref_at].to_vec();
1259    out.extend_from_slice(b"xref\n");
1260    let mut i = xref_at + 4;
1261    let skip_ws = |i: &mut usize| {
1262        while matches!(bytes.get(*i), Some(b'\r' | b'\n' | b' ')) {
1263            *i += 1;
1264        }
1265    };
1266    loop {
1267        skip_ws(&mut i);
1268        // Either the next subsection header ("first count") or the trailer.
1269        if bytes[i..].starts_with(b"trailer") {
1270            out.extend_from_slice(&bytes[i..]);
1271            return Ok(out);
1272        }
1273        let header_end = i + bytes[i..]
1274            .iter()
1275            .position(|c| matches!(c, b'\n' | b'\r'))
1276            .ok_or("subsection header runs off the end")?;
1277        let header = std::str::from_utf8(&bytes[i..header_end])
1278            .map_err(|_| "subsection header is not text")?
1279            .trim();
1280        let mut parts = header.split_whitespace();
1281        let count: usize = parts
1282            .nth(1)
1283            .and_then(|c| c.parse().ok())
1284            .ok_or("unparseable subsection header")?;
1285        if parts.next().is_some() || count == 0 {
1286            return Err("unexpected subsection header shape");
1287        }
1288        out.extend_from_slice(header.as_bytes());
1289        out.push(b'\n');
1290        i = header_end;
1291        for _ in 0..count {
1292            skip_ws(&mut i);
1293            // `nnnnnnnnnn ggggg n` — the 18 bytes before whatever EOL follows.
1294            let entry = bytes.get(i..i + 18).ok_or("xref entry runs off the end")?;
1295            let well_formed = entry[..10].iter().all(u8::is_ascii_digit)
1296                && entry[10] == b' '
1297                && entry[11..16].iter().all(u8::is_ascii_digit)
1298                && entry[16] == b' '
1299                && matches!(entry[17], b'n' | b'f');
1300            if !well_formed {
1301                return Err("xref entry is not `nnnnnnnnnn ggggg n`");
1302            }
1303            out.extend_from_slice(entry);
1304            out.extend_from_slice(b" \n"); // the spec's 2-byte EOL -> 20 bytes
1305            i += 18;
1306        }
1307    }
1308}
1309
1310/// Debug: raw glyph stream `(ch, ll, lr, lb, lt)` (native coords) for page
1311/// `index`, before the sanitizer. For comparing char cells to docling-parse.
1312pub fn debug_glyphs(bytes: &[u8], index: usize) -> Vec<(char, f32, f32, f32, f32)> {
1313    let Some(doc) = load_document(bytes) else {
1314        return Vec::new();
1315    };
1316    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1317    pages.sort_by_key(|(n, _)| *n);
1318    let Some((_, pid)) = pages.get(index) else {
1319        return Vec::new();
1320    };
1321    page_glyphs(&doc, *pid)
1322        .into_iter()
1323        .map(|g| (g.ch, g.ll, g.lr, g.lb, g.lt))
1324        .collect()
1325}
1326
1327/// Public entry: per-page (width, height, line cells) for a PDF, via the Rust
1328/// text parser + the docling-parse line sanitizer. Used by the pipeline and the
1329/// `textparse_dump` example.
1330pub fn pdf_textlines(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1331    let Some(doc) = load_document(bytes) else {
1332        return Vec::new();
1333    };
1334    let mut caches = DocCaches::default();
1335    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1336    pages.sort_by_key(|(n, _)| *n);
1337    pages
1338        .into_iter()
1339        .map(|(_, pid)| {
1340            let (w, h) = page_size(&doc, pid);
1341            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1342            let cells = crate::dp_lines::line_cells(&glyphs, h, true);
1343            (w, h, cells)
1344        })
1345        .collect()
1346}
1347
1348/// Debug/diagnostic entry: per-page (width, height, word cells) for a PDF, via
1349/// the Rust parser glyphs run through the docling-parse word grouping. Used to
1350/// compare parser word cells against docling-parse's `word_cells` oracle (roadmap
1351/// item 6).
1352pub fn pdf_words(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1353    let Some(doc) = load_document(bytes) else {
1354        return Vec::new();
1355    };
1356    let mut caches = DocCaches::default();
1357    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1358    pages.sort_by_key(|(n, _)| *n);
1359    pages
1360        .into_iter()
1361        .map(|(_, pid)| {
1362            let (w, h) = page_size(&doc, pid);
1363            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1364            let cells = crate::dp_lines::word_cells(&glyphs, h, true);
1365            (w, h, cells)
1366        })
1367        .collect()
1368}
1369
1370/// One page's text cells from the pure-Rust parser: prose line cells, per-word
1371/// cells, and code line cells — all from a single glyph parse. Replaces the
1372/// pdfium text path (roadmap item 6) when the parser drop is enabled.
1373#[derive(Default)]
1374pub struct PageParserCells {
1375    pub prose: Vec<crate::pdfium_backend::TextCell>,
1376    pub words: Vec<crate::pdfium_backend::TextCell>,
1377    pub code: Vec<crate::pdfium_backend::TextCell>,
1378    /// Drawn checkbox squares (#609), in the cells' frame.
1379    pub checkboxes: Vec<crate::checkbox::CheckBox>,
1380    /// Image placements (#999), in the cells' frame — see
1381    /// [`crate::pdfium_backend::PdfPage::bitmaps`].
1382    pub bitmaps: Vec<[f32; 4]>,
1383}
1384
1385/// The parser text layer, driven one page at a time: the document is loaded
1386/// (and repaired, see [`load_document`]) once, the font/form caches persist
1387/// across pages, and each page's glyphs are parsed only when asked for.
1388///
1389/// The eager whole-document walk this replaces ran *before* the first page
1390/// was rendered, so on a long PDF it was a serial prefix the page-worker pool
1391/// sat idle through — 6.2 s on the 1913-page .NET reference, in front of a
1392/// pipeline that otherwise overlaps parsing with inference — and a `--pages`
1393/// window still paid for every page in the file. Pulling pages on demand
1394/// keeps the parse on the producer thread but interleaved with rendering,
1395/// and skips unselected pages entirely. Output per page is unchanged: same
1396/// glyph walk, same shared caches, same contraction.
1397pub struct PageTextParser {
1398    doc: Document,
1399    caches: DocCaches,
1400    /// Page object ids in document order (page 1 first).
1401    pages: Vec<lopdf::ObjectId>,
1402    /// How words split where no space glyph does (#921); from
1403    /// `DOCLING_RS_PDF_WORD_GAP` unless [`Self::set_word_gap`] says.
1404    word_gap: crate::dp_lines::WordGap,
1405    /// Whether [`Self::text_page`]'s page keeps typographic punctuation as
1406    /// drawn (#921); `DOCLING_RS_PDF_KEEP_TYPOGRAPHY` unless
1407    /// [`Self::set_keep_typography`] says.
1408    keep_typography: bool,
1409}
1410
1411impl PageTextParser {
1412    /// Load the document; `None` when lopdf cannot read it at all.
1413    pub fn open(bytes: &[u8]) -> Option<Self> {
1414        Self::open_with_password(bytes, None)
1415    }
1416
1417    /// [`open`](Self::open) with the document's password (an encrypted file
1418    /// the password does not open reads as `None` here; the pipeline reports
1419    /// it through [`crate::pdf_meta::PdfMeta::open_with_password`] first).
1420    pub fn open_with_password(bytes: &[u8], password: Option<&str>) -> Option<Self> {
1421        let doc = open_document(bytes, password).ok()?;
1422        let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1423        pages.sort_by_key(|(n, _)| *n);
1424        Some(Self {
1425            doc,
1426            caches: DocCaches::default(),
1427            pages: pages.into_iter().map(|(_, pid)| pid).collect(),
1428            word_gap: crate::dp_lines::WordGap::from_env(),
1429            keep_typography: crate::assemble::keep_typography_from_env(),
1430        })
1431    }
1432
1433    /// The word-gap rule the line and word cells are built with (#921);
1434    /// `None` keeps the environment's (`DOCLING_RS_PDF_WORD_GAP`, else
1435    /// `adaptive`).
1436    pub fn set_word_gap(&mut self, gap: Option<crate::dp_lines::WordGap>) {
1437        if let Some(g) = gap {
1438            self.word_gap = g;
1439        }
1440    }
1441
1442    /// Whether the pages [`Self::text_page`] builds keep typographic
1443    /// punctuation as drawn (#921); `None` keeps the environment's
1444    /// (`DOCLING_RS_PDF_KEEP_TYPOGRAPHY`, else folding).
1445    pub fn set_keep_typography(&mut self, keep: Option<bool>) {
1446        if let Some(k) = keep {
1447            self.keep_typography = k;
1448        }
1449    }
1450
1451    /// The glyph boxes and font styles of the 0-based page `index` — the
1452    /// heading-hierarchy stage's style signal (#302): every non-space glyph's
1453    /// box (font ascent + descent at its size — the font-size proxy pdfium's
1454    /// loose char box also gave) in top-left coordinates, with the weight
1455    /// class and slant its `/BaseFont` name declares. Empty for a page
1456    /// without a text layer (a scan), and the stage falls back to its other
1457    /// signals.
1458    pub(crate) fn glyph_styles(
1459        &mut self,
1460        index: usize,
1461    ) -> Vec<crate::heading_hierarchy::GlyphStyle> {
1462        let Some(&pid) = self.pages.get(index) else {
1463            return Vec::new();
1464        };
1465        let (_w, h) = page_size(&self.doc, pid);
1466        let glyphs = page_glyphs_cached(&self.doc, pid, &mut self.caches);
1467        // Font hash → style, over the fonts the walk just parsed (an inline,
1468        // uncached font dictionary reads as unstyled).
1469        let styles: HashMap<u64, crate::font_style::FontStyle> = self
1470            .caches
1471            .fonts
1472            .values()
1473            .map(|f| (f.hash, f.style))
1474            .collect();
1475        glyphs
1476            .iter()
1477            .filter(|g| !g.ch.is_whitespace() && g.ll.is_finite())
1478            .map(|g| {
1479                let st = styles.get(&g.font).copied().unwrap_or_default();
1480                crate::heading_hierarchy::GlyphStyle {
1481                    l: g.ll,
1482                    t: h - g.lt,
1483                    r: g.lr,
1484                    b: h - g.lb,
1485                    height: g.height(),
1486                    weight_cls: crate::font_style::weight_class(st.weight),
1487                    italic: st.italic,
1488                    styled: st.known,
1489                }
1490            })
1491            .collect()
1492    }
1493
1494    /// The characters of the 0-based page `index` as the script-position
1495    /// stage measures them (#1005, docling's `read_page_chars`): every
1496    /// visible, non-blank glyph with its font-metric box, baseline and drawn
1497    /// size in the unrotated page frame (y up), and beside it the same box in
1498    /// the display frame (`/Rotate` applied, top-left origin: the frame the
1499    /// items' boxes live in). Invisible text (`Tr` 3 / 7, an OCR layer over a
1500    /// scan) is left out, as docling leaves it: it follows the scanned lines,
1501    /// not a typeset baseline.
1502    pub(crate) fn script_chars(
1503        &mut self,
1504        index: usize,
1505    ) -> Vec<(crate::script_position::ScriptChar, [f32; 4])> {
1506        let Some(&pid) = self.pages.get(index) else {
1507            return Vec::new();
1508        };
1509        let (w, h) = page_size(&self.doc, pid);
1510        let rotation = page_rotation(&self.doc, pid);
1511        page_glyphs_cached(&self.doc, pid, &mut self.caches)
1512            .iter()
1513            .filter(|g| !g.invisible && !g.ch.is_whitespace() && g.ll.is_finite())
1514            .filter_map(|g| {
1515                let [_, y, size] = g.pen?;
1516                let c = crate::script_position::ScriptChar {
1517                    text: g.ch,
1518                    left: g.ll,
1519                    bottom: g.lb,
1520                    right: g.lr,
1521                    top: g.lt,
1522                    baseline: y,
1523                    size,
1524                    horizontal: g.quad.is_none(),
1525                };
1526                let (l, t, r, b) = crate::pdfium_backend::to_display_frame(
1527                    (g.ll, h - g.lt, g.lr, h - g.lb),
1528                    rotation,
1529                    w,
1530                    h,
1531                );
1532                Some((c, [l, t, r, b]))
1533            })
1534            .collect()
1535    }
1536
1537    /// Prose, word and code cells of the 0-based page `index` — empty for an
1538    /// index the parser's page tree doesn't have (a damaged file whose page
1539    /// tree disagrees with the object model's count).
1540    pub fn cells(&mut self, index: usize) -> PageParserCells {
1541        let Some(&pid) = self.pages.get(index) else {
1542            return PageParserCells::default();
1543        };
1544        let (_w, h) = page_size(&self.doc, pid);
1545        let (glyphs, inks, bitmaps) = page_glyphs_and_inks(&self.doc, pid, &mut self.caches);
1546        let (prose, words) =
1547            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1548        PageParserCells {
1549            prose,
1550            words,
1551            code: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1552            checkboxes: crate::checkbox::find(&inks, h),
1553            bitmaps,
1554        }
1555    }
1556
1557    /// Pages in the document's page tree.
1558    pub fn page_count(&self) -> usize {
1559        self.pages.len()
1560    }
1561
1562    /// The 0-based page `index` as the text-layer-only conversion sees it —
1563    /// one element of [`pdf_text_pages`], parsed on demand so a caller can
1564    /// drop each page once it is assembled instead of holding the whole
1565    /// document's cells. A blank page (no cells) for an index the page tree
1566    /// doesn't have.
1567    pub fn text_page(&mut self, index: usize) -> crate::pdfium_backend::PdfPage {
1568        let (w, h, (glyphs, inks, _)) = match self.pages.get(index) {
1569            Some(&pid) => {
1570                let (w, h) = page_size(&self.doc, pid);
1571                (w, h, page_glyphs_and_inks(&self.doc, pid, &mut self.caches))
1572            }
1573            None => (0.0, 0.0, Default::default()),
1574        };
1575        let (mut prose, mut words) =
1576            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1577        drop_overpainted_cells(&mut prose);
1578        drop_overpainted_cells(&mut words);
1579        crate::pdfium_backend::PdfPage {
1580            #[cfg(feature = "ocr-prep")]
1581            image_layout: None,
1582            width: w,
1583            height: h,
1584            // Cells are native PDF points; there is no rendered bitmap.
1585            scale: 1.0,
1586            cells: prose,
1587            code_cells: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1588            word_cells: words,
1589            checkboxes: crate::checkbox::find(&inks, h),
1590            bitmaps: Vec::new(),
1591            #[cfg(feature = "ocr-prep")]
1592            image: image::RgbImage::new(1, 1),
1593            links: Vec::new(),
1594            rotation: 0,
1595            keep_typography: self.keep_typography,
1596            #[cfg(feature = "render")]
1597            embedded_images: Vec::new(),
1598            #[cfg(feature = "render")]
1599            pending_image: None,
1600        }
1601    }
1602}
1603
1604/// Full parser text layer: prose + word + code cells per page, glyphs parsed once.
1605/// `prose`/`words` come from the docling-parse contraction ([`crate::dp_lines`]);
1606/// `code` splits only at the parser's own space glyphs (monospace keeps its
1607/// source spacing). The eager form of [`PageTextParser`].
1608pub fn pdf_all_cells(bytes: &[u8]) -> Vec<PageParserCells> {
1609    let Some(mut parser) = PageTextParser::open(bytes) else {
1610        return Vec::new();
1611    };
1612    (0..parser.pages.len()).map(|i| parser.cells(i)).collect()
1613}
1614
1615/// Whole pages for the text-layer-only conversion ([`crate::convert_text_layer`]):
1616/// the parser's prose/word/code cells plus page geometry, assembled into
1617/// [`PdfPage`]s with no rendered image and no link annotations. Everything here
1618/// is pure Rust (lopdf), so it compiles without the `ml` feature — including on
1619/// wasm32. A page the parser can't read (no text layer) comes back with empty
1620/// cells; there is no pdfium fallback on this path.
1621pub fn pdf_text_pages(bytes: &[u8]) -> Vec<crate::pdfium_backend::PdfPage> {
1622    let Some(mut parser) = PageTextParser::open(bytes) else {
1623        return Vec::new();
1624    };
1625    (0..parser.page_count())
1626        .map(|i| parser.text_page(i))
1627        .collect()
1628}
1629
1630/// Drop line cells that are *painted over each other* — glyphs used as artwork.
1631///
1632/// Some generators draw their logo with a symbol font: on the reporting
1633/// invoice, a `TeleLogo` Type1 paints the T-Mobile mark by stacking the glyphs
1634/// encoded as `"` and `==` on top of one another, and the flat text-layer
1635/// output opened with that garbage. Nothing in the font metadata gives it away
1636/// (the *text* fonts in the same file are also flagged symbolic, and the logo
1637/// font names its glyphs `quotedbl` &c.), but the geometry does: two cells with
1638/// different text where one lies inside the other on the same line is
1639/// physically impossible for prose — ink from two words never occupies the
1640/// same box. Both cells of such a pair are paint, not text.
1641///
1642/// Containment (not mere overlap) keeps this narrow: adjacent words touch but
1643/// never contain each other, and a same-text near-duplicate (double-draw faux
1644/// bold) is left alone for the sanitizer's usual handling. Applied on the
1645/// flat/browser path only — the ML pipeline's text layer is byte-pinned by the
1646/// PDF corpus, and there the layout model already sinks logo marks into
1647/// `picture` regions.
1648fn drop_overpainted_cells(cells: &mut Vec<crate::pdfium_backend::TextCell>) {
1649    let mut paint = vec![false; cells.len()];
1650    for i in 0..cells.len() {
1651        for j in 0..cells.len() {
1652            if i == j || cells[i].text == cells[j].text {
1653                continue;
1654            }
1655            let (a, b) = (&cells[i], &cells[j]);
1656            // Same line band: the vertical overlap covers most of the shorter.
1657            let vo = (a.b.min(b.b) - a.t.max(b.t)).max(0.0);
1658            if vo < 0.6 * (a.b - a.t).min(b.b - b.t) {
1659                continue;
1660            }
1661            // `a` horizontally inside `b` (with a small tolerance).
1662            let ho = (a.r.min(b.r) - a.l.max(b.l)).max(0.0);
1663            if ho >= 0.8 * (a.r - a.l) && (a.r - a.l) <= (b.r - b.l) {
1664                paint[i] = true;
1665                paint[j] = true;
1666            }
1667        }
1668    }
1669    let mut keep = paint.iter().map(|p| !p);
1670    cells.retain(|_| keep.next().unwrap());
1671}
1672
1673/// The text-state scalars inherited by a Form XObject when it is invoked via
1674/// `Do` (the PDF graphics state includes the text parameters, but not the text
1675/// matrices, which a form re-establishes inside its own `BT`/`ET`).
1676#[derive(Clone, Copy)]
1677struct TextState {
1678    tc: f64,
1679    tw: f64,
1680    th: f64,
1681    tl: f64,
1682    trise: f64,
1683    fsize: f64,
1684    /// The text rendering mode (`Tr`).
1685    tr: i64,
1686}
1687
1688impl TextState {
1689    const INIT: TextState = TextState {
1690        tc: 0.0,
1691        tw: 0.0,
1692        th: 1.0,
1693        tl: 0.0,
1694        trise: 0.0,
1695        fsize: 0.0,
1696        tr: 0,
1697    };
1698}
1699
1700/// pdfium's `CPDF_Page::GetPageRotation` — the inherited `/Rotate`, in
1701/// quarter turns, as degrees (`pdf_meta`'s rule).
1702fn page_rotation(doc: &Document, pid: lopdf::ObjectId) -> u16 {
1703    let mut id = pid;
1704    for _ in 0..32 {
1705        let Ok(dict) = doc.get_dictionary(id) else {
1706            return 0;
1707        };
1708        if let Some(obj) = dict.get(b"Rotate").ok().and_then(|o| deref(doc, o)) {
1709            let quarter = (num(obj).unwrap_or(0.0) as i64 / 90).rem_euclid(4);
1710            return (quarter * 90) as u16;
1711        }
1712        match dict.get(b"Parent").ok().and_then(|o| o.as_reference().ok()) {
1713            Some(parent) => id = parent,
1714            None => return 0,
1715        }
1716    }
1717    0
1718}
1719
1720/// The effective `/Resources` dictionary for a page (inline or via reference,
1721/// falling back to an inherited one from a `/Parent`).
1722fn page_res(doc: &Document, page_id: lopdf::ObjectId) -> Option<&Dictionary> {
1723    let (inline, ids) = doc.get_page_resources(page_id).ok()?;
1724    if let Some(d) = inline {
1725        return Some(d);
1726    }
1727    ids.into_iter().find_map(|id| doc.get_dictionary(id).ok())
1728}
1729
1730/// Build the code→[`Font`] map for a resources dictionary's `/Font` sub-dict,
1731/// reusing the per-document cache for fonts referenced indirectly (the common
1732/// case — the same font objects recur on every page).
1733fn fonts_from_res(
1734    doc: &Document,
1735    res: &Dictionary,
1736    caches: &mut DocCaches,
1737) -> HashMap<Vec<u8>, Arc<Font>> {
1738    let mut map = HashMap::new();
1739    let font_dict = res
1740        .get(b"Font")
1741        .ok()
1742        .and_then(|o| deref(doc, o))
1743        .and_then(|o| o.as_dict().ok());
1744    if let Some(fd) = font_dict {
1745        for (name, value) in fd.iter() {
1746            let font = match value {
1747                Object::Reference(id) => {
1748                    let key = (*id, name.clone());
1749                    if let Some(f) = caches.fonts.get(&key) {
1750                        Arc::clone(f)
1751                    } else if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
1752                        let f = Arc::new(parse_font(doc, name, fdict));
1753                        caches.fonts.insert(key, Arc::clone(&f));
1754                        f
1755                    } else {
1756                        continue;
1757                    }
1758                }
1759                _ => {
1760                    if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
1761                        Arc::new(parse_font(doc, name, fdict))
1762                    } else {
1763                        continue;
1764                    }
1765                }
1766            };
1767            map.insert(name.clone(), font);
1768        }
1769    }
1770    map
1771}
1772
1773/// Extract every glyph on a page as a native-coordinate [`Glyph`].
1774/// [`PageTextParser::glyph_styles`] for the given **1-based** pages of a
1775/// document, keyed by page number — a separate, on-demand pass (no
1776/// rendering), so the extraction pipeline stays byte-identical whether or not
1777/// the heading-hierarchy stage runs. Empty when lopdf cannot open the file.
1778pub(crate) fn glyph_styles(
1779    bytes: &[u8],
1780    pages: &[usize],
1781) -> HashMap<usize, Vec<crate::heading_hierarchy::GlyphStyle>> {
1782    let mut out = HashMap::new();
1783    let Some(mut parser) = PageTextParser::open(bytes) else {
1784        return out;
1785    };
1786    for &page_no in pages {
1787        if page_no == 0 {
1788            continue;
1789        }
1790        let styles = parser.glyph_styles(page_no - 1);
1791        if !styles.is_empty() {
1792            out.insert(page_no, styles);
1793        }
1794    }
1795    out
1796}
1797
1798pub(crate) fn page_glyphs(doc: &Document, page_id: lopdf::ObjectId) -> Vec<Glyph> {
1799    page_glyphs_cached(doc, page_id, &mut DocCaches::default())
1800}
1801
1802/// [`page_glyphs`] with an explicit per-document cache, so a multi-page walk
1803/// parses each font / decodes each form once instead of once per page.
1804fn page_glyphs_cached(
1805    doc: &Document,
1806    page_id: lopdf::ObjectId,
1807    caches: &mut DocCaches,
1808) -> Vec<Glyph> {
1809    page_glyphs_and_inks(doc, page_id, caches).0
1810}
1811
1812/// [`page_glyphs_cached`] plus the small painted path pieces the same walk
1813/// passed — what [`crate::checkbox::find`] looks for checkbox squares in
1814/// (#609) — and the page's image placements (#999, `[l, t, r, b]` top-left
1815/// page points clipped to the page, the cells' frame).
1816fn page_glyphs_and_inks(
1817    doc: &Document,
1818    page_id: lopdf::ObjectId,
1819    caches: &mut DocCaches,
1820) -> (Vec<Glyph>, Vec<crate::checkbox::Ink>, Vec<[f32; 4]>) {
1821    let mut out = Vec::new();
1822    let mut inks = Vec::new();
1823    // get_page_content returns the assembled content-stream bytes directly
1824    // (an empty Vec when the page has none).
1825    let content_bytes = doc.get_page_content(page_id);
1826    let Ok(content) = lopdf::content::Content::decode(&content_bytes) else {
1827        return (out, inks, Vec::new());
1828    };
1829    let mut bitmaps = Vec::new();
1830    if let Some(res) = page_res(doc, page_id) {
1831        // pdfium's page matrix: user space translated so the display box's
1832        // lower-left corner is the origin (see [`PageBox`]).
1833        let pb = page_box(doc, page_id);
1834        let base = Mat {
1835            e: -(pb.l as f64),
1836            f: -(pb.b as f64),
1837            ..Mat::ID
1838        };
1839        run_content(
1840            doc,
1841            res,
1842            &content,
1843            base,
1844            TextState::INIT,
1845            0,
1846            caches,
1847            &mut out,
1848            &mut inks,
1849        );
1850        out.retain(|g| on_page(g, pb.w, pb.h));
1851        drop_overprinted_spaces(&mut out);
1852        let (w, h) = (f64::from(pb.w), f64::from(pb.h));
1853        inks.retain(|ink| match *ink {
1854            crate::checkbox::Ink::Bitmap { l, b, r, t } => {
1855                let (l, r, b, t) = (l.max(0.0), r.min(w), b.max(0.0), t.min(h));
1856                if r > l && t > b {
1857                    bitmaps.push([l as f32, (h - t) as f32, r as f32, (h - b) as f32]);
1858                }
1859                false
1860            }
1861            _ => true,
1862        });
1863    }
1864    (out, inks, bitmaps)
1865}
1866
1867/// Drop a space glyph the next glyph is drawn over (#992). A typesetter
1868/// that sets a superscript footnote marker as `( 1 )` — space glyphs around
1869/// the raised digit — and then cancels each space's advance with a negative
1870/// `TJ` offset (Arbortext on EUR-Lex: the `1` starts 0.05 pt *before* its
1871/// space begins) renders `(1)`; the spaces exist in the content stream
1872/// only. docling-parse keeps them as characters and its line cell reads
1873/// `( 1 )`; pdftotext and pdfium, which break words by geometry alone,
1874/// read `(1)`, and so does the page. A deliberate divergence
1875/// (`docs/MIGRATION.md` §4 item 22): a space whose successor in stream
1876/// order sits on the same line and starts within the first 30 % of the
1877/// space's advance — the pen went back to (or before) where the space
1878/// began, so it left no gap — is not a character of the text. A space
1879/// followed by a glyph on another line (a trailing space before the
1880/// cursor jumps to the next line's left margin, far to the left of it) or
1881/// by one that starts past that mark (a kerned pair, a justified word gap
1882/// pulled in a little) is kept; the word-gap rules then see the geometry
1883/// as before.
1884fn drop_overprinted_spaces(glyphs: &mut Vec<Glyph>) {
1885    if glyphs.len() < 2 {
1886        return;
1887    }
1888    let overprinted = |sp: &Glyph, next: &Glyph| {
1889        if sp.ch != ' ' || next.ch == ' ' {
1890            return false;
1891        }
1892        // Upright glyphs only: a rotated run's loose boxes are the extents
1893        // of its quads, every glyph of a 90° line shares one left edge and
1894        // the advance runs along y, so the test below would read every
1895        // space of a landscape slide drawn through a 90° `cm` as
1896        // overprinted (`Nullammollis`, odf_presentation_02).
1897        if sp.quad.is_some() || next.quad.is_some() {
1898            return false;
1899        }
1900        let width = sp.lr - sp.ll;
1901        if !width.is_finite() || width <= 0.0 || !next.ll.is_finite() {
1902            return false;
1903        }
1904        // Same line: the loose boxes overlap vertically (a raised digit's box
1905        // still overlaps its neighbours' em boxes).
1906        let same_line = next.lb < sp.lt && next.lt > sp.lb;
1907        same_line && next.ll >= sp.ll - 0.1 * width && next.ll <= sp.ll + 0.3 * width
1908    };
1909    let mut keep = vec![true; glyphs.len()];
1910    for i in 0..glyphs.len() - 1 {
1911        if overprinted(&glyphs[i], &glyphs[i + 1]) {
1912            keep[i] = false;
1913        }
1914    }
1915    let mut it = keep.into_iter();
1916    glyphs.retain(|_| it.next().unwrap_or(true));
1917}
1918
1919/// Whether a glyph is on the page (#529): docling-parse keeps a character
1920/// only when its whole box lies inside the display box — the CropBox, or
1921/// the MediaBox without one — edges included, so a FrameMaker print slug
1922/// drawn beside the CropBox or a tiled page's neighbouring text beyond the
1923/// MediaBox never becomes a cell, and a line crossing the edge is cut at the
1924/// last glyph that fits (`Crossing the rig`). The box is docling-parse's
1925/// char box — the advance by the font's ascent/descent, the loose box here
1926/// (its axis-aligned extent for rotated text) — in the frame `page_glyphs`
1927/// already moved to the display box's corner, so the page is `[0, w] × [0,
1928/// h]`. A glyph without a finite box is kept, as before. (A standard-14 font
1929/// without a FontDescriptor gets the 750 / −250 default ascent / descent
1930/// here where docling-parse reads the AFM's — Helvetica's 718 / −207 — so
1931/// for those a glyph within ~0.04 em of the top or bottom edge can fall the
1932/// other way; horizontally the advance is the same.)
1933fn on_page(g: &Glyph, w: f32, h: f32) -> bool {
1934    // f32 noise only: docling-parse already drops a glyph 0.01 pt over.
1935    const EPS: f32 = 1e-3;
1936    let (l, b, r, t) = if [g.ll, g.lb, g.lr, g.lt].iter().all(|v| v.is_finite()) {
1937        (g.ll, g.lb, g.lr, g.lt)
1938    } else {
1939        (g.l, g.b, g.r, g.t)
1940    };
1941    if ![l, b, r, t].iter().all(|v| v.is_finite()) {
1942        return true;
1943    }
1944    l >= -EPS && b >= -EPS && r <= w + EPS && t <= h + EPS
1945}
1946
1947/// Run a content stream's operators, emitting glyphs into `out`. Recurses into
1948/// Form XObjects on `Do` (bulk body text in heavy PDFs lives inside a form, not
1949/// the page content stream). `res` is the resources dict in scope (the page's,
1950/// or the form's own); `base_ctm` is the CTM at the point of invocation.
1951#[allow(clippy::too_many_arguments)]
1952fn run_content(
1953    doc: &Document,
1954    res: &Dictionary,
1955    content: &lopdf::content::Content,
1956    base_ctm: Mat,
1957    init: TextState,
1958    depth: u32,
1959    caches: &mut DocCaches,
1960    out: &mut Vec<Glyph>,
1961    inks: &mut Vec<crate::checkbox::Ink>,
1962) {
1963    let fonts = fonts_from_res(doc, res, caches);
1964    let xobjects = res
1965        .get(b"XObject")
1966        .ok()
1967        .and_then(|o| deref(doc, o))
1968        .and_then(|o| o.as_dict().ok());
1969
1970    // Graphics + text state. `q`/`Q` save and restore the whole graphics state,
1971    // which includes the text parameters (Tc, Tw, Tz, TL, Tfs, Trise, font) —
1972    // *not* the text matrix (that is reset by BT). Saving only the CTM let a Tc
1973    // set inside a `q…Q` block leak out and drift every later glyph.
1974    #[allow(clippy::type_complexity)]
1975    let mut gstate_stack: Vec<(Mat, f64, f64, f64, f64, f64, f64, i64, Option<&Arc<Font>>)> =
1976        Vec::new();
1977    let mut ctm = base_ctm;
1978    let mut tm = Mat::ID;
1979    let mut tlm = Mat::ID;
1980    let mut font: Option<&Arc<Font>> = None;
1981    let mut fsize = init.fsize;
1982    let mut tc = init.tc; // char spacing
1983    let mut tw = init.tw; // word spacing
1984    let mut th = init.th; // horizontal scale (Tz/100)
1985    let mut tl = init.tl; // leading
1986    let mut trise = init.trise;
1987    let mut tr = init.tr; // rendering mode
1988
1989    let op_f = |operands: &[Object], i: usize| operands.get(i).and_then(num).unwrap_or(0.0);
1990    // The path under construction, already in page space (a path is built
1991    // under one CTM — `cm` is not allowed inside it), for the checkbox
1992    // squares (#609). Text extraction never reads it.
1993    let mut path: Vec<PathPiece> = Vec::new();
1994    let (mut cur, mut start) = ((0.0, 0.0), (0.0, 0.0));
1995    // Whether the fill colour is (near) white, saved/restored with `q`/`Q`:
1996    // a white-filled stroked box is a checkbox outline, a coloured one a
1997    // chart legend's swatch. The initial fill colour is black.
1998    let mut fill_light = false;
1999    let mut fill_stack: Vec<bool> = Vec::new();
2000
2001    for op in &content.operations {
2002        let operands = &op.operands;
2003        let shown = out.len();
2004        match op.operator.as_str() {
2005            "q" => {
2006                gstate_stack.push((ctm, tc, tw, th, tl, trise, fsize, tr, font));
2007                fill_stack.push(fill_light);
2008            }
2009            "Q" => {
2010                fill_light = fill_stack.pop().unwrap_or(false);
2011                if let Some((c, a, b, h, l, r, fs, mode, f)) = gstate_stack.pop() {
2012                    ctm = c;
2013                    tc = a;
2014                    tw = b;
2015                    th = h;
2016                    tl = l;
2017                    trise = r;
2018                    fsize = fs;
2019                    tr = mode;
2020                    font = f;
2021                }
2022            }
2023            "cm" => {
2024                let m = Mat {
2025                    a: op_f(operands, 0),
2026                    b: op_f(operands, 1),
2027                    c: op_f(operands, 2),
2028                    d: op_f(operands, 3),
2029                    e: op_f(operands, 4),
2030                    f: op_f(operands, 5),
2031                };
2032                ctm = m.then(ctm);
2033            }
2034            "BT" => {
2035                tm = Mat::ID;
2036                tlm = Mat::ID;
2037            }
2038            "ET" => {}
2039            "Tf" => {
2040                if let Some(Object::Name(n)) = operands.first() {
2041                    font = fonts.get(n.as_slice());
2042                }
2043                fsize = op_f(operands, 1);
2044            }
2045            "Td" => {
2046                tlm = Mat {
2047                    a: 1.0,
2048                    b: 0.0,
2049                    c: 0.0,
2050                    d: 1.0,
2051                    e: op_f(operands, 0),
2052                    f: op_f(operands, 1),
2053                }
2054                .then(tlm);
2055                tm = tlm;
2056            }
2057            "TD" => {
2058                tl = -op_f(operands, 1);
2059                tlm = Mat {
2060                    a: 1.0,
2061                    b: 0.0,
2062                    c: 0.0,
2063                    d: 1.0,
2064                    e: op_f(operands, 0),
2065                    f: op_f(operands, 1),
2066                }
2067                .then(tlm);
2068                tm = tlm;
2069            }
2070            "Tm" => {
2071                tlm = Mat {
2072                    a: op_f(operands, 0),
2073                    b: op_f(operands, 1),
2074                    c: op_f(operands, 2),
2075                    d: op_f(operands, 3),
2076                    e: op_f(operands, 4),
2077                    f: op_f(operands, 5),
2078                };
2079                tm = tlm;
2080            }
2081            "T*" => {
2082                tlm = Mat {
2083                    a: 1.0,
2084                    b: 0.0,
2085                    c: 0.0,
2086                    d: 1.0,
2087                    e: 0.0,
2088                    f: -tl,
2089                }
2090                .then(tlm);
2091                tm = tlm;
2092            }
2093            "Tc" => tc = op_f(operands, 0),
2094            "Tw" => tw = op_f(operands, 0),
2095            "Tz" => th = op_f(operands, 0) / 100.0,
2096            "TL" => tl = op_f(operands, 0),
2097            "Ts" => trise = op_f(operands, 0),
2098            "Tr" => tr = op_f(operands, 0) as i64,
2099            "Tj" | "'" | "\"" => {
2100                if op.operator == "'" || op.operator == "\"" {
2101                    // move to next line first
2102                    tlm = Mat {
2103                        a: 1.0,
2104                        b: 0.0,
2105                        c: 0.0,
2106                        d: 1.0,
2107                        e: 0.0,
2108                        f: -tl,
2109                    }
2110                    .then(tlm);
2111                    tm = tlm;
2112                }
2113                if op.operator == "\"" {
2114                    // `aw ac string "` sets word- and char-spacing before
2115                    // showing the string (PDF 32000-1 §9.4.3), persisting after.
2116                    tw = op_f(operands, 0);
2117                    tc = op_f(operands, 1);
2118                }
2119                if let (Some(f), Some(Object::String(s, _))) = (font, operands.last()) {
2120                    show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out);
2121                }
2122            }
2123            "TJ" => {
2124                if let (Some(f), Some(Object::Array(arr))) = (font, operands.first()) {
2125                    for el in arr {
2126                        match el {
2127                            Object::String(s, _) => {
2128                                show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out)
2129                            }
2130                            other => {
2131                                if let Some(adj) = num(other) {
2132                                    // negative number moves text right (PDF: subtract)
2133                                    let tx = -adj / 1000.0 * fsize * th;
2134                                    tm = Mat {
2135                                        a: 1.0,
2136                                        b: 0.0,
2137                                        c: 0.0,
2138                                        d: 1.0,
2139                                        e: tx,
2140                                        f: 0.0,
2141                                    }
2142                                    .then(tm);
2143                                }
2144                            }
2145                        }
2146                    }
2147                }
2148            }
2149            "m" => {
2150                cur = ctm.apply(op_f(operands, 0), op_f(operands, 1));
2151                start = cur;
2152            }
2153            "l" => {
2154                let p = ctm.apply(op_f(operands, 0), op_f(operands, 1));
2155                path.push(PathPiece::Line(cur, p));
2156                cur = p;
2157            }
2158            "c" | "v" | "y" => {
2159                let pts: Vec<(f64, f64)> = (0..operands.len() / 2)
2160                    .map(|k| ctm.apply(op_f(operands, 2 * k), op_f(operands, 2 * k + 1)))
2161                    .collect();
2162                if let Some(&end) = pts.last() {
2163                    path.push(PathPiece::Curve(
2164                        std::iter::once(cur).chain(pts.iter().copied()).collect(),
2165                    ));
2166                    cur = end;
2167                }
2168            }
2169            "h" => {
2170                if cur != start {
2171                    path.push(PathPiece::Line(cur, start));
2172                }
2173                cur = start;
2174            }
2175            "re" => {
2176                let (x, y, w, h) = (
2177                    op_f(operands, 0),
2178                    op_f(operands, 1),
2179                    op_f(operands, 2),
2180                    op_f(operands, 3),
2181                );
2182                path.push(PathPiece::Rect([
2183                    ctm.apply(x, y),
2184                    ctm.apply(x + w, y),
2185                    ctm.apply(x + w, y + h),
2186                    ctm.apply(x, y + h),
2187                ]));
2188                cur = ctm.apply(x, y);
2189                start = cur;
2190            }
2191            "S" | "s" | "f" | "F" | "f*" | "B" | "B*" | "b" | "b*" => {
2192                let op = op.operator.as_str();
2193                if matches!(op, "s" | "b" | "b*") && cur != start {
2194                    path.push(PathPiece::Line(cur, start));
2195                }
2196                let stroke = matches!(op, "S" | "s" | "B" | "B*" | "b" | "b*");
2197                let fill = !matches!(op, "S" | "s");
2198                paint_path(&path, stroke, fill, fill_light, inks);
2199                path.clear();
2200            }
2201            "n" => path.clear(),
2202            // The fill colour, by its components: gray `g`, `rg`, CMYK `k`,
2203            // and `sc`/`scn` by operand count (a pattern name is not light).
2204            "g" | "rg" | "k" | "sc" | "scn" => {
2205                let v: Vec<f64> = operands
2206                    .iter()
2207                    .map(num)
2208                    .collect::<Option<_>>()
2209                    .unwrap_or_default();
2210                fill_light = match v.as_slice() {
2211                    [g] => *g >= 0.9,
2212                    [r, g, b] => r.min(*g).min(*b) >= 0.9,
2213                    [c, m, y, k] => c.max(*m).max(*y).max(*k) <= 0.1,
2214                    _ => false,
2215                };
2216            }
2217            "cs" => fill_light = false,
2218            // An inline image: the unit square under the CTM (#999).
2219            "BI" => inks.push(image_ink(ctm)),
2220            "Do" => {
2221                // Invoke a Form XObject: bulk body text in many PDFs lives inside
2222                // a form, reached only here. An image XObject carries no text,
2223                // only its placement (#999).
2224                if depth >= 8 {
2225                    continue;
2226                }
2227                let Some(Object::Name(n)) = operands.first() else {
2228                    continue;
2229                };
2230                let obj = xobjects.and_then(|d| d.get(n.as_slice()).ok());
2231                let form_id = match obj {
2232                    Some(Object::Reference(id)) => Some(*id),
2233                    _ => None,
2234                };
2235                let stream = obj
2236                    .and_then(|o| deref(doc, o))
2237                    .and_then(|o| o.as_stream().ok());
2238                let Some(stream) = stream else { continue };
2239                let is_form = stream
2240                    .dict
2241                    .get(b"Subtype")
2242                    .ok()
2243                    .and_then(|o| o.as_name().ok())
2244                    == Some(b"Form".as_slice());
2245                if !is_form {
2246                    let is_image = stream
2247                        .dict
2248                        .get(b"Subtype")
2249                        .ok()
2250                        .and_then(|o| o.as_name().ok())
2251                        == Some(b"Image".as_slice());
2252                    if is_image {
2253                        inks.push(image_ink(ctm));
2254                    }
2255                    continue;
2256                }
2257                // Decode a reused form's content once per document
2258                // (headers/footers and bulk body text invoke the same form on
2259                // every page); see `DocCaches::forms_seen` for single-use ones.
2260                let cached = form_id.and_then(|id| caches.forms.get(&id).cloned());
2261                let form_content = match cached {
2262                    Some(c) => c,
2263                    None => {
2264                        let Ok(data) = stream.decompressed_content() else {
2265                            continue;
2266                        };
2267                        let Ok(c) = lopdf::content::Content::decode(&data) else {
2268                            continue;
2269                        };
2270                        let c = Arc::new(c);
2271                        if let Some(id) = form_id {
2272                            if !caches.forms_seen.insert(id) {
2273                                caches.forms.insert(id, Arc::clone(&c));
2274                            }
2275                        }
2276                        c
2277                    }
2278                };
2279                // The form's /Matrix maps form space into the CTM at invocation.
2280                let form_mat = match stream.dict.get(b"Matrix").ok() {
2281                    Some(Object::Array(a)) if a.len() == 6 => {
2282                        let v: Vec<f64> = a.iter().filter_map(num).collect();
2283                        if v.len() == 6 {
2284                            Mat {
2285                                a: v[0],
2286                                b: v[1],
2287                                c: v[2],
2288                                d: v[3],
2289                                e: v[4],
2290                                f: v[5],
2291                            }
2292                        } else {
2293                            Mat::ID
2294                        }
2295                    }
2296                    _ => Mat::ID,
2297                };
2298                // The form's own /Resources, falling back to the inherited ones.
2299                let form_res = stream
2300                    .dict
2301                    .get(b"Resources")
2302                    .ok()
2303                    .and_then(|o| deref(doc, o))
2304                    .and_then(|o| o.as_dict().ok())
2305                    .unwrap_or(res);
2306                let state = TextState {
2307                    tc,
2308                    tw,
2309                    th,
2310                    tl,
2311                    trise,
2312                    fsize,
2313                    tr,
2314                };
2315                run_content(
2316                    doc,
2317                    form_res,
2318                    &form_content,
2319                    form_mat.then(ctm),
2320                    state,
2321                    depth + 1,
2322                    caches,
2323                    out,
2324                    inks,
2325                );
2326            }
2327            _ => {}
2328        }
2329        // Invisible text (`Tr` 3, or 7: clip only) — an OCR layer over a scan
2330        // follows the scanned lines, not a typeset baseline (#1005).
2331        if matches!(tr, 3 | 7) {
2332            for g in &mut out[shown..] {
2333                g.invisible = true;
2334            }
2335        }
2336    }
2337}
2338
2339/// An image placement's box — the unit square under `ctm`, y-up page
2340/// points — as an [`Ink::Bitmap`](crate::checkbox::Ink::Bitmap).
2341fn image_ink(ctm: Mat) -> crate::checkbox::Ink {
2342    let pts = [(0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0)].map(|(x, y)| ctm.apply(x, y));
2343    let (mut l, mut b, mut r, mut t) = (
2344        f64::INFINITY,
2345        f64::INFINITY,
2346        f64::NEG_INFINITY,
2347        f64::NEG_INFINITY,
2348    );
2349    for (x, y) in pts {
2350        (l, b, r, t) = (l.min(x), b.min(y), r.max(x), t.max(y));
2351    }
2352    crate::checkbox::Ink::Bitmap { l, b, r, t }
2353}
2354
2355/// One piece of a path under construction, page space (see `run_content`).
2356enum PathPiece {
2357    Line((f64, f64), (f64, f64)),
2358    /// An `re`: its four corners, counter-clockwise from `(x, y)`.
2359    Rect([(f64, f64); 4]),
2360    /// A Bézier segment: its start and control/end points (their hull).
2361    Curve(Vec<(f64, f64)>),
2362}
2363
2364/// Record a painted path's pieces for the checkbox finder (#609): a stroked
2365/// line is a [`Seg`](crate::checkbox::Ink::Seg), a stroked rectangle that
2366/// stays axis-aligned a [`Rect`](crate::checkbox::Ink::Rect) (one filled
2367/// white too — still an outline), and a coloured fill or a curve only its
2368/// bounding box, a [`Blot`](crate::checkbox::Ink::Blot): a possible tick
2369/// mark, or a swatch. White fills draw nothing and are dropped. Only pieces small
2370/// enough to be a checkbox edge or a mark inside one are kept.
2371fn paint_path(
2372    path: &[PathPiece],
2373    stroke: bool,
2374    fill: bool,
2375    fill_light: bool,
2376    inks: &mut Vec<crate::checkbox::Ink>,
2377) {
2378    use crate::checkbox::Ink;
2379    let bbox = |pts: &mut dyn Iterator<Item = (f64, f64)>| {
2380        pts.fold(
2381            (
2382                f64::INFINITY,
2383                f64::INFINITY,
2384                f64::NEG_INFINITY,
2385                f64::NEG_INFINITY,
2386            ),
2387            |(l, b, r, t), (x, y)| (l.min(x), b.min(y), r.max(x), t.max(y)),
2388        )
2389    };
2390    let mut keep = |ink: Ink| {
2391        if ink.worth_keeping() {
2392            inks.push(ink);
2393        }
2394    };
2395    if fill && !fill_light {
2396        // A coloured fill is a blot — a tick mark, or (the size of a square)
2397        // a legend swatch, which `checkbox::find` then rules out. A white
2398        // fill is no ink on paper: it only leaves the stroke, if any, below.
2399        let (l, b, r, t) = bbox(&mut path.iter().flat_map(|p| match p {
2400            PathPiece::Line(a, z) => vec![*a, *z],
2401            PathPiece::Rect(c) => c.to_vec(),
2402            PathPiece::Curve(c) => c.clone(),
2403        }));
2404        if l.is_finite() {
2405            keep(Ink::Blot { l, b, r, t });
2406        }
2407        return;
2408    }
2409    if !stroke {
2410        return;
2411    }
2412    for piece in path {
2413        match piece {
2414            PathPiece::Line(a, z) => keep(Ink::Seg {
2415                x0: a.0,
2416                y0: a.1,
2417                x1: z.0,
2418                y1: z.1,
2419            }),
2420            PathPiece::Rect(c) => {
2421                let axis = ((c[0].1 - c[1].1).abs() < 1e-6 && (c[1].0 - c[2].0).abs() < 1e-6)
2422                    || ((c[0].0 - c[1].0).abs() < 1e-6 && (c[1].1 - c[2].1).abs() < 1e-6);
2423                if axis {
2424                    let (l, b, r, t) = bbox(&mut c.iter().copied());
2425                    keep(Ink::Rect { l, b, r, t });
2426                } else {
2427                    for k in 0..4 {
2428                        let (a, z) = (c[k], c[(k + 1) % 4]);
2429                        keep(Ink::Seg {
2430                            x0: a.0,
2431                            y0: a.1,
2432                            x1: z.0,
2433                            y1: z.1,
2434                        });
2435                    }
2436                }
2437            }
2438            PathPiece::Curve(c) => {
2439                let (l, b, r, t) = bbox(&mut c.iter().copied());
2440                keep(Ink::Blot { l, b, r, t });
2441            }
2442        }
2443    }
2444}
2445
2446#[allow(clippy::too_many_arguments)]
2447fn show_text(
2448    font: &Font,
2449    bytes: &[u8],
2450    fsize: f64,
2451    tc: f64,
2452    tw: f64,
2453    th: f64,
2454    trise: f64,
2455    tm: &mut Mat,
2456    ctm: Mat,
2457    out: &mut Vec<Glyph>,
2458) {
2459    for code in codes(font, bytes) {
2460        let (text, w, blank) = font.decode_code(code);
2461        let w0 = w / 1000.0; // advance in text-space (em) units
2462                             // The glyph→user transform: scale glyph space by font size, then Tm, CTM.
2463        let scale = Mat {
2464            a: fsize * th,
2465            b: 0.0,
2466            c: 0.0,
2467            d: fsize,
2468            e: 0.0,
2469            f: trise,
2470        };
2471        let trm = scale.then(*tm).then(ctm);
2472        // Box in glyph space (1000-unit em): x 0..w, y descent..ascent.
2473        // A blank unmapped glyph keeps its advance but no ink height
2474        // (docling-parse collapses its ascent and descent).
2475        let (desc, asc) = if blank {
2476            (0.0, 0.0)
2477        } else {
2478            (font.descent / 1000.0, font.ascent / 1000.0)
2479        };
2480        let (x0, y0) = trm.apply(0.0, desc);
2481        let (x1, y1) = trm.apply(w0, desc);
2482        let (x2, y2) = trm.apply(w0, asc);
2483        let (x3, y3) = trm.apply(0.0, asc);
2484        // Upright = the baseline runs left-to-right along +x. Only then is the
2485        // loose box the rectangle spanned by the baseline's x and the em's y
2486        // (a synthetic oblique's skew is ignored, as it always was). A rotated
2487        // matrix — the `0 s -s 0 tx ty Tm` landscape pages are built with
2488        // (#528) — collapsed that rectangle to zero width, and every such run
2489        // vanished; those glyphs keep their real quad (docling-parse's char
2490        // rect) and its extent instead.
2491        let upright = trm.a > 0.0 && trm.b.abs() <= 1e-6 * trm.a;
2492        let (left, bot, right, top, quad) = if upright {
2493            (x0.min(x1), y0.min(y3), x0.max(x1), y0.max(y3), None)
2494        } else {
2495            let (xs, ys) = ([x0, x1, x2, x3], [y0, y1, y2, y3]);
2496            let fold = |v: [f64; 4], f: fn(f64, f64) -> f64| v.into_iter().reduce(f).unwrap();
2497            let q = [x0, y0, x1, y1, x2, y2, x3, y3].map(|v| v as f32);
2498            (
2499                fold(xs, f64::min),
2500                fold(ys, f64::min),
2501                fold(xs, f64::max),
2502                fold(ys, f64::max),
2503                Some(q),
2504            )
2505        };
2506        // The pen's origin (rise included) and the font size's extent from
2507        // it, as docling-parse maps them (`map_text_point(0, rise)` and
2508        // `(0, rise + font_size)`).
2509        let (ox, oy) = trm.apply(0.0, 0.0);
2510        let (sx, sy) = trm.apply(0.0, 1.0);
2511        let pen = Some([ox as f32, oy as f32, (sx - ox).hypot(sy - oy) as f32]);
2512        if let Some(s) = text {
2513            // A run may map one code to multiple chars (ligature/fraction); share box.
2514            for ch in s.chars() {
2515                // A NUL from the `/ToUnicode` is an unmapped glyph too
2516                // (docling#5070): U+FFFD.
2517                let ch = if ch == '\u{0}' { '\u{FFFD}' } else { ch };
2518                out.push(Glyph {
2519                    ch,
2520                    l: left as f32,
2521                    b: bot as f32,
2522                    r: right as f32,
2523                    t: top as f32,
2524                    ll: left as f32,
2525                    lb: bot as f32,
2526                    lr: right as f32,
2527                    lt: top as f32,
2528                    font: font.hash,
2529                    quad,
2530                    pen,
2531                    math_font: font.math,
2532                    invisible: false,
2533                });
2534            }
2535        }
2536        // Advance the text matrix. Word spacing applies to single-byte code 32.
2537        let is_space = !font.two_byte && code == 32;
2538        let tx = (w0 * fsize + tc + if is_space { tw } else { 0.0 }) * th;
2539        *tm = Mat {
2540            a: 1.0,
2541            b: 0.0,
2542            c: 0.0,
2543            d: 1.0,
2544            e: tx,
2545            f: 0.0,
2546        }
2547        .then(*tm);
2548    }
2549}
2550
2551/// Build a simple font's code→char table from its `/Encoding`: the base
2552/// encoding (WinAnsi / MacRoman) plus any `/Differences` overrides (glyph names
2553/// resolved through a small Adobe-glyph-name subset).
2554fn simple_encoding_table(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
2555    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o));
2556    let base_name = match enc {
2557        Some(Object::Name(n)) => n.clone(),
2558        Some(Object::Dictionary(d)) => d
2559            .get(b"BaseEncoding")
2560            .ok()
2561            .and_then(|o| o.as_name().ok())
2562            .map(|n| n.to_vec())
2563            .unwrap_or_default(),
2564        _ => Vec::new(),
2565    };
2566    let mut m = if base_name == b"MacRomanEncoding" {
2567        macroman_table()
2568    } else if base_name.is_empty() {
2569        // No PDF /Encoding at all: the font's *built-in* encoding applies. For
2570        // the standard TeX math fonts that is their fixed TeX layout — falling
2571        // back to StandardEncoding read CMSY's braces as `f`/`g`, `→` as `!`,
2572        // `∈` as `2` (2203's `{ahn,…}` author line). The font program (often
2573        // CFF, which this parser does not read) carries the same mapping;
2574        // docling-parse decodes it from there.
2575        tex_math_builtin(fdict).unwrap_or_else(winansi_table)
2576    } else {
2577        winansi_table()
2578    };
2579    // Apply /Differences: `code /glyphname /glyphname ... code ...`.
2580    if let Some(Object::Dictionary(d)) = enc {
2581        if let Some(Object::Array(diffs)) = d.get(b"Differences").ok().and_then(|o| deref(doc, o)) {
2582            let mut code = 0u8;
2583            for el in diffs {
2584                match el {
2585                    Object::Integer(i) => code = *i as u8,
2586                    Object::Name(name) => {
2587                        if let Some(ch) = glyph_name_to_char(name) {
2588                            m.insert(code, ch);
2589                        }
2590                        code = code.wrapping_add(1);
2591                    }
2592                    _ => {}
2593                }
2594            }
2595        }
2596    }
2597    m
2598}
2599
2600/// The fixed built-in encodings of the standard TeX math fonts (TeXbook
2601/// Appendix F), keyed off the base font name: `CMSY*` (symbols; `CMBSY` is its
2602/// bold) and `CMMI*` (math italic). These fonts ship no PDF `/Encoding` and no
2603/// ToUnicode, and their program is usually CFF — without this table the codes
2604/// fell through to StandardEncoding and rendered as the wrong ASCII.
2605fn tex_math_builtin(fdict: &Dictionary) -> Option<HashMap<u8, char>> {
2606    const CMSY: [char; 128] = [
2607        '−', '·', '×', '∗', '÷', '⋄', '±', '∓', '⊕', '⊖', '⊗', '⊘', '⊙', '◯', '∘', '•', '≍', '≡',
2608        '⊆', '⊇', '≤', '≥', '≼', '≽', '∼', '≈', '⊂', '⊃', '≪', '≫', '≺', '≻', '←', '→', '↑', '↓',
2609        '↔', '↗', '↘', '≃', '⇐', '⇒', '⇑', '⇓', '⇔', '↖', '↙', '∝', '′', '∞', '∈', '∋', '△', '▽',
2610        '\u{338}', '↦', '∀', '∃', '¬', '∅', 'ℜ', 'ℑ', '⊤', '⊥', 'ℵ', 'A', 'B', 'C', 'D', 'E', 'F',
2611        'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
2612        'Y', 'Z', '∪', '∩', '⊎', '∧', '∨', '⊢', '⊣', '⌊', '⌋', '⌈', '⌉', '{', '}', '⟨', '⟩', '|',
2613        '∥', '↕', '⇕', '\\', '≀', '√', '∐', '∇', '∫', '⊔', '⊓', '⊑', '⊒', '§', '†', '‡', '¶', '♣',
2614        '♢', '♡', '♠',
2615    ];
2616    const CMMI: [char; 128] = [
2617        'Γ', 'Δ', 'Θ', 'Λ', 'Ξ', 'Π', 'Σ', 'Υ', 'Φ', 'Ψ', 'Ω', 'α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η',
2618        'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'π', 'ρ', 'σ', 'τ', 'υ', 'φ', 'χ', 'ψ', 'ω', 'ϵ', 'ϑ',
2619        'ϖ', 'ϱ', 'ς', 'ϕ', '↼', '↽', '⇀', '⇁', '↩', '↪', '▷', '◁', '0', '1', '2', '3', '4', '5',
2620        '6', '7', '8', '9', '.', ',', '<', '/', '>', '⋆', '∂', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
2621        'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y',
2622        'Z', '♭', '♮', '♯', '⌣', '⌢', 'ℓ', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
2623        'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'ı', 'ȷ', '℘',
2624        '\u{20d7}', '⁀',
2625    ];
2626    let name = base_font_name(fdict)?;
2627    let up = name.to_ascii_uppercase();
2628    let table: &[char; 128] = if up.starts_with(b"CMSY") || up.starts_with(b"CMBSY") {
2629        &CMSY
2630    } else if up.starts_with(b"CMMI") {
2631        &CMMI
2632    } else {
2633        return None;
2634    };
2635    Some(
2636        table
2637            .iter()
2638            .enumerate()
2639            .map(|(i, &c)| (i as u8, c))
2640            .collect(),
2641    )
2642}
2643
2644/// The Adobe Glyph List's names for Latin-1 Supplement and Latin Extended-A
2645/// (plus `Euro`, `florin`, the comma-accent pairs): what a Type1 font's
2646/// `/Differences` or builtin encoding calls an accented or stroked letter.
2647/// 2203.01017v2 names `Ł` as `/Lslash` in a NimbusRomNo9L subset with no
2648/// ToUnicode, and the resolver, which knew only the ASCII, ligature and symbol
2649/// names, dropped the glyph outright (`Ł ukasz Kaiser` read `ukasz Kaiser`).
2650/// Sorted by name for the binary search in [`glyph_name_to_char`].
2651const AGL_LATIN: &[(&str, char)] = &[
2652    ("AE", '\u{00C6}'),
2653    ("Aacute", '\u{00C1}'),
2654    ("Abreve", '\u{0102}'),
2655    ("Acircumflex", '\u{00C2}'),
2656    ("Adieresis", '\u{00C4}'),
2657    ("Agrave", '\u{00C0}'),
2658    ("Amacron", '\u{0100}'),
2659    ("Aogonek", '\u{0104}'),
2660    ("Aring", '\u{00C5}'),
2661    ("Atilde", '\u{00C3}'),
2662    ("Cacute", '\u{0106}'),
2663    ("Ccaron", '\u{010C}'),
2664    ("Ccedilla", '\u{00C7}'),
2665    ("Ccircumflex", '\u{0108}'),
2666    ("Cdotaccent", '\u{010A}'),
2667    ("Dcaron", '\u{010E}'),
2668    ("Dcroat", '\u{0110}'),
2669    ("Eacute", '\u{00C9}'),
2670    ("Ebreve", '\u{0114}'),
2671    ("Ecaron", '\u{011A}'),
2672    ("Ecircumflex", '\u{00CA}'),
2673    ("Edieresis", '\u{00CB}'),
2674    ("Edotaccent", '\u{0116}'),
2675    ("Egrave", '\u{00C8}'),
2676    ("Emacron", '\u{0112}'),
2677    ("Eng", '\u{014A}'),
2678    ("Eogonek", '\u{0118}'),
2679    ("Eth", '\u{00D0}'),
2680    ("Euro", '\u{20AC}'),
2681    ("Gbreve", '\u{011E}'),
2682    ("Gcircumflex", '\u{011C}'),
2683    ("Gcommaaccent", '\u{0122}'),
2684    ("Gdotaccent", '\u{0120}'),
2685    ("Hbar", '\u{0126}'),
2686    ("Hcircumflex", '\u{0124}'),
2687    ("IJ", '\u{0132}'),
2688    ("Iacute", '\u{00CD}'),
2689    ("Ibreve", '\u{012C}'),
2690    ("Icircumflex", '\u{00CE}'),
2691    ("Idieresis", '\u{00CF}'),
2692    ("Idotaccent", '\u{0130}'),
2693    ("Igrave", '\u{00CC}'),
2694    ("Imacron", '\u{012A}'),
2695    ("Iogonek", '\u{012E}'),
2696    ("Itilde", '\u{0128}'),
2697    ("Jcircumflex", '\u{0134}'),
2698    ("Kcommaaccent", '\u{0136}'),
2699    ("Lacute", '\u{0139}'),
2700    ("Lcaron", '\u{013D}'),
2701    ("Lcommaaccent", '\u{013B}'),
2702    ("Ldot", '\u{013F}'),
2703    ("Lslash", '\u{0141}'),
2704    ("Nacute", '\u{0143}'),
2705    ("Ncaron", '\u{0147}'),
2706    ("Ncommaaccent", '\u{0145}'),
2707    ("Ntilde", '\u{00D1}'),
2708    ("OE", '\u{0152}'),
2709    ("Oacute", '\u{00D3}'),
2710    ("Obreve", '\u{014E}'),
2711    ("Ocircumflex", '\u{00D4}'),
2712    ("Odieresis", '\u{00D6}'),
2713    ("Ograve", '\u{00D2}'),
2714    ("Ohungarumlaut", '\u{0150}'),
2715    ("Omacron", '\u{014C}'),
2716    ("Oslash", '\u{00D8}'),
2717    ("Otilde", '\u{00D5}'),
2718    ("Racute", '\u{0154}'),
2719    ("Rcaron", '\u{0158}'),
2720    ("Rcommaaccent", '\u{0156}'),
2721    ("Sacute", '\u{015A}'),
2722    ("Scaron", '\u{0160}'),
2723    ("Scedilla", '\u{015E}'),
2724    ("Scircumflex", '\u{015C}'),
2725    ("Scommaaccent", '\u{0218}'),
2726    ("Tbar", '\u{0166}'),
2727    ("Tcaron", '\u{0164}'),
2728    ("Tcommaaccent", '\u{0162}'),
2729    ("Thorn", '\u{00DE}'),
2730    ("Uacute", '\u{00DA}'),
2731    ("Ubreve", '\u{016C}'),
2732    ("Ucircumflex", '\u{00DB}'),
2733    ("Udieresis", '\u{00DC}'),
2734    ("Ugrave", '\u{00D9}'),
2735    ("Uhungarumlaut", '\u{0170}'),
2736    ("Umacron", '\u{016A}'),
2737    ("Uogonek", '\u{0172}'),
2738    ("Uring", '\u{016E}'),
2739    ("Utilde", '\u{0168}'),
2740    ("Wcircumflex", '\u{0174}'),
2741    ("Yacute", '\u{00DD}'),
2742    ("Ycircumflex", '\u{0176}'),
2743    ("Ydieresis", '\u{0178}'),
2744    ("Zacute", '\u{0179}'),
2745    ("Zcaron", '\u{017D}'),
2746    ("Zdotaccent", '\u{017B}'),
2747    ("aacute", '\u{00E1}'),
2748    ("abreve", '\u{0103}'),
2749    ("acircumflex", '\u{00E2}'),
2750    ("acute", '\u{00B4}'),
2751    ("adieresis", '\u{00E4}'),
2752    ("ae", '\u{00E6}'),
2753    ("agrave", '\u{00E0}'),
2754    ("amacron", '\u{0101}'),
2755    ("aogonek", '\u{0105}'),
2756    ("aring", '\u{00E5}'),
2757    ("atilde", '\u{00E3}'),
2758    ("brokenbar", '\u{00A6}'),
2759    ("cacute", '\u{0107}'),
2760    ("ccaron", '\u{010D}'),
2761    ("ccedilla", '\u{00E7}'),
2762    ("ccircumflex", '\u{0109}'),
2763    ("cdotaccent", '\u{010B}'),
2764    ("cedilla", '\u{00B8}'),
2765    ("cent", '\u{00A2}'),
2766    ("currency", '\u{00A4}'),
2767    ("dcaron", '\u{010F}'),
2768    ("dcroat", '\u{0111}'),
2769    ("dieresis", '\u{00A8}'),
2770    ("dotlessi", '\u{0131}'),
2771    ("eacute", '\u{00E9}'),
2772    ("ebreve", '\u{0115}'),
2773    ("ecaron", '\u{011B}'),
2774    ("ecircumflex", '\u{00EA}'),
2775    ("edieresis", '\u{00EB}'),
2776    ("edotaccent", '\u{0117}'),
2777    ("egrave", '\u{00E8}'),
2778    ("emacron", '\u{0113}'),
2779    ("eng", '\u{014B}'),
2780    ("eogonek", '\u{0119}'),
2781    ("eth", '\u{00F0}'),
2782    ("exclamdown", '\u{00A1}'),
2783    ("florin", '\u{0192}'),
2784    ("gbreve", '\u{011F}'),
2785    ("gcircumflex", '\u{011D}'),
2786    ("gcommaaccent", '\u{0123}'),
2787    ("gdotaccent", '\u{0121}'),
2788    ("germandbls", '\u{00DF}'),
2789    ("guillemotleft", '\u{00AB}'),
2790    ("guillemotright", '\u{00BB}'),
2791    ("hbar", '\u{0127}'),
2792    ("hcircumflex", '\u{0125}'),
2793    ("iacute", '\u{00ED}'),
2794    ("ibreve", '\u{012D}'),
2795    ("icircumflex", '\u{00EE}'),
2796    ("idieresis", '\u{00EF}'),
2797    ("igrave", '\u{00EC}'),
2798    ("ij", '\u{0133}'),
2799    ("imacron", '\u{012B}'),
2800    ("iogonek", '\u{012F}'),
2801    ("itilde", '\u{0129}'),
2802    ("jcircumflex", '\u{0135}'),
2803    ("kcommaaccent", '\u{0137}'),
2804    ("kgreenlandic", '\u{0138}'),
2805    ("lacute", '\u{013A}'),
2806    ("lcaron", '\u{013E}'),
2807    ("lcommaaccent", '\u{013C}'),
2808    ("ldot", '\u{0140}'),
2809    ("logicalnot", '\u{00AC}'),
2810    ("longs", '\u{017F}'),
2811    ("lslash", '\u{0142}'),
2812    ("macron", '\u{00AF}'),
2813    ("nacute", '\u{0144}'),
2814    ("napostrophe", '\u{0149}'),
2815    ("ncaron", '\u{0148}'),
2816    ("ncommaaccent", '\u{0146}'),
2817    ("ntilde", '\u{00F1}'),
2818    ("oacute", '\u{00F3}'),
2819    ("obreve", '\u{014F}'),
2820    ("ocircumflex", '\u{00F4}'),
2821    ("odieresis", '\u{00F6}'),
2822    ("oe", '\u{0153}'),
2823    ("ograve", '\u{00F2}'),
2824    ("ohungarumlaut", '\u{0151}'),
2825    ("omacron", '\u{014D}'),
2826    ("onehalf", '\u{00BD}'),
2827    ("onequarter", '\u{00BC}'),
2828    ("onesuperior", '\u{00B9}'),
2829    ("ordfeminine", '\u{00AA}'),
2830    ("ordmasculine", '\u{00BA}'),
2831    ("oslash", '\u{00F8}'),
2832    ("otilde", '\u{00F5}'),
2833    ("paragraph", '\u{00B6}'),
2834    ("questiondown", '\u{00BF}'),
2835    ("racute", '\u{0155}'),
2836    ("rcaron", '\u{0159}'),
2837    ("rcommaaccent", '\u{0157}'),
2838    ("sacute", '\u{015B}'),
2839    ("scaron", '\u{0161}'),
2840    ("scedilla", '\u{015F}'),
2841    ("scircumflex", '\u{015D}'),
2842    ("scommaaccent", '\u{0219}'),
2843    ("section", '\u{00A7}'),
2844    ("sfthyphen", '\u{00AD}'),
2845    ("sterling", '\u{00A3}'),
2846    ("tbar", '\u{0167}'),
2847    ("tcaron", '\u{0165}'),
2848    ("tcommaaccent", '\u{0163}'),
2849    ("thorn", '\u{00FE}'),
2850    ("threequarters", '\u{00BE}'),
2851    ("threesuperior", '\u{00B3}'),
2852    ("twosuperior", '\u{00B2}'),
2853    ("uacute", '\u{00FA}'),
2854    ("ubreve", '\u{016D}'),
2855    ("ucircumflex", '\u{00FB}'),
2856    ("udieresis", '\u{00FC}'),
2857    ("ugrave", '\u{00F9}'),
2858    ("uhungarumlaut", '\u{0171}'),
2859    ("umacron", '\u{016B}'),
2860    ("uogonek", '\u{0173}'),
2861    ("uring", '\u{016F}'),
2862    ("utilde", '\u{0169}'),
2863    ("wcircumflex", '\u{0175}'),
2864    ("yacute", '\u{00FD}'),
2865    ("ycircumflex", '\u{0177}'),
2866    ("ydieresis", '\u{00FF}'),
2867    ("yen", '\u{00A5}'),
2868    ("zacute", '\u{017A}'),
2869    ("zcaron", '\u{017E}'),
2870    ("zdotaccent", '\u{017C}'),
2871];
2872
2873/// Resolve an Adobe glyph name to Unicode: `uniXXXX`, single ASCII letters, the
2874/// digit/punctuation names from the Adobe Glyph List, and common typographic
2875/// names. A `.suffix` (`one.taboldstyle`, `a.sc`) is stripped and the base name
2876/// retried — docling renders these as the base character.
2877pub(crate) fn glyph_name_to_char(name: &[u8]) -> Option<char> {
2878    let s = std::str::from_utf8(name).ok()?;
2879    if let Some(hex) = s.strip_prefix("uni") {
2880        if let Ok(cp) = u32::from_str_radix(hex.get(0..4)?, 16) {
2881            return char::from_u32(cp);
2882        }
2883    }
2884    // Single ASCII letter names (`A`, `m`) map to themselves.
2885    if s.len() == 1 {
2886        let b = s.as_bytes()[0];
2887        if b.is_ascii_alphabetic() {
2888            return Some(b as char);
2889        }
2890    }
2891    let resolved = match s {
2892        "space" => ' ',
2893        "exclam" => '!',
2894        "quotedbl" => '"',
2895        "numbersign" => '#',
2896        "dollar" => '$',
2897        "percent" => '%',
2898        "ampersand" => '&',
2899        "quotesingle" => '\'',
2900        "parenleft" => '(',
2901        "parenright" => ')',
2902        "asterisk" => '*',
2903        "plus" => '+',
2904        "comma" => ',',
2905        "hyphen" => '-',
2906        "period" => '.',
2907        "slash" => '/',
2908        "zero" => '0',
2909        "one" => '1',
2910        "two" => '2',
2911        "three" => '3',
2912        "four" => '4',
2913        "five" => '5',
2914        "six" => '6',
2915        "seven" => '7',
2916        "eight" => '8',
2917        "nine" => '9',
2918        "colon" => ':',
2919        "semicolon" => ';',
2920        "less" => '<',
2921        "equal" => '=',
2922        "greater" => '>',
2923        "question" => '?',
2924        "at" => '@',
2925        "bracketleft" => '[',
2926        "backslash" => '\\',
2927        "bracketright" => ']',
2928        "asciicircum" => '^',
2929        "underscore" => '_',
2930        "grave" => '`',
2931        "braceleft" => '{',
2932        "bar" => '|',
2933        "braceright" => '}',
2934        "asciitilde" => '~',
2935        "bullet" => '\u{2022}',
2936        "periodcentered" => '\u{00B7}',
2937        "endash" => '\u{2013}',
2938        "emdash" => '\u{2014}',
2939        "quoteright" => '\u{2019}',
2940        "quoteleft" => '\u{2018}',
2941        "quotedblleft" => '\u{201C}',
2942        "quotedblright" => '\u{201D}',
2943        "quotedblbase" => '\u{201E}',
2944        "quotesinglbase" => '\u{201A}',
2945        // Latin f-ligatures named in `/Differences` (e.g. 2305's body font). These
2946        // map to the presentation-form code points, which `decompose_ligatures`
2947        // then spells back out (`ff`→"ff") — without them the glyph decodes to
2948        // nothing and the sanitizer fills the gap with a space (`di erences`).
2949        "ff" => '\u{FB00}',
2950        "fi" => '\u{FB01}',
2951        "fl" => '\u{FB02}',
2952        "ffi" => '\u{FB03}',
2953        "ffl" => '\u{FB04}',
2954        "ft" => '\u{FB05}',
2955        "st" => '\u{FB06}',
2956        "degree" => '\u{00B0}',
2957        "trademark" => '\u{2122}',
2958        "registered" => '\u{00AE}',
2959        "copyright" => '\u{00A9}',
2960        "ellipsis" => '\u{2026}',
2961        "minus" => '\u{2212}',
2962        "fraction" => '\u{2044}',
2963        "nbspace" => '\u{00A0}',
2964        // Greek + math glyph names (standard Adobe Glyph List). Standard TeX math
2965        // fonts (CMMI/CMSY/…) name their glyphs this way in the embedded font
2966        // program's `/Encoding`; without these a `λ`/`≤` decodes to nothing and is
2967        // dropped from body text (`and λ set to 0.5` → `and set to 0.5`).
2968        "alpha" => '\u{03B1}',
2969        "beta" => '\u{03B2}',
2970        "gamma" => '\u{03B3}',
2971        "delta" => '\u{03B4}',
2972        "epsilon" | "epsilon1" => '\u{03B5}',
2973        "zeta" => '\u{03B6}',
2974        "eta" => '\u{03B7}',
2975        "theta" | "theta1" => '\u{03B8}',
2976        "iota" => '\u{03B9}',
2977        "kappa" => '\u{03BA}',
2978        "lambda" => '\u{03BB}',
2979        "mu" => '\u{03BC}',
2980        "nu" => '\u{03BD}',
2981        "xi" => '\u{03BE}',
2982        "omicron" => '\u{03BF}',
2983        "pi" | "pi1" => '\u{03C0}',
2984        "rho" | "rho1" => '\u{03C1}',
2985        "sigma" => '\u{03C3}',
2986        "sigma1" => '\u{03C2}',
2987        "tau" => '\u{03C4}',
2988        "upsilon" => '\u{03C5}',
2989        "phi" | "phi1" => '\u{03C6}',
2990        "chi" => '\u{03C7}',
2991        "psi" => '\u{03C8}',
2992        "omega" | "omega1" => '\u{03C9}',
2993        "Gamma" => '\u{0393}',
2994        "Delta" => '\u{0394}',
2995        "Theta" => '\u{0398}',
2996        "Lambda" => '\u{039B}',
2997        "Xi" => '\u{039E}',
2998        "Pi" => '\u{03A0}',
2999        "Sigma" => '\u{03A3}',
3000        "Upsilon" => '\u{03A5}',
3001        "Phi" => '\u{03A6}',
3002        "Psi" => '\u{03A8}',
3003        "Omega" => '\u{03A9}',
3004        "lessequal" => '\u{2264}',
3005        "greaterequal" => '\u{2265}',
3006        "notequal" => '\u{2260}',
3007        "approxequal" => '\u{2248}',
3008        "equivalence" => '\u{2261}',
3009        "element" => '\u{2208}',
3010        "plusminus" => '\u{00B1}',
3011        "multiply" => '\u{00D7}',
3012        "divide" => '\u{00F7}',
3013        "infinity" => '\u{221E}',
3014        "partialdiff" => '\u{2202}',
3015        "gradient" => '\u{2207}',
3016        "summation" => '\u{2211}',
3017        "product" => '\u{220F}',
3018        "integral" => '\u{222B}',
3019        "radical" => '\u{221A}',
3020        "proportional" => '\u{221D}',
3021        "arrowright" => '\u{2192}',
3022        "arrowleft" => '\u{2190}',
3023        "arrowup" => '\u{2191}',
3024        "arrowdown" => '\u{2193}',
3025        "arrowboth" => '\u{2194}',
3026        "arrowdblright" => '\u{21D2}',
3027        "logicaland" => '\u{2227}',
3028        "logicalor" => '\u{2228}',
3029        "intersection" => '\u{2229}',
3030        "union" => '\u{222A}',
3031        "similar" => '\u{223C}',
3032        "congruent" => '\u{2245}',
3033        "dotmath" => '\u{22C5}',
3034        "asteriskmath" => '\u{2217}',
3035        _ => {
3036            if let Ok(i) = AGL_LATIN.binary_search_by_key(&s, |(n, _)| n) {
3037                return Some(AGL_LATIN[i].1);
3038            }
3039            // Strip an AGL `.suffix` (oldstyle/small-cap variant) and retry.
3040            if let Some((base, _)) = s.split_once('.') {
3041                if !base.is_empty() {
3042                    return glyph_name_to_char(base.as_bytes());
3043                }
3044            }
3045            // An AGL ligature name joins its components with `_` (`f_f_i`,
3046            // `f_l` — what a re-encoded Libertine subset calls them); joined,
3047            // the presentation-form arms above resolve them.
3048            if s.contains('_') {
3049                let joined: String = s.chars().filter(|&c| c != '_').collect();
3050                if !joined.is_empty() && joined.len() < s.len() {
3051                    return glyph_name_to_char(joined.as_bytes());
3052                }
3053            }
3054            return None;
3055        }
3056    };
3057    Some(resolved)
3058}
3059
3060/// Minimal WinAnsiEncoding (Latin-1-ish) for simple fonts lacking ToUnicode.
3061fn winansi_table() -> HashMap<u8, char> {
3062    let mut m = HashMap::new();
3063    for b in 0x20u8..=0x7e {
3064        m.insert(b, b as char);
3065    }
3066    // High range: Windows-1252 printable points that differ from Latin-1.
3067    let extra: &[(u8, char)] = &[
3068        (0x91, '\u{2018}'),
3069        (0x92, '\u{2019}'),
3070        (0x93, '\u{201C}'),
3071        (0x94, '\u{201D}'),
3072        (0x95, '\u{2022}'),
3073        (0x96, '\u{2013}'),
3074        (0x97, '\u{2014}'),
3075        (0x85, '\u{2026}'),
3076        (0xA0, '\u{00A0}'),
3077        // The rest of Windows-1252's 0x80–0x9F row. Left out, the German
3078        // opening quote `„` (0x84) vanished from the text layer while its
3079        // closing `“` (0x93) stayed (#922); `€ ‚ ƒ † ‡ ˆ ‰ Š ‹ Œ Ž ˜ ™ š › œ ž Ÿ`
3080        // were lost the same way. 0x81/0x8D/0x8F/0x90/0x9D stay unmapped
3081        // (PDF's WinAnsi leaves them undefined).
3082        (0x80, '\u{20AC}'),
3083        (0x82, '\u{201A}'),
3084        (0x83, '\u{0192}'),
3085        (0x84, '\u{201E}'),
3086        (0x86, '\u{2020}'),
3087        (0x87, '\u{2021}'),
3088        (0x88, '\u{02C6}'),
3089        (0x89, '\u{2030}'),
3090        (0x8A, '\u{0160}'),
3091        (0x8B, '\u{2039}'),
3092        (0x8C, '\u{0152}'),
3093        (0x8E, '\u{017D}'),
3094        (0x98, '\u{02DC}'),
3095        (0x99, '\u{2122}'),
3096        (0x9A, '\u{0161}'),
3097        (0x9B, '\u{203A}'),
3098        (0x9C, '\u{0153}'),
3099        (0x9E, '\u{017E}'),
3100        (0x9F, '\u{0178}'),
3101    ];
3102    for &(b, c) in extra {
3103        m.insert(b, c);
3104    }
3105    for b in 0xA1u8..=0xFF {
3106        m.entry(b).or_insert(b as char);
3107    }
3108    m
3109}
3110
3111/// MacRomanEncoding: ASCII plus the whole high range, 0x80–0xFF, as Mac OS
3112/// Roman maps it (#744) — the table pdfium, pdftotext and pdf.js decode with.
3113/// Two codes follow the PDF reference instead of Apple's later revisions:
3114/// 0xDB is `currency` (¤, not €), and 0xF0 (`apple`, a private-use glyph) is
3115/// left unmapped. 0xCA, the PDF table's second `space`, reads as a no-break
3116/// space.
3117fn macroman_table() -> HashMap<u8, char> {
3118    #[rustfmt::skip]
3119    const HIGH: [char; 128] = [
3120        'Ä', 'Å', 'Ç', 'É', 'Ñ', 'Ö', 'Ü', 'á',  // 0x80
3121        'à', 'â', 'ä', 'ã', 'å', 'ç', 'é', 'è',  // 0x88
3122        'ê', 'ë', 'í', 'ì', 'î', 'ï', 'ñ', 'ó',  // 0x90
3123        'ò', 'ô', 'ö', 'õ', 'ú', 'ù', 'û', 'ü',  // 0x98
3124        '†', '°', '¢', '£', '§', '•', '¶', 'ß',  // 0xA0
3125        '®', '©', '™', '´', '¨', '≠', 'Æ', 'Ø',  // 0xA8
3126        '∞', '±', '≤', '≥', '¥', 'µ', '∂', '∑',  // 0xB0
3127        '∏', 'π', '∫', 'ª', 'º', 'Ω', 'æ', 'ø',  // 0xB8
3128        '¿', '¡', '¬', '√', 'ƒ', '≈', '∆', '«',  // 0xC0
3129        '»', '…', '\u{00A0}', 'À', 'Ã', 'Õ', 'Œ', 'œ',  // 0xC8
3130        '–', '—', '“', '”', '‘', '’', '÷', '◊',  // 0xD0
3131        'ÿ', 'Ÿ', '⁄', '¤', '‹', '›', 'fi', 'fl',  // 0xD8
3132        '‡', '·', '‚', '„', '‰', 'Â', 'Ê', 'Á',  // 0xE0
3133        'Ë', 'È', 'Í', 'Î', 'Ï', 'Ì', 'Ó', 'Ô',  // 0xE8
3134        '\0', 'Ò', 'Ú', 'Û', 'Ù', 'ı', 'ˆ', '˜',  // 0xF0
3135        '¯', '˘', '˙', '˚', '¸', '˝', '˛', 'ˇ',  // 0xF8
3136    ];
3137    let mut m = HashMap::new();
3138    for b in 0x20u8..=0x7e {
3139        m.insert(b, b as char);
3140    }
3141    for (b, &c) in (0x80u8..=0xFF).zip(HIGH.iter()) {
3142        if c != '\0' {
3143            m.insert(b, c);
3144        }
3145    }
3146    m
3147}
3148
3149#[cfg(test)]
3150mod glyph_names {
3151    /// `/Lslash` and the other Latin-1 / Latin Extended-A glyph names a Type1
3152    /// `/Differences` uses resolve (2203.01017v2's `Łukasz`); the table the
3153    /// lookup bisects is sorted and the ligature / symbol arms still win.
3154    #[test]
3155    fn latin_glyph_names_resolve() {
3156        for w in super::AGL_LATIN.windows(2) {
3157            assert!(
3158                w[0].0 < w[1].0,
3159                "AGL_LATIN not sorted at {:?} / {:?}",
3160                w[0].0,
3161                w[1].0
3162            );
3163        }
3164        for (name, ch) in [
3165            ("Lslash", 'Ł'),
3166            ("lslash", 'ł'),
3167            ("Aacute", 'Á'),
3168            ("aacute", 'á'),
3169            ("Scaron", 'Š'),
3170            ("germandbls", 'ß'),
3171            ("oe", 'œ'),
3172            ("Ydieresis", 'Ÿ'),
3173            ("Euro", '€'),
3174            ("Lslash.sc", 'Ł'),
3175            ("fi", '\u{FB01}'),
3176            ("Omega", '\u{03A9}'),
3177        ] {
3178            assert_eq!(
3179                super::glyph_name_to_char(name.as_bytes()),
3180                Some(ch),
3181                "{name}"
3182            );
3183        }
3184        assert_eq!(super::glyph_name_to_char(b"g123"), None);
3185        // AGL ligature names: components joined with `_`.
3186        assert_eq!(super::glyph_name_to_char(b"f_f_i"), Some('\u{FB03}'));
3187        assert_eq!(super::glyph_name_to_char(b"f_l"), Some('\u{FB02}'));
3188        assert_eq!(super::glyph_name_to_char(b"a_b"), None);
3189    }
3190
3191    /// The stale-CMap case: ToUnicode maps a code to its own identity while
3192    /// `/Differences` names a ligature there — the name wins; a CMap that
3193    /// maps the code anywhere else, or a code with no entry, is untouched.
3194    #[test]
3195    fn a_stale_tounicode_yields_to_differences() {
3196        use lopdf::Object;
3197        use std::collections::HashMap;
3198        let mut tu: HashMap<u32, String> = HashMap::new();
3199        tu.insert(33, "!".into()); // identity — stale
3200        tu.insert(34, "x".into()); // the producer meant it
3201        let diffs = vec![
3202            Object::Integer(33),
3203            Object::Name(b"f_f_i".to_vec()),
3204            Object::Name(b"f_i".to_vec()),
3205            Object::Name(b"f_f".to_vec()),
3206            Object::Integer(65),
3207            Object::Name(b"A".to_vec()), // identity on both sides: nothing to do
3208        ];
3209        super::reconcile_stale_tounicode(&mut tu, &diffs);
3210        assert_eq!(tu.get(&33).map(String::as_str), Some("\u{FB03}"));
3211        assert_eq!(tu.get(&34).map(String::as_str), Some("x"));
3212        assert_eq!(tu.get(&35), None);
3213        assert_eq!(tu.get(&65), None);
3214    }
3215}
3216
3217#[cfg(test)]
3218mod page_box_frame {
3219    use super::*;
3220    use lopdf::dictionary;
3221
3222    /// One page, `boxes` spliced into the page dictionary verbatim, one text
3223    /// run at user-space `(x, y)`.
3224    fn pdf(boxes: &str, x: f32, y: f32) -> Vec<u8> {
3225        pdf_content(
3226            boxes,
3227            &format!("BT /F1 12 Tf {x} {y} Td (First printing) Tj ET\n"),
3228        )
3229    }
3230
3231    /// One page, `boxes` spliced into the page dictionary, `content` as its
3232    /// content stream (font `/F1` = Helvetica).
3233    fn pdf_content(boxes: &str, content: &str) -> Vec<u8> {
3234        let objs: Vec<String> = vec![
3235            "<</Type/Catalog/Pages 2 0 R>>".into(),
3236            format!("<</Type/Pages/Kids[3 0 R]/Count 1{boxes}>>"),
3237            "<</Type/Page/Parent 2 0 R/Contents 4 0 R/Resources<</Font<</F1 5 0 R>>>>>>".into(),
3238            format!("<</Length {}>>stream\n{content}endstream", content.len()),
3239            "<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".into(),
3240        ];
3241        let mut out = b"%PDF-1.4\n".to_vec();
3242        let mut offsets = Vec::new();
3243        for (i, body) in objs.iter().enumerate() {
3244            offsets.push(out.len());
3245            out.extend_from_slice(format!("{} 0 obj{body}endobj\n", i + 1).as_bytes());
3246        }
3247        let xref_at = out.len();
3248        out.extend_from_slice(
3249            format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes(),
3250        );
3251        for off in &offsets {
3252            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
3253        }
3254        out.extend_from_slice(
3255            format!(
3256                "trailer<</Size {}/Root 1 0 R>>\nstartxref\n{xref_at}\n%%EOF\n",
3257                objs.len() + 1
3258            )
3259            .as_bytes(),
3260        );
3261        out
3262    }
3263
3264    /// One page whose `/Contents` is an array of separate streams.
3265    fn pdf_content_streams(contents: &[&[u8]]) -> Vec<u8> {
3266        let mut doc = Document::with_version("1.5");
3267        let streams = contents
3268            .iter()
3269            .map(|content| {
3270                Object::Reference(
3271                    doc.add_object(lopdf::Stream::new(Dictionary::new(), content.to_vec())),
3272                )
3273            })
3274            .collect::<Vec<_>>();
3275        let font_id = doc.add_object(dictionary! {
3276            "Type" => "Font",
3277            "Subtype" => "Type1",
3278            "BaseFont" => "Helvetica",
3279        });
3280        let resources_id = doc.add_object(dictionary! {
3281            "Font" => dictionary! { "F1" => Object::Reference(font_id) },
3282        });
3283        let pages_id = doc.new_object_id();
3284        let page_id = doc.add_object(dictionary! {
3285            "Type" => "Page",
3286            "Parent" => Object::Reference(pages_id),
3287            "MediaBox" => vec![0.into(), 0.into(), 300.into(), 400.into()],
3288            "Contents" => streams,
3289            "Resources" => Object::Reference(resources_id),
3290        });
3291        doc.objects.insert(
3292            pages_id,
3293            Object::Dictionary(dictionary! {
3294                "Type" => "Pages",
3295                "Kids" => vec![Object::Reference(page_id)],
3296                "Count" => 1,
3297            }),
3298        );
3299        let catalog_id = doc.add_object(dictionary! {
3300            "Type" => "Catalog",
3301            "Pages" => Object::Reference(pages_id),
3302        });
3303        doc.trailer.set("Root", Object::Reference(catalog_id));
3304        let mut out = Vec::new();
3305        doc.save_to(&mut out).expect("save synthetic PDF");
3306        out
3307    }
3308
3309    fn only_page(bytes: &[u8]) -> (PageBox, Vec<Glyph>) {
3310        let doc = load_document(bytes).expect("loads");
3311        let pid = *doc.get_pages().values().next().expect("one page");
3312        (page_box(&doc, pid), page_glyphs(&doc, pid))
3313    }
3314
3315    /// The trimmed-book-page shape: MediaBox and CropBox both away from the
3316    /// origin (inherited from `/Pages`). The display box is the CropBox, and a
3317    /// glyph's coordinates count from its lower-left corner — the same numbers
3318    /// the same text gets on a page whose CropBox *is* `[0 0 w h]`.
3319    #[test]
3320    fn glyphs_count_from_the_cropbox_corner_like_pdfium() {
3321        let (pb, shifted) = only_page(&pdf(
3322            "/MediaBox[-56.505 -58.25 576.303 723.31]/CropBox[1.095 -0.65 518.703 665.71]",
3323            37.0 + 1.095,
3324            58.0 - 0.65,
3325        ));
3326        assert!((pb.l - 1.095).abs() < 1e-3 && (pb.b + 0.65).abs() < 1e-3);
3327        assert!(
3328            (pb.w - 517.608).abs() < 1e-3 && (pb.h - 666.36).abs() < 1e-3,
3329            "{pb:?}"
3330        );
3331        let (pb0, plain) = only_page(&pdf("/MediaBox[0 0 517.608 666.36]", 37.0, 58.0));
3332        assert!((pb0.w - pb.w).abs() < 1e-3 && (pb0.h - pb.h).abs() < 1e-3);
3333        assert_eq!(shifted.len(), plain.len());
3334        assert!(!plain.is_empty());
3335        for (a, b) in shifted.iter().zip(&plain) {
3336            assert!(
3337                (a.l - b.l).abs() < 1e-3 && (a.b - b.b).abs() < 1e-3,
3338                "{:?} vs {:?}",
3339                (a.l, a.b),
3340                (b.l, b.b)
3341            );
3342        }
3343        assert!((plain[0].l - 37.0).abs() < 1e-3, "{}", plain[0].l);
3344    }
3345
3346    fn text(glyphs: &[Glyph]) -> String {
3347        glyphs.iter().map(|g| g.ch).collect()
3348    }
3349
3350    /// #529: text drawn outside the display box — a print slug beside the
3351    /// CropBox, a tiled page's neighbour beyond the MediaBox — is dropped,
3352    /// glyph by glyph like docling-parse, instead of being clamped onto the
3353    /// page edge; a line crossing the edge keeps the glyphs that fit.
3354    #[test]
3355    fn glyphs_outside_the_display_box_are_dropped() {
3356        let (_, g) = only_page(&pdf_content(
3357            "/MediaBox[0 0 400 400]/CropBox[100 100 400 400]",
3358            "BT /F1 14 Tf 120 300 Td (Visible.) Tj ET\nBT /F1 8 Tf 5 40 Td (Slug) Tj ET\n",
3359        ));
3360        assert_eq!(text(&g), "Visible.");
3361        let (_, g) = only_page(&pdf_content(
3362            "/MediaBox[0 0 300 400]",
3363            "BT /F1 14 Tf 40 300 Td (On page) Tj ET\nBT /F1 14 Tf 400 250 Td (Beyond) Tj ET\n",
3364        ));
3365        assert_eq!(text(&g), "On page");
3366        // docling-parse: "Crossing the right edge" on a 300 pt page → "Crossing the rig".
3367        let (_, g) = only_page(&pdf_content(
3368            "/MediaBox[0 0 300 400]",
3369            "BT /F1 14 Tf 200 250 Td (Crossing the right edge) Tj ET\n",
3370        ));
3371        assert_eq!(text(&g), "Crossing the rig");
3372    }
3373
3374    #[test]
3375    fn leading_comment_content_stream_does_not_hide_later_text() {
3376        // The comment has its own EOL; lopdf inserts another while assembling
3377        // the streams, which is the boundary its 0.44 parser stopped at.
3378        let (_, glyphs) = only_page(&pdf_content_streams(&[
3379            b"% CANON_PFINF_TYPE2_TEXTOFF\n",
3380            b"BT /F1 14 Tf 40 300 Td (Second stream text) Tj ET",
3381        ]));
3382        assert_eq!(text(&glyphs), "Second stream text");
3383    }
3384
3385    /// The containment test is docling-parse's: the whole char box (advance
3386    /// × the font's ascent / descent) inside the display box, edges
3387    /// included — a Helvetica `W` at 50 pt (47.2 wide) ending exactly on the
3388    /// right edge stays, 0.01 pt further it goes; likewise on the left.
3389    #[test]
3390    fn a_glyph_on_the_edge_stays_one_over_it_goes() {
3391        let w_at = |x: f32, y: f32| {
3392            let (_, g) = only_page(&pdf_content(
3393                "/MediaBox[0 0 300 400]",
3394                &format!("BT /F1 50 Tf {x} {y} Td (W) Tj ET\n"),
3395            ));
3396            text(&g)
3397        };
3398        assert_eq!(w_at(252.8, 200.0), "W");
3399        assert_eq!(w_at(252.81, 200.0), "");
3400        assert_eq!(w_at(0.0, 200.0), "W");
3401        assert_eq!(w_at(-0.01, 200.0), "");
3402        // Vertically the box is the font's ascent / descent: a baseline far
3403        // enough up keeps the descender on the page, one at 0 does not.
3404        assert_eq!(w_at(100.0, 20.0), "W");
3405        assert_eq!(w_at(100.0, 0.0), "");
3406        assert_eq!(w_at(100.0, 380.0), "");
3407    }
3408
3409    /// pdfium's fallbacks: no MediaBox → Letter; a CropBox is clipped to the
3410    /// MediaBox, and one that misses it entirely is ignored.
3411    #[test]
3412    fn page_box_follows_pdfium_fallbacks() {
3413        let (pb, _) = only_page(&pdf("", 10.0, 10.0));
3414        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 612.0, 792.0));
3415        let (pb, _) = only_page(&pdf(
3416            "/MediaBox[0 0 500 700]/CropBox[-100 100 600 900]",
3417            10.0,
3418            10.0,
3419        ));
3420        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 100.0, 500.0, 600.0));
3421        let (pb, _) = only_page(&pdf(
3422            "/MediaBox[0 0 500 700]/CropBox[800 800 900 900]",
3423            10.0,
3424            10.0,
3425        ));
3426        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
3427        // Reversed corners normalize.
3428        let (pb, _) = only_page(&pdf("/MediaBox[500 700 0 0]", 10.0, 10.0));
3429        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
3430    }
3431}
3432
3433#[cfg(test)]
3434mod xref_repair {
3435    /// Build a tiny one-page PDF whose cross-reference entries are either the
3436    /// spec's 20 bytes (`two_byte_eol`) or the 19-byte form some generators
3437    /// emit — everything else about the two files is identical.
3438    fn pdf_with_xref(two_byte_eol: bool) -> Vec<u8> {
3439        let content = b"BT /F1 12 Tf 72 700 Td (Invoice 922769430725) Tj ET\n";
3440        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
3441        let objs: Vec<Vec<u8>> = vec![
3442            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
3443            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
3444            b"<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
3445               /Resources<</Font<</F1 5 0 R>>>>>>"
3446                .to_vec(),
3447            [stream.as_slice(), content.as_slice(), b"endstream"].concat(),
3448            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".to_vec(),
3449        ];
3450
3451        let mut out = b"%PDF-1.4\n".to_vec();
3452        let mut offsets = Vec::new();
3453        for (i, body) in objs.iter().enumerate() {
3454            offsets.push(out.len());
3455            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
3456            out.extend_from_slice(body);
3457            out.extend_from_slice(b"endobj\n");
3458        }
3459        let xref_at = out.len();
3460        let eol: &[u8] = if two_byte_eol { b" \n" } else { b"\n" };
3461        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
3462        out.extend_from_slice(b"0000000000 65535 f");
3463        out.extend_from_slice(eol);
3464        for off in &offsets {
3465            out.extend_from_slice(format!("{off:010} 00000 n").as_bytes());
3466            out.extend_from_slice(eol);
3467        }
3468        out.extend_from_slice(
3469            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
3470        );
3471        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
3472        out
3473    }
3474
3475    /// A 19-byte cross-reference entry (a bare LF where the spec wants a
3476    /// two-byte EOL) used to make lopdf reject the whole file. Lopdf 0.45
3477    /// accepts it directly, so exercise the fallback itself and prove it still
3478    /// produces the exact 20-byte entries lopdf 0.44 needed.
3479    #[test]
3480    fn short_xref_entries_are_padded_for_fallback() {
3481        let good = pdf_with_xref(true);
3482        let broken = pdf_with_xref(false);
3483        assert!(
3484            broken.len() < good.len(),
3485            "the broken file is the shorter one"
3486        );
3487        assert!(
3488            lopdf::Document::load_mem(&good).is_ok(),
3489            "the control file must load unaided"
3490        );
3491        assert!(
3492            lopdf::Document::load_mem(&broken).is_ok(),
3493            "lopdf 0.45 should accept 19-byte entries"
3494        );
3495
3496        let repaired = super::pad_short_xref_entries(&broken).expect("repair short entries");
3497        assert_eq!(repaired, good, "repair should recreate the valid xref");
3498        assert!(
3499            lopdf::Document::load_mem(&repaired).is_ok(),
3500            "the bytes produced for lopdf 0.44 must load"
3501        );
3502
3503        let cells = |b: &[u8]| -> Vec<String> {
3504            super::pdf_textlines(b)
3505                .into_iter()
3506                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
3507                .collect()
3508        };
3509        let from_good = cells(&good);
3510        assert!(
3511            from_good.iter().any(|t| t.contains("922769430725")),
3512            "control text: {from_good:?}"
3513        );
3514        assert_eq!(
3515            cells(&repaired),
3516            from_good,
3517            "repair must match the good parse"
3518        );
3519    }
3520
3521    /// The same generator overstates `/Length`. Lopdf 0.45 recovers at
3522    /// `endstream`, so exercise our fallback directly and prove it still writes
3523    /// the exact length lopdf 0.44 needed without moving any bytes.
3524    #[test]
3525    fn overstated_stream_length_is_fixed_for_fallback() {
3526        let good = pdf_with_xref(true);
3527        // Inflate the content stream's /Length by one, exactly as the invoice
3528        // that prompted this does.
3529        let broken = {
3530            let at = good
3531                .windows(8)
3532                .position(|w| w == b"/Length ")
3533                .expect("a /Length")
3534                + 8;
3535            let digits = good[at..].iter().take_while(|c| c.is_ascii_digit()).count();
3536            let n: usize = std::str::from_utf8(&good[at..at + digits])
3537                .unwrap()
3538                .parse()
3539                .unwrap();
3540            let inflated = (n + 1).to_string();
3541            assert_eq!(inflated.len(), digits, "keep the digit count");
3542            let mut b = good.clone();
3543            b[at..at + digits].copy_from_slice(inflated.as_bytes());
3544            b
3545        };
3546        assert_eq!(broken.len(), good.len(), "the defect must not move bytes");
3547        // Lopdf 0.45 keeps the stream despite the overstated length.
3548        let raw = lopdf::Document::load_mem(&broken).expect("still loads");
3549        assert!(
3550            raw.get_pages()
3551                .into_values()
3552                .any(|p| !raw.get_page_content(p).is_empty()),
3553            "lopdf 0.45 should recover content at endstream"
3554        );
3555
3556        let repaired = super::fix_stream_lengths(&broken);
3557        assert_eq!(repaired.len(), broken.len(), "repair must preserve offsets");
3558        assert_eq!(repaired, good, "repair should restore the true length");
3559        let repaired_doc = lopdf::Document::load_mem(&repaired).expect("repaired PDF loads");
3560        assert!(
3561            repaired_doc
3562                .get_pages()
3563                .into_values()
3564                .any(|p| !repaired_doc.get_page_content(p).is_empty()),
3565            "the bytes produced for lopdf 0.44 must retain page content"
3566        );
3567
3568        // Ours recovers the same text the well-formed file gives.
3569        let text = |b: &[u8]| -> Vec<String> {
3570            super::pdf_textlines(b)
3571                .into_iter()
3572                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
3573                .collect()
3574        };
3575        let expected = text(&good);
3576        assert!(!expected.is_empty(), "control must produce text");
3577        assert_eq!(text(&repaired), expected);
3578    }
3579
3580    /// The repair only fires where padding cannot move an object: it declines a
3581    /// file whose xref precedes an object (an incremental update), rather than
3582    /// shifting every offset the table records.
3583    #[test]
3584    fn repair_declines_when_padding_would_move_objects() {
3585        let mut incremental = pdf_with_xref(false);
3586        incremental.extend_from_slice(b"6 0 obj<</Type/Whatever>>endobj\n");
3587        let declined = super::pad_short_xref_entries(&incremental).unwrap_err();
3588        assert!(
3589            declined.contains("object follows the xref"),
3590            "reason: {declined}"
3591        );
3592    }
3593}
3594
3595/// #187: standard-14 fonts referenced without an embedded program (and thus
3596/// usually without `/Widths` or a `/FontDescriptor`) must decode with the
3597/// built-in Adobe Core 14 metrics instead of collapsing every cell to zero
3598/// width — the failure mode where a valid text layer was silently dropped
3599/// while pdfium read the same file fine.
3600#[cfg(test)]
3601mod base14_fonts {
3602    /// A one-page PDF whose single `Tj` uses `fontdict` (no embedded program).
3603    fn pdf_with_font(fontdict: &[u8], text: &[u8]) -> Vec<u8> {
3604        let content = [b"BT /F1 12 Tf 72 700 Td (".as_slice(), text, b") Tj ET\n"].concat();
3605        pdf_with_content(fontdict, &content)
3606    }
3607
3608    /// A one-page A4 PDF drawing `content` with `fontdict` as `/F1`.
3609    pub(super) fn pdf_with_content(fontdict: &[u8], content: &[u8]) -> Vec<u8> {
3610        pdf_with_fonts(&[fontdict], content)
3611    }
3612
3613    /// A one-page A4 PDF drawing `content` with `fontdicts[i]` bound as
3614    /// `/F{i+1}` (objects 5, 6, …).
3615    fn pdf_with_fonts(fontdicts: &[&[u8]], content: &[u8]) -> Vec<u8> {
3616        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
3617        let font_res: String = fontdicts
3618            .iter()
3619            .enumerate()
3620            .map(|(i, _)| format!("/F{} {} 0 R", i + 1, i + 5))
3621            .collect();
3622        let mut objs: Vec<Vec<u8>> = vec![
3623            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
3624            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
3625            format!(
3626                "<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
3627                 /Resources<</Font<<{font_res}>>>>>>"
3628            )
3629            .into_bytes(),
3630            [stream.as_slice(), content, b"endstream"].concat(),
3631        ];
3632        objs.extend(fontdicts.iter().map(|d| d.to_vec()));
3633        let mut out = b"%PDF-1.4\n".to_vec();
3634        let mut offsets = Vec::new();
3635        for (i, body) in objs.iter().enumerate() {
3636            offsets.push(out.len());
3637            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
3638            out.extend_from_slice(body);
3639            out.extend_from_slice(b"endobj\n");
3640        }
3641        let xref_at = out.len();
3642        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
3643        out.extend_from_slice(b"0000000000 65535 f \n");
3644        for off in &offsets {
3645            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
3646        }
3647        out.extend_from_slice(
3648            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
3649        );
3650        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
3651        out
3652    }
3653
3654    /// The parsed cells of the only page.
3655    pub(super) fn cells(pdf: &[u8]) -> Vec<crate::pdfium_backend::TextCell> {
3656        super::pdf_textlines(pdf)
3657            .into_iter()
3658            .flat_map(|(_, _, c)| c)
3659            .collect()
3660    }
3661
3662    /// Text drawn with a rotated text matrix reads as one line in its own
3663    /// reading order, boxed by the run's axis-aligned extent (#528). The
3664    /// landscape case is `0 s -s 0 tx ty Tm` on a `/Rotate 90` page; every
3665    /// such run used to collapse to zero width and vanish (90°/270°/tilted)
3666    /// or read backwards (180°). Expected boxes are docling-parse 7.22's line
3667    /// cells for the same runs (y-up, PDF points).
3668    #[test]
3669    fn rotated_text_matrix_reads_in_order() {
3670        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
3671        const TEXT: &str = "Upright text on a rotated page.";
3672        for (tm, [l, b, r, t]) in [
3673            ("0 14 -14 0 200 100", [189.9, 100.0, 202.9, 289.1]), // 90°
3674            ("-14 0 0 -14 350 300", [160.9, 289.9, 350.0, 302.9]), // 180°
3675            ("0 -14 14 0 200 500", [197.1, 310.9, 210.1, 500.0]), // 270°
3676            (
3677                "9.8995 9.8995 -9.8995 9.8995 100 100",
3678                [92.9, 98.0, 235.8, 240.8],
3679            ), // 45°
3680        ] {
3681            let content = format!("BT /F1 1 Tf {tm} Tm ({TEXT}) Tj ET\n");
3682            let pdf = pdf_with_content(HELV, content.as_bytes());
3683            let cs = cells(&pdf);
3684            assert_eq!(cs.len(), 1, "{tm}: {cs:?}");
3685            let c = &cs[0];
3686            assert_eq!(c.text, TEXT, "{tm}");
3687            // `cells` is top-left-origin on the 842 pt page. The descriptor-
3688            // less base-14 face gets the parser's 1-em box where docling-parse
3689            // reads Helvetica's AFM (718/−207) — the same ≤ 0.6 pt
3690            // across-the-baseline offset upright text has — so allow 1 pt.
3691            let got = [c.l, 842.0 - c.b, c.r, 842.0 - c.t];
3692            for (g, w) in got.iter().zip([l, b, r, t]) {
3693                assert!(
3694                    (g - w).abs() < 1.0,
3695                    "{tm}: box {got:?} vs docling-parse {:?}",
3696                    [l, b, r, t]
3697                );
3698            }
3699            let words: Vec<String> = super::pdf_words(&pdf)
3700                .into_iter()
3701                .flat_map(|(_, _, c)| c)
3702                .map(|c| c.text)
3703                .collect();
3704            assert_eq!(words, TEXT.split(' ').collect::<Vec<_>>(), "{tm}");
3705        }
3706    }
3707
3708    /// A rotated run past the page edge (a cover's spine title) is dropped
3709    /// by the on-page test (#529) — measured on the quad's extent, not the
3710    /// zero-width box rotated glyphs used to get — instead of clamping onto
3711    /// the edge as a zero-width cell. One inside the page is kept.
3712    #[test]
3713    fn rotated_text_off_the_page_is_dropped() {
3714        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
3715        let off = pdf_with_content(
3716            HELV,
3717            b"BT /F1 1 Tf 0 14 -14 0 -4 200 Tm (Spine title) Tj ET\n",
3718        );
3719        assert!(cells(&off).is_empty(), "{:?}", cells(&off));
3720        let on = pdf_with_content(
3721            HELV,
3722            b"BT /F1 1 Tf 0 14 -14 0 30 200 Tm (Margin note) Tj ET\n",
3723        );
3724        assert_eq!(cells(&on).len(), 1);
3725    }
3726
3727    /// Upright text keeps the plain loose rectangle — no quad, and the
3728    /// sanitizer path is the one every pinned PDF baseline was made with.
3729    #[test]
3730    fn upright_glyphs_carry_no_quad() {
3731        let only_page = |pdf: &[u8]| {
3732            let doc = super::load_document(pdf).expect("loads");
3733            let pid = *doc.get_pages().values().next().expect("one page");
3734            super::page_glyphs(&doc, pid)
3735        };
3736        let pdf = pdf_with_font(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", b"Plain");
3737        let glyphs = only_page(&pdf);
3738        assert!(!glyphs.is_empty());
3739        assert!(glyphs.iter().all(|g| g.quad.is_none() && g.lr > g.ll));
3740        // A 90° glyph's style height is its font size, not its advance.
3741        let pdf = pdf_with_content(
3742            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>",
3743            b"BT /F1 1 Tf 0 14 -14 0 200 100 Tm (W) Tj ET\n",
3744        );
3745        let glyphs = only_page(&pdf);
3746        let g = &glyphs[0];
3747        assert!(g.quad.is_some());
3748        // 14 pt × the face's 1-em box (no descriptor: ascent − descent = 1).
3749        assert!((g.height() - 14.0).abs() < 0.05, "{}", g.height());
3750    }
3751
3752    /// #609: ReportLab's chart axis sets each tick label with the *same* `Tm`
3753    /// inside its own `q … cm … Q`, so only the CTM stacks `3`…`8` one under
3754    /// another at one x. Each tick is its own line cell with its own box (as
3755    /// docling-parse reads it), not one `345678` cell boxed like the first.
3756    #[test]
3757    fn ticks_stacked_by_cm_are_separate_line_cells() {
3758        let mut content = String::new();
3759        for (k, tick) in "345678".chars().enumerate() {
3760            let y = 300.0 + 25.92 * k as f64;
3761            content.push_str(&format!(
3762                "q 1 0 0 1 125 {y} cm BT /F1 10 Tf 1 0 0 1 -5 -4 Tm ({tick}) Tj ET Q\n"
3763            ));
3764        }
3765        let pdf = pdf_with_content(
3766            b"<</Type/Font/Subtype/Type1/BaseFont/Times-Roman>>",
3767            content.as_bytes(),
3768        );
3769        let cells = cells(&pdf);
3770        let texts: Vec<&str> = cells.iter().map(|c| c.text.as_str()).collect();
3771        assert_eq!(texts, ["3", "4", "5", "6", "7", "8"]);
3772        for (k, c) in cells.iter().enumerate() {
3773            // Top-left y of a tick whose baseline is 4 pt under its cm origin.
3774            let base = 842.0 - (300.0 + 25.92 * k as f32 - 4.0);
3775            assert!((c.l - 120.0).abs() < 0.1, "{k}: l {}", c.l);
3776            assert!(
3777                c.t < base && base - c.t < 10.0,
3778                "{k}: t {} base {base}",
3779                c.t
3780            );
3781        }
3782    }
3783
3784    /// #609: the path walk finds drawn checkbox squares — ReportLab's four
3785    /// `m … l S` edges under a `cm`, and a stroked `re` with a tick inside
3786    /// (checked) — in the text cells' top-left frame, and a stroked box
3787    /// filled white; a filled bar, a big frame and a colour-filled legend
3788    /// swatch are not checkboxes.
3789    #[test]
3790    fn drawn_checkbox_squares_are_found() {
3791        let content = b"q 1 0 0 1 119.52 449.04 cm \
3792            n 0 12.96 m 12.96 12.96 l S n 0 0 m 12.96 0 l S \
3793            n 0 0 m 0 12.96 l S n 12.96 0 m 12.96 12.96 l S Q\n\
3794            200 400 10 10 re S 202 405 m 204.5 402 l 208.5 408.5 l S\n\
3795            300 400 12 12 re f 50 50 100 100 re S\n\
3796            q 0.2 0.4 0.8 rg 400 400 12 12 re B Q q 1 g 500 400 12 12 re B Q\n";
3797        let pdf = pdf_with_content(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", content);
3798        let mut parser = super::PageTextParser::open(&pdf).expect("parses");
3799        let mut boxes = parser.cells(0).checkboxes;
3800        boxes.sort_by(|a, b| a.l.total_cmp(&b.l));
3801        let got: Vec<([i32; 4], bool)> = boxes
3802            .iter()
3803            .map(|c| {
3804                let r = |v: f32| (v * 100.0).round() as i32;
3805                ([r(c.l), r(c.t), r(c.r), r(c.b)], c.checked)
3806            })
3807            .collect();
3808        // Page height 842: y-up 449.04..462.0 → top-left 380.0..392.96.
3809        assert_eq!(
3810            got,
3811            [
3812                ([11952, 38000, 13248, 39296], false),
3813                ([20000, 43200, 21000, 44200], true),
3814                ([50000, 43000, 51200, 44200], false),
3815            ]
3816        );
3817    }
3818
3819    /// Every standard-14 alias/style decodes with real (positive-width) boxes.
3820    #[test]
3821    fn standard14_faces_get_builtin_widths() {
3822        for fontdict in [
3823            // ReportLab's default: base-14 Helvetica, WinAnsi, nothing else.
3824            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>".as_slice(),
3825            // No /Encoding at all (StandardEncoding-ish default).
3826            b"<</Type/Font/Subtype/Type1/BaseFont/Times-BoldItalic>>",
3827            // Substitution aliases + a subset prefix.
3828            b"<</Type/Font/Subtype/TrueType/BaseFont/Arial,Bold>>",
3829            b"<</Type/Font/Subtype/Type1/BaseFont/ABCDEF+Courier-Oblique>>",
3830        ] {
3831            let pdf = pdf_with_font(fontdict, b"Words have width now");
3832            let cs = cells(&pdf);
3833            let text: String = cs
3834                .iter()
3835                .map(|c| c.text.as_str())
3836                .collect::<Vec<_>>()
3837                .join(" ");
3838            assert!(
3839                text.contains("Words have width now"),
3840                "{}: text lost: {text:?}",
3841                String::from_utf8_lossy(fontdict)
3842            );
3843            assert!(
3844                cs.iter().all(|c| c.r > c.l),
3845                "{}: zero-width cells: {cs:?}",
3846                String::from_utf8_lossy(fontdict)
3847            );
3848        }
3849    }
3850
3851    /// MacRomanEncoding without a ToUnicode CMap (#744) — Word for Mac's
3852    /// Quartz output: the whole high range decodes, Danish æ ø å, é, ü, ß
3853    /// and § included, through the plain encoding name and as the base of a
3854    /// `/Differences` dictionary alike.
3855    #[test]
3856    fn macroman_fonts_keep_accented_letters() {
3857        let sentence = b"Bl\x8cb\xbergr\xbfd p\x8c \xbfen \xd0 caf\x8e M\x9fller, Stra\xa7e \xa4 3";
3858        for (fontdict, want) in [
3859            (
3860                b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/MacRomanEncoding>>"
3861                    .as_slice(),
3862                "Blåbærgrød på øen – café Müller, Straße § 3",
3863            ),
3864            (
3865                b"<</Type/Font/Subtype/TrueType/BaseFont/ABCDEF+Arial\
3866                   /Encoding<</Type/Encoding/BaseEncoding/MacRomanEncoding/Differences[51/four]>>>>",
3867                "Blåbærgrød på øen – café Müller, Straße § 4",
3868            ),
3869        ] {
3870            let text: String = cells(&pdf_with_font(fontdict, sentence))
3871                .iter()
3872                .map(|c| c.text.as_str())
3873                .collect::<Vec<_>>()
3874                .join(" ");
3875            assert_eq!(text, want, "{}", String::from_utf8_lossy(fontdict));
3876        }
3877
3878        // The table itself: all of 0x80–0xFF but the Apple logo, ¤ at 0xDB.
3879        let table = super::macroman_table();
3880        assert_eq!(
3881            (0x80u8..=0xFF).filter(|b| table.contains_key(b)).count(),
3882            127
3883        );
3884        assert_eq!(table.get(&0xF0), None);
3885        assert_eq!(table[&0xDB], '¤');
3886        assert_eq!(
3887            [0x80, 0x8C, 0xAE, 0xAF, 0xBE, 0xBF, 0xFF].map(|b| table[&b]),
3888            ['Ä', 'å', 'Æ', 'Ø', 'æ', 'ø', 'ˇ']
3889        );
3890    }
3891
3892    /// Unmapped glyphs read U+FFFD (#1003, docling#5070): a ToUnicode NUL —
3893    /// docling's `test_threaded_page_backend_replaces_nul_with_replacement_char`
3894    /// case, `mg.mL \x00 1` — and a bare glyph-index `/Differences` name,
3895    /// while an unknown code-like name keeps its text without the slash
3896    /// (docling-parse 7.24's `SM590000`).
3897    #[test]
3898    fn unmapped_glyphs_read_as_the_replacement_char() {
3899        let cmap = b"/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n\
3900            1 begincodespacerange <00> <FF> endcodespacerange\n\
3901            1 beginbfchar <7e> <0000> endbfchar\n\
3902            endcmap CMapName currentdict /CMap defineresource pop end end";
3903        let stream = [
3904            format!("<</Length {}>>stream\n", cmap.len()).as_bytes(),
3905            cmap.as_slice(),
3906            b"\nendstream",
3907        ]
3908        .concat();
3909        let pdf = pdf_with_fonts(
3910            &[
3911                b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/ToUnicode 6 0 R\
3912                   /Encoding<</Type/Encoding/Differences[1/g115 2/SM590000]>>>>",
3913                &stream,
3914            ],
3915            b"BT /F1 12 Tf 72 700 Td (mg.mL ~ 1 \x01 x \x02 y) Tj ET\n",
3916        );
3917        let text: String = cells(&pdf)
3918            .iter()
3919            .map(|c| c.text.as_str())
3920            .collect::<Vec<_>>()
3921            .join(" ");
3922        assert_eq!(text, "mg.mL \u{FFFD} 1 \u{FFFD} x SM590000 y");
3923    }
3924
3925    /// redp5110's subset font names every glyph by index (`/g115/g3 /g40…`):
3926    /// the painted ones read U+FFFD, the blank `/g3` program is a space, so
3927    /// the line keeps its words (docling-parse 7.24's line cell).
3928    #[cfg(feature = "render")]
3929    #[test]
3930    fn a_blank_glyph_program_is_a_space() {
3931        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
3932            .join("../../tests/data/pdf/sources/redp5110_sampled.pdf");
3933        let Ok(bytes) = std::fs::read(path) else {
3934            return;
3935        };
3936        let lines: Vec<String> = super::pdf_textlines(&bytes)
3937            .into_iter()
3938            .flat_map(|(_, _, c)| c)
3939            .map(|c| c.text)
3940            .collect();
3941        let want = "\u{FFFD} \u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD} \
3942                    \u{FFFD}\u{FFFD}\u{FFFD} ";
3943        assert!(
3944            lines.iter().any(|l| l.starts_with(want)),
3945            "no `{want}` line in {:?}",
3946            lines
3947                .iter()
3948                .filter(|l| l.contains('\u{FFFD}'))
3949                .collect::<Vec<_>>()
3950        );
3951    }
3952
3953    /// The page's image placements reach the cells (#999): the scan under
3954    /// the #999 fixture's one native word is the whole page.
3955    #[test]
3956    fn image_placements_are_the_pages_bitmaps() {
3957        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
3958            .join("../docling/tests/fixtures/partial_text_layer/scan_with_native_word.pdf");
3959        let bytes = std::fs::read(path).expect("fixture");
3960        let mut parser = super::PageTextParser::open(&bytes).expect("parser");
3961        let page = parser.cells(0);
3962        assert_eq!(page.bitmaps, vec![[0.0, 0.0, 612.0, 792.0]]);
3963        assert_eq!(page.words.len(), 1);
3964        assert!(page.checkboxes.is_empty());
3965    }
3966
3967    /// A `bfrange` whose destination is a surrogate pair steps its last code
3968    /// unit (#1003: equations.pdf's `H𝛂`, `<1d59> <1d5a> <d835dec2>`); a
3969    /// single-unit base keeps counting in code points.
3970    #[test]
3971    fn surrogate_pair_bfrange_maps_each_code() {
3972        let map = super::parse_tounicode(
3973            b"1 beginbfrange <1d59> <1d5a> <d835dec2> endbfrange\n\
3974              1 beginbfrange <0010> <0011> <0041> endbfrange",
3975        );
3976        assert_eq!(map.get(&0x1d59).map(String::as_str), Some("\u{1D6C2}"));
3977        assert_eq!(map.get(&0x1d5a).map(String::as_str), Some("\u{1D6C3}"));
3978        assert_eq!(map.get(&0x11).map(String::as_str), Some("B"));
3979    }
3980
3981    /// Which unknown glyph names are text and which are bare indexes
3982    /// (docling-parse 7.24's `resolve_unknown_name`).
3983    #[test]
3984    fn unknown_glyph_names_follow_docling_parse() {
3985        for name in ["g115", "gid00043", "Glyph7", "cid42", "index9", "g3.alt"] {
3986            assert_eq!(super::unknown_name_text(name), None, "{name}");
3987        }
3988        assert_eq!(
3989            super::unknown_name_text("SM590000").as_deref(),
3990            Some("SM590000")
3991        );
3992        assert_eq!(
3993            super::unknown_name_text("SM590000.sc").as_deref(),
3994            Some("SM590000")
3995        );
3996    }
3997
3998    /// An explicit `/Widths` array always wins over the built-in metrics, and a
3999    /// non-standard face without `/Widths` stays as before (no invented boxes).
4000    #[test]
4001    fn explicit_widths_win_and_unknown_faces_are_untouched() {
4002        // Helvetica with explicit 100/1000-em widths: the word's box must be
4003        // ~4×100 units at 12pt = 4.8pt wide — far narrower than the ~2.7×
4004        // wider built-in Helvetica advances would make it.
4005        let explicit = pdf_with_font(
4006            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/FirstChar 65\
4007               /Widths[100 100 100 100]/Encoding/WinAnsiEncoding>>",
4008            b"ABBA",
4009        );
4010        let builtin = pdf_with_font(
4011            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>",
4012            b"ABBA",
4013        );
4014        let w = |pdf: &[u8]| {
4015            let cs = cells(pdf);
4016            assert_eq!(cs.len(), 1, "one word cell: {cs:?}");
4017            cs[0].r - cs[0].l
4018        };
4019        let (we, wb) = (w(&explicit), w(&builtin));
4020        assert!(
4021            (we - 4.8).abs() < 0.1,
4022            "explicit widths must win: got {we}, want 4×100×12/1000"
4023        );
4024        assert!(
4025            wb > 2.0 * we,
4026            "built-in Helvetica is much wider: {wb} vs {we}"
4027        );
4028
4029        // An unknown face with no /Widths: still parses (text kept), but no
4030        // built-in table applies — the old zero-width behavior is preserved
4031        // rather than inventing Helvetica metrics for an arbitrary font.
4032        let unknown = pdf_with_font(
4033            b"<</Type/Font/Subtype/Type1/BaseFont/FancyCorp-Display>>",
4034            b"Mystery",
4035        );
4036        let cs = cells(&unknown);
4037        let text: String = cs.iter().map(|c| c.text.as_str()).collect();
4038        assert!(text.contains("Mystery"), "text still decodes: {cs:?}");
4039    }
4040    /// #728: one face bound under two resource keys — `/F1` Helvetica in
4041    /// WinAnsi, `/F2` the same Helvetica re-encoded so code 0x81 is the `fi`
4042    /// glyph — with every fi-word drawn in `/F2` right after an `/F1` run.
4043    /// docling-parse names a font by `/FontDescriptor/FontName` → `/Name` →
4044    /// `/BaseFont`, never by the resource key, so both resources are the same
4045    /// font and the line reads `financial`. Keyed by resource key the `fi`
4046    /// ligature (font-neutral in `applicable_for_merge`) joined the `/F1`
4047    /// run and the font barrier then split it from `nancial`.
4048    #[test]
4049    fn same_face_under_two_resource_keys_is_one_font_for_the_sanitizer() {
4050        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4051        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding<</Type/Encoding\
4052                   /BaseEncoding/WinAnsiEncoding/Differences[129/fi]>>>>";
4053        let content = b"BT 12 TL 72 720 Td \
4054            /F1 12 Tf (The ) Tj /F2 12 Tf (\\201nancial position) Tj \
4055            /F1 12 Tf ( of the company is shown in ) Tj /F2 12 Tf (\\201gure 3) Tj \
4056            /F1 12 Tf ( with the ) Tj /F2 12 Tf (\\201rst results.) Tj \
4057            T* T* /F2 12 Tf (Same words in /F2 only: \\201nancial, \\201gure, \\201rst.) Tj ET\n";
4058        let cs = cells(&pdf_with_fonts(&[f1, f2], content));
4059        let lines: Vec<&str> = cs.iter().map(|c| c.text.as_str()).collect();
4060        assert_eq!(
4061            lines,
4062            [
4063                "The financial position of the company is shown in figure 3 with the first results.",
4064                "Same words in /F2 only: financial, figure, first.",
4065            ],
4066            "{cs:?}"
4067        );
4068    }
4069
4070    /// Under `legacy` the font barrier itself is untouched: two *different*
4071    /// faces (regular and bold Helvetica, which docling-parse names apart) set
4072    /// flush against each other stay separate line cells, as
4073    /// `enforce_same_font` demands. docling-parse 7 dropped the barrier — its
4074    /// line runs follow geometry alone — so under the default `adaptive` the
4075    /// flush pair is one line and one word.
4076    #[test]
4077    fn different_faces_keep_the_font_barrier_only_under_legacy() {
4078        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4079        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica-Bold/Encoding/WinAnsiEncoding>>";
4080        let content = b"BT 72 720 Td /F2 12 Tf (Total:) Tj /F1 12 Tf (42) Tj ET\n";
4081        let pdf = pdf_with_fonts(&[f1, f2], content);
4082        let lines = |gap| {
4083            let mut p = super::PageTextParser::open(&pdf).expect("opens");
4084            p.set_word_gap(Some(gap));
4085            p.cells(0)
4086                .prose
4087                .into_iter()
4088                .map(|c| c.text)
4089                .collect::<Vec<_>>()
4090        };
4091        assert_eq!(lines(crate::dp_lines::WordGap::Legacy), ["Total:", "42"]);
4092        assert_eq!(lines(crate::dp_lines::WordGap::Adaptive), ["Total:42"]);
4093    }
4094
4095    /// #992: a space glyph whose advance the next `TJ` offset takes back —
4096    /// EUR-Lex's superscript footnote marker `( 1 )`, set with spaces the
4097    /// digit is then drawn over — is not a character of the text: the page
4098    /// shows `(1)`, pdftotext and pdfium read `(1)`, and so does the line
4099    /// cell here (docling-parse keeps the spaces, `( 1 )`). A space the next
4100    /// glyph only kerns into (−100/1000 em, well past the space's start) is
4101    /// a real gap and stays, as does one before a jump to the next line.
4102    #[test]
4103    fn a_space_the_next_glyph_is_drawn_over_is_not_a_character() {
4104        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4105        let lines = |content: &[u8]| {
4106            let pdf = pdf_with_content(f1, content);
4107            super::PageTextParser::open(&pdf)
4108                .expect("opens")
4109                .cells(0)
4110                .prose
4111                .into_iter()
4112                .map(|c| c.text)
4113                .collect::<Vec<_>>()
4114        };
4115        // Helvetica's space is 278/1000 em: an offset of 300 moves the pen
4116        // back past where the space began, 100 leaves two thirds of it.
4117        assert_eq!(
4118            lines(b"BT /F1 12 Tf 72 700 Td [(EG \\() ( ) 300 (1) ( ) 300 (\\) erf)] TJ ET\n"),
4119            ["EG (1) erf"]
4120        );
4121        assert_eq!(
4122            lines(b"BT /F1 12 Tf 72 700 Td [(EG \\() ( ) 100 (1) ( ) 100 (\\) erf)] TJ ET\n"),
4123            ["EG ( 1 ) erf"]
4124        );
4125        // A trailing space, then the next line starts at the left margin.
4126        assert_eq!(
4127            lines(b"BT /F1 12 Tf 72 700 Td (first line ) Tj 0 -14 Td (second) Tj ET\n"),
4128            ["first line", "second"]
4129        );
4130        // A line drawn rotated 90° (a landscape slide on a portrait page,
4131        // #528): its glyphs' loose boxes stack on one left edge, which is
4132        // not an overprint — the space stays.
4133        assert_eq!(
4134            lines(b"BT 0 1 -1 0 300 100 Tm /F1 12 Tf (Nullam mollis) Tj ET\n"),
4135            ["Nullam mollis"]
4136        );
4137    }
4138
4139    /// The identity follows docling-parse's precedence: a `/FontDescriptor/FontName`
4140    /// wins over `/BaseFont`, so two dictionaries that share a `/BaseFont` but
4141    /// describe different programs are two fonts, and two that share the
4142    /// descriptor name are one whatever their `/BaseFont` says.
4143    #[test]
4144    fn math_faces_are_named_like_docling_parse_names_them() {
4145        for name in [
4146            "ABCDEF+CMR7",
4147            "CMMI10",
4148            "MSBM10",
4149            "Cambria-Math",
4150            "Symbol",
4151            "MTExtra",
4152        ] {
4153            assert!(super::is_math_font_name(name.as_bytes()), "{name}");
4154        }
4155        for name in ["Helvetica", "ABCDEF+TimesNewRomanPSMT", "Arial-BoldMT"] {
4156            assert!(!super::is_math_font_name(name.as_bytes()), "{name}");
4157        }
4158    }
4159
4160    #[test]
4161    fn sanitizer_font_name_follows_docling_parse_precedence() {
4162        use lopdf::{dictionary, Document, Object};
4163        let mut doc = Document::with_version("1.4");
4164        let fd_a =
4165            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "AAAAAA+Face" });
4166        let fd_b =
4167            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "BBBBBB+Face" });
4168        let with = |fd: Option<lopdf::ObjectId>, base: &str| {
4169            let mut d = dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => base };
4170            if let Some(id) = fd {
4171                d.set("FontDescriptor", Object::Reference(id));
4172            }
4173            d
4174        };
4175        let name = |d: &lopdf::Dictionary| {
4176            String::from_utf8(super::sanitizer_font_name(&doc, d, false)).unwrap()
4177        };
4178        assert_eq!(name(&with(Some(fd_a), "Face")), "AAAAAA+Face");
4179        assert_eq!(name(&with(Some(fd_b), "Face")), "BBBBBB+Face");
4180        assert_eq!(name(&with(None, "ABCDEF+Face")), "ABCDEF+Face");
4181        assert_eq!(
4182            name(&dictionary! { "Type" => "Font", "Name" => "F7" }),
4183            "F7"
4184        );
4185        assert_eq!(name(&dictionary! { "Type" => "Font" }), "null");
4186    }
4187}
4188
4189#[cfg(test)]
4190mod overpainted {
4191    use crate::pdfium_backend::TextCell;
4192
4193    fn cell(text: &str, l: f32, t: f32, r: f32, b: f32) -> TextCell {
4194        TextCell {
4195            text: text.into(),
4196            l,
4197            t,
4198            r,
4199            b,
4200        }
4201    }
4202
4203    /// The reporting invoice's logo: a `"` painted inside a `==` on one band —
4204    /// artwork drawn with glyphs. Both cells go; the real text on the next
4205    /// band stays.
4206    #[test]
4207    fn stacked_logo_glyphs_are_dropped() {
4208        let mut cells = vec![
4209            cell("\"", 72.7, 21.5, 86.4, 31.5),
4210            cell("==", 59.4, 21.5, 99.6, 31.5),
4211            cell("Herr", 65.2, 151.3, 81.7, 161.3),
4212        ];
4213        super::drop_overpainted_cells(&mut cells);
4214        assert_eq!(cells.len(), 1, "cells: {cells:?}");
4215        assert_eq!(cells[0].text, "Herr");
4216    }
4217
4218    /// Adjacent words on a line touch but never contain each other — prose is
4219    /// untouched, and so is a same-text near-duplicate (double-drawn faux
4220    /// bold), which is not evidence of artwork.
4221    #[test]
4222    fn prose_and_double_draw_are_kept() {
4223        let mut cells = vec![
4224            cell("Telefon", 354.3, 133.2, 381.5, 143.2),
4225            cell("0676/2000", 387.3, 133.2, 428.7, 143.2),
4226            cell("Bold", 100.0, 50.0, 130.0, 60.0),
4227            cell("Bold", 100.3, 50.0, 130.3, 60.0),
4228        ];
4229        super::drop_overpainted_cells(&mut cells);
4230        assert_eq!(cells.len(), 4);
4231    }
4232}
4233
4234#[cfg(test)]
4235mod vestigial_layer {
4236    use crate::pdfium_backend::{PdfPage, TextCell};
4237
4238    fn page_with(texts: &[&str]) -> PdfPage {
4239        let cells = texts
4240            .iter()
4241            .enumerate()
4242            .map(|(i, t)| TextCell {
4243                text: t.to_string(),
4244                l: 10.0,
4245                t: 10.0 + 12.0 * i as f32,
4246                r: 90.0,
4247                b: 20.0 + 12.0 * i as f32,
4248            })
4249            .collect();
4250        PdfPage::from_cells(595.0, 842.0, 1.0, cells)
4251    }
4252
4253    /// The reported scanned form: three typed-in field values ("03", "05",
4254    /// "2025") over three image pages. That must read as *no usable layer*,
4255    /// so the browser routes the document to OCR instead of extracting
4256    /// thirteen characters and skipping the letter entirely.
4257    #[test]
4258    fn typed_in_form_fields_are_not_a_text_layer() {
4259        let pages = vec![
4260            page_with(&["03", "05", "2025"]),
4261            page_with(&[]),
4262            page_with(&[]),
4263        ];
4264        assert!(super::text_layer_is_vestigial(&pages));
4265        assert!(super::text_layer_is_vestigial(&[page_with(&[])]));
4266    }
4267
4268    /// A short but genuine digital document — one page, a few real lines —
4269    /// keeps the fast text path.
4270    #[test]
4271    fn sparse_but_real_documents_pass() {
4272        let one_pager = vec![page_with(&[
4273            "Confidential briefing",
4274            "Prepared for the board meeting",
4275            "Do not distribute",
4276        ])];
4277        assert!(!super::text_layer_is_vestigial(&one_pager));
4278    }
4279}
4280
4281/// #994: decoded Form XObject content is cached per document only from a
4282/// form's second sighting, so one-form-per-page documents do not hold every
4283/// page's decoded form until the document ends.
4284#[cfg(test)]
4285mod form_cache {
4286    use lopdf::content::{Content, Operation};
4287    use lopdf::{dictionary, Document, Object, Stream};
4288
4289    fn form(doc: &mut Document, text: &str) -> lopdf::ObjectId {
4290        let ops = vec![
4291            Operation::new("BT", vec![]),
4292            Operation::new("Tf", vec!["F1".into(), 12.into()]),
4293            Operation::new("Td", vec![40.into(), 500.into()]),
4294            Operation::new("Tj", vec![Object::string_literal(text)]),
4295            Operation::new("ET", vec![]),
4296        ];
4297        let data = Content { operations: ops }.encode().unwrap();
4298        doc.add_object(Stream::new(
4299            dictionary! { "Type" => "XObject", "Subtype" => "Form",
4300            "BBox" => vec![0.into(), 0.into(), 595.into(), 842.into()] },
4301            data,
4302        ))
4303    }
4304
4305    #[test]
4306    fn single_use_forms_are_not_retained_shared_ones_are() {
4307        let mut doc = Document::with_version("1.5");
4308        let pages_id = doc.new_object_id();
4309        let font = doc.add_object(
4310            dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => "Helvetica" },
4311        );
4312        let header = form(&mut doc, "Header");
4313        let mut kids = Vec::new();
4314        let mut own_forms = Vec::new();
4315        for p in 0..3 {
4316            let own = form(&mut doc, &format!("Body{p}"));
4317            own_forms.push(own);
4318            let content = doc.add_object(Stream::new(
4319                dictionary! {},
4320                b"q /Hd Do Q q /Fx Do Q".to_vec(),
4321            ));
4322            kids.push(Object::Reference(doc.add_object(dictionary! {
4323                "Type" => "Page", "Parent" => pages_id,
4324                "MediaBox" => vec![0.into(), 0.into(), 595.into(), 842.into()],
4325                "Contents" => content,
4326                "Resources" => dictionary! {
4327                    "Font" => dictionary! { "F1" => font },
4328                    "XObject" => dictionary! { "Hd" => header, "Fx" => own },
4329                },
4330            })));
4331        }
4332        doc.objects.insert(
4333            pages_id,
4334            Object::Dictionary(dictionary! { "Type" => "Pages", "Kids" => kids, "Count" => 3 }),
4335        );
4336        let catalog = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages_id });
4337        doc.trailer.set("Root", catalog);
4338
4339        let mut caches = super::DocCaches::default();
4340        let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
4341        pages.sort_by_key(|(n, _)| *n);
4342        for (i, (_, pid)) in pages.iter().enumerate() {
4343            let text: String = super::page_glyphs_cached(&doc, *pid, &mut caches)
4344                .iter()
4345                .map(|g| g.ch)
4346                .collect();
4347            // Both forms' text is still read, cached or not.
4348            assert!(text.contains("Header"), "page {i}: {text:?}");
4349            assert!(text.contains(&format!("Body{i}")), "page {i}: {text:?}");
4350        }
4351        // The shared header form is cached from its second page on; the
4352        // per-page forms, each used once, are not held.
4353        assert!(caches.forms.contains_key(&header));
4354        for own in &own_forms {
4355            assert!(!caches.forms.contains_key(own));
4356        }
4357        assert_eq!(caches.forms.len(), 1);
4358    }
4359}
4360
4361#[cfg(test)]
4362mod cid_collections {
4363    use super::base14_fonts::{cells, pdf_with_content};
4364
4365    fn type0(encoding: &str) -> Vec<u8> {
4366        format!(
4367            "<</Type/Font/Subtype/Type0/BaseFont/Test-Font-Light/Encoding/{encoding}\
4368             /DescendantFonts[<</Type/Font/Subtype/CIDFontType0/BaseFont/Test-Font-Light\
4369             /CIDSystemInfo<</Registry(Adobe)/Ordering(Japan1)/Supplement 7>>/DW 1000>>]>>"
4370        )
4371        .into_bytes()
4372    }
4373
4374    fn text(fontdict: &[u8], hex: &str) -> String {
4375        let content = format!("BT /F1 12 Tf 72 700 Td <{hex}> Tj ET\n");
4376        cells(&pdf_with_content(fontdict, content.as_bytes()))
4377            .iter()
4378            .map(|c| c.text.as_str())
4379            .collect()
4380    }
4381
4382    #[test]
4383    #[cfg(feature = "cid-collections")]
4384    fn identity_codes_read_through_the_collection() {
4385        assert_eq!(text(&type0("Identity-H"), "0CD40E8A07A0"), "日本語");
4386    }
4387
4388    #[test]
4389    fn unicode_keyed_cmaps_read_the_code() {
4390        assert_eq!(text(&type0("UniJIS-UCS2-H"), "65E5672C8A9E"), "日本語");
4391    }
4392
4393    #[test]
4394    #[cfg(feature = "cid-collections")]
4395    fn collection_ranges_keep_supplementary_characters() {
4396        assert_eq!(text(&type0("Identity-H"), "0CD44507450807A0"), "日𢭏𢭐語");
4397        let text = |o: &[u8], cid| super::collection_text(super::collection_table(o).unwrap(), cid);
4398        assert_eq!(text(b"Japan1", 0x2714).as_deref(), Some("🄐"));
4399        assert_eq!(text(b"CNS1", 0x3a32).as_deref(), Some("𡤃"));
4400    }
4401
4402    #[test]
4403    #[cfg(feature = "cid-collections")]
4404    fn collection_entries_are_plain_characters() {
4405        assert_eq!(text(&type0("Identity-H"), "046D"), "逢");
4406        let text = |o: &[u8], cid| super::collection_text(super::collection_table(o).unwrap(), cid);
4407        assert_eq!(text(b"Korea1", 8752), None);
4408        assert_eq!(text(b"KR", 11704), None);
4409        assert_eq!(text(b"Japan1", 0x1ffd).as_deref(), Some("sec"));
4410    }
4411
4412    #[test]
4413    fn tounicode_ranges_increment_the_utf16_string() {
4414        let map = super::parse_tounicode(
4415            b"3 beginbfrange\n\
4416              <01> <02> <d842dfb7>\n\
4417              <03> <04> <00660069>\n\
4418              <05> <06> <0061>\n\
4419              endbfrange",
4420        );
4421        assert_eq!(map[&1], "𠮷");
4422        assert_eq!(map[&2], "𠮸");
4423        assert_eq!(map[&3], "fi");
4424        assert_eq!(map[&4], "fj");
4425        assert_eq!(map[&5], "a");
4426        assert_eq!(map[&6], "b");
4427    }
4428
4429    #[test]
4430    fn tounicode_skips_lone_surrogates_overflows_and_huge_ranges() {
4431        let map = super::parse_tounicode(
4432            b"2 beginbfchar\n<01> <d842>\n<02> <0041>\nendbfchar\n\
4433              4 beginbfrange\n\
4434              <10> <11> [<dc00> <0042>]\n\
4435              <20> <22> <0041fffe>\n\
4436              <30> <31> <d842dfff>\n\
4437              <00010000> <ffffffff> <>\n\
4438              endbfrange",
4439        );
4440        assert!(!map.contains_key(&0x01));
4441        assert_eq!(map[&0x02], "A");
4442        assert!(!map.contains_key(&0x10));
4443        assert_eq!(map[&0x11], "B");
4444        assert_eq!(map[&0x20], "A\u{fffe}");
4445        assert_eq!(map[&0x21], "A\u{ffff}");
4446        assert!(!map.contains_key(&0x22));
4447        assert_eq!(map[&0x30], "\u{20bff}");
4448        assert!(!map.contains_key(&0x31));
4449        assert_eq!(map[&0x1ffff], "");
4450        assert_eq!(map.len(), 5 + 0x10000);
4451    }
4452
4453    #[test]
4454    fn utf16_pairs_are_one_glyph_and_one_advance() {
4455        let font = type0("UniJIS-UTF16-H");
4456        assert_eq!(text(&font, "65E5D842DFB78A9E"), "日𠮷語");
4457        let content = b"BT /F1 12 Tf 72 700 Td <65E5D842DFB78A9E> Tj ET\n";
4458        let pdf = pdf_with_content(&font, content);
4459        let doc = lopdf::Document::load_mem(&pdf).unwrap();
4460        let glyphs = super::page_glyphs(&doc, doc.get_pages()[&1]);
4461        assert_eq!(glyphs.len(), 3);
4462        assert!((glyphs[2].l - glyphs[0].l - 24.0).abs() < 0.001);
4463    }
4464
4465    #[test]
4466    fn utf16_codes_preserve_tounicode_keys_and_identity_cids() {
4467        use lopdf::{dictionary, Document};
4468        let doc = Document::with_version("1.4");
4469        let dict = dictionary! {"Subtype" => "Type0", "Encoding" => "UniJIS-UTF16-H"};
4470        let mut font = super::parse_font(&doc, b"F1", &dict);
4471        assert_eq!(
4472            super::codes(&font, &[0xd8, 0x42, 0xdf, 0xb7]),
4473            vec![0xd842dfb7]
4474        );
4475        font.to_unicode.insert(0xd842dfb7, "override".into());
4476        assert_eq!(font.decode_code(0xd842dfb7).0.as_deref(), Some("override"));
4477        assert_eq!(
4478            super::codes(&font, &[0xd8, 0x42, 0x65, 0xe5]),
4479            vec![0xd842, 0x65e5]
4480        );
4481        let dict = dictionary! {"Subtype" => "Type0", "Encoding" => "Identity-H"};
4482        let font = super::parse_font(&doc, b"F1", &dict);
4483        assert_eq!(
4484            super::codes(&font, &[0xd8, 0x42, 0xdf, 0xb7]),
4485            vec![0xd842, 0xdfb7]
4486        );
4487    }
4488}