Skip to main content

docling_pdf/
textparse.rs

1//! Pure-Rust PDF text extraction (replacing pdfium's glyph layer).
2//!
3//! pdfium reports *rendered* glyph boxes, which diverge from docling's
4//! `docling-parse` C++ parser at exactly the points that drive conformance:
5//! generated spaces get a zero-width box, combining diacritics get a real-width
6//! box, and ligature/fraction glyphs land at different x. This module instead
7//! reconstructs each glyph's box from the **font's own advance widths** and the
8//! PDF text/graphics matrices — the same information docling-parse uses — so a
9//! space is as wide as the font says and a combining mark has zero advance.
10//!
11//! The output is the same [`Glyph`] stream pdfium produces (native PDF
12//! coordinates, y-up), fed straight into the existing docling-parse line
13//! sanitizer ([`crate::dp_lines`]). Only the digital text layer is handled here;
14//! pages without one still fall back to OCR upstream.
15
16use std::collections::{HashMap, HashSet};
17use std::sync::Arc;
18
19use lopdf::{Dictionary, Document, Object};
20
21use crate::pdfium_backend::Glyph;
22use crate::tounicode::parse_tounicode;
23
24/// Per-document caches for the content-stream interpreter. Fonts are indirect
25/// objects shared by many pages, but were fully re-parsed — ToUnicode CMap
26/// decompression + tokenization, embedded Type1 program scan, width tables —
27/// for **every page and every Form XObject invocation**; decoded form content
28/// streams were likewise re-inflated on every `Do`. Cached per document,
29/// keyed by the referenced object id (fonts also by resource name — the
30/// `/Font` entry a page binds the object to). Inline (non-reference) dicts
31/// are rare and stay uncached.
32#[derive(Default)]
33struct DocCaches {
34    fonts: HashMap<(lopdf::ObjectId, Vec<u8>), Arc<Font>>,
35    forms: HashMap<lopdf::ObjectId, Arc<lopdf::content::Content>>,
36    /// Forms decoded once and not (yet) cached (#994). A form enters
37    /// [`Self::forms`] on its *second* sighting: a shared header/footer form
38    /// is decoded twice and then hits the cache on every later page, while
39    /// a generator's one-form-per-page body (each page's chart in its own
40    /// form, used once) is dropped with its page instead of every decoded
41    /// `Content` — many times its stream's size — living until the document
42    /// ends. Caching on first use made peak RSS grow ~45 MB per page on such
43    /// files (7 GiB at 60 pages of the issue's repro, flat ~0.6 GiB now).
44    /// Only ids are kept here, so the set itself stays tiny.
45    forms_seen: HashSet<lopdf::ObjectId>,
46}
47
48/// A 2×3 affine matrix `[a b c d e f]`: maps `(x,y)` → `(a·x+c·y+e, b·x+d·y+f)`.
49#[derive(Clone, Copy)]
50struct Mat {
51    a: f64,
52    b: f64,
53    c: f64,
54    d: f64,
55    e: f64,
56    f: f64,
57}
58
59impl Mat {
60    const ID: Mat = Mat {
61        a: 1.0,
62        b: 0.0,
63        c: 0.0,
64        d: 1.0,
65        e: 0.0,
66        f: 0.0,
67    };
68
69    /// `self ∘ m`: the matrix that applies `self` first, then `m`.
70    fn then(self, m: Mat) -> Mat {
71        Mat {
72            a: self.a * m.a + self.b * m.c,
73            b: self.a * m.b + self.b * m.d,
74            c: self.c * m.a + self.d * m.c,
75            d: self.c * m.b + self.d * m.d,
76            e: self.e * m.a + self.f * m.c + m.e,
77            f: self.e * m.b + self.f * m.d + m.f,
78        }
79    }
80
81    fn apply(self, x: f64, y: f64) -> (f64, f64) {
82        (
83            self.a * x + self.c * y + self.e,
84            self.b * x + self.d * y + self.f,
85        )
86    }
87}
88
89/// A parsed font: how to turn raw string bytes into (unicode, advance) pairs.
90struct Font {
91    /// 2-byte codes (Type0 / Identity-H) vs 1-byte (simple fonts).
92    two_byte: bool,
93    /// code → Unicode string (from ToUnicode; may be multi-char, e.g. ligatures).
94    to_unicode: HashMap<u32, String>,
95    /// For codes ToUnicode lacks (ISO 32000-1, 9.10.2).
96    cid_fallback: CidFallback,
97    /// code → glyph advance, in 1000-unit glyph space.
98    widths: HashMap<u32, f64>,
99    default_width: f64,
100    /// 1-byte fallback decoding when ToUnicode lacks a code (WinAnsi-ish).
101    simple_encoding: Option<HashMap<u8, char>>,
102    /// code → raw `/Differences` glyph name, for placeholder names (`g115`,
103    /// `SM590000`) that have no Unicode mapping; they override the base
104    /// encoding (see [`unknown_name_text`]).
105    fallback_names: HashMap<u8, String>,
106    /// The unmapped codes whose embedded glyph paints nothing — semantic
107    /// spaces (docling-parse 7.24's blank glyph programs: the redp5110
108    /// list's `/g3`). Every other unmapped code reads U+FFFD.
109    blank_codes: HashSet<u32>,
110    /// code → char from the embedded Type1 font program's own `/Encoding` vector,
111    /// used only as a last resort for glyphs the base encoding leaves unmapped
112    /// (standard TeX math fonts: `λ`, `≤`, …).
113    program_encoding: HashMap<u8, char>,
114    ascent: f64,
115    descent: f64,
116    hash: u64,
117    /// Whether the font's docling-parse name marks a math face
118    /// ([`is_math_font_name`]) — for the word contraction's local
119    /// attachments (#921).
120    math: bool,
121    /// Weight and slant read from the `/BaseFont` name (the heading
122    /// hierarchy's style signal, #302).
123    style: crate::font_style::FontStyle,
124}
125
126impl Font {
127    /// The text of `code` and its advance; the flag marks a blank unmapped
128    /// glyph, whose box has no height.
129    ///
130    /// A code no source maps to Unicode is visibly painted but carries no
131    /// text: docling-parse 7.24 emits U+FFFD for it, and docling turns a NUL
132    /// into U+FFFD too (docling#5070), so the loss stays visible and counted
133    /// by the text-quality check instead of silently dropping the glyph
134    /// (`mL⁻¹` reading `mL1`). A glyph whose embedded program paints nothing
135    /// is a space — a word boundary — instead.
136    fn decode_code(&self, code: u32) -> (Option<String>, f64, bool) {
137        let w = self
138            .widths
139            .get(&code)
140            .copied()
141            .unwrap_or(self.default_width);
142        if let Some(s) = self.to_unicode.get(&code) {
143            return (Some(decompose_ligatures(s)), w, false);
144        }
145        let unmapped = |font: &Font| {
146            let blank = font.blank_codes.contains(&code);
147            (
148                Some(if blank { " " } else { "\u{FFFD}" }.to_string()),
149                w,
150                blank,
151            )
152        };
153        if !self.two_byte {
154            // A placeholder `/Differences` name (no Unicode) overrides the
155            // base encoding. docling-parse 7.24 reads a bare glyph index
156            // (`g115`, `gid00043`) as unmapped and any other unknown name as
157            // its text (`SM590000`, the redp5110 bullets).
158            if let Some(name) = self.fallback_names.get(&(code as u8)) {
159                return match unknown_name_text(name) {
160                    Some(text) => (Some(text), w, false),
161                    None => unmapped(self),
162                };
163            }
164            if let Some(enc) = &self.simple_encoding {
165                if let Some(&ch) = enc.get(&(code as u8)) {
166                    return (Some(decompose_ligatures(&ch.to_string())), w, false);
167                }
168            }
169            // Last resort: the embedded Type1 font program's own `/Encoding`
170            // vector (`dup N /glyphname put`). Standard TeX math fonts (CMMI, CMSY,
171            // …) ship no PDF `/Encoding` and no ToUnicode, so a glyph like `λ`
172            // (CMMI code 21 → `/lambda`) or `≤` (CMSY code 20 → `/lessequal`) has
173            // no other mapping and would otherwise be silently dropped. docling
174            // recovers these from the same font program. This only fills codes the
175            // base encoding left unmapped, so it never changes an existing decode.
176            if let Some(&ch) = self.program_encoding.get(&(code as u8)) {
177                return (Some(decompose_ligatures(&ch.to_string())), w, false);
178            }
179        } else {
180            match self.cid_fallback {
181                CidFallback::None => {}
182                CidFallback::UnicodeCodes { .. } => {
183                    if let Some(ch) = unicode_cmap_char(code).filter(|c| !c.is_control()) {
184                        return (Some(ch.to_string()), w, false);
185                    }
186                }
187                CidFallback::Collection(table) => {
188                    if let Some(s) = collection_text(table, code) {
189                        return (Some(decompose_ligatures(&s)), w, false);
190                    }
191                }
192            }
193        }
194        unmapped(self)
195    }
196}
197
198/// Spell out Latin presentation-form ligatures (`fi`→`fi`, `ffi`→`ffi`, …) the way
199/// docling does, so `configuration`/`difficult` don't keep the ligature glyph.
200/// The chars share the ligature's box, so the line sanitizer recomposes them.
201fn decompose_ligatures(s: &str) -> String {
202    if !s.chars().any(|c| ('\u{FB00}'..='\u{FB06}').contains(&c)) {
203        return s.to_string();
204    }
205    s.chars()
206        .map(|c| {
207            match c {
208                '\u{FB00}' => "ff",
209                '\u{FB01}' => "fi",
210                '\u{FB02}' => "fl",
211                '\u{FB03}' => "ffi",
212                '\u{FB04}' => "ffl",
213                // U+FB05 is LATIN SMALL LIGATURE LONG S T (`ſt`): `st`, as
214                // docling's ligature map and Unicode's decomposition (#1001).
215                '\u{FB05}' => "st",
216                '\u{FB06}' => "st",
217                _ => return c.to_string(),
218            }
219            .to_string()
220        })
221        .collect()
222}
223
224fn hash_name(name: &[u8]) -> u64 {
225    use std::hash::{Hash, Hasher};
226    let mut h = std::collections::hash_map::DefaultHasher::new();
227    name.hash(&mut h);
228    h.finish()
229}
230
231/// Resolve a possibly-indirect object to a dictionary.
232fn as_dict<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Dictionary> {
233    match obj {
234        Object::Dictionary(d) => Some(d),
235        Object::Reference(id) => doc.get_object(*id).ok().and_then(|o| o.as_dict().ok()),
236        _ => None,
237    }
238}
239
240fn deref<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Object> {
241    match obj {
242        Object::Reference(id) => doc.get_object(*id).ok(),
243        other => Some(other),
244    }
245}
246
247/// The font identity the line sanitizer's `enforce_same_font` compares —
248/// docling-parse's `page_font::init_font_name`: `/FontDescriptor/FontName`
249/// (the descendant CIDFont's for a Type0), else `/Name`, else `/BaseFont`
250/// (its `base_font`), else the literal `"null"`. The resource key (`/F1`,
251/// `/F2`) is deliberately **not** part of it: a producer that references one
252/// face twice — the same Helvetica under two keys, one re-encoded so a code
253/// maps to the `fi` glyph — draws one word across the two resources, and
254/// docling-parse reads both as the same font. Keyed by resource name (as
255/// this port was until #728) the `fi` ligature, which bridges fonts, glued
256/// itself to the preceding run while the font barrier then cut it off from
257/// its own word (`fi nancial`); docling's `financial` needs the two
258/// resources to compare equal.
259/// docling-parse's `has_math_positioning_semantics`, the font-name half: a
260/// TeX math or symbol face (`cmmi`, `cmsy`, `cmex`, `cmr`, `cmbx`, `msam`,
261/// `msbm`) or any name saying `math`, `symbol` or `mtextra`.
262fn is_math_font_name(name: &[u8]) -> bool {
263    let lower = name.to_ascii_lowercase();
264    [
265        "cmmi", "cmsy", "cmex", "cmr", "cmbx", "msam", "msbm", "math", "symbol", "mtextra",
266    ]
267    .iter()
268    .any(|t| lower.windows(t.len()).any(|w| w == t.as_bytes()))
269}
270
271fn sanitizer_font_name(doc: &Document, fdict: &Dictionary, two_byte: bool) -> Vec<u8> {
272    let descriptor_font_name = |owner: &Dictionary| {
273        owner
274            .get(b"FontDescriptor")
275            .ok()
276            .and_then(|o| as_dict(doc, o))
277            .and_then(|fd| fd.get(b"FontName").ok())
278            .and_then(|o| deref(doc, o))
279            .and_then(|o| o.as_name().ok().map(<[u8]>::to_vec))
280    };
281    if let Some(n) = descriptor_font_name(fdict) {
282        return n;
283    }
284    if two_byte {
285        let descendant = fdict
286            .get(b"DescendantFonts")
287            .ok()
288            .and_then(|o| deref(doc, o))
289            .and_then(|o| match o {
290                Object::Array(a) => a.first(),
291                _ => None,
292            })
293            .and_then(|o| as_dict(doc, o));
294        if let Some(n) = descendant.and_then(descriptor_font_name) {
295            return n;
296        }
297    }
298    if let Some(n) = fdict
299        .get(b"Name")
300        .ok()
301        .and_then(|o| deref(doc, o))
302        .and_then(|o| o.as_name().ok())
303    {
304        return n.to_vec();
305    }
306    // docling-parse's `base_font` keeps the subset prefix (`ABCDEF+Helvetica`):
307    // two subsets of one face are two fonts there, and so here.
308    if let Some(n) = fdict.get(b"BaseFont").ok().and_then(|o| o.as_name().ok()) {
309        return n.to_vec();
310    }
311    b"null".to_vec()
312}
313
314/// Parse one font dictionary into a [`Font`].
315fn parse_font(doc: &Document, name: &[u8], fdict: &Dictionary) -> Font {
316    let subtype: &[u8] = fdict
317        .get(b"Subtype")
318        .ok()
319        .and_then(|o| o.as_name().ok())
320        .unwrap_or(&[]);
321    let two_byte = subtype == b"Type0".as_slice();
322
323    let mut to_unicode = fdict
324        .get(b"ToUnicode")
325        .ok()
326        .and_then(|o| deref(doc, o))
327        .and_then(|o| o.as_stream().ok())
328        .and_then(|s| s.decompressed_content().ok())
329        .map(|data| parse_tounicode(&data))
330        .unwrap_or_default();
331    if !two_byte {
332        if let Some(diffs) = differences_array(doc, fdict) {
333            reconcile_stale_tounicode(&mut to_unicode, &diffs);
334        }
335    }
336
337    let cid_fallback = if two_byte {
338        cid_fallback(doc, fdict)
339    } else {
340        CidFallback::None
341    };
342
343    let (mut widths, mut default_width) = if two_byte {
344        cid_widths(doc, fdict)
345    } else {
346        simple_widths(doc, fdict)
347    };
348
349    let simple_encoding = if two_byte {
350        None
351    } else {
352        Some(simple_encoding_table(doc, fdict))
353    };
354
355    // A standard-14 font referenced without an embedded program usually ships
356    // no `/Widths` and no `/FontDescriptor` either (ReportLab's default, #187).
357    // Every advance then resolved to 0, the cells collapsed to zero width, and
358    // the page's whole text layer was silently dropped — while pdfium, with
359    // its built-in metrics, reads the same file fine. Fill the widths from the
360    // built-in Adobe Core 14 AFM tables via the font's own code→char decode
361    // (base encoding + `/Differences`), so an explicit `/Widths` always wins.
362    if !two_byte && widths.is_empty() && default_width == 0.0 {
363        if let Some(std14) = base_font_name(fdict).and_then(|n| crate::std14::widths_for(&n)) {
364            if let Some(enc) = &simple_encoding {
365                for (&code, &ch) in enc {
366                    if let Some(w) = std14.width(ch) {
367                        widths.insert(u32::from(code), w);
368                    }
369                }
370            }
371            // Codes the table misses still advance a typical width instead of
372            // stacking at x=0 (the failure mode this whole branch fixes).
373            default_width = 500.0;
374        }
375    }
376    let fallback_names = if two_byte {
377        HashMap::new()
378    } else {
379        differences_gid_names(doc, fdict)
380    };
381    let program_encoding = if two_byte {
382        HashMap::new()
383    } else {
384        type1_program_encoding(doc, fdict)
385    };
386
387    let (ascent, descent) = font_ascent_descent(doc, fdict, two_byte);
388    let sanitizer_name = sanitizer_font_name(doc, fdict, two_byte);
389    let mut font = Font {
390        two_byte,
391        to_unicode,
392        cid_fallback,
393        widths,
394        default_width,
395        simple_encoding,
396        fallback_names,
397        blank_codes: HashSet::new(),
398        program_encoding,
399        ascent,
400        descent,
401        hash: hash_name(&sanitizer_name),
402        math: is_math_font_name(&sanitizer_name),
403        style: crate::font_style::parse_font_style(&String::from_utf8_lossy(
404            &base_font_name(fdict).unwrap_or_else(|| name.to_vec()),
405        )),
406    };
407    font.blank_codes = blank_codes(doc, fdict, &font);
408    font
409}
410
411/// The codes `font` leaves unmapped whose embedded glyph paints nothing
412/// ([`Font::blank_codes`]). The program is loaded only when the font has
413/// unmapped codes among those it declares widths for; without the
414/// renderer (the wasm text-layer build) there is no program to ask, and
415/// every unmapped glyph reads U+FFFD.
416#[cfg(feature = "render")]
417fn blank_codes(doc: &Document, fdict: &Dictionary, font: &Font) -> HashSet<u32> {
418    // A CID font can declare tens of thousands of widths: past this many
419    // unmapped codes nothing is classified (the glyphs read U+FFFD).
420    const MAX_CANDIDATES: usize = 4096;
421    let unmapped: Vec<u32> = font
422        .widths
423        .keys()
424        .copied()
425        .filter(|&c| font.decode_code(c).0.as_deref() == Some("\u{FFFD}"))
426        .take(MAX_CANDIDATES + 1)
427        .collect();
428    if unmapped.is_empty() || unmapped.len() > MAX_CANDIDATES {
429        return HashSet::new();
430    }
431    let loaded = crate::render::font::LoadedFont::load(doc, fdict);
432    unmapped
433        .into_iter()
434        .filter(|&c| loaded.glyph_has_ink(c) == Some(false))
435        .collect()
436}
437
438#[cfg(not(feature = "render"))]
439fn blank_codes(_doc: &Document, _fdict: &Dictionary, _font: &Font) -> HashSet<u32> {
440    HashSet::new()
441}
442
443#[derive(Clone, Copy)]
444enum CidFallback {
445    None,
446    UnicodeCodes { utf16: bool },
447    Collection(&'static [u8]),
448}
449
450/// A Unicode CMap code retains its original bytes for ToUnicode lookups;
451/// a four-byte UTF-16 code is a packed surrogate pair, not a scalar value.
452pub(crate) fn unicode_cmap_char(code: u32) -> Option<char> {
453    if code <= 0xffff {
454        char::from_u32(code)
455    } else {
456        let units = [(code >> 16) as u16, code as u16];
457        let mut chars = char::decode_utf16(units);
458        let ch = chars.next()?.ok()?;
459        chars.next().is_none().then_some(ch)
460    }
461}
462
463#[cfg(not(feature = "cid-collections"))]
464fn collection_table(_ordering: &[u8]) -> Option<&'static [u8]> {
465    None
466}
467
468#[cfg(feature = "cid-collections")]
469fn collection_table(ordering: &[u8]) -> Option<&'static [u8]> {
470    macro_rules! table {
471        ($file:literal) => {
472            include_bytes!(concat!(env!("OUT_DIR"), "/", $file))
473        };
474    }
475    Some(match ordering {
476        b"Japan1" => table!("Adobe-Japan1-UCS2"),
477        b"GB1" => table!("Adobe-GB1-UCS2"),
478        b"CNS1" => table!("Adobe-CNS1-UCS2"),
479        b"Korea1" => table!("Adobe-Korea1-UCS2"),
480        b"KR" => table!("Adobe-KR-UCS2"),
481        _ => return None,
482    })
483}
484
485/// The text of `cid` in a table build.rs wrote: the CID count, then three
486/// bytes per CID (0 for none, a code point, or 0x110000 + the offset of a
487/// length-prefixed UTF-8 string after the entries).
488pub(crate) fn collection_text(table: &[u8], cid: u32) -> Option<String> {
489    let n = u32::from_le_bytes(table.get(..4)?.try_into().ok()?) as usize;
490    let at = 4 + 3 * usize::try_from(cid).ok().filter(|&c| c < n)?;
491    let &[a, b, c] = table.get(at..at + 3)? else {
492        return None;
493    };
494    match u32::from_be_bytes([0, a, b, c]) {
495        0 => None,
496        v @ ..0x110000 => char::from_u32(v).map(String::from),
497        v => {
498            let at = 4 + 3 * n + (v - 0x110000) as usize;
499            let len = *table.get(at)? as usize;
500            std::str::from_utf8(table.get(at + 1..at + 1 + len)?)
501                .ok()
502                .map(str::to_owned)
503        }
504    }
505}
506
507fn cid_fallback(doc: &Document, fdict: &Dictionary) -> CidFallback {
508    let encoding = fdict
509        .get(b"Encoding")
510        .ok()
511        .and_then(|o| deref(doc, o))
512        .and_then(|o| o.as_name().ok())
513        .unwrap_or(&[]);
514    if encoding.starts_with(b"Uni")
515        && (encoding.windows(4).any(|w| w == b"UCS2") || encoding.windows(5).any(|w| w == b"UTF16"))
516    {
517        return CidFallback::UnicodeCodes {
518            utf16: encoding.windows(5).any(|w| w == b"UTF16"),
519        };
520    }
521    if encoding != b"Identity-H" && encoding != b"Identity-V" {
522        // Other CMaps need code → CID tables that are not bundled.
523        return CidFallback::None;
524    }
525    cid_collection(doc, fdict).map_or(CidFallback::None, CidFallback::Collection)
526}
527
528/// The CID → Unicode table of the character collection a Type0 font uses.
529pub(crate) fn cid_collection(doc: &Document, fdict: &Dictionary) -> Option<&'static [u8]> {
530    let Object::Array(descendants) = deref(doc, fdict.get(b"DescendantFonts").ok()?)? else {
531        return None;
532    };
533    let descendant = as_dict(doc, descendants.first()?)?;
534    let info = as_dict(doc, descendant.get(b"CIDSystemInfo").ok()?)?;
535    let registry = deref(doc, info.get(b"Registry").ok()?)?.as_str().ok()?;
536    let ordering = deref(doc, info.get(b"Ordering").ok()?)?.as_str().ok()?;
537    (registry == b"Adobe").then(|| collection_table(ordering))?
538}
539
540/// A simple font's `/Encoding` `/Differences` array, dereferenced.
541fn differences_array(doc: &Document, fdict: &Dictionary) -> Option<Vec<Object>> {
542    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))?;
543    let diffs = enc.as_dict().ok()?.get(b"Differences").ok()?;
544    match deref(doc, diffs)? {
545        Object::Array(a) => Some(a.clone()),
546        _ => None,
547    }
548}
549
550/// A ToUnicode CMap copied from the *original* font while the glyphs were
551/// re-encoded: the encrypted `2206.01062_pg3` fixture (a Quartz re-save of
552/// pdfTeX output) keeps the T1-encoding CMap, keyed on the old codes, next
553/// to a `/Differences [33 /f_f_i /f_i /f_f /f_l]` that names the ligatures at
554/// new ones, so the CMap maps code 33 to `!` — its own identity — and
555/// `difficulties` read `di ! culties`, in docling's pdfium chain too. A
556/// ToUnicode entry that is the code's identity, for a code whose
557/// `/Differences` name resolves to something else, is that stale copy: the
558/// name wins. A CMap that disagrees any other way is left alone — ToUnicode
559/// is the producer's word on what the glyph means.
560fn reconcile_stale_tounicode(to_unicode: &mut HashMap<u32, String>, diffs: &[Object]) {
561    let mut code = 0u8;
562    for el in diffs {
563        match el {
564            Object::Integer(i) => code = *i as u8,
565            Object::Name(name) => {
566                if let Some(ch) = glyph_name_to_char(name) {
567                    let identity = (code as char).to_string();
568                    if ch.to_string() != identity
569                        && to_unicode.get(&u32::from(code)) == Some(&identity)
570                    {
571                        to_unicode.insert(u32::from(code), ch.to_string());
572                    }
573                }
574                code = code.wrapping_add(1);
575            }
576            _ => {}
577        }
578    }
579}
580
581/// Collect `/Differences` entries whose glyph name is a GID placeholder
582/// (`g115`, `cid42`, `glyph7`, `index9`, `SM590000`) with no Unicode mapping
583/// (subsetted fonts with no ToUnicode); [`unknown_name_text`] decides what
584/// each reads as. The placeholder-name restriction keeps real Adobe glyph
585/// names on the normal path so this never invents garbage on the clean files.
586fn differences_gid_names(doc: &Document, fdict: &Dictionary) -> HashMap<u8, String> {
587    let mut map = HashMap::new();
588    let Some(Object::Dictionary(enc)) = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))
589    else {
590        return map;
591    };
592    let Some(Object::Array(diffs)) = enc.get(b"Differences").ok().and_then(|o| deref(doc, o))
593    else {
594        return map;
595    };
596    let mut code = 0u8;
597    for el in diffs {
598        match el {
599            Object::Integer(i) => code = *i as u8,
600            Object::Name(name) => {
601                if glyph_name_to_char(name).is_none() && is_gid_name(name) {
602                    map.insert(code, String::from_utf8_lossy(name).into_owned());
603                }
604                code = code.wrapping_add(1);
605            }
606            _ => {}
607        }
608    }
609    map
610}
611
612/// Parse the embedded Type1 font program's built-in `/Encoding` vector
613/// (`dup <code> /<glyphname> put` entries in the clear-text header before
614/// `eexec`) into `code → char`. This is how docling recovers glyphs from
615/// standard TeX math fonts (CMMI/CMSY/…) that carry no PDF `/Encoding` and no
616/// ToUnicode — e.g. CMMI's `dup 21 /lambda` or CMSY's `dup 20 /lessequal`.
617/// Only `FontFile` (Type1) is parsed; CFF (`FontFile3`) and TrueType
618/// (`FontFile2`) store their encoding in a binary table and are left alone.
619fn type1_program_encoding(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
620    let mut map = HashMap::new();
621    let Some(desc) = fdict
622        .get(b"FontDescriptor")
623        .ok()
624        .and_then(|o| deref(doc, o))
625        .and_then(|o| o.as_dict().ok())
626    else {
627        return map;
628    };
629    let Some(data) = desc
630        .get(b"FontFile")
631        .ok()
632        .and_then(|o| deref(doc, o))
633        .and_then(|o| o.as_stream().ok())
634        .and_then(|s| s.decompressed_content().ok())
635    else {
636        return map;
637    };
638    // The clear-text header (PostScript) ends at `eexec`; the rest is encrypted.
639    let head_end = data
640        .windows(5)
641        .position(|w| w == b"eexec")
642        .unwrap_or(data.len());
643    let head = String::from_utf8_lossy(&data[..head_end]);
644    // Scan for `dup <code> /<name> put` tokens.
645    let toks: Vec<&str> = head.split_whitespace().collect();
646    for w in toks.windows(4) {
647        if w[0] == "dup" && w[3] == "put" {
648            if let (Ok(code), Some(name)) = (w[1].parse::<u32>(), w[2].strip_prefix('/')) {
649                if code <= 255 {
650                    if let Some(ch) = glyph_name_to_char(name.as_bytes()) {
651                        map.insert(code as u8, ch);
652                    }
653                }
654            }
655        }
656    }
657    map
658}
659
660/// The reading text docling-parse 7.24 keeps for a `/Differences` glyph name
661/// no table resolves (`resolve_unknown_name`): the name without any
662/// `.suffix` — `None` when that is a bare glyph index
663/// (`(gid|glyph|g|cid|index)\d+`, any case), which names a glyph of the
664/// font program but carries no text (docling-parse#238), so it is unmapped.
665fn unknown_name_text(name: &str) -> Option<String> {
666    let stem = name.rsplit_once('.').map_or(name, |(s, _)| s);
667    let lower = stem.to_ascii_lowercase();
668    let index = ["gid", "glyph", "g", "cid", "index"].iter().any(|p| {
669        lower
670            .strip_prefix(p)
671            .is_some_and(|d| !d.is_empty() && d.bytes().all(|b| b.is_ascii_digit()))
672    });
673    (!index).then(|| stem.to_string())
674}
675
676/// A glyph name that is a synthetic placeholder, not a real Adobe name:
677/// `g115`, `cid42`, `glyph7`, `index9`, `G12`, or a short-prefix code name like
678/// `SM590000` (IBM BookMaster). These carry no Unicode meaning (docling-parse
679/// keeps the code-like names as text, `SM590000`, and reads the glyph indexes
680/// as unmapped, see [`unknown_name_text`]). `afii####` / `uni####` are real Adobe names
681/// and excluded. The restriction keeps genuine glyph names on the Unicode path.
682fn is_gid_name(name: &[u8]) -> bool {
683    let Ok(s) = std::str::from_utf8(name) else {
684        return false;
685    };
686    if s.starts_with("afii") || s.starts_with("uni") {
687        return false;
688    }
689    for prefix in ["g", "G", "cid", "CID", "glyph", "index"] {
690        if let Some(rest) = s.strip_prefix(prefix) {
691            if !rest.is_empty() && rest.bytes().all(|b| b.is_ascii_digit()) {
692                return true;
693            }
694        }
695    }
696    // Short alpha prefix (≤3 letters) followed by a run of ≥3 digits — synthetic
697    // code names like `SM590000`, distinct from real Adobe names (whole words or
698    // letter+`.suffix` variants).
699    let alpha = s.bytes().take_while(|b| b.is_ascii_alphabetic()).count();
700    let digits = s.len() - alpha;
701    (1..=3).contains(&alpha)
702        && digits >= 3
703        && s.as_bytes()[alpha..].iter().all(|b| b.is_ascii_digit())
704}
705
706fn font_ascent_descent(doc: &Document, fdict: &Dictionary, two_byte: bool) -> (f64, f64) {
707    // For Type0, the descriptor lives on the descendant CIDFont.
708    let descr_owner = if two_byte {
709        fdict
710            .get(b"DescendantFonts")
711            .ok()
712            .and_then(|o| deref(doc, o))
713            .and_then(|o| match o {
714                Object::Array(a) => a.first(),
715                _ => None,
716            })
717            .and_then(|o| as_dict(doc, o))
718    } else {
719        Some(fdict)
720    };
721    let fd = descr_owner
722        .and_then(|d| d.get(b"FontDescriptor").ok())
723        .and_then(|o| as_dict(doc, o));
724    let asc = fd
725        .and_then(|d| d.get(b"Ascent").ok())
726        .and_then(|o| {
727            o.as_float()
728                .ok()
729                .or_else(|| o.as_i64().ok().map(|i| i as f32))
730        })
731        .unwrap_or(750.0) as f64;
732    let desc = fd
733        .and_then(|d| d.get(b"Descent").ok())
734        .and_then(|o| {
735            o.as_float()
736                .ok()
737                .or_else(|| o.as_i64().ok().map(|i| i as f32))
738        })
739        .unwrap_or(-250.0) as f64;
740    // Some subsetted fonts carry a degenerate FontDescriptor (`/Ascent 0
741    // /Descent 0`) — the real metrics live in the font program. That collapses
742    // the loose box to zero height, so the line cells get zero area and the
743    // layout's region/text assignment drops them (2305's References list lost
744    // every prose line, keeping only the URLs). Fall back to typical text metrics
745    // so the box has height.
746    if asc - desc <= 1.0 {
747        return (750.0, -250.0);
748    }
749    (asc, desc)
750}
751
752/// The `/BaseFont` name with any `ABCDEF+` subset prefix stripped (#187).
753fn base_font_name(fdict: &Dictionary) -> Option<Vec<u8>> {
754    let name = fdict.get(b"BaseFont").ok()?.as_name().ok()?;
755    let stripped = match name.iter().position(|&b| b == b'+') {
756        Some(i) if i == 6 => &name[i + 1..],
757        _ => name,
758    };
759    Some(stripped.to_vec())
760}
761
762/// Simple-font widths: `/FirstChar` + `/Widths` array, `/MissingWidth` default.
763fn simple_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
764    let mut map = HashMap::new();
765    let first = fdict
766        .get(b"FirstChar")
767        .ok()
768        .and_then(|o| o.as_i64().ok())
769        .unwrap_or(0) as u32;
770    if let Some(Object::Array(arr)) = fdict.get(b"Widths").ok().and_then(|o| deref(doc, o)) {
771        for (i, w) in arr.iter().enumerate() {
772            if let Some(w) = num(w) {
773                map.insert(first + i as u32, w);
774            }
775        }
776    }
777    let dw = fdict
778        .get(b"FontDescriptor")
779        .ok()
780        .and_then(|o| as_dict(doc, o))
781        .and_then(|d| d.get(b"MissingWidth").ok())
782        .and_then(num)
783        .unwrap_or(0.0);
784    (map, dw)
785}
786
787/// CIDFont widths: the `/W` array on the descendant font (`/DW` default = 1000).
788fn cid_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
789    let mut map = HashMap::new();
790    let Some(desc) = fdict
791        .get(b"DescendantFonts")
792        .ok()
793        .and_then(|o| deref(doc, o))
794        .and_then(|o| match o {
795            Object::Array(a) => a.first(),
796            _ => None,
797        })
798        .and_then(|o| as_dict(doc, o))
799    else {
800        return (map, 1000.0);
801    };
802    let dw = desc.get(b"DW").ok().and_then(num).unwrap_or(1000.0);
803    if let Some(Object::Array(w)) = desc.get(b"W").ok().and_then(|o| deref(doc, o)) {
804        let mut i = 0;
805        while i < w.len() {
806            let c = w.get(i).and_then(num);
807            match (c, w.get(i + 1)) {
808                // `c [w1 w2 ...]`: consecutive CIDs starting at c.
809                (Some(c), Some(Object::Array(list))) => {
810                    for (k, wv) in list.iter().enumerate() {
811                        if let Some(wv) = num(wv) {
812                            map.insert(c as u32 + k as u32, wv);
813                        }
814                    }
815                    i += 2;
816                }
817                // `c_first c_last w`: a run all of width w.
818                (Some(c1), Some(o2)) => {
819                    if let (Some(c2), Some(wv)) = (num(o2), w.get(i + 2).and_then(num)) {
820                        for cid in c1 as u32..=c2 as u32 {
821                            map.insert(cid, wv);
822                        }
823                    }
824                    i += 3;
825                }
826                _ => break,
827            }
828        }
829    }
830    (map, dw)
831}
832
833fn num(o: &Object) -> Option<f64> {
834    match o {
835        Object::Integer(i) => Some(*i as f64),
836        Object::Real(r) => Some(*r as f64),
837        _ => None,
838    }
839}
840
841/// Decode a PDF string literal into codes, retaining packed UTF-16 pairs
842/// as the original four-byte keys used by ToUnicode.
843fn codes(font: &Font, bytes: &[u8]) -> Vec<u32> {
844    if font.two_byte {
845        let mut units = bytes
846            .chunks(2)
847            .map(|c| {
848                if c.len() == 2 {
849                    ((c[0] as u32) << 8) | c[1] as u32
850                } else {
851                    c[0] as u32
852                }
853            })
854            .peekable();
855        let utf16 = matches!(font.cid_fallback, CidFallback::UnicodeCodes { utf16: true });
856        let mut out = Vec::with_capacity(bytes.len() / 2 + 1);
857        while let Some(mut code) = units.next() {
858            if utf16 && (0xd800..=0xdbff).contains(&code) {
859                if let Some(low) = units.next_if(|u| (0xdc00..=0xdfff).contains(u)) {
860                    code = (code << 16) | low;
861                }
862            }
863            out.push(code);
864        }
865        out
866    } else {
867        bytes.iter().map(|&b| b as u32).collect()
868    }
869}
870
871/// A page's display box, in PDF user space: the `/CropBox` clipped to the
872/// `/MediaBox`, both inherited through the page tree and normalized — a
873/// missing or empty MediaBox is US Letter, an empty CropBox is the MediaBox
874/// (pdfium's `CPDF_Page::UpdateDimensions`). pdfium reports the page size from
875/// this box, renders exactly it, and translates every content coordinate so
876/// its lower-left corner is the origin (`m_PageMatrix`); docling's backends
877/// inherit that frame, so text cells, `prov` boxes and destinations all count
878/// from the CropBox corner, not the MediaBox one. The parser used to flip
879/// glyphs with the MediaBox *height* and no translation at all, so a page
880/// whose boxes do not start at (0, 0) — a trimmed book page with
881/// `MediaBox [-56 -58 576 723]` / `CropBox [1 -0.6 519 666]`, or a LaTeX
882/// figure cropped to `[156 147 637 391]` — had its text displaced against the
883/// rendered bitmap by the box offset, the bottom lines pushed past the page
884/// edge and clamped to `t = b`.
885#[derive(Debug, Clone, Copy, PartialEq)]
886pub(crate) struct PageBox {
887    /// Left edge, user space.
888    pub l: f32,
889    /// Bottom edge, user space.
890    pub b: f32,
891    pub w: f32,
892    pub h: f32,
893}
894
895impl PageBox {
896    /// Top edge, user space — the y that becomes `0` in the y-down frame.
897    pub fn top(&self) -> f32 {
898        self.b + self.h
899    }
900}
901
902/// A page-tree rect attribute (`/MediaBox`, `/CropBox`), inherited from the
903/// nearest ancestor that sets it, as normalized `(l, b, r, t)`.
904fn inherited_rect(
905    doc: &Document,
906    page_id: lopdf::ObjectId,
907    key: &[u8],
908) -> Option<(f32, f32, f32, f32)> {
909    let mut id = page_id;
910    for _ in 0..32 {
911        let dict = doc.get_object(id).ok()?.as_dict().ok()?;
912        if let Some(Object::Array(a)) = dict.get(key).ok().and_then(|o| deref(doc, o)) {
913            let v: Vec<f32> = a.iter().filter_map(|o| num(o).map(|x| x as f32)).collect();
914            if v.len() == 4 && v.iter().all(|x| x.is_finite()) {
915                return Some((
916                    v[0].min(v[2]),
917                    v[1].min(v[3]),
918                    v[0].max(v[2]),
919                    v[1].max(v[3]),
920                ));
921            }
922            return None;
923        }
924        id = dict.get(b"Parent").ok()?.as_reference().ok()?;
925    }
926    None
927}
928
929pub(crate) fn page_box(doc: &Document, page_id: lopdf::ObjectId) -> PageBox {
930    let nonempty = |r: &(f32, f32, f32, f32)| r.2 > r.0 && r.3 > r.1;
931    let media = inherited_rect(doc, page_id, b"MediaBox")
932        .filter(nonempty)
933        .unwrap_or((0.0, 0.0, 612.0, 792.0));
934    let crop = inherited_rect(doc, page_id, b"CropBox")
935        .map(|c| {
936            (
937                c.0.max(media.0),
938                c.1.max(media.1),
939                c.2.min(media.2),
940                c.3.min(media.3),
941            )
942        })
943        .filter(nonempty)
944        .unwrap_or(media);
945    PageBox {
946        l: crop.0,
947        b: crop.1,
948        w: crop.2 - crop.0,
949        h: crop.3 - crop.1,
950    }
951}
952
953/// Page size (width, height) in PDF points — the display box's, like pdfium's
954/// `FPDF_GetPageWidthF/HeightF`.
955fn page_size(doc: &Document, page_id: lopdf::ObjectId) -> (f32, f32) {
956    let pb = page_box(doc, page_id);
957    (pb.w, pb.h)
958}
959
960/// Localize where a page's text is lost, for the `text_layer` diagnostic.
961/// Extraction can come up empty at three different points — no content stream
962/// reached the parser, the stream did not decode into operators, or it ran but
963/// produced no glyphs (fonts/encodings) — and from the outside all three look
964/// the same. Report them per page.
965pub fn content_diagnosis(bytes: &[u8]) -> String {
966    let Some(doc) = load_document(bytes) else {
967        return "document does not load".into();
968    };
969    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
970    pages.sort_by_key(|(n, _)| *n);
971    let mut out = String::new();
972    let mut caches = DocCaches::default();
973    for (n, pid) in pages.into_iter().take(4) {
974        let content_bytes = doc.get_page_content(pid);
975        let ops = lopdf::content::Content::decode(&content_bytes)
976            .map(|c| c.operations.len())
977            .ok();
978        let res = page_res(&doc, pid);
979        let fonts = res.map(|r| fonts_from_res(&doc, r, &mut caches).len());
980        let glyphs = page_glyphs_cached(&doc, pid, &mut caches).len();
981        out.push_str(&format!(
982            "\n   page {n}: content {} B, ops {}, resources {}, fonts {}, glyphs {}",
983            content_bytes.len(),
984            ops.map_or("UNDECODABLE".to_string(), |n| n.to_string()),
985            if res.is_some() { "ok" } else { "MISSING" },
986            fonts.map_or("-".to_string(), |n| n.to_string()),
987            glyphs,
988        ));
989    }
990    out
991}
992
993/// Is this "text layer" a vestige rather than the document's text?
994///
995/// Scanned forms often carry a handful of typed-in strings — a date filled
996/// into three form fields, say — on top of pages that are otherwise images.
997/// Treating that as a real text layer is the worst of both worlds: the text
998/// path proudly extracts thirteen characters, and no OCR ever runs on the
999/// letter the pages actually show. The reported form did exactly this (3
1000/// lines, 13 chars, 3 pages).
1001///
1002/// The rule is deliberately tight so genuinely sparse *digital* documents are
1003/// not misrouted into OCR: only a document averaging at most one line per page
1004/// **and** totalling fewer than 32 characters is called vestigial.
1005pub fn text_layer_is_vestigial(pages: &[crate::pdfium_backend::PdfPage]) -> bool {
1006    let mut tally = TextLayerTally::default();
1007    pages.iter().for_each(|p| tally.add(p));
1008    tally.is_vestigial()
1009}
1010
1011/// [`text_layer_is_vestigial`]'s counts, accumulated a page at a time so a
1012/// streaming caller need not keep the pages around to decide.
1013#[derive(Debug, Default, Clone, Copy)]
1014pub struct TextLayerTally {
1015    pages: usize,
1016    lines: usize,
1017    chars: usize,
1018}
1019
1020impl TextLayerTally {
1021    pub fn add(&mut self, page: &crate::pdfium_backend::PdfPage) {
1022        self.pages += 1;
1023        self.lines += page.cells.len();
1024        self.chars += page
1025            .cells
1026            .iter()
1027            .map(|c| c.text.chars().count())
1028            .sum::<usize>();
1029    }
1030
1031    /// The verdict over the pages added so far.
1032    pub fn is_vestigial(&self) -> bool {
1033        self.lines == 0 || (self.lines <= self.pages && self.chars < 32)
1034    }
1035
1036    /// Whether the pages added so far already rule vestigial out for a
1037    /// document of `total_pages`, whatever the remaining pages hold — both
1038    /// counts only grow, so a caller may stop parsing pages it does not need.
1039    pub fn proves_text(&self, total_pages: usize) -> bool {
1040        self.lines > 0 && (self.lines > total_pages || self.chars >= 32)
1041    }
1042}
1043
1044/// Why the cross-reference repair did or did not fire, for the `text_layer`
1045/// diagnostic. A PDF that will not load is indistinguishable from a scan in
1046/// production (both convert to nothing), so the reason has to be askable.
1047pub fn xref_repair_status(bytes: &[u8]) -> String {
1048    if Document::load_mem(bytes).is_ok() {
1049        return "loads unaided; no repair needed".into();
1050    }
1051    match pad_short_xref_entries(bytes) {
1052        Ok(fixed) => match Document::load_mem(&fixed) {
1053            Ok(_) => "repaired: cross-reference entries padded to 20 bytes".into(),
1054            Err(e) => format!("padded the entries, but it still will not load: {e}"),
1055        },
1056        Err(why) => format!("repair declined — {why}"),
1057    }
1058}
1059
1060/// Load a PDF, repairing the one malformation that otherwise costs us the whole
1061/// document: **19-byte cross-reference entries**.
1062///
1063/// The spec fixes an xref entry at 20 bytes — `nnnnnnnnnn ggggg n` plus a
1064/// *two*-byte EOL. Some generators (an Austrian telecom's invoices, for one)
1065/// emit a bare LF instead, making each entry 19 bytes. lopdf rejects the file
1066/// outright (`invalid file trailer`) where pdfium reads it happily, so a
1067/// perfectly good text layer looked to the browser exactly like a scan and cost
1068/// ten seconds of OCR.
1069///
1070/// Padding is only attempted when it cannot move anything the xref points at:
1071/// a single `xref` section that begins after the last object. The repair then
1072/// has to prove itself — the padded bytes are used only if they load — so a
1073/// mis-repair degrades to today's behaviour rather than to silent garbage.
1074pub(crate) fn load_document(bytes: &[u8]) -> Option<Document> {
1075    open_document(bytes, None).ok()
1076}
1077
1078/// Why [`open_document`] could not hand back a readable document.
1079#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1080pub(crate) enum OpenError {
1081    /// lopdf cannot read the file even after the repairs.
1082    Unreadable,
1083    /// The file is encrypted and the password given (or none) does not open it.
1084    Password,
1085}
1086
1087/// Load `bytes` with the document's password. lopdf decrypts while reading —
1088/// with `password`, or the empty user password most "protected" PDFs carry
1089/// (what every viewer opens silently) — so every reader downstream sees plain
1090/// streams; a file the password does not open loads with its streams still
1091/// encrypted, which is reported as [`OpenError::Password`] rather than handed
1092/// on as a document whose every stream decodes to nothing.
1093pub(crate) fn open_document(bytes: &[u8], password: Option<&str>) -> Result<Document, OpenError> {
1094    let Some(doc) = load_document_raw(bytes, password) else {
1095        // lopdf refuses to load at all under a *wrong* password (a missing
1096        // one loads the file with its streams still encrypted); tell the two
1097        // apart by loading without it.
1098        if password.is_some() && load_document_raw(bytes, None).is_some_and(|d| d.is_encrypted()) {
1099            return Err(OpenError::Password);
1100        }
1101        return Err(OpenError::Unreadable);
1102    };
1103    if doc.is_encrypted() {
1104        return Err(OpenError::Password);
1105    }
1106    Ok(doc)
1107}
1108
1109fn load_options(password: Option<&str>) -> lopdf::LoadOptions {
1110    lopdf::LoadOptions {
1111        password: password.map(str::to_string),
1112        ..lopdf::LoadOptions::default()
1113    }
1114}
1115
1116fn load_document_raw(bytes: &[u8], password: Option<&str>) -> Option<Document> {
1117    // Try progressively more repair, and accept a candidate only once the pages
1118    // actually carry content — a document whose streams were dropped still
1119    // "loads", so loading alone is not evidence the repair helped. A
1120    // well-formed file returns on the first attempt and pays for nothing.
1121    let mut fallback = None;
1122    if let Some(doc) = best_effort_load(bytes, password, &mut fallback) {
1123        return Some(doc);
1124    }
1125    let xref_fixed = pad_short_xref_entries(bytes).ok();
1126    if let Some(fixed) = &xref_fixed {
1127        if let Some(doc) = best_effort_load(fixed, password, &mut fallback) {
1128            return Some(doc);
1129        }
1130    }
1131    // Both defects can coexist, and the second only becomes visible once the
1132    // first is repaired, so build on whatever the previous step produced.
1133    let lengths_fixed = fix_stream_lengths(xref_fixed.as_deref().unwrap_or(bytes));
1134    if let Some(doc) = best_effort_load(&lengths_fixed, password, &mut fallback) {
1135        return Some(doc);
1136    }
1137    fallback
1138}
1139
1140/// Load `data`, returning it only when its pages carry content; a document that
1141/// merely parses is remembered as the fallback for when nothing does better.
1142fn best_effort_load(
1143    data: &[u8],
1144    password: Option<&str>,
1145    fallback: &mut Option<Document>,
1146) -> Option<Document> {
1147    match Document::load_mem_with_options(data, load_options(password)) {
1148        Ok(doc) if has_page_content(&doc) => Some(doc),
1149        Ok(doc) => {
1150            fallback.get_or_insert(doc);
1151            None
1152        }
1153        Err(_) => None,
1154    }
1155}
1156
1157/// Does any page actually hand us a content stream? A document whose streams
1158/// were dropped still parses — it simply has nothing to read — so this is what
1159/// tells a successful repair from a pointless one.
1160fn has_page_content(doc: &Document) -> bool {
1161    doc.get_pages()
1162        .into_values()
1163        .take(4)
1164        .any(|pid| !doc.get_page_content(pid).is_empty())
1165}
1166
1167/// Correct `/Length` values that disagree with where `endstream` actually is.
1168///
1169/// The same generator that writes short xref entries also overstates its
1170/// content-stream lengths by a byte or two. lopdf trusts `/Length`, reads past
1171/// the data, fails to find `endstream` there and drops the stream — the object
1172/// comes back as a bare dictionary, so the page has no content at all and the
1173/// document looks like a scan. pdfium instead trusts `endstream`, which is what
1174/// this does.
1175///
1176/// The rewrite is length-preserving: the corrected number is written over the
1177/// old digits and padded with spaces, so every byte offset in the file — and
1178/// therefore the whole cross-reference table — stays valid.
1179fn fix_stream_lengths(bytes: &[u8]) -> Vec<u8> {
1180    let mut out = bytes.to_vec();
1181    let mut i = 0;
1182    while let Some(rel) = find(&out[i..], b"stream") {
1183        let kw = i + rel;
1184        i = kw + 6;
1185        // Skip `endstream` (the keyword we are measuring *to*).
1186        if kw >= 3 && &out[kw - 3..kw] == b"end" {
1187            continue;
1188        }
1189        // The stream data starts after the EOL that follows the keyword.
1190        let mut data = kw + 6;
1191        if out.get(data..data + 2) == Some(b"\r\n".as_slice()) {
1192            data += 2;
1193        } else if matches!(out.get(data), Some(b'\n' | b'\r')) {
1194            data += 1;
1195        }
1196        let Some(end) = find(&out[data..], b"endstream").map(|r| data + r) else {
1197            continue;
1198        };
1199        // `/Length <digits>` in the dictionary just before the keyword.
1200        let dict_start = out[..kw].iter().rposition(|&c| c == b'<').unwrap_or(0);
1201        let Some(lrel) = find(&out[dict_start..kw], b"/Length") else {
1202            continue;
1203        };
1204        let mut d = dict_start + lrel + 7;
1205        while matches!(out.get(d), Some(b' ')) {
1206            d += 1;
1207        }
1208        let digits = out[d..].iter().take_while(|c| c.is_ascii_digit()).count();
1209        if digits == 0 {
1210            continue;
1211        }
1212        let declared: usize = match std::str::from_utf8(&out[d..d + digits])
1213            .ok()
1214            .and_then(|s| s.parse().ok())
1215        {
1216            Some(v) => v,
1217            None => continue,
1218        };
1219        let actual = end - data;
1220        // Only shrink, and only when the new value fits the space the old one
1221        // occupied — growing the number would move every following byte.
1222        let replacement = actual.to_string();
1223        if actual == declared || replacement.len() > digits {
1224            continue;
1225        }
1226        out[d..d + digits].fill(b' ');
1227        out[d..d + replacement.len()].copy_from_slice(replacement.as_bytes());
1228    }
1229    out
1230}
1231
1232fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
1233    haystack.windows(needle.len()).position(|w| w == needle)
1234}
1235
1236/// Rewrite a classic cross-reference table's entries to the spec's 20 bytes,
1237/// or `None` when the file's shape makes that unsafe (see [`load_document`]).
1238pub(crate) fn pad_short_xref_entries(bytes: &[u8]) -> Result<Vec<u8>, &'static str> {
1239    // Exactly one xref section, and it must start after every object, so that
1240    // growing it shifts nothing the table's offsets refer to.
1241    let is_boundary = |i: usize| i == 0 || matches!(bytes[i - 1], b'\n' | b'\r');
1242    let mut starts = (0..bytes.len().saturating_sub(4))
1243        .filter(|&i| &bytes[i..i + 4] == b"xref" && is_boundary(i));
1244    let xref_at = starts
1245        .next()
1246        .ok_or("no classic `xref` section (an xref stream?)")?;
1247    if starts.next().is_some() {
1248        return Err("more than one xref section (incremental update)");
1249    }
1250    let last_obj = bytes
1251        .windows(3)
1252        .rposition(|w| w == b"obj")
1253        .ok_or("no objects found")?;
1254    if last_obj > xref_at {
1255        return Err("an object follows the xref — padding would move it");
1256    }
1257
1258    let mut out = bytes[..xref_at].to_vec();
1259    out.extend_from_slice(b"xref\n");
1260    let mut i = xref_at + 4;
1261    let skip_ws = |i: &mut usize| {
1262        while matches!(bytes.get(*i), Some(b'\r' | b'\n' | b' ')) {
1263            *i += 1;
1264        }
1265    };
1266    loop {
1267        skip_ws(&mut i);
1268        // Either the next subsection header ("first count") or the trailer.
1269        if bytes[i..].starts_with(b"trailer") {
1270            out.extend_from_slice(&bytes[i..]);
1271            return Ok(out);
1272        }
1273        let header_end = i + bytes[i..]
1274            .iter()
1275            .position(|c| matches!(c, b'\n' | b'\r'))
1276            .ok_or("subsection header runs off the end")?;
1277        let header = std::str::from_utf8(&bytes[i..header_end])
1278            .map_err(|_| "subsection header is not text")?
1279            .trim();
1280        let mut parts = header.split_whitespace();
1281        let count: usize = parts
1282            .nth(1)
1283            .and_then(|c| c.parse().ok())
1284            .ok_or("unparseable subsection header")?;
1285        if parts.next().is_some() || count == 0 {
1286            return Err("unexpected subsection header shape");
1287        }
1288        out.extend_from_slice(header.as_bytes());
1289        out.push(b'\n');
1290        i = header_end;
1291        for _ in 0..count {
1292            skip_ws(&mut i);
1293            // `nnnnnnnnnn ggggg n` — the 18 bytes before whatever EOL follows.
1294            let entry = bytes.get(i..i + 18).ok_or("xref entry runs off the end")?;
1295            let well_formed = entry[..10].iter().all(u8::is_ascii_digit)
1296                && entry[10] == b' '
1297                && entry[11..16].iter().all(u8::is_ascii_digit)
1298                && entry[16] == b' '
1299                && matches!(entry[17], b'n' | b'f');
1300            if !well_formed {
1301                return Err("xref entry is not `nnnnnnnnnn ggggg n`");
1302            }
1303            out.extend_from_slice(entry);
1304            out.extend_from_slice(b" \n"); // the spec's 2-byte EOL -> 20 bytes
1305            i += 18;
1306        }
1307    }
1308}
1309
1310/// Debug: raw glyph stream `(ch, ll, lr, lb, lt)` (native coords) for page
1311/// `index`, before the sanitizer. For comparing char cells to docling-parse.
1312pub fn debug_glyphs(bytes: &[u8], index: usize) -> Vec<(char, f32, f32, f32, f32)> {
1313    let Some(doc) = load_document(bytes) else {
1314        return Vec::new();
1315    };
1316    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1317    pages.sort_by_key(|(n, _)| *n);
1318    let Some((_, pid)) = pages.get(index) else {
1319        return Vec::new();
1320    };
1321    page_glyphs(&doc, *pid)
1322        .into_iter()
1323        .map(|g| (g.ch, g.ll, g.lr, g.lb, g.lt))
1324        .collect()
1325}
1326
1327/// Public entry: per-page (width, height, line cells) for a PDF, via the Rust
1328/// text parser + the docling-parse line sanitizer. Used by the pipeline and the
1329/// `textparse_dump` example.
1330pub fn pdf_textlines(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1331    let Some(doc) = load_document(bytes) else {
1332        return Vec::new();
1333    };
1334    let mut caches = DocCaches::default();
1335    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1336    pages.sort_by_key(|(n, _)| *n);
1337    pages
1338        .into_iter()
1339        .map(|(_, pid)| {
1340            let (w, h) = page_size(&doc, pid);
1341            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1342            let cells = crate::dp_lines::line_cells(&glyphs, h, true);
1343            (w, h, cells)
1344        })
1345        .collect()
1346}
1347
1348/// Debug/diagnostic entry: per-page (width, height, word cells) for a PDF, via
1349/// the Rust parser glyphs run through the docling-parse word grouping. Used to
1350/// compare parser word cells against docling-parse's `word_cells` oracle (roadmap
1351/// item 6).
1352pub fn pdf_words(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1353    let Some(doc) = load_document(bytes) else {
1354        return Vec::new();
1355    };
1356    let mut caches = DocCaches::default();
1357    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1358    pages.sort_by_key(|(n, _)| *n);
1359    pages
1360        .into_iter()
1361        .map(|(_, pid)| {
1362            let (w, h) = page_size(&doc, pid);
1363            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1364            let cells = crate::dp_lines::word_cells(&glyphs, h, true);
1365            (w, h, cells)
1366        })
1367        .collect()
1368}
1369
1370/// One page's text cells from the pure-Rust parser: prose line cells, per-word
1371/// cells, and code line cells — all from a single glyph parse. Replaces the
1372/// pdfium text path (roadmap item 6) when the parser drop is enabled.
1373#[derive(Default)]
1374pub struct PageParserCells {
1375    pub prose: Vec<crate::pdfium_backend::TextCell>,
1376    pub words: Vec<crate::pdfium_backend::TextCell>,
1377    pub code: Vec<crate::pdfium_backend::TextCell>,
1378    /// Drawn checkbox squares (#609), in the cells' frame.
1379    pub checkboxes: Vec<crate::checkbox::CheckBox>,
1380    /// Image placements (#999), in the cells' frame — see
1381    /// [`crate::pdfium_backend::PdfPage::bitmaps`].
1382    pub bitmaps: Vec<[f32; 4]>,
1383    /// The page's shapes for the reading-order separators (#1092).
1384    pub shapes: PageShapes,
1385}
1386
1387/// The parser text layer, driven one page at a time: the document is loaded
1388/// (and repaired, see [`load_document`]) once, the font/form caches persist
1389/// across pages, and each page's glyphs are parsed only when asked for.
1390///
1391/// The eager whole-document walk this replaces ran *before* the first page
1392/// was rendered, so on a long PDF it was a serial prefix the page-worker pool
1393/// sat idle through — 6.2 s on the 1913-page .NET reference, in front of a
1394/// pipeline that otherwise overlaps parsing with inference — and a `--pages`
1395/// window still paid for every page in the file. Pulling pages on demand
1396/// keeps the parse on the producer thread but interleaved with rendering,
1397/// and skips unselected pages entirely. Output per page is unchanged: same
1398/// glyph walk, same shared caches, same contraction.
1399pub struct PageTextParser {
1400    doc: Document,
1401    caches: DocCaches,
1402    /// Page object ids in document order (page 1 first).
1403    pages: Vec<lopdf::ObjectId>,
1404    /// How words split where no space glyph does (#921); from
1405    /// `DOCLING_RS_PDF_WORD_GAP` unless [`Self::set_word_gap`] says.
1406    word_gap: crate::dp_lines::WordGap,
1407    /// Whether [`Self::text_page`]'s page keeps typographic punctuation as
1408    /// drawn (#921); `DOCLING_RS_PDF_KEEP_TYPOGRAPHY` unless
1409    /// [`Self::set_keep_typography`] says.
1410    keep_typography: bool,
1411}
1412
1413impl PageTextParser {
1414    /// Load the document; `None` when lopdf cannot read it at all.
1415    pub fn open(bytes: &[u8]) -> Option<Self> {
1416        Self::open_with_password(bytes, None)
1417    }
1418
1419    /// [`open`](Self::open) with the document's password (an encrypted file
1420    /// the password does not open reads as `None` here; the pipeline reports
1421    /// it through [`crate::pdf_meta::PdfMeta::open_with_password`] first).
1422    pub fn open_with_password(bytes: &[u8], password: Option<&str>) -> Option<Self> {
1423        let doc = open_document(bytes, password).ok()?;
1424        let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1425        pages.sort_by_key(|(n, _)| *n);
1426        Some(Self {
1427            doc,
1428            caches: DocCaches::default(),
1429            pages: pages.into_iter().map(|(_, pid)| pid).collect(),
1430            word_gap: crate::dp_lines::WordGap::from_env(),
1431            keep_typography: crate::assemble::keep_typography_from_env(),
1432        })
1433    }
1434
1435    /// The word-gap rule the line and word cells are built with (#921);
1436    /// `None` keeps the environment's (`DOCLING_RS_PDF_WORD_GAP`, else
1437    /// `adaptive`).
1438    pub fn set_word_gap(&mut self, gap: Option<crate::dp_lines::WordGap>) {
1439        if let Some(g) = gap {
1440            self.word_gap = g;
1441        }
1442    }
1443
1444    /// Whether the pages [`Self::text_page`] builds keep typographic
1445    /// punctuation as drawn (#921); `None` keeps the environment's
1446    /// (`DOCLING_RS_PDF_KEEP_TYPOGRAPHY`, else folding).
1447    pub fn set_keep_typography(&mut self, keep: Option<bool>) {
1448        if let Some(k) = keep {
1449            self.keep_typography = k;
1450        }
1451    }
1452
1453    /// The glyph boxes and font styles of the 0-based page `index` — the
1454    /// heading-hierarchy stage's style signal (#302): every non-space glyph's
1455    /// box (font ascent + descent at its size — the font-size proxy pdfium's
1456    /// loose char box also gave) in top-left coordinates, with the weight
1457    /// class and slant its `/BaseFont` name declares. Empty for a page
1458    /// without a text layer (a scan), and the stage falls back to its other
1459    /// signals.
1460    pub(crate) fn glyph_styles(
1461        &mut self,
1462        index: usize,
1463    ) -> Vec<crate::heading_hierarchy::GlyphStyle> {
1464        let Some(&pid) = self.pages.get(index) else {
1465            return Vec::new();
1466        };
1467        let (_w, h) = page_size(&self.doc, pid);
1468        let glyphs = page_glyphs_cached(&self.doc, pid, &mut self.caches);
1469        // Font hash → style, over the fonts the walk just parsed (an inline,
1470        // uncached font dictionary reads as unstyled).
1471        let styles: HashMap<u64, crate::font_style::FontStyle> = self
1472            .caches
1473            .fonts
1474            .values()
1475            .map(|f| (f.hash, f.style))
1476            .collect();
1477        glyphs
1478            .iter()
1479            .filter(|g| !g.ch.is_whitespace() && g.ll.is_finite())
1480            .map(|g| {
1481                let st = styles.get(&g.font).copied().unwrap_or_default();
1482                crate::heading_hierarchy::GlyphStyle {
1483                    l: g.ll,
1484                    t: h - g.lt,
1485                    r: g.lr,
1486                    b: h - g.lb,
1487                    height: g.height(),
1488                    weight_cls: crate::font_style::weight_class(st.weight),
1489                    italic: st.italic,
1490                    styled: st.known,
1491                }
1492            })
1493            .collect()
1494    }
1495
1496    /// The characters of the 0-based page `index` as the script-position
1497    /// stage measures them (#1005, docling's `read_page_chars`): every
1498    /// visible, non-blank glyph with its font-metric box, baseline and drawn
1499    /// size in the unrotated page frame (y up), and beside it the same box in
1500    /// the display frame (`/Rotate` applied, top-left origin: the frame the
1501    /// items' boxes live in). Invisible text (`Tr` 3 / 7, an OCR layer over a
1502    /// scan) is left out, as docling leaves it: it follows the scanned lines,
1503    /// not a typeset baseline.
1504    pub(crate) fn script_chars(
1505        &mut self,
1506        index: usize,
1507    ) -> Vec<(crate::script_position::ScriptChar, [f32; 4])> {
1508        let Some(&pid) = self.pages.get(index) else {
1509            return Vec::new();
1510        };
1511        let (w, h) = page_size(&self.doc, pid);
1512        let rotation = page_rotation(&self.doc, pid);
1513        page_glyphs_cached(&self.doc, pid, &mut self.caches)
1514            .iter()
1515            .filter(|g| !g.invisible && !g.ch.is_whitespace() && g.ll.is_finite())
1516            .filter_map(|g| {
1517                let [_, y, size] = g.pen?;
1518                let c = crate::script_position::ScriptChar {
1519                    text: g.ch,
1520                    left: g.ll,
1521                    bottom: g.lb,
1522                    right: g.lr,
1523                    top: g.lt,
1524                    baseline: y,
1525                    size,
1526                    horizontal: g.quad.is_none(),
1527                };
1528                let (l, t, r, b) = crate::pdfium_backend::to_display_frame(
1529                    (g.ll, h - g.lt, g.lr, h - g.lb),
1530                    rotation,
1531                    w,
1532                    h,
1533                );
1534                Some((c, [l, t, r, b]))
1535            })
1536            .collect()
1537    }
1538
1539    /// Prose, word and code cells of the 0-based page `index` — empty for an
1540    /// index the parser's page tree doesn't have (a damaged file whose page
1541    /// tree disagrees with the object model's count).
1542    pub fn cells(&mut self, index: usize) -> PageParserCells {
1543        let Some(&pid) = self.pages.get(index) else {
1544            return PageParserCells::default();
1545        };
1546        let (_w, h) = page_size(&self.doc, pid);
1547        let (glyphs, inks, bitmaps, shapes) =
1548            page_glyphs_and_inks(&self.doc, pid, &mut self.caches);
1549        let (prose, words) =
1550            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1551        PageParserCells {
1552            prose,
1553            words,
1554            code: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1555            checkboxes: crate::checkbox::find(&inks, h),
1556            bitmaps,
1557            shapes,
1558        }
1559    }
1560
1561    /// Pages in the document's page tree.
1562    pub fn page_count(&self) -> usize {
1563        self.pages.len()
1564    }
1565
1566    /// The 0-based page `index` as the text-layer-only conversion sees it —
1567    /// one element of [`pdf_text_pages`], parsed on demand so a caller can
1568    /// drop each page once it is assembled instead of holding the whole
1569    /// document's cells. A blank page (no cells) for an index the page tree
1570    /// doesn't have.
1571    pub fn text_page(&mut self, index: usize) -> crate::pdfium_backend::PdfPage {
1572        let (w, h, (glyphs, inks, _, shapes)) = match self.pages.get(index) {
1573            Some(&pid) => {
1574                let (w, h) = page_size(&self.doc, pid);
1575                (w, h, page_glyphs_and_inks(&self.doc, pid, &mut self.caches))
1576            }
1577            None => (0.0, 0.0, Default::default()),
1578        };
1579        let (mut prose, mut words) =
1580            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1581        drop_overpainted_cells(&mut prose);
1582        drop_overpainted_cells(&mut words);
1583        crate::pdfium_backend::PdfPage {
1584            #[cfg(feature = "ocr-prep")]
1585            image_layout: None,
1586            width: w,
1587            height: h,
1588            // Cells are native PDF points; there is no rendered bitmap.
1589            scale: 1.0,
1590            cells: prose,
1591            code_cells: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1592            word_cells: words,
1593            checkboxes: crate::checkbox::find(&inks, h),
1594            bitmaps: Vec::new(),
1595            shapes,
1596            #[cfg(feature = "ocr-prep")]
1597            image: image::RgbImage::new(1, 1),
1598            links: Vec::new(),
1599            rotation: 0,
1600            keep_typography: self.keep_typography,
1601            #[cfg(feature = "render")]
1602            embedded_images: Vec::new(),
1603            #[cfg(feature = "render")]
1604            pending_image: None,
1605        }
1606    }
1607}
1608
1609/// Full parser text layer: prose + word + code cells per page, glyphs parsed once.
1610/// `prose`/`words` come from the docling-parse contraction ([`crate::dp_lines`]);
1611/// `code` splits only at the parser's own space glyphs (monospace keeps its
1612/// source spacing). The eager form of [`PageTextParser`].
1613pub fn pdf_all_cells(bytes: &[u8]) -> Vec<PageParserCells> {
1614    let Some(mut parser) = PageTextParser::open(bytes) else {
1615        return Vec::new();
1616    };
1617    (0..parser.pages.len()).map(|i| parser.cells(i)).collect()
1618}
1619
1620/// Whole pages for the text-layer-only conversion ([`crate::convert_text_layer`]):
1621/// the parser's prose/word/code cells plus page geometry, assembled into
1622/// [`PdfPage`]s with no rendered image and no link annotations. Everything here
1623/// is pure Rust (lopdf), so it compiles without the `ml` feature — including on
1624/// wasm32. A page the parser can't read (no text layer) comes back with empty
1625/// cells; there is no pdfium fallback on this path.
1626pub fn pdf_text_pages(bytes: &[u8]) -> Vec<crate::pdfium_backend::PdfPage> {
1627    let Some(mut parser) = PageTextParser::open(bytes) else {
1628        return Vec::new();
1629    };
1630    (0..parser.page_count())
1631        .map(|i| parser.text_page(i))
1632        .collect()
1633}
1634
1635/// Drop line cells that are *painted over each other* — glyphs used as artwork.
1636///
1637/// Some generators draw their logo with a symbol font: on the reporting
1638/// invoice, a `TeleLogo` Type1 paints the T-Mobile mark by stacking the glyphs
1639/// encoded as `"` and `==` on top of one another, and the flat text-layer
1640/// output opened with that garbage. Nothing in the font metadata gives it away
1641/// (the *text* fonts in the same file are also flagged symbolic, and the logo
1642/// font names its glyphs `quotedbl` &c.), but the geometry does: two cells with
1643/// different text where one lies inside the other on the same line is
1644/// physically impossible for prose — ink from two words never occupies the
1645/// same box. Both cells of such a pair are paint, not text.
1646///
1647/// Containment (not mere overlap) keeps this narrow: adjacent words touch but
1648/// never contain each other, and a same-text near-duplicate (double-draw faux
1649/// bold) is left alone for the sanitizer's usual handling. Applied on the
1650/// flat/browser path only — the ML pipeline's text layer is byte-pinned by the
1651/// PDF corpus, and there the layout model already sinks logo marks into
1652/// `picture` regions.
1653fn drop_overpainted_cells(cells: &mut Vec<crate::pdfium_backend::TextCell>) {
1654    let mut paint = vec![false; cells.len()];
1655    for i in 0..cells.len() {
1656        for j in 0..cells.len() {
1657            if i == j || cells[i].text == cells[j].text {
1658                continue;
1659            }
1660            let (a, b) = (&cells[i], &cells[j]);
1661            // Same line band: the vertical overlap covers most of the shorter.
1662            let vo = (a.b.min(b.b) - a.t.max(b.t)).max(0.0);
1663            if vo < 0.6 * (a.b - a.t).min(b.b - b.t) {
1664                continue;
1665            }
1666            // `a` horizontally inside `b` (with a small tolerance).
1667            let ho = (a.r.min(b.r) - a.l.max(b.l)).max(0.0);
1668            if ho >= 0.8 * (a.r - a.l) && (a.r - a.l) <= (b.r - b.l) {
1669                paint[i] = true;
1670                paint[j] = true;
1671            }
1672        }
1673    }
1674    let mut keep = paint.iter().map(|p| !p);
1675    cells.retain(|_| keep.next().unwrap());
1676}
1677
1678/// The text-state scalars inherited by a Form XObject when it is invoked via
1679/// `Do` (the PDF graphics state includes the text parameters, but not the text
1680/// matrices, which a form re-establishes inside its own `BT`/`ET`).
1681#[derive(Clone, Copy)]
1682struct TextState {
1683    tc: f64,
1684    tw: f64,
1685    th: f64,
1686    tl: f64,
1687    trise: f64,
1688    fsize: f64,
1689    /// The text rendering mode (`Tr`).
1690    tr: i64,
1691}
1692
1693impl TextState {
1694    const INIT: TextState = TextState {
1695        tc: 0.0,
1696        tw: 0.0,
1697        th: 1.0,
1698        tl: 0.0,
1699        trise: 0.0,
1700        fsize: 0.0,
1701        tr: 0,
1702    };
1703}
1704
1705/// pdfium's `CPDF_Page::GetPageRotation` — the inherited `/Rotate`, in
1706/// quarter turns, as degrees (`pdf_meta`'s rule).
1707fn page_rotation(doc: &Document, pid: lopdf::ObjectId) -> u16 {
1708    let mut id = pid;
1709    for _ in 0..32 {
1710        let Ok(dict) = doc.get_dictionary(id) else {
1711            return 0;
1712        };
1713        if let Some(obj) = dict.get(b"Rotate").ok().and_then(|o| deref(doc, o)) {
1714            let quarter = (num(obj).unwrap_or(0.0) as i64 / 90).rem_euclid(4);
1715            return (quarter * 90) as u16;
1716        }
1717        match dict.get(b"Parent").ok().and_then(|o| o.as_reference().ok()) {
1718            Some(parent) => id = parent,
1719            None => return 0,
1720        }
1721    }
1722    0
1723}
1724
1725/// The effective `/Resources` dictionary for a page (inline or via reference,
1726/// falling back to an inherited one from a `/Parent`).
1727fn page_res(doc: &Document, page_id: lopdf::ObjectId) -> Option<&Dictionary> {
1728    let (inline, ids) = doc.get_page_resources(page_id).ok()?;
1729    if let Some(d) = inline {
1730        return Some(d);
1731    }
1732    ids.into_iter().find_map(|id| doc.get_dictionary(id).ok())
1733}
1734
1735/// Build the code→[`Font`] map for a resources dictionary's `/Font` sub-dict,
1736/// reusing the per-document cache for fonts referenced indirectly (the common
1737/// case — the same font objects recur on every page).
1738fn fonts_from_res(
1739    doc: &Document,
1740    res: &Dictionary,
1741    caches: &mut DocCaches,
1742) -> HashMap<Vec<u8>, Arc<Font>> {
1743    let mut map = HashMap::new();
1744    let font_dict = res
1745        .get(b"Font")
1746        .ok()
1747        .and_then(|o| deref(doc, o))
1748        .and_then(|o| o.as_dict().ok());
1749    if let Some(fd) = font_dict {
1750        for (name, value) in fd.iter() {
1751            let font = match value {
1752                Object::Reference(id) => {
1753                    let key = (*id, name.clone());
1754                    if let Some(f) = caches.fonts.get(&key) {
1755                        Arc::clone(f)
1756                    } else if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
1757                        let f = Arc::new(parse_font(doc, name, fdict));
1758                        caches.fonts.insert(key, Arc::clone(&f));
1759                        f
1760                    } else {
1761                        continue;
1762                    }
1763                }
1764                _ => {
1765                    if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
1766                        Arc::new(parse_font(doc, name, fdict))
1767                    } else {
1768                        continue;
1769                    }
1770                }
1771            };
1772            map.insert(name.clone(), font);
1773        }
1774    }
1775    map
1776}
1777
1778/// Extract every glyph on a page as a native-coordinate [`Glyph`].
1779/// [`PageTextParser::glyph_styles`] for the given **1-based** pages of a
1780/// document, keyed by page number — a separate, on-demand pass (no
1781/// rendering), so the extraction pipeline stays byte-identical whether or not
1782/// the heading-hierarchy stage runs. Empty when lopdf cannot open the file.
1783pub(crate) fn glyph_styles(
1784    bytes: &[u8],
1785    pages: &[usize],
1786) -> HashMap<usize, Vec<crate::heading_hierarchy::GlyphStyle>> {
1787    let mut out = HashMap::new();
1788    let Some(mut parser) = PageTextParser::open(bytes) else {
1789        return out;
1790    };
1791    for &page_no in pages {
1792        if page_no == 0 {
1793            continue;
1794        }
1795        let styles = parser.glyph_styles(page_no - 1);
1796        if !styles.is_empty() {
1797            out.insert(page_no, styles);
1798        }
1799    }
1800    out
1801}
1802
1803pub(crate) fn page_glyphs(doc: &Document, page_id: lopdf::ObjectId) -> Vec<Glyph> {
1804    page_glyphs_cached(doc, page_id, &mut DocCaches::default())
1805}
1806
1807/// [`page_glyphs`] with an explicit per-document cache, so a multi-page walk
1808/// parses each font / decodes each form once instead of once per page.
1809fn page_glyphs_cached(
1810    doc: &Document,
1811    page_id: lopdf::ObjectId,
1812    caches: &mut DocCaches,
1813) -> Vec<Glyph> {
1814    page_glyphs_and_inks(doc, page_id, caches).0
1815}
1816
1817/// [`page_glyphs_cached`] plus the small painted path pieces the same walk
1818/// passed — what [`crate::checkbox::find`] looks for checkbox squares in
1819/// (#609) — the page's image placements (#999, `[l, t, r, b]` top-left
1820/// page points clipped to the page, the cells' frame), and its shapes for the
1821/// reading-order separators (#1092, [`PageShapes`]).
1822fn page_glyphs_and_inks(
1823    doc: &Document,
1824    page_id: lopdf::ObjectId,
1825    caches: &mut DocCaches,
1826) -> (
1827    Vec<Glyph>,
1828    Vec<crate::checkbox::Ink>,
1829    Vec<[f32; 4]>,
1830    PageShapes,
1831) {
1832    let mut out = Vec::new();
1833    let mut inks = Vec::new();
1834    // get_page_content returns the assembled content-stream bytes directly
1835    // (an empty Vec when the page has none).
1836    let content_bytes = doc.get_page_content(page_id);
1837    let Ok(content) = lopdf::content::Content::decode(&content_bytes) else {
1838        return (out, inks, Vec::new(), PageShapes::default());
1839    };
1840    let mut bitmaps = Vec::new();
1841    let mut page_shapes = PageShapes::default();
1842    if let Some(res) = page_res(doc, page_id) {
1843        // pdfium's page matrix: user space translated so the display box's
1844        // lower-left corner is the origin (see [`PageBox`]).
1845        let pb = page_box(doc, page_id);
1846        let base = Mat {
1847            e: -(pb.l as f64),
1848            f: -(pb.b as f64),
1849            ..Mat::ID
1850        };
1851        let mut shapes = Shapes::default();
1852        run_content(
1853            doc,
1854            res,
1855            &content,
1856            base,
1857            TextState::INIT,
1858            0,
1859            caches,
1860            &mut out,
1861            &mut inks,
1862            ShapeGs::INIT,
1863            &mut shapes,
1864        );
1865        out.retain(|g| on_page(g, pb.w, pb.h));
1866        drop_overprinted_spaces(&mut out);
1867        drop_shadow_runs(&mut out);
1868        let (w, h) = (f64::from(pb.w), f64::from(pb.h));
1869        inks.retain(|ink| match *ink {
1870            crate::checkbox::Ink::Bitmap { l, b, r, t } => {
1871                let (l, r, b, t) = (l.max(0.0), r.min(w), b.max(0.0), t.min(h));
1872                if r > l && t > b {
1873                    bitmaps.push([l as f32, (h - t) as f32, r as f32, (h - b) as f32]);
1874                }
1875                false
1876            }
1877            _ => true,
1878        });
1879        let top_left =
1880            |[l, b, r, t]: [f64; 4]| [l as f32, (h - t) as f32, r as f32, (h - b) as f32];
1881        page_shapes.lines = shapes.lines.into_iter().map(top_left).collect();
1882        if shapes.boxes.len() <= MAX_SHAPE_BOXES {
1883            page_shapes.boxes = connected_boxes(&shapes.boxes)
1884                .into_iter()
1885                .map(top_left)
1886                .collect();
1887        } else {
1888            docling_core::debug_log!(
1889                "textparse: {} painted paths on the page; their boxes are not merged",
1890                shapes.boxes.len()
1891            );
1892        }
1893    }
1894    (out, inks, bitmaps, page_shapes)
1895}
1896
1897/// A page's shapes for the reading-order separators (#1092), in the cells'
1898/// frame (`[l, t, r, b]`, top-left page points): docling-parse's
1899/// `get_shape_lines` and `get_connected_shape_bounding_boxes`.
1900#[derive(Default, Clone, Debug, PartialEq)]
1901pub struct PageShapes {
1902    pub lines: Vec<[f32; 4]>,
1903    pub boxes: Vec<[f32; 4]>,
1904}
1905
1906/// Drop a run of glyphs painted again over itself — a text shadow. A word
1907/// processor exports a heading with a shadow effect as the same run drawn
1908/// twice: the shadow, then the text over it, shifted by a fraction of a point
1909/// (0.5–0.8 pt up and left, black under red, in the manuals that showed it).
1910/// docling-parse keeps both copies and reads `Safety instructions Safety
1911/// instructions`; pdfium's text page, behind docling's pypdfium2 backend,
1912/// drops the second, and the page shows the heading once. A deliberate
1913/// divergence (`docs/MIGRATION.md` §4 item 25): a glyph that repeats an
1914/// earlier one — same character, font and size, its pen shifted from the
1915/// earlier pen by at most 0.15 em on each axis, and by less than the earlier
1916/// glyph's step to its own successor — starts a copy when the glyphs after it
1917/// repeat the earlier one's successors by the same offset and size, and the
1918/// whole stretch that keeps the offset is dropped. Two glyphs at least: a lone
1919/// mark stamped twice is left alone, and so is the per-glyph re-stamp the
1920/// sanitizer already folds (`dp_lines`, the overprint duplicate). Text that
1921/// really repeats sits an advance apart or more — a doubled letter (`ll`,
1922/// 0.22 em in Helvetica), a word set twice (`very very`), the same text on
1923/// the next line or in the next table cell — and is kept, and so is a run of
1924/// one character set tighter than a shadow's shift (dot leaders squeezed by
1925/// `Tc`): its glyph two on is farther than the next one. A line stamped two
1926/// or three times at a small offset to fake bold reads once the same way. The
1927/// original must start within the 1024 glyphs before its copy, so a shadowed
1928/// block longer than that is not caught; a shadowed heading is drawn right
1929/// after itself. A rotated run, whose pen positions are not compared here, is
1930/// left alone.
1931fn drop_shadow_runs(glyphs: &mut Vec<Glyph>) {
1932    // How far back a copy's original may start.
1933    const LOOKBACK: usize = 1024;
1934    // The largest shift of a copy, per axis, in ems of the glyph's size.
1935    const NEAR_EM: f32 = 0.15;
1936    // How much the shift and the size may vary along a copy, in ems.
1937    const SAME_EM: f32 = 0.01;
1938    let pen = |g: &Glyph| if g.quad.is_none() { g.pen } else { None };
1939    let n = glyphs.len();
1940    let mut keep = vec![true; n];
1941    // The glyphs a copy may start on, by character and font, in stream
1942    // order: kept (not part of a copy), upright, sized and not whitespace. A
1943    // copy repeats both, so a glyph walks only its own list back through the
1944    // window; scanning the whole window nearly doubled a `--text-layer-only`
1945    // read of pages without a shadow.
1946    let mut earlier: HashMap<(char, u64), Vec<usize>> = HashMap::new();
1947    let mut j = 0;
1948    while j < n {
1949        let b = &glyphs[j];
1950        let start = pen(b).filter(|p| p[2] > 0.0 && !b.ch.is_whitespace());
1951        let mut copy = 0;
1952        if let Some([xj, yj, sj]) = start {
1953            let lo = j.saturating_sub(LOOKBACK);
1954            let candidates = earlier.entry((b.ch, b.font)).or_default();
1955            for &i in candidates.iter().rev().take_while(|&&i| i >= lo) {
1956                let Some([xi, yi, si]) = pen(&glyphs[i]) else {
1957                    continue;
1958                };
1959                if (si - sj).abs() > SAME_EM * sj {
1960                    continue;
1961                }
1962                let (dx, dy) = (xj - xi, yj - yi);
1963                if dx.abs() > NEAR_EM * sj || dy.abs() > NEAR_EM * sj {
1964                    continue;
1965                }
1966                // A shadow is shifted: the characters one glyph expands to
1967                // (a code-like name read as `SM590000`, a decomposed
1968                // ligature) share the glyph's pen and are not a copy.
1969                if dx.abs() + dy.abs() <= SAME_EM * sj {
1970                    continue;
1971                }
1972                // And it lies over its original, nearer to it than the
1973                // original's next glyph is: one character set tighter than
1974                // half the largest shift (dot leaders squeezed by `Tc`) meets
1975                // its own successors at one offset two glyphs on.
1976                if let Some([xn, yn, _]) = glyphs.get(i + 1).and_then(pen) {
1977                    let step = (xn - xi).abs().max((yn - yi).abs());
1978                    if step > SAME_EM * sj && dx.abs().max(dy.abs()) >= step {
1979                        continue;
1980                    }
1981                }
1982                // The copy goes on while each next glyph repeats the
1983                // original's next one by the same shift; the original
1984                // ends before the copy starts.
1985                let mut m = 1;
1986                while j + m < n && i + m < j {
1987                    let (a, b) = (&glyphs[i + m], &glyphs[j + m]);
1988                    let (Some([xa, ya, sa]), Some([xb, yb, sb])) = (pen(a), pen(b)) else {
1989                        break;
1990                    };
1991                    if a.ch != b.ch
1992                        || a.font != b.font
1993                        || (sa - sb).abs() > SAME_EM * sj
1994                        || ((xb - xa) - dx).abs() > SAME_EM * sj
1995                        || ((yb - ya) - dy).abs() > SAME_EM * sj
1996                    {
1997                        break;
1998                    }
1999                    m += 1;
2000                }
2001                if m >= 2 {
2002                    copy = m;
2003                    break;
2004                }
2005            }
2006            if copy == 0 {
2007                candidates.push(j);
2008            }
2009        }
2010        if copy > 0 {
2011            keep[j..j + copy].iter_mut().for_each(|k| *k = false);
2012            j += copy;
2013        } else {
2014            j += 1;
2015        }
2016    }
2017    let mut it = keep.into_iter();
2018    glyphs.retain(|_| it.next().unwrap_or(true));
2019}
2020
2021/// How many marked-content spans may be open for one's `/ActualText` to be
2022/// read (see [`apply_actual_text`]); real documents nest a few.
2023const MAX_ACTUAL_TEXT_DEPTH: usize = 64;
2024
2025/// Put a marked-content span's `/ActualText` in place of the glyphs drawn
2026/// inside it, as docling-parse does since 7.9 (`apply_actual_text`,
2027/// docling-parse#300, on by default; PDF 32000-1 §14.9.4). A writer draws a
2028/// glyph cluster whose characters the font cannot spell this way: a base
2029/// letter and a zero-advance accent whose code collides with a Latin letter
2030/// (`Oreši` read as `OresMi`, docling#3783), or the Arabic marks of a
2031/// LibreOffice font subset whose `/ToUnicode` skips them, so the byte reads as
2032/// WinAnsi (`منعًا` as `منع8ا`). docling-parse's rules, cell for cell: a span
2033/// with no glyphs (over graphics) is left alone, and so is a text implausibly
2034/// long for it (more than 8 characters and 4 per cell: `/Alt` written into
2035/// the wrong key); an empty text removes the glyphs; as many characters as
2036/// cells replace them one by one, keeping each cell's geometry; any other
2037/// count puts the whole text on the span's first cell, over the union of the
2038/// span's boxes, its baseline running to the cell that reaches furthest along
2039/// it (an accent drawn after its base letter trails behind it). Nested spans
2040/// apply innermost first, so the outer text wins. docling-parse has a cell per
2041/// character code; a code whose `/ToUnicode` spells several characters is
2042/// several glyphs here, all on the code's pen, and counts as one cell.
2043///
2044/// Each `EMC` rewrites the glyphs under its span, so a crafted nest of spans
2045/// over the same glyphs would cost glyphs × depth: past
2046/// [`MAX_ACTUAL_TEXT_DEPTH`] open spans, a span still pairs with its `EMC` but
2047/// its `/ActualText` is not read.
2048fn apply_actual_text(out: &mut Vec<Glyph>, begin: usize, text: &str) {
2049    let begin = begin.min(out.len());
2050    let mut cells: Vec<std::ops::Range<usize>> = Vec::new();
2051    for i in begin..out.len() {
2052        match cells.last_mut() {
2053            Some(cell) if out[i].pen.is_some() && out[i].pen == out[cell.end - 1].pen => {
2054                cell.end = i + 1;
2055            }
2056            _ => cells.push(i..i + 1),
2057        }
2058    }
2059    if cells.is_empty() {
2060        return;
2061    }
2062    let chars: Vec<char> = text.chars().collect();
2063    if chars.len() > (4 * cells.len()).max(8) {
2064        return;
2065    }
2066    if chars.is_empty() {
2067        out.truncate(begin);
2068        return;
2069    }
2070    if chars.len() == cells.len() {
2071        // Each cell's first glyph takes its character, the rest of the cell
2072        // goes: one pass, writing behind the cells read.
2073        for (w, (cell, &ch)) in cells.iter().zip(&chars).enumerate() {
2074            out[begin + w] = Glyph {
2075                ch,
2076                ..out[cell.start]
2077            };
2078        }
2079        out.truncate(begin + cells.len());
2080        return;
2081    }
2082    // The leading edge (quad corners 0 and 3) stays the first cell's; the
2083    // trailing one (1 and 2) is the cell's that reaches furthest along the
2084    // baseline, projected on it so a rotated run keeps its direction.
2085    let quad_of = |g: &Glyph| {
2086        g.quad
2087            .unwrap_or([g.ll, g.lb, g.lr, g.lb, g.lr, g.lt, g.ll, g.lt])
2088    };
2089    let mut merged = Glyph { ..out[begin] };
2090    let mut quad = quad_of(&merged);
2091    let dir = (quad[2] - quad[0], quad[3] - quad[1]);
2092    let along = |q: &[f32; 8]| q[2] * dir.0 + q[3] * dir.1;
2093    let mut reach = along(&quad);
2094    for g in &out[begin + 1..] {
2095        merged.l = merged.l.min(g.l);
2096        merged.b = merged.b.min(g.b);
2097        merged.r = merged.r.max(g.r);
2098        merged.t = merged.t.max(g.t);
2099        merged.ll = merged.ll.min(g.ll);
2100        merged.lb = merged.lb.min(g.lb);
2101        merged.lr = merged.lr.max(g.lr);
2102        merged.lt = merged.lt.max(g.lt);
2103        let q = quad_of(g);
2104        if along(&q) > reach {
2105            reach = along(&q);
2106            quad[2..6].copy_from_slice(&q[2..6]);
2107        }
2108    }
2109    if merged.quad.is_some() {
2110        merged.quad = Some(quad);
2111    }
2112    out.truncate(begin);
2113    // The characters of one cell share its box, as a ligature's do.
2114    out.extend(chars.iter().map(|&ch| Glyph { ch, ..merged }));
2115}
2116
2117/// Drop a space glyph the next glyph is drawn over (#992). A typesetter
2118/// that sets a superscript footnote marker as `( 1 )` — space glyphs around
2119/// the raised digit — and then cancels each space's advance with a negative
2120/// `TJ` offset (Arbortext on EUR-Lex: the `1` starts 0.05 pt *before* its
2121/// space begins) renders `(1)`; the spaces exist in the content stream
2122/// only. docling-parse keeps them as characters and its line cell reads
2123/// `( 1 )`; pdftotext and pdfium, which break words by geometry alone,
2124/// read `(1)`, and so does the page. A deliberate divergence
2125/// (`docs/MIGRATION.md` §4 item 22): a space whose successor in stream
2126/// order sits on the same line and starts within the first 30 % of the
2127/// space's advance — the pen went back to (or before) where the space
2128/// began, so it left no gap — is not a character of the text. A space
2129/// followed by a glyph on another line (a trailing space before the
2130/// cursor jumps to the next line's left margin, far to the left of it) or
2131/// by one that starts past that mark (a kerned pair, a justified word gap
2132/// pulled in a little) is kept; the word-gap rules then see the geometry
2133/// as before.
2134fn drop_overprinted_spaces(glyphs: &mut Vec<Glyph>) {
2135    if glyphs.len() < 2 {
2136        return;
2137    }
2138    let overprinted = |sp: &Glyph, next: &Glyph| {
2139        if sp.ch != ' ' || next.ch == ' ' {
2140            return false;
2141        }
2142        // Upright glyphs only: a rotated run's loose boxes are the extents
2143        // of its quads, every glyph of a 90° line shares one left edge and
2144        // the advance runs along y, so the test below would read every
2145        // space of a landscape slide drawn through a 90° `cm` as
2146        // overprinted (`Nullammollis`, odf_presentation_02).
2147        if sp.quad.is_some() || next.quad.is_some() {
2148            return false;
2149        }
2150        let width = sp.lr - sp.ll;
2151        if !width.is_finite() || width <= 0.0 || !next.ll.is_finite() {
2152            return false;
2153        }
2154        // Same line: the loose boxes overlap vertically (a raised digit's box
2155        // still overlaps its neighbours' em boxes).
2156        let same_line = next.lb < sp.lt && next.lt > sp.lb;
2157        same_line && next.ll >= sp.ll - 0.1 * width && next.ll <= sp.ll + 0.3 * width
2158    };
2159    let mut keep = vec![true; glyphs.len()];
2160    for i in 0..glyphs.len() - 1 {
2161        if overprinted(&glyphs[i], &glyphs[i + 1]) {
2162            keep[i] = false;
2163        }
2164    }
2165    let mut it = keep.into_iter();
2166    glyphs.retain(|_| it.next().unwrap_or(true));
2167}
2168
2169/// Whether a glyph is on the page (#529): docling-parse keeps a character
2170/// only when its whole box lies inside the display box — the CropBox, or
2171/// the MediaBox without one — edges included, so a FrameMaker print slug
2172/// drawn beside the CropBox or a tiled page's neighbouring text beyond the
2173/// MediaBox never becomes a cell, and a line crossing the edge is cut at the
2174/// last glyph that fits (`Crossing the rig`). The box is docling-parse's
2175/// char box — the advance by the font's ascent/descent, the loose box here
2176/// (its axis-aligned extent for rotated text) — in the frame `page_glyphs`
2177/// already moved to the display box's corner, so the page is `[0, w] × [0,
2178/// h]`. A glyph without a finite box is kept, as before. (A standard-14 font
2179/// without a FontDescriptor gets the 750 / −250 default ascent / descent
2180/// here where docling-parse reads the AFM's — Helvetica's 718 / −207 — so
2181/// for those a glyph within ~0.04 em of the top or bottom edge can fall the
2182/// other way; horizontally the advance is the same.)
2183fn on_page(g: &Glyph, w: f32, h: f32) -> bool {
2184    // f32 noise only: docling-parse already drops a glyph 0.01 pt over.
2185    const EPS: f32 = 1e-3;
2186    let (l, b, r, t) = if [g.ll, g.lb, g.lr, g.lt].iter().all(|v| v.is_finite()) {
2187        (g.ll, g.lb, g.lr, g.lt)
2188    } else {
2189        (g.l, g.b, g.r, g.t)
2190    };
2191    if ![l, b, r, t].iter().all(|v| v.is_finite()) {
2192        return true;
2193    }
2194    l >= -EPS && b >= -EPS && r <= w + EPS && t <= h + EPS
2195}
2196
2197/// Run a content stream's operators, emitting glyphs into `out`. Recurses into
2198/// Form XObjects on `Do` (bulk body text in heavy PDFs lives inside a form, not
2199/// the page content stream). `res` is the resources dict in scope (the page's,
2200/// or the form's own); `base_ctm` is the CTM at the point of invocation.
2201#[allow(clippy::too_many_arguments)]
2202fn run_content(
2203    doc: &Document,
2204    res: &Dictionary,
2205    content: &lopdf::content::Content,
2206    base_ctm: Mat,
2207    init: TextState,
2208    depth: u32,
2209    caches: &mut DocCaches,
2210    out: &mut Vec<Glyph>,
2211    inks: &mut Vec<crate::checkbox::Ink>,
2212    shape_init: ShapeGs,
2213    shapes: &mut Shapes,
2214) {
2215    let fonts = fonts_from_res(doc, res, caches);
2216    let xobjects = res
2217        .get(b"XObject")
2218        .ok()
2219        .and_then(|o| deref(doc, o))
2220        .and_then(|o| o.as_dict().ok());
2221
2222    // Graphics + text state. `q`/`Q` save and restore the whole graphics state,
2223    // which includes the text parameters (Tc, Tw, Tz, TL, Tfs, Trise, font) —
2224    // *not* the text matrix (that is reset by BT). Saving only the CTM let a Tc
2225    // set inside a `q…Q` block leak out and drift every later glyph.
2226    #[allow(clippy::type_complexity)]
2227    let mut gstate_stack: Vec<(Mat, f64, f64, f64, f64, f64, f64, i64, Option<&Arc<Font>>)> =
2228        Vec::new();
2229    let mut ctm = base_ctm;
2230    let mut tm = Mat::ID;
2231    let mut tlm = Mat::ID;
2232    let mut font: Option<&Arc<Font>> = None;
2233    let mut fsize = init.fsize;
2234    let mut tc = init.tc; // char spacing
2235    let mut tw = init.tw; // word spacing
2236    let mut th = init.th; // horizontal scale (Tz/100)
2237    let mut tl = init.tl; // leading
2238    let mut trise = init.trise;
2239    let mut tr = init.tr; // rendering mode
2240
2241    let op_f = |operands: &[Object], i: usize| operands.get(i).and_then(num).unwrap_or(0.0);
2242    // The path under construction, already in page space (a path is built
2243    // under one CTM — `cm` is not allowed inside it), for the checkbox
2244    // squares (#609). Text extraction never reads it.
2245    let mut path: Vec<PathPiece> = Vec::new();
2246    let (mut cur, mut start) = ((0.0, 0.0), (0.0, 0.0));
2247    // Whether the fill colour is (near) white, saved/restored with `q`/`Q`:
2248    // a white-filled stroked box is a checkbox outline, a coloured one a
2249    // chart legend's swatch. The initial fill colour is black.
2250    let mut fill_light = false;
2251    let mut fill_stack: Vec<bool> = Vec::new();
2252    // Open marked-content sequences (`BMC`/`BDC` … `EMC`): where each one's
2253    // glyphs start in `out`, and its inline `/ActualText`, if any.
2254    let mut marked: Vec<(usize, Option<String>)> = Vec::new();
2255    // The reading-order shapes (#1092): docling-parse's view of the path
2256    // under construction, its graphics state and the pending clip.
2257    let mut shape_gs = shape_init;
2258    let mut shape_stack: Vec<ShapeGs> = Vec::new();
2259    let mut shape_path: Vec<ShapeSub> = Vec::new();
2260    let mut clip_pending = false;
2261    let ext_gstates = res
2262        .get(b"ExtGState")
2263        .ok()
2264        .and_then(|o| deref(doc, o))
2265        .and_then(|o| o.as_dict().ok());
2266    let colorspaces = res
2267        .get(b"ColorSpace")
2268        .ok()
2269        .and_then(|o| deref(doc, o))
2270        .and_then(|o| o.as_dict().ok());
2271
2272    for op in &content.operations {
2273        let operands = &op.operands;
2274        let shown = out.len();
2275        match op.operator.as_str() {
2276            "q" => {
2277                gstate_stack.push((ctm, tc, tw, th, tl, trise, fsize, tr, font));
2278                fill_stack.push(fill_light);
2279                shape_stack.push(shape_gs.clone());
2280            }
2281            "BMC" => marked.push((out.len(), None)),
2282            // Only an inline property list is read: a named one (`/OC /MC0`)
2283            // would need the `/Properties` resource, as in docling-parse.
2284            "BDC" => {
2285                let actual = operands
2286                    .get(1)
2287                    .filter(|_| marked.len() < MAX_ACTUAL_TEXT_DEPTH)
2288                    .and_then(|o| o.as_dict().ok())
2289                    .and_then(|d| d.get(b"ActualText").ok())
2290                    .and_then(|o| match o {
2291                        Object::String(bytes, _) => Some(crate::info::decode_text(bytes)),
2292                        _ => None,
2293                    });
2294                marked.push((out.len(), actual));
2295            }
2296            "EMC" => {
2297                if let Some((begin, Some(text))) = marked.pop() {
2298                    apply_actual_text(out, begin, &text);
2299                }
2300                // Shows no text: the invisible-mode stamp below has nothing
2301                // new to mark, and the span may have shortened `out`.
2302                continue;
2303            }
2304            "Q" => {
2305                fill_light = fill_stack.pop().unwrap_or(false);
2306                if let Some(gs) = shape_stack.pop() {
2307                    shape_gs = gs;
2308                }
2309                if let Some((c, a, b, h, l, r, fs, mode, f)) = gstate_stack.pop() {
2310                    ctm = c;
2311                    tc = a;
2312                    tw = b;
2313                    th = h;
2314                    tl = l;
2315                    trise = r;
2316                    fsize = fs;
2317                    tr = mode;
2318                    font = f;
2319                }
2320            }
2321            "cm" => {
2322                let m = Mat {
2323                    a: op_f(operands, 0),
2324                    b: op_f(operands, 1),
2325                    c: op_f(operands, 2),
2326                    d: op_f(operands, 3),
2327                    e: op_f(operands, 4),
2328                    f: op_f(operands, 5),
2329                };
2330                ctm = m.then(ctm);
2331            }
2332            "BT" => {
2333                tm = Mat::ID;
2334                tlm = Mat::ID;
2335            }
2336            "ET" => {}
2337            "Tf" => {
2338                if let Some(Object::Name(n)) = operands.first() {
2339                    font = fonts.get(n.as_slice());
2340                }
2341                fsize = op_f(operands, 1);
2342            }
2343            "Td" => {
2344                tlm = Mat {
2345                    a: 1.0,
2346                    b: 0.0,
2347                    c: 0.0,
2348                    d: 1.0,
2349                    e: op_f(operands, 0),
2350                    f: op_f(operands, 1),
2351                }
2352                .then(tlm);
2353                tm = tlm;
2354            }
2355            "TD" => {
2356                tl = -op_f(operands, 1);
2357                tlm = Mat {
2358                    a: 1.0,
2359                    b: 0.0,
2360                    c: 0.0,
2361                    d: 1.0,
2362                    e: op_f(operands, 0),
2363                    f: op_f(operands, 1),
2364                }
2365                .then(tlm);
2366                tm = tlm;
2367            }
2368            "Tm" => {
2369                tlm = Mat {
2370                    a: op_f(operands, 0),
2371                    b: op_f(operands, 1),
2372                    c: op_f(operands, 2),
2373                    d: op_f(operands, 3),
2374                    e: op_f(operands, 4),
2375                    f: op_f(operands, 5),
2376                };
2377                tm = tlm;
2378            }
2379            "T*" => {
2380                tlm = Mat {
2381                    a: 1.0,
2382                    b: 0.0,
2383                    c: 0.0,
2384                    d: 1.0,
2385                    e: 0.0,
2386                    f: -tl,
2387                }
2388                .then(tlm);
2389                tm = tlm;
2390            }
2391            "Tc" => tc = op_f(operands, 0),
2392            "Tw" => tw = op_f(operands, 0),
2393            "Tz" => th = op_f(operands, 0) / 100.0,
2394            "TL" => tl = op_f(operands, 0),
2395            "Ts" => trise = op_f(operands, 0),
2396            "Tr" => tr = op_f(operands, 0) as i64,
2397            "Tj" | "'" | "\"" => {
2398                if op.operator == "'" || op.operator == "\"" {
2399                    // move to next line first
2400                    tlm = Mat {
2401                        a: 1.0,
2402                        b: 0.0,
2403                        c: 0.0,
2404                        d: 1.0,
2405                        e: 0.0,
2406                        f: -tl,
2407                    }
2408                    .then(tlm);
2409                    tm = tlm;
2410                }
2411                if op.operator == "\"" {
2412                    // `aw ac string "` sets word- and char-spacing before
2413                    // showing the string (PDF 32000-1 §9.4.3), persisting after.
2414                    tw = op_f(operands, 0);
2415                    tc = op_f(operands, 1);
2416                }
2417                if let (Some(f), Some(Object::String(s, _))) = (font, operands.last()) {
2418                    show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out);
2419                }
2420            }
2421            "TJ" => {
2422                if let (Some(f), Some(Object::Array(arr))) = (font, operands.first()) {
2423                    for el in arr {
2424                        match el {
2425                            Object::String(s, _) => {
2426                                show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out)
2427                            }
2428                            other => {
2429                                if let Some(adj) = num(other) {
2430                                    // negative number moves text right (PDF: subtract)
2431                                    let tx = -adj / 1000.0 * fsize * th;
2432                                    tm = Mat {
2433                                        a: 1.0,
2434                                        b: 0.0,
2435                                        c: 0.0,
2436                                        d: 1.0,
2437                                        e: tx,
2438                                        f: 0.0,
2439                                    }
2440                                    .then(tm);
2441                                }
2442                            }
2443                        }
2444                    }
2445                }
2446            }
2447            "m" => {
2448                cur = ctm.apply(op_f(operands, 0), op_f(operands, 1));
2449                start = cur;
2450                shape_path.push(ShapeSub::at(cur));
2451            }
2452            "l" => {
2453                let p = ctm.apply(op_f(operands, 0), op_f(operands, 1));
2454                path.push(PathPiece::Line(cur, p));
2455                cur = p;
2456                if let Some(sub) = shape_path.last_mut() {
2457                    sub.lines.push((sub.cur, p));
2458                    sub.pts.push(p);
2459                    sub.cur = p;
2460                }
2461            }
2462            "c" | "v" | "y" => {
2463                let pts: Vec<(f64, f64)> = (0..operands.len() / 2)
2464                    .map(|k| ctm.apply(op_f(operands, 2 * k), op_f(operands, 2 * k + 1)))
2465                    .collect();
2466                if let Some(&end) = pts.last() {
2467                    path.push(PathPiece::Curve(
2468                        std::iter::once(cur).chain(pts.iter().copied()).collect(),
2469                    ));
2470                    cur = end;
2471                    if let Some(sub) = shape_path.last_mut() {
2472                        sub.pts.extend(pts.iter().copied());
2473                        sub.cur = end;
2474                    }
2475                }
2476            }
2477            "h" => {
2478                if cur != start {
2479                    path.push(PathPiece::Line(cur, start));
2480                }
2481                cur = start;
2482                // docling-parse closes the subpath and opens the next one at
2483                // its start, where a following `l` continues.
2484                if let Some(sub) = shape_path.last_mut() {
2485                    sub.closed = true;
2486                    sub.pts.push(sub.start);
2487                    let at = sub.start;
2488                    shape_path.push(ShapeSub::at(at));
2489                }
2490            }
2491            "re" => {
2492                let (x, y, w, h) = (
2493                    op_f(operands, 0),
2494                    op_f(operands, 1),
2495                    op_f(operands, 2),
2496                    op_f(operands, 3),
2497                );
2498                path.push(PathPiece::Rect([
2499                    ctm.apply(x, y),
2500                    ctm.apply(x + w, y),
2501                    ctm.apply(x + w, y + h),
2502                    ctm.apply(x, y + h),
2503                ]));
2504                cur = ctm.apply(x, y);
2505                start = cur;
2506                // `m` + three `l` + `h`, as docling-parse builds it.
2507                let c = [
2508                    ctm.apply(x, y),
2509                    ctm.apply(x + w, y),
2510                    ctm.apply(x + w, y + h),
2511                    ctm.apply(x, y + h),
2512                ];
2513                shape_path.push(ShapeSub {
2514                    start: c[0],
2515                    cur: c[3],
2516                    pts: vec![c[0], c[1], c[2], c[3], c[0]],
2517                    lines: vec![(c[0], c[1]), (c[1], c[2]), (c[2], c[3])],
2518                    closed: true,
2519                });
2520                shape_path.push(ShapeSub::at(c[0]));
2521            }
2522            "S" | "s" | "f" | "F" | "f*" | "B" | "B*" | "b" | "b*" => {
2523                let op = op.operator.as_str();
2524                if matches!(op, "s" | "b" | "b*") && cur != start {
2525                    path.push(PathPiece::Line(cur, start));
2526                }
2527                let stroke = matches!(op, "S" | "s" | "B" | "B*" | "b" | "b*");
2528                let fill = !matches!(op, "S" | "s");
2529                paint_path(&path, stroke, fill, fill_light, inks);
2530                path.clear();
2531                // `s`, `f`, `f*`, `b`, `b*` close the last subpath first.
2532                if !matches!(op, "S" | "B" | "B*") {
2533                    if let Some(sub) = shape_path.last_mut() {
2534                        sub.closed = true;
2535                    }
2536                }
2537                let scale = (ctm.a * ctm.d - ctm.b * ctm.c).abs().sqrt();
2538                record_shape(
2539                    &shape_path,
2540                    stroke,
2541                    fill,
2542                    &shape_gs,
2543                    if scale > 0.0 { scale } else { 1.0 },
2544                    shapes,
2545                );
2546                // A pending `W` takes effect after the painting operator.
2547                if std::mem::take(&mut clip_pending) {
2548                    capture_clip(&shape_path, &mut shape_gs.clips);
2549                }
2550                shape_path.clear();
2551            }
2552            "n" => {
2553                path.clear();
2554                if std::mem::take(&mut clip_pending) {
2555                    capture_clip(&shape_path, &mut shape_gs.clips);
2556                }
2557                shape_path.clear();
2558            }
2559            "W" | "W*" => clip_pending = true,
2560            "w" => shape_gs.line_width = op_f(operands, 0),
2561            "gs" => {
2562                let params = operands
2563                    .first()
2564                    .and_then(|o| o.as_name().ok())
2565                    .and_then(|n| ext_gstates?.get(n).ok())
2566                    .and_then(|o| deref(doc, o))
2567                    .and_then(|o| o.as_dict().ok());
2568                if let Some(params) = params {
2569                    let get = |k: &[u8]| params.get(k).ok().and_then(num);
2570                    if let Some(lw) = get(b"LW") {
2571                        shape_gs.line_width = lw;
2572                    }
2573                    if let Some(a) = get(b"CA") {
2574                        shape_gs.stroke_alpha = a;
2575                    }
2576                    if let Some(a) = get(b"ca") {
2577                        shape_gs.fill_alpha = a;
2578                    }
2579                }
2580            }
2581            // The fill colour, by its components: gray `g`, `rg`, CMYK `k`,
2582            // and `sc`/`scn` by operand count (a pattern name is not light).
2583            "g" | "rg" | "k" | "sc" | "scn" => {
2584                // docling-parse: a colour resolves a pattern fill; a pattern
2585                // name in `scn` makes it one.
2586                if matches!(op.operator.as_str(), "scn")
2587                    && matches!(operands.last(), Some(Object::Name(_)))
2588                {
2589                    shape_gs.fill_pattern = true;
2590                } else if !operands.is_empty() {
2591                    shape_gs.fill_pattern = false;
2592                }
2593                let v: Vec<f64> = operands
2594                    .iter()
2595                    .map(num)
2596                    .collect::<Option<_>>()
2597                    .unwrap_or_default();
2598                fill_light = match v.as_slice() {
2599                    [g] => *g >= 0.9,
2600                    [r, g, b] => r.min(*g).min(*b) >= 0.9,
2601                    [c, m, y, k] => c.max(*m).max(*y).max(*k) <= 0.1,
2602                    _ => false,
2603                };
2604            }
2605            "cs" => {
2606                fill_light = false;
2607                // A Pattern space carries no colour until `scn` names one.
2608                let pattern = match operands.first() {
2609                    Some(Object::Name(n)) if n.as_slice() == b"Pattern" => true,
2610                    Some(Object::Name(n)) => colorspaces
2611                        .and_then(|cs| cs.get(n).ok())
2612                        .and_then(|o| deref(doc, o))
2613                        .is_some_and(|o| match o {
2614                            Object::Name(f) => f.as_slice() == b"Pattern",
2615                            Object::Array(a) => a
2616                                .first()
2617                                .and_then(|f| f.as_name().ok())
2618                                .is_some_and(|f| f == b"Pattern"),
2619                            _ => false,
2620                        }),
2621                    _ => false,
2622                };
2623                if pattern {
2624                    shape_gs.fill_pattern = true;
2625                }
2626            }
2627            // An inline image: the unit square under the CTM (#999).
2628            "BI" => inks.push(image_ink(ctm)),
2629            "Do" => {
2630                // Invoke a Form XObject: bulk body text in many PDFs lives inside
2631                // a form, reached only here. An image XObject carries no text,
2632                // only its placement (#999).
2633                if depth >= 8 {
2634                    continue;
2635                }
2636                let Some(Object::Name(n)) = operands.first() else {
2637                    continue;
2638                };
2639                let obj = xobjects.and_then(|d| d.get(n.as_slice()).ok());
2640                let form_id = match obj {
2641                    Some(Object::Reference(id)) => Some(*id),
2642                    _ => None,
2643                };
2644                let stream = obj
2645                    .and_then(|o| deref(doc, o))
2646                    .and_then(|o| o.as_stream().ok());
2647                let Some(stream) = stream else { continue };
2648                let is_form = stream
2649                    .dict
2650                    .get(b"Subtype")
2651                    .ok()
2652                    .and_then(|o| o.as_name().ok())
2653                    == Some(b"Form".as_slice());
2654                if !is_form {
2655                    let is_image = stream
2656                        .dict
2657                        .get(b"Subtype")
2658                        .ok()
2659                        .and_then(|o| o.as_name().ok())
2660                        == Some(b"Image".as_slice());
2661                    if is_image {
2662                        inks.push(image_ink(ctm));
2663                    }
2664                    continue;
2665                }
2666                // Decode a reused form's content once per document
2667                // (headers/footers and bulk body text invoke the same form on
2668                // every page); see `DocCaches::forms_seen` for single-use ones.
2669                let cached = form_id.and_then(|id| caches.forms.get(&id).cloned());
2670                let form_content = match cached {
2671                    Some(c) => c,
2672                    None => {
2673                        let Ok(data) = stream.decompressed_content() else {
2674                            continue;
2675                        };
2676                        let Ok(c) = lopdf::content::Content::decode(&data) else {
2677                            continue;
2678                        };
2679                        let c = Arc::new(c);
2680                        if let Some(id) = form_id {
2681                            if !caches.forms_seen.insert(id) {
2682                                caches.forms.insert(id, Arc::clone(&c));
2683                            }
2684                        }
2685                        c
2686                    }
2687                };
2688                // The form's /Matrix maps form space into the CTM at invocation.
2689                let form_mat = match stream.dict.get(b"Matrix").ok() {
2690                    Some(Object::Array(a)) if a.len() == 6 => {
2691                        let v: Vec<f64> = a.iter().filter_map(num).collect();
2692                        if v.len() == 6 {
2693                            Mat {
2694                                a: v[0],
2695                                b: v[1],
2696                                c: v[2],
2697                                d: v[3],
2698                                e: v[4],
2699                                f: v[5],
2700                            }
2701                        } else {
2702                            Mat::ID
2703                        }
2704                    }
2705                    _ => Mat::ID,
2706                };
2707                // The form's own /Resources, falling back to the inherited ones.
2708                let form_res = stream
2709                    .dict
2710                    .get(b"Resources")
2711                    .ok()
2712                    .and_then(|o| deref(doc, o))
2713                    .and_then(|o| o.as_dict().ok())
2714                    .unwrap_or(res);
2715                let state = TextState {
2716                    tc,
2717                    tw,
2718                    th,
2719                    tl,
2720                    trise,
2721                    fsize,
2722                    tr,
2723                };
2724                run_content(
2725                    doc,
2726                    form_res,
2727                    &form_content,
2728                    form_mat.then(ctm),
2729                    state,
2730                    depth + 1,
2731                    caches,
2732                    out,
2733                    inks,
2734                    shape_gs.clone(),
2735                    shapes,
2736                );
2737            }
2738            _ => {}
2739        }
2740        // Invisible text (`Tr` 3, or 7: clip only) — an OCR layer over a scan
2741        // follows the scanned lines, not a typeset baseline (#1005).
2742        if matches!(tr, 3 | 7) {
2743            for g in &mut out[shown..] {
2744                g.invisible = true;
2745            }
2746        }
2747    }
2748}
2749
2750/// An image placement's box — the unit square under `ctm`, y-up page
2751/// points — as an [`Ink::Bitmap`](crate::checkbox::Ink::Bitmap).
2752fn image_ink(ctm: Mat) -> crate::checkbox::Ink {
2753    let pts = [(0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0)].map(|(x, y)| ctm.apply(x, y));
2754    let (mut l, mut b, mut r, mut t) = (
2755        f64::INFINITY,
2756        f64::INFINITY,
2757        f64::NEG_INFINITY,
2758        f64::NEG_INFINITY,
2759    );
2760    for (x, y) in pts {
2761        (l, b, r, t) = (l.min(x), b.min(y), r.max(x), t.max(y));
2762    }
2763    crate::checkbox::Ink::Bitmap { l, b, r, t }
2764}
2765
2766/// The page's vector shapes as docling-parse's page decoder reports them to
2767/// docling's reading order (#1092): `get_shape_lines` — every stroked,
2768/// axis-aligned straight segment, clipped — and the boxes
2769/// `get_connected_shape_bounding_boxes` merges — every visible painted path,
2770/// its stroke's half width included, clipped. `[l, b, r, t]`, page points,
2771/// bottom-left origin, in the frame `run_content` walks (the display box's
2772/// lower-left corner at the origin, before `/Rotate`).
2773#[derive(Default, Clone, Debug, PartialEq)]
2774pub(crate) struct Shapes {
2775    pub lines: Vec<[f64; 4]>,
2776    pub boxes: Vec<[f64; 4]>,
2777}
2778
2779/// More painted paths than this on a page and their boxes are not merged —
2780/// docling-parse's merge is quadratic in them, and a page that dense (a map,
2781/// a seat plan, text drawn as outlines) yields no trustworthy rule anyway;
2782/// the stroked lines still count.
2783const MAX_SHAPE_BOXES: usize = 10_000;
2784
2785/// One subpath of the shape path under construction, page space: what
2786/// docling-parse's `page_item<PAGE_SHAPE>` keeps of it.
2787struct ShapeSub {
2788    start: (f64, f64),
2789    cur: (f64, f64),
2790    /// Every point the path passes or is controlled by — a curve's control
2791    /// points included, as docling-parse's segment data has them.
2792    pts: Vec<(f64, f64)>,
2793    /// Its straight segments, in order.
2794    lines: Vec<((f64, f64), (f64, f64))>,
2795    closed: bool,
2796}
2797
2798impl ShapeSub {
2799    fn at(p: (f64, f64)) -> Self {
2800        ShapeSub {
2801            start: p,
2802            cur: p,
2803            pts: vec![p],
2804            lines: Vec::new(),
2805            closed: false,
2806        }
2807    }
2808
2809    /// `keep_shape`: two points at least, spanning more than a speck (the
2810    /// box diagonal, squared, over 1e-3).
2811    fn kept(&self) -> bool {
2812        let Some((l, b, r, t)) = bbox_of(&self.pts) else {
2813            return false;
2814        };
2815        self.pts.len() >= 2 && (r - l).powi(2) + (t - b).powi(2) >= 1e-3
2816    }
2817}
2818
2819fn bbox_of(pts: &[(f64, f64)]) -> Option<(f64, f64, f64, f64)> {
2820    let (&first, rest) = pts.split_first()?;
2821    Some(rest.iter().fold(
2822        (first.0, first.1, first.0, first.1),
2823        |(l, b, r, t), &(x, y)| (l.min(x), b.min(y), r.max(x), t.max(y)),
2824    ))
2825}
2826
2827/// The graphics state docling-parse's shapes read: line width (user space),
2828/// stroke and fill alpha (`CA` / `ca`), whether the fill colour is a pattern
2829/// not yet resolved to a paint, and the clip paths' boxes. Saved and
2830/// restored with `q` / `Q`.
2831#[derive(Clone)]
2832struct ShapeGs {
2833    line_width: f64,
2834    stroke_alpha: f64,
2835    fill_alpha: f64,
2836    fill_pattern: bool,
2837    clips: Vec<[f64; 4]>,
2838}
2839
2840impl ShapeGs {
2841    const INIT: ShapeGs = ShapeGs {
2842        line_width: 1.0,
2843        stroke_alpha: 1.0,
2844        fill_alpha: 1.0,
2845        fill_pattern: false,
2846        clips: Vec::new(),
2847    };
2848}
2849
2850/// docling-parse's `bbox_intersects`: the boxes share area (a degenerate box
2851/// lying inside the other counts).
2852fn boxes_intersect(a: [f64; 4], b: [f64; 4]) -> bool {
2853    a[0] < b[2] && a[2] > b[0] && a[1] < b[3] && a[3] > b[1]
2854}
2855
2856/// `clipped_bbox`: a box cut to every clip path's box in turn; `None` once
2857/// one of them misses it.
2858fn clipped(mut bbox: [f64; 4], clips: &[[f64; 4]]) -> Option<[f64; 4]> {
2859    for c in clips {
2860        if !boxes_intersect(bbox, *c) {
2861            return None;
2862        }
2863        bbox = [
2864            bbox[0].max(c[0]),
2865            bbox[1].max(c[1]),
2866            bbox[2].min(c[2]),
2867            bbox[3].min(c[3]),
2868        ];
2869    }
2870    Some(bbox)
2871}
2872
2873/// `get_shape_lines`' `maybe_add_line` + `clip_axis_aligned_segment`: a
2874/// straight segment that is horizontal or vertical within 1e-3 (not both: a
2875/// point), cut to every clip path's box along its axis and dropped when one
2876/// misses it.
2877fn shape_line(a: (f64, f64), z: (f64, f64), clips: &[[f64; 4]], out: &mut Vec<[f64; 4]>) {
2878    const TOL: f64 = 1e-3;
2879    let horizontal = (z.1 - a.1).abs() <= TOL;
2880    let vertical = (z.0 - a.0).abs() <= TOL;
2881    if horizontal == vertical {
2882        return;
2883    }
2884    let mut bbox = [a.0.min(z.0), a.1.min(z.1), a.0.max(z.0), a.1.max(z.1)];
2885    for c in clips {
2886        if horizontal {
2887            if a.1 < c[1] - TOL || a.1 > c[3] + TOL {
2888                return;
2889            }
2890            bbox[0] = bbox[0].max(c[0]);
2891            bbox[2] = bbox[2].min(c[2]);
2892            if bbox[0] > bbox[2] + TOL {
2893                return;
2894            }
2895        } else {
2896            if a.0 < c[0] - TOL || a.0 > c[2] + TOL {
2897                return;
2898            }
2899            bbox[1] = bbox[1].max(c[1]);
2900            bbox[3] = bbox[3].min(c[3]);
2901            if bbox[1] > bbox[3] + TOL {
2902                return;
2903            }
2904        }
2905    }
2906    out.push(bbox);
2907}
2908
2909/// `capture_pending_clip`: each subpath of the clipping path with a segment
2910/// (two points or more) bounds what later operators paint — appended, which
2911/// approximates the intersection the spec asks for, as docling-parse does.
2912fn capture_clip(subs: &[ShapeSub], clips: &mut Vec<[f64; 4]>) {
2913    for sub in subs.iter().filter(|s| s.pts.len() >= 2) {
2914        if let Some((l, b, r, t)) = bbox_of(&sub.pts) {
2915            clips.push([l, b, r, t]);
2916        }
2917    }
2918}
2919
2920/// docling-parse's `register_paths` for the reading-order geometry: a painted
2921/// path's kept subpaths become shape lines (when its stroke shows) and a
2922/// shape box (when anything of it shows). A fill whose colour is an
2923/// unresolved pattern paints nothing, so a fill-only path is no shape and a
2924/// fill-and-stroke one keeps its stroke. `scale` maps the user-space line
2925/// width to page space (the CTM's `sqrt|det|`; a hairline stays 0).
2926fn record_shape(
2927    subs: &[ShapeSub],
2928    stroke: bool,
2929    fill: bool,
2930    gs: &ShapeGs,
2931    scale: f64,
2932    shapes: &mut Shapes,
2933) {
2934    let kept: Vec<&ShapeSub> = subs.iter().filter(|s| s.kept()).collect();
2935    if kept.is_empty() {
2936        return;
2937    }
2938    let fill = fill && !gs.fill_pattern;
2939    if gs.fill_pattern && !stroke {
2940        return;
2941    }
2942    let strokes_visible = stroke && gs.stroke_alpha > 0.0;
2943    let visible = strokes_visible || (fill && gs.fill_alpha > 0.0);
2944    if strokes_visible {
2945        for sub in &kept {
2946            for &(a, z) in &sub.lines {
2947                shape_line(a, z, &gs.clips, &mut shapes.lines);
2948            }
2949            if sub.closed {
2950                shape_line(sub.cur, sub.start, &gs.clips, &mut shapes.lines);
2951            }
2952        }
2953    }
2954    if visible {
2955        let pts: Vec<(f64, f64)> = kept.iter().flat_map(|s| s.pts.iter().copied()).collect();
2956        if let Some((l, b, r, t)) = bbox_of(&pts) {
2957            let pad = if strokes_visible {
2958                (gs.line_width * scale).max(0.0) * 0.5
2959            } else {
2960                0.0
2961            };
2962            if let Some(bbox) = clipped([l - pad, b - pad, r + pad, t + pad], &gs.clips) {
2963                shapes.boxes.push(bbox);
2964            }
2965        }
2966    }
2967}
2968
2969/// `get_connected_shape_bounding_boxes`: each box in turn (unless already
2970/// taken) grows by every untaken box its growing union touches, until none
2971/// does — docling-parse's greedy pass, so a component finished early keeps a
2972/// box a later one would also have reached.
2973fn connected_boxes(boxes: &[[f64; 4]]) -> Vec<[f64; 4]> {
2974    let touches =
2975        |a: [f64; 4], b: [f64; 4]| a[0] <= b[2] && a[2] >= b[0] && a[1] <= b[3] && a[3] >= b[1];
2976    let mut taken = vec![false; boxes.len()];
2977    let mut out = Vec::new();
2978    for i in 0..boxes.len() {
2979        if taken[i] {
2980            continue;
2981        }
2982        taken[i] = true;
2983        let mut component = boxes[i];
2984        let mut changed = true;
2985        while changed {
2986            changed = false;
2987            for j in 0..boxes.len() {
2988                if !taken[j] && touches(component, boxes[j]) {
2989                    let b = boxes[j];
2990                    component = [
2991                        component[0].min(b[0]),
2992                        component[1].min(b[1]),
2993                        component[2].max(b[2]),
2994                        component[3].max(b[3]),
2995                    ];
2996                    taken[j] = true;
2997                    changed = true;
2998                }
2999            }
3000        }
3001        out.push(component);
3002    }
3003    out
3004}
3005
3006/// One piece of a path under construction, page space (see `run_content`).
3007enum PathPiece {
3008    Line((f64, f64), (f64, f64)),
3009    /// An `re`: its four corners, counter-clockwise from `(x, y)`.
3010    Rect([(f64, f64); 4]),
3011    /// A Bézier segment: its start and control/end points (their hull).
3012    Curve(Vec<(f64, f64)>),
3013}
3014
3015/// Record a painted path's pieces for the checkbox finder (#609): a stroked
3016/// line is a [`Seg`](crate::checkbox::Ink::Seg), a stroked rectangle that
3017/// stays axis-aligned a [`Rect`](crate::checkbox::Ink::Rect) (one filled
3018/// white too — still an outline), and a coloured fill or a curve only its
3019/// bounding box, a [`Blot`](crate::checkbox::Ink::Blot): a possible tick
3020/// mark, or a swatch. White fills draw nothing and are dropped. Only pieces small
3021/// enough to be a checkbox edge or a mark inside one are kept.
3022fn paint_path(
3023    path: &[PathPiece],
3024    stroke: bool,
3025    fill: bool,
3026    fill_light: bool,
3027    inks: &mut Vec<crate::checkbox::Ink>,
3028) {
3029    use crate::checkbox::Ink;
3030    let bbox = |pts: &mut dyn Iterator<Item = (f64, f64)>| {
3031        pts.fold(
3032            (
3033                f64::INFINITY,
3034                f64::INFINITY,
3035                f64::NEG_INFINITY,
3036                f64::NEG_INFINITY,
3037            ),
3038            |(l, b, r, t), (x, y)| (l.min(x), b.min(y), r.max(x), t.max(y)),
3039        )
3040    };
3041    let mut keep = |ink: Ink| {
3042        if ink.worth_keeping() {
3043            inks.push(ink);
3044        }
3045    };
3046    if fill && !fill_light {
3047        // A coloured fill is a blot — a tick mark, or (the size of a square)
3048        // a legend swatch, which `checkbox::find` then rules out. A white
3049        // fill is no ink on paper: it only leaves the stroke, if any, below.
3050        let (l, b, r, t) = bbox(&mut path.iter().flat_map(|p| match p {
3051            PathPiece::Line(a, z) => vec![*a, *z],
3052            PathPiece::Rect(c) => c.to_vec(),
3053            PathPiece::Curve(c) => c.clone(),
3054        }));
3055        if l.is_finite() {
3056            keep(Ink::Blot { l, b, r, t });
3057        }
3058        return;
3059    }
3060    if !stroke {
3061        return;
3062    }
3063    for piece in path {
3064        match piece {
3065            PathPiece::Line(a, z) => keep(Ink::Seg {
3066                x0: a.0,
3067                y0: a.1,
3068                x1: z.0,
3069                y1: z.1,
3070            }),
3071            PathPiece::Rect(c) => {
3072                let axis = ((c[0].1 - c[1].1).abs() < 1e-6 && (c[1].0 - c[2].0).abs() < 1e-6)
3073                    || ((c[0].0 - c[1].0).abs() < 1e-6 && (c[1].1 - c[2].1).abs() < 1e-6);
3074                if axis {
3075                    let (l, b, r, t) = bbox(&mut c.iter().copied());
3076                    keep(Ink::Rect { l, b, r, t });
3077                } else {
3078                    for k in 0..4 {
3079                        let (a, z) = (c[k], c[(k + 1) % 4]);
3080                        keep(Ink::Seg {
3081                            x0: a.0,
3082                            y0: a.1,
3083                            x1: z.0,
3084                            y1: z.1,
3085                        });
3086                    }
3087                }
3088            }
3089            PathPiece::Curve(c) => {
3090                let (l, b, r, t) = bbox(&mut c.iter().copied());
3091                keep(Ink::Blot { l, b, r, t });
3092            }
3093        }
3094    }
3095}
3096
3097#[allow(clippy::too_many_arguments)]
3098fn show_text(
3099    font: &Font,
3100    bytes: &[u8],
3101    fsize: f64,
3102    tc: f64,
3103    tw: f64,
3104    th: f64,
3105    trise: f64,
3106    tm: &mut Mat,
3107    ctm: Mat,
3108    out: &mut Vec<Glyph>,
3109) {
3110    for code in codes(font, bytes) {
3111        let (text, w, blank) = font.decode_code(code);
3112        let w0 = w / 1000.0; // advance in text-space (em) units
3113                             // The glyph→user transform: scale glyph space by font size, then Tm, CTM.
3114        let scale = Mat {
3115            a: fsize * th,
3116            b: 0.0,
3117            c: 0.0,
3118            d: fsize,
3119            e: 0.0,
3120            f: trise,
3121        };
3122        let trm = scale.then(*tm).then(ctm);
3123        // Box in glyph space (1000-unit em): x 0..w, y descent..ascent.
3124        // A blank unmapped glyph keeps its advance but no ink height
3125        // (docling-parse collapses its ascent and descent).
3126        let (desc, asc) = if blank {
3127            (0.0, 0.0)
3128        } else {
3129            (font.descent / 1000.0, font.ascent / 1000.0)
3130        };
3131        let (x0, y0) = trm.apply(0.0, desc);
3132        let (x1, y1) = trm.apply(w0, desc);
3133        let (x2, y2) = trm.apply(w0, asc);
3134        let (x3, y3) = trm.apply(0.0, asc);
3135        // Upright = the baseline runs left-to-right along +x. Only then is the
3136        // loose box the rectangle spanned by the baseline's x and the em's y
3137        // (a synthetic oblique's skew is ignored, as it always was). A rotated
3138        // matrix — the `0 s -s 0 tx ty Tm` landscape pages are built with
3139        // (#528) — collapsed that rectangle to zero width, and every such run
3140        // vanished; those glyphs keep their real quad (docling-parse's char
3141        // rect) and its extent instead.
3142        let upright = trm.a > 0.0 && trm.b.abs() <= 1e-6 * trm.a;
3143        let (left, bot, right, top, quad) = if upright {
3144            (x0.min(x1), y0.min(y3), x0.max(x1), y0.max(y3), None)
3145        } else {
3146            let (xs, ys) = ([x0, x1, x2, x3], [y0, y1, y2, y3]);
3147            let fold = |v: [f64; 4], f: fn(f64, f64) -> f64| v.into_iter().reduce(f).unwrap();
3148            let q = [x0, y0, x1, y1, x2, y2, x3, y3].map(|v| v as f32);
3149            (
3150                fold(xs, f64::min),
3151                fold(ys, f64::min),
3152                fold(xs, f64::max),
3153                fold(ys, f64::max),
3154                Some(q),
3155            )
3156        };
3157        // The pen's origin (rise included) and the font size's extent from
3158        // it, as docling-parse maps them (`map_text_point(0, rise)` and
3159        // `(0, rise + font_size)`).
3160        let (ox, oy) = trm.apply(0.0, 0.0);
3161        let (sx, sy) = trm.apply(0.0, 1.0);
3162        let pen = Some([ox as f32, oy as f32, (sx - ox).hypot(sy - oy) as f32]);
3163        if let Some(s) = text {
3164            // A run may map one code to multiple chars (ligature/fraction); share box.
3165            for ch in s.chars() {
3166                // A NUL from the `/ToUnicode` is an unmapped glyph too
3167                // (docling#5070): U+FFFD.
3168                let ch = if ch == '\u{0}' { '\u{FFFD}' } else { ch };
3169                out.push(Glyph {
3170                    ch,
3171                    l: left as f32,
3172                    b: bot as f32,
3173                    r: right as f32,
3174                    t: top as f32,
3175                    ll: left as f32,
3176                    lb: bot as f32,
3177                    lr: right as f32,
3178                    lt: top as f32,
3179                    font: font.hash,
3180                    quad,
3181                    pen,
3182                    math_font: font.math,
3183                    invisible: false,
3184                });
3185            }
3186        }
3187        // Advance the text matrix. Word spacing applies to single-byte code 32.
3188        let is_space = !font.two_byte && code == 32;
3189        let tx = (w0 * fsize + tc + if is_space { tw } else { 0.0 }) * th;
3190        *tm = Mat {
3191            a: 1.0,
3192            b: 0.0,
3193            c: 0.0,
3194            d: 1.0,
3195            e: tx,
3196            f: 0.0,
3197        }
3198        .then(*tm);
3199    }
3200}
3201
3202/// Build a simple font's code→char table from its `/Encoding`: the base
3203/// encoding (WinAnsi / MacRoman) plus any `/Differences` overrides (glyph names
3204/// resolved through a small Adobe-glyph-name subset).
3205fn simple_encoding_table(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
3206    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o));
3207    let base_name = match enc {
3208        Some(Object::Name(n)) => n.clone(),
3209        Some(Object::Dictionary(d)) => d
3210            .get(b"BaseEncoding")
3211            .ok()
3212            .and_then(|o| o.as_name().ok())
3213            .map(|n| n.to_vec())
3214            .unwrap_or_default(),
3215        _ => Vec::new(),
3216    };
3217    let mut m = if base_name == b"MacRomanEncoding" {
3218        macroman_table()
3219    } else if base_name.is_empty() {
3220        // No PDF /Encoding at all: the font's *built-in* encoding applies. For
3221        // the standard TeX math fonts that is their fixed TeX layout — falling
3222        // back to StandardEncoding read CMSY's braces as `f`/`g`, `→` as `!`,
3223        // `∈` as `2` (2203's `{ahn,…}` author line). The font program (often
3224        // CFF, which this parser does not read) carries the same mapping;
3225        // docling-parse decodes it from there.
3226        tex_math_builtin(fdict).unwrap_or_else(winansi_table)
3227    } else {
3228        winansi_table()
3229    };
3230    // Apply /Differences: `code /glyphname /glyphname ... code ...`.
3231    if let Some(Object::Dictionary(d)) = enc {
3232        if let Some(Object::Array(diffs)) = d.get(b"Differences").ok().and_then(|o| deref(doc, o)) {
3233            let mut code = 0u8;
3234            for el in diffs {
3235                match el {
3236                    Object::Integer(i) => code = *i as u8,
3237                    Object::Name(name) => {
3238                        if let Some(ch) = glyph_name_to_char(name) {
3239                            m.insert(code, ch);
3240                        }
3241                        code = code.wrapping_add(1);
3242                    }
3243                    _ => {}
3244                }
3245            }
3246        }
3247    }
3248    m
3249}
3250
3251/// The fixed built-in encodings of the standard TeX math fonts (TeXbook
3252/// Appendix F), keyed off the base font name: `CMSY*` (symbols; `CMBSY` is its
3253/// bold) and `CMMI*` (math italic). These fonts ship no PDF `/Encoding` and no
3254/// ToUnicode, and their program is usually CFF — without this table the codes
3255/// fell through to StandardEncoding and rendered as the wrong ASCII.
3256fn tex_math_builtin(fdict: &Dictionary) -> Option<HashMap<u8, char>> {
3257    const CMSY: [char; 128] = [
3258        '−', '·', '×', '∗', '÷', '⋄', '±', '∓', '⊕', '⊖', '⊗', '⊘', '⊙', '◯', '∘', '•', '≍', '≡',
3259        '⊆', '⊇', '≤', '≥', '≼', '≽', '∼', '≈', '⊂', '⊃', '≪', '≫', '≺', '≻', '←', '→', '↑', '↓',
3260        '↔', '↗', '↘', '≃', '⇐', '⇒', '⇑', '⇓', '⇔', '↖', '↙', '∝', '′', '∞', '∈', '∋', '△', '▽',
3261        '\u{338}', '↦', '∀', '∃', '¬', '∅', 'ℜ', 'ℑ', '⊤', '⊥', 'ℵ', 'A', 'B', 'C', 'D', 'E', 'F',
3262        'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
3263        'Y', 'Z', '∪', '∩', '⊎', '∧', '∨', '⊢', '⊣', '⌊', '⌋', '⌈', '⌉', '{', '}', '⟨', '⟩', '|',
3264        '∥', '↕', '⇕', '\\', '≀', '√', '∐', '∇', '∫', '⊔', '⊓', '⊑', '⊒', '§', '†', '‡', '¶', '♣',
3265        '♢', '♡', '♠',
3266    ];
3267    const CMMI: [char; 128] = [
3268        'Γ', 'Δ', 'Θ', 'Λ', 'Ξ', 'Π', 'Σ', 'Υ', 'Φ', 'Ψ', 'Ω', 'α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η',
3269        'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'π', 'ρ', 'σ', 'τ', 'υ', 'φ', 'χ', 'ψ', 'ω', 'ϵ', 'ϑ',
3270        'ϖ', 'ϱ', 'ς', 'ϕ', '↼', '↽', '⇀', '⇁', '↩', '↪', '▷', '◁', '0', '1', '2', '3', '4', '5',
3271        '6', '7', '8', '9', '.', ',', '<', '/', '>', '⋆', '∂', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
3272        'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y',
3273        'Z', '♭', '♮', '♯', '⌣', '⌢', 'ℓ', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
3274        'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'ı', 'ȷ', '℘',
3275        '\u{20d7}', '⁀',
3276    ];
3277    let name = base_font_name(fdict)?;
3278    let up = name.to_ascii_uppercase();
3279    let table: &[char; 128] = if up.starts_with(b"CMSY") || up.starts_with(b"CMBSY") {
3280        &CMSY
3281    } else if up.starts_with(b"CMMI") {
3282        &CMMI
3283    } else {
3284        return None;
3285    };
3286    Some(
3287        table
3288            .iter()
3289            .enumerate()
3290            .map(|(i, &c)| (i as u8, c))
3291            .collect(),
3292    )
3293}
3294
3295/// The Adobe Glyph List's names for Latin-1 Supplement and Latin Extended-A
3296/// (plus `Euro`, `florin`, the comma-accent pairs): what a Type1 font's
3297/// `/Differences` or builtin encoding calls an accented or stroked letter.
3298/// 2203.01017v2 names `Ł` as `/Lslash` in a NimbusRomNo9L subset with no
3299/// ToUnicode, and the resolver, which knew only the ASCII, ligature and symbol
3300/// names, dropped the glyph outright (`Ł ukasz Kaiser` read `ukasz Kaiser`).
3301/// Sorted by name for the binary search in [`glyph_name_to_char`].
3302const AGL_LATIN: &[(&str, char)] = &[
3303    ("AE", '\u{00C6}'),
3304    ("Aacute", '\u{00C1}'),
3305    ("Abreve", '\u{0102}'),
3306    ("Acircumflex", '\u{00C2}'),
3307    ("Adieresis", '\u{00C4}'),
3308    ("Agrave", '\u{00C0}'),
3309    ("Amacron", '\u{0100}'),
3310    ("Aogonek", '\u{0104}'),
3311    ("Aring", '\u{00C5}'),
3312    ("Atilde", '\u{00C3}'),
3313    ("Cacute", '\u{0106}'),
3314    ("Ccaron", '\u{010C}'),
3315    ("Ccedilla", '\u{00C7}'),
3316    ("Ccircumflex", '\u{0108}'),
3317    ("Cdotaccent", '\u{010A}'),
3318    ("Dcaron", '\u{010E}'),
3319    ("Dcroat", '\u{0110}'),
3320    ("Eacute", '\u{00C9}'),
3321    ("Ebreve", '\u{0114}'),
3322    ("Ecaron", '\u{011A}'),
3323    ("Ecircumflex", '\u{00CA}'),
3324    ("Edieresis", '\u{00CB}'),
3325    ("Edotaccent", '\u{0116}'),
3326    ("Egrave", '\u{00C8}'),
3327    ("Emacron", '\u{0112}'),
3328    ("Eng", '\u{014A}'),
3329    ("Eogonek", '\u{0118}'),
3330    ("Eth", '\u{00D0}'),
3331    ("Euro", '\u{20AC}'),
3332    ("Gbreve", '\u{011E}'),
3333    ("Gcircumflex", '\u{011C}'),
3334    ("Gcommaaccent", '\u{0122}'),
3335    ("Gdotaccent", '\u{0120}'),
3336    ("Hbar", '\u{0126}'),
3337    ("Hcircumflex", '\u{0124}'),
3338    ("IJ", '\u{0132}'),
3339    ("Iacute", '\u{00CD}'),
3340    ("Ibreve", '\u{012C}'),
3341    ("Icircumflex", '\u{00CE}'),
3342    ("Idieresis", '\u{00CF}'),
3343    ("Idotaccent", '\u{0130}'),
3344    ("Igrave", '\u{00CC}'),
3345    ("Imacron", '\u{012A}'),
3346    ("Iogonek", '\u{012E}'),
3347    ("Itilde", '\u{0128}'),
3348    ("Jcircumflex", '\u{0134}'),
3349    ("Kcommaaccent", '\u{0136}'),
3350    ("Lacute", '\u{0139}'),
3351    ("Lcaron", '\u{013D}'),
3352    ("Lcommaaccent", '\u{013B}'),
3353    ("Ldot", '\u{013F}'),
3354    ("Lslash", '\u{0141}'),
3355    ("Nacute", '\u{0143}'),
3356    ("Ncaron", '\u{0147}'),
3357    ("Ncommaaccent", '\u{0145}'),
3358    ("Ntilde", '\u{00D1}'),
3359    ("OE", '\u{0152}'),
3360    ("Oacute", '\u{00D3}'),
3361    ("Obreve", '\u{014E}'),
3362    ("Ocircumflex", '\u{00D4}'),
3363    ("Odieresis", '\u{00D6}'),
3364    ("Ograve", '\u{00D2}'),
3365    ("Ohungarumlaut", '\u{0150}'),
3366    ("Omacron", '\u{014C}'),
3367    ("Oslash", '\u{00D8}'),
3368    ("Otilde", '\u{00D5}'),
3369    ("Racute", '\u{0154}'),
3370    ("Rcaron", '\u{0158}'),
3371    ("Rcommaaccent", '\u{0156}'),
3372    ("Sacute", '\u{015A}'),
3373    ("Scaron", '\u{0160}'),
3374    ("Scedilla", '\u{015E}'),
3375    ("Scircumflex", '\u{015C}'),
3376    ("Scommaaccent", '\u{0218}'),
3377    ("Tbar", '\u{0166}'),
3378    ("Tcaron", '\u{0164}'),
3379    ("Tcommaaccent", '\u{0162}'),
3380    ("Thorn", '\u{00DE}'),
3381    ("Uacute", '\u{00DA}'),
3382    ("Ubreve", '\u{016C}'),
3383    ("Ucircumflex", '\u{00DB}'),
3384    ("Udieresis", '\u{00DC}'),
3385    ("Ugrave", '\u{00D9}'),
3386    ("Uhungarumlaut", '\u{0170}'),
3387    ("Umacron", '\u{016A}'),
3388    ("Uogonek", '\u{0172}'),
3389    ("Uring", '\u{016E}'),
3390    ("Utilde", '\u{0168}'),
3391    ("Wcircumflex", '\u{0174}'),
3392    ("Yacute", '\u{00DD}'),
3393    ("Ycircumflex", '\u{0176}'),
3394    ("Ydieresis", '\u{0178}'),
3395    ("Zacute", '\u{0179}'),
3396    ("Zcaron", '\u{017D}'),
3397    ("Zdotaccent", '\u{017B}'),
3398    ("aacute", '\u{00E1}'),
3399    ("abreve", '\u{0103}'),
3400    ("acircumflex", '\u{00E2}'),
3401    ("acute", '\u{00B4}'),
3402    ("adieresis", '\u{00E4}'),
3403    ("ae", '\u{00E6}'),
3404    ("agrave", '\u{00E0}'),
3405    ("amacron", '\u{0101}'),
3406    ("aogonek", '\u{0105}'),
3407    ("aring", '\u{00E5}'),
3408    ("atilde", '\u{00E3}'),
3409    ("brokenbar", '\u{00A6}'),
3410    ("cacute", '\u{0107}'),
3411    ("ccaron", '\u{010D}'),
3412    ("ccedilla", '\u{00E7}'),
3413    ("ccircumflex", '\u{0109}'),
3414    ("cdotaccent", '\u{010B}'),
3415    ("cedilla", '\u{00B8}'),
3416    ("cent", '\u{00A2}'),
3417    ("currency", '\u{00A4}'),
3418    ("dcaron", '\u{010F}'),
3419    ("dcroat", '\u{0111}'),
3420    ("dieresis", '\u{00A8}'),
3421    ("dotlessi", '\u{0131}'),
3422    ("eacute", '\u{00E9}'),
3423    ("ebreve", '\u{0115}'),
3424    ("ecaron", '\u{011B}'),
3425    ("ecircumflex", '\u{00EA}'),
3426    ("edieresis", '\u{00EB}'),
3427    ("edotaccent", '\u{0117}'),
3428    ("egrave", '\u{00E8}'),
3429    ("emacron", '\u{0113}'),
3430    ("eng", '\u{014B}'),
3431    ("eogonek", '\u{0119}'),
3432    ("eth", '\u{00F0}'),
3433    ("exclamdown", '\u{00A1}'),
3434    ("florin", '\u{0192}'),
3435    ("gbreve", '\u{011F}'),
3436    ("gcircumflex", '\u{011D}'),
3437    ("gcommaaccent", '\u{0123}'),
3438    ("gdotaccent", '\u{0121}'),
3439    ("germandbls", '\u{00DF}'),
3440    ("guillemotleft", '\u{00AB}'),
3441    ("guillemotright", '\u{00BB}'),
3442    ("hbar", '\u{0127}'),
3443    ("hcircumflex", '\u{0125}'),
3444    ("iacute", '\u{00ED}'),
3445    ("ibreve", '\u{012D}'),
3446    ("icircumflex", '\u{00EE}'),
3447    ("idieresis", '\u{00EF}'),
3448    ("igrave", '\u{00EC}'),
3449    ("ij", '\u{0133}'),
3450    ("imacron", '\u{012B}'),
3451    ("iogonek", '\u{012F}'),
3452    ("itilde", '\u{0129}'),
3453    ("jcircumflex", '\u{0135}'),
3454    ("kcommaaccent", '\u{0137}'),
3455    ("kgreenlandic", '\u{0138}'),
3456    ("lacute", '\u{013A}'),
3457    ("lcaron", '\u{013E}'),
3458    ("lcommaaccent", '\u{013C}'),
3459    ("ldot", '\u{0140}'),
3460    ("logicalnot", '\u{00AC}'),
3461    ("longs", '\u{017F}'),
3462    ("lslash", '\u{0142}'),
3463    ("macron", '\u{00AF}'),
3464    ("nacute", '\u{0144}'),
3465    ("napostrophe", '\u{0149}'),
3466    ("ncaron", '\u{0148}'),
3467    ("ncommaaccent", '\u{0146}'),
3468    ("ntilde", '\u{00F1}'),
3469    ("oacute", '\u{00F3}'),
3470    ("obreve", '\u{014F}'),
3471    ("ocircumflex", '\u{00F4}'),
3472    ("odieresis", '\u{00F6}'),
3473    ("oe", '\u{0153}'),
3474    ("ograve", '\u{00F2}'),
3475    ("ohungarumlaut", '\u{0151}'),
3476    ("omacron", '\u{014D}'),
3477    ("onehalf", '\u{00BD}'),
3478    ("onequarter", '\u{00BC}'),
3479    ("onesuperior", '\u{00B9}'),
3480    ("ordfeminine", '\u{00AA}'),
3481    ("ordmasculine", '\u{00BA}'),
3482    ("oslash", '\u{00F8}'),
3483    ("otilde", '\u{00F5}'),
3484    ("paragraph", '\u{00B6}'),
3485    ("questiondown", '\u{00BF}'),
3486    ("racute", '\u{0155}'),
3487    ("rcaron", '\u{0159}'),
3488    ("rcommaaccent", '\u{0157}'),
3489    ("sacute", '\u{015B}'),
3490    ("scaron", '\u{0161}'),
3491    ("scedilla", '\u{015F}'),
3492    ("scircumflex", '\u{015D}'),
3493    ("scommaaccent", '\u{0219}'),
3494    ("section", '\u{00A7}'),
3495    ("sfthyphen", '\u{00AD}'),
3496    ("sterling", '\u{00A3}'),
3497    ("tbar", '\u{0167}'),
3498    ("tcaron", '\u{0165}'),
3499    ("tcommaaccent", '\u{0163}'),
3500    ("thorn", '\u{00FE}'),
3501    ("threequarters", '\u{00BE}'),
3502    ("threesuperior", '\u{00B3}'),
3503    ("twosuperior", '\u{00B2}'),
3504    ("uacute", '\u{00FA}'),
3505    ("ubreve", '\u{016D}'),
3506    ("ucircumflex", '\u{00FB}'),
3507    ("udieresis", '\u{00FC}'),
3508    ("ugrave", '\u{00F9}'),
3509    ("uhungarumlaut", '\u{0171}'),
3510    ("umacron", '\u{016B}'),
3511    ("uogonek", '\u{0173}'),
3512    ("uring", '\u{016F}'),
3513    ("utilde", '\u{0169}'),
3514    ("wcircumflex", '\u{0175}'),
3515    ("yacute", '\u{00FD}'),
3516    ("ycircumflex", '\u{0177}'),
3517    ("ydieresis", '\u{00FF}'),
3518    ("yen", '\u{00A5}'),
3519    ("zacute", '\u{017A}'),
3520    ("zcaron", '\u{017E}'),
3521    ("zdotaccent", '\u{017C}'),
3522];
3523
3524/// Resolve an Adobe glyph name to Unicode: `uniXXXX`, single ASCII letters, the
3525/// digit/punctuation names from the Adobe Glyph List, and common typographic
3526/// names. A `.suffix` (`one.taboldstyle`, `a.sc`) is stripped and the base name
3527/// retried — docling renders these as the base character.
3528pub(crate) fn glyph_name_to_char(name: &[u8]) -> Option<char> {
3529    let s = std::str::from_utf8(name).ok()?;
3530    if let Some(hex) = s.strip_prefix("uni") {
3531        if let Ok(cp) = u32::from_str_radix(hex.get(0..4)?, 16) {
3532            return char::from_u32(cp);
3533        }
3534    }
3535    // Single ASCII letter names (`A`, `m`) map to themselves.
3536    if s.len() == 1 {
3537        let b = s.as_bytes()[0];
3538        if b.is_ascii_alphabetic() {
3539            return Some(b as char);
3540        }
3541    }
3542    let resolved = match s {
3543        "space" => ' ',
3544        "exclam" => '!',
3545        "quotedbl" => '"',
3546        "numbersign" => '#',
3547        "dollar" => '$',
3548        "percent" => '%',
3549        "ampersand" => '&',
3550        "quotesingle" => '\'',
3551        "parenleft" => '(',
3552        "parenright" => ')',
3553        "asterisk" => '*',
3554        "plus" => '+',
3555        "comma" => ',',
3556        "hyphen" => '-',
3557        "period" => '.',
3558        "slash" => '/',
3559        "zero" => '0',
3560        "one" => '1',
3561        "two" => '2',
3562        "three" => '3',
3563        "four" => '4',
3564        "five" => '5',
3565        "six" => '6',
3566        "seven" => '7',
3567        "eight" => '8',
3568        "nine" => '9',
3569        "colon" => ':',
3570        "semicolon" => ';',
3571        "less" => '<',
3572        "equal" => '=',
3573        "greater" => '>',
3574        "question" => '?',
3575        "at" => '@',
3576        "bracketleft" => '[',
3577        "backslash" => '\\',
3578        "bracketright" => ']',
3579        "asciicircum" => '^',
3580        "underscore" => '_',
3581        "grave" => '`',
3582        "braceleft" => '{',
3583        "bar" => '|',
3584        "braceright" => '}',
3585        "asciitilde" => '~',
3586        "bullet" => '\u{2022}',
3587        "periodcentered" => '\u{00B7}',
3588        "endash" => '\u{2013}',
3589        "emdash" => '\u{2014}',
3590        "quoteright" => '\u{2019}',
3591        "quoteleft" => '\u{2018}',
3592        "quotedblleft" => '\u{201C}',
3593        "quotedblright" => '\u{201D}',
3594        "quotedblbase" => '\u{201E}',
3595        "quotesinglbase" => '\u{201A}',
3596        // Latin f-ligatures named in `/Differences` (e.g. 2305's body font). These
3597        // map to the presentation-form code points, which `decompose_ligatures`
3598        // then spells back out (`ff`→"ff") — without them the glyph decodes to
3599        // nothing and the sanitizer fills the gap with a space (`di erences`).
3600        "ff" => '\u{FB00}',
3601        "fi" => '\u{FB01}',
3602        "fl" => '\u{FB02}',
3603        "ffi" => '\u{FB03}',
3604        "ffl" => '\u{FB04}',
3605        "ft" => '\u{FB05}',
3606        "st" => '\u{FB06}',
3607        "degree" => '\u{00B0}',
3608        "trademark" => '\u{2122}',
3609        "registered" => '\u{00AE}',
3610        "copyright" => '\u{00A9}',
3611        "ellipsis" => '\u{2026}',
3612        "minus" => '\u{2212}',
3613        "fraction" => '\u{2044}',
3614        "nbspace" => '\u{00A0}',
3615        // Greek + math glyph names (standard Adobe Glyph List). Standard TeX math
3616        // fonts (CMMI/CMSY/…) name their glyphs this way in the embedded font
3617        // program's `/Encoding`; without these a `λ`/`≤` decodes to nothing and is
3618        // dropped from body text (`and λ set to 0.5` → `and set to 0.5`).
3619        "alpha" => '\u{03B1}',
3620        "beta" => '\u{03B2}',
3621        "gamma" => '\u{03B3}',
3622        "delta" => '\u{03B4}',
3623        "epsilon" | "epsilon1" => '\u{03B5}',
3624        "zeta" => '\u{03B6}',
3625        "eta" => '\u{03B7}',
3626        "theta" | "theta1" => '\u{03B8}',
3627        "iota" => '\u{03B9}',
3628        "kappa" => '\u{03BA}',
3629        "lambda" => '\u{03BB}',
3630        "mu" => '\u{03BC}',
3631        "nu" => '\u{03BD}',
3632        "xi" => '\u{03BE}',
3633        "omicron" => '\u{03BF}',
3634        "pi" | "pi1" => '\u{03C0}',
3635        "rho" | "rho1" => '\u{03C1}',
3636        "sigma" => '\u{03C3}',
3637        "sigma1" => '\u{03C2}',
3638        "tau" => '\u{03C4}',
3639        "upsilon" => '\u{03C5}',
3640        "phi" | "phi1" => '\u{03C6}',
3641        "chi" => '\u{03C7}',
3642        "psi" => '\u{03C8}',
3643        "omega" | "omega1" => '\u{03C9}',
3644        "Gamma" => '\u{0393}',
3645        "Delta" => '\u{0394}',
3646        "Theta" => '\u{0398}',
3647        "Lambda" => '\u{039B}',
3648        "Xi" => '\u{039E}',
3649        "Pi" => '\u{03A0}',
3650        "Sigma" => '\u{03A3}',
3651        "Upsilon" => '\u{03A5}',
3652        "Phi" => '\u{03A6}',
3653        "Psi" => '\u{03A8}',
3654        "Omega" => '\u{03A9}',
3655        "lessequal" => '\u{2264}',
3656        "greaterequal" => '\u{2265}',
3657        "notequal" => '\u{2260}',
3658        "approxequal" => '\u{2248}',
3659        "equivalence" => '\u{2261}',
3660        "element" => '\u{2208}',
3661        "plusminus" => '\u{00B1}',
3662        "multiply" => '\u{00D7}',
3663        "divide" => '\u{00F7}',
3664        "infinity" => '\u{221E}',
3665        "partialdiff" => '\u{2202}',
3666        "gradient" => '\u{2207}',
3667        "summation" => '\u{2211}',
3668        "product" => '\u{220F}',
3669        "integral" => '\u{222B}',
3670        "radical" => '\u{221A}',
3671        "proportional" => '\u{221D}',
3672        "arrowright" => '\u{2192}',
3673        "arrowleft" => '\u{2190}',
3674        "arrowup" => '\u{2191}',
3675        "arrowdown" => '\u{2193}',
3676        "arrowboth" => '\u{2194}',
3677        "arrowdblright" => '\u{21D2}',
3678        "logicaland" => '\u{2227}',
3679        "logicalor" => '\u{2228}',
3680        "intersection" => '\u{2229}',
3681        "union" => '\u{222A}',
3682        "similar" => '\u{223C}',
3683        "congruent" => '\u{2245}',
3684        "dotmath" => '\u{22C5}',
3685        "asteriskmath" => '\u{2217}',
3686        _ => {
3687            if let Ok(i) = AGL_LATIN.binary_search_by_key(&s, |(n, _)| n) {
3688                return Some(AGL_LATIN[i].1);
3689            }
3690            // Strip an AGL `.suffix` (oldstyle/small-cap variant) and retry.
3691            if let Some((base, _)) = s.split_once('.') {
3692                if !base.is_empty() {
3693                    return glyph_name_to_char(base.as_bytes());
3694                }
3695            }
3696            // An AGL ligature name joins its components with `_` (`f_f_i`,
3697            // `f_l` — what a re-encoded Libertine subset calls them); joined,
3698            // the presentation-form arms above resolve them.
3699            if s.contains('_') {
3700                let joined: String = s.chars().filter(|&c| c != '_').collect();
3701                if !joined.is_empty() && joined.len() < s.len() {
3702                    return glyph_name_to_char(joined.as_bytes());
3703                }
3704            }
3705            return None;
3706        }
3707    };
3708    Some(resolved)
3709}
3710
3711/// Minimal WinAnsiEncoding (Latin-1-ish) for simple fonts lacking ToUnicode.
3712fn winansi_table() -> HashMap<u8, char> {
3713    let mut m = HashMap::new();
3714    for b in 0x20u8..=0x7e {
3715        m.insert(b, b as char);
3716    }
3717    // High range: Windows-1252 printable points that differ from Latin-1.
3718    let extra: &[(u8, char)] = &[
3719        (0x91, '\u{2018}'),
3720        (0x92, '\u{2019}'),
3721        (0x93, '\u{201C}'),
3722        (0x94, '\u{201D}'),
3723        (0x95, '\u{2022}'),
3724        (0x96, '\u{2013}'),
3725        (0x97, '\u{2014}'),
3726        (0x85, '\u{2026}'),
3727        (0xA0, '\u{00A0}'),
3728        // The rest of Windows-1252's 0x80–0x9F row. Left out, the German
3729        // opening quote `„` (0x84) vanished from the text layer while its
3730        // closing `“` (0x93) stayed (#922); `€ ‚ ƒ † ‡ ˆ ‰ Š ‹ Œ Ž ˜ ™ š › œ ž Ÿ`
3731        // were lost the same way. 0x81/0x8D/0x8F/0x90/0x9D stay unmapped
3732        // (PDF's WinAnsi leaves them undefined).
3733        (0x80, '\u{20AC}'),
3734        (0x82, '\u{201A}'),
3735        (0x83, '\u{0192}'),
3736        (0x84, '\u{201E}'),
3737        (0x86, '\u{2020}'),
3738        (0x87, '\u{2021}'),
3739        (0x88, '\u{02C6}'),
3740        (0x89, '\u{2030}'),
3741        (0x8A, '\u{0160}'),
3742        (0x8B, '\u{2039}'),
3743        (0x8C, '\u{0152}'),
3744        (0x8E, '\u{017D}'),
3745        (0x98, '\u{02DC}'),
3746        (0x99, '\u{2122}'),
3747        (0x9A, '\u{0161}'),
3748        (0x9B, '\u{203A}'),
3749        (0x9C, '\u{0153}'),
3750        (0x9E, '\u{017E}'),
3751        (0x9F, '\u{0178}'),
3752    ];
3753    for &(b, c) in extra {
3754        m.insert(b, c);
3755    }
3756    for b in 0xA1u8..=0xFF {
3757        m.entry(b).or_insert(b as char);
3758    }
3759    m
3760}
3761
3762/// MacRomanEncoding: ASCII plus the whole high range, 0x80–0xFF, as Mac OS
3763/// Roman maps it (#744) — the table pdfium, pdftotext and pdf.js decode with.
3764/// Two codes follow the PDF reference instead of Apple's later revisions:
3765/// 0xDB is `currency` (¤, not €), and 0xF0 (`apple`, a private-use glyph) is
3766/// left unmapped. 0xCA, the PDF table's second `space`, reads as a no-break
3767/// space.
3768fn macroman_table() -> HashMap<u8, char> {
3769    #[rustfmt::skip]
3770    const HIGH: [char; 128] = [
3771        'Ä', 'Å', 'Ç', 'É', 'Ñ', 'Ö', 'Ü', 'á',  // 0x80
3772        'à', 'â', 'ä', 'ã', 'å', 'ç', 'é', 'è',  // 0x88
3773        'ê', 'ë', 'í', 'ì', 'î', 'ï', 'ñ', 'ó',  // 0x90
3774        'ò', 'ô', 'ö', 'õ', 'ú', 'ù', 'û', 'ü',  // 0x98
3775        '†', '°', '¢', '£', '§', '•', '¶', 'ß',  // 0xA0
3776        '®', '©', '™', '´', '¨', '≠', 'Æ', 'Ø',  // 0xA8
3777        '∞', '±', '≤', '≥', '¥', 'µ', '∂', '∑',  // 0xB0
3778        '∏', 'π', '∫', 'ª', 'º', 'Ω', 'æ', 'ø',  // 0xB8
3779        '¿', '¡', '¬', '√', 'ƒ', '≈', '∆', '«',  // 0xC0
3780        '»', '…', '\u{00A0}', 'À', 'Ã', 'Õ', 'Œ', 'œ',  // 0xC8
3781        '–', '—', '“', '”', '‘', '’', '÷', '◊',  // 0xD0
3782        'ÿ', 'Ÿ', '⁄', '¤', '‹', '›', 'fi', 'fl',  // 0xD8
3783        '‡', '·', '‚', '„', '‰', 'Â', 'Ê', 'Á',  // 0xE0
3784        'Ë', 'È', 'Í', 'Î', 'Ï', 'Ì', 'Ó', 'Ô',  // 0xE8
3785        '\0', 'Ò', 'Ú', 'Û', 'Ù', 'ı', 'ˆ', '˜',  // 0xF0
3786        '¯', '˘', '˙', '˚', '¸', '˝', '˛', 'ˇ',  // 0xF8
3787    ];
3788    let mut m = HashMap::new();
3789    for b in 0x20u8..=0x7e {
3790        m.insert(b, b as char);
3791    }
3792    for (b, &c) in (0x80u8..=0xFF).zip(HIGH.iter()) {
3793        if c != '\0' {
3794            m.insert(b, c);
3795        }
3796    }
3797    m
3798}
3799
3800#[cfg(test)]
3801mod glyph_names {
3802    /// `/Lslash` and the other Latin-1 / Latin Extended-A glyph names a Type1
3803    /// `/Differences` uses resolve (2203.01017v2's `Łukasz`); the table the
3804    /// lookup bisects is sorted and the ligature / symbol arms still win.
3805    #[test]
3806    fn latin_glyph_names_resolve() {
3807        for w in super::AGL_LATIN.windows(2) {
3808            assert!(
3809                w[0].0 < w[1].0,
3810                "AGL_LATIN not sorted at {:?} / {:?}",
3811                w[0].0,
3812                w[1].0
3813            );
3814        }
3815        for (name, ch) in [
3816            ("Lslash", 'Ł'),
3817            ("lslash", 'ł'),
3818            ("Aacute", 'Á'),
3819            ("aacute", 'á'),
3820            ("Scaron", 'Š'),
3821            ("germandbls", 'ß'),
3822            ("oe", 'œ'),
3823            ("Ydieresis", 'Ÿ'),
3824            ("Euro", '€'),
3825            ("Lslash.sc", 'Ł'),
3826            ("fi", '\u{FB01}'),
3827            ("Omega", '\u{03A9}'),
3828        ] {
3829            assert_eq!(
3830                super::glyph_name_to_char(name.as_bytes()),
3831                Some(ch),
3832                "{name}"
3833            );
3834        }
3835        assert_eq!(super::glyph_name_to_char(b"g123"), None);
3836        // AGL ligature names: components joined with `_`.
3837        assert_eq!(super::glyph_name_to_char(b"f_f_i"), Some('\u{FB03}'));
3838        assert_eq!(super::glyph_name_to_char(b"f_l"), Some('\u{FB02}'));
3839        assert_eq!(super::glyph_name_to_char(b"a_b"), None);
3840    }
3841
3842    /// The stale-CMap case: ToUnicode maps a code to its own identity while
3843    /// `/Differences` names a ligature there — the name wins; a CMap that
3844    /// maps the code anywhere else, or a code with no entry, is untouched.
3845    #[test]
3846    fn a_stale_tounicode_yields_to_differences() {
3847        use lopdf::Object;
3848        use std::collections::HashMap;
3849        let mut tu: HashMap<u32, String> = HashMap::new();
3850        tu.insert(33, "!".into()); // identity — stale
3851        tu.insert(34, "x".into()); // the producer meant it
3852        let diffs = vec![
3853            Object::Integer(33),
3854            Object::Name(b"f_f_i".to_vec()),
3855            Object::Name(b"f_i".to_vec()),
3856            Object::Name(b"f_f".to_vec()),
3857            Object::Integer(65),
3858            Object::Name(b"A".to_vec()), // identity on both sides: nothing to do
3859        ];
3860        super::reconcile_stale_tounicode(&mut tu, &diffs);
3861        assert_eq!(tu.get(&33).map(String::as_str), Some("\u{FB03}"));
3862        assert_eq!(tu.get(&34).map(String::as_str), Some("x"));
3863        assert_eq!(tu.get(&35), None);
3864        assert_eq!(tu.get(&65), None);
3865    }
3866}
3867
3868#[cfg(test)]
3869mod page_box_frame {
3870    use super::*;
3871    use lopdf::dictionary;
3872
3873    /// One page, `boxes` spliced into the page dictionary verbatim, one text
3874    /// run at user-space `(x, y)`.
3875    fn pdf(boxes: &str, x: f32, y: f32) -> Vec<u8> {
3876        pdf_content(
3877            boxes,
3878            &format!("BT /F1 12 Tf {x} {y} Td (First printing) Tj ET\n"),
3879        )
3880    }
3881
3882    /// One page, `boxes` spliced into the page dictionary, `content` as its
3883    /// content stream (font `/F1` = Helvetica).
3884    fn pdf_content(boxes: &str, content: &str) -> Vec<u8> {
3885        let objs: Vec<String> = vec![
3886            "<</Type/Catalog/Pages 2 0 R>>".into(),
3887            format!("<</Type/Pages/Kids[3 0 R]/Count 1{boxes}>>"),
3888            "<</Type/Page/Parent 2 0 R/Contents 4 0 R/Resources<</Font<</F1 5 0 R>>>>>>".into(),
3889            format!("<</Length {}>>stream\n{content}endstream", content.len()),
3890            "<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".into(),
3891        ];
3892        let mut out = b"%PDF-1.4\n".to_vec();
3893        let mut offsets = Vec::new();
3894        for (i, body) in objs.iter().enumerate() {
3895            offsets.push(out.len());
3896            out.extend_from_slice(format!("{} 0 obj{body}endobj\n", i + 1).as_bytes());
3897        }
3898        let xref_at = out.len();
3899        out.extend_from_slice(
3900            format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes(),
3901        );
3902        for off in &offsets {
3903            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
3904        }
3905        out.extend_from_slice(
3906            format!(
3907                "trailer<</Size {}/Root 1 0 R>>\nstartxref\n{xref_at}\n%%EOF\n",
3908                objs.len() + 1
3909            )
3910            .as_bytes(),
3911        );
3912        out
3913    }
3914
3915    /// One page whose `/Contents` is an array of separate streams.
3916    fn pdf_content_streams(contents: &[&[u8]]) -> Vec<u8> {
3917        let mut doc = Document::with_version("1.5");
3918        let streams = contents
3919            .iter()
3920            .map(|content| {
3921                Object::Reference(
3922                    doc.add_object(lopdf::Stream::new(Dictionary::new(), content.to_vec())),
3923                )
3924            })
3925            .collect::<Vec<_>>();
3926        let font_id = doc.add_object(dictionary! {
3927            "Type" => "Font",
3928            "Subtype" => "Type1",
3929            "BaseFont" => "Helvetica",
3930        });
3931        let resources_id = doc.add_object(dictionary! {
3932            "Font" => dictionary! { "F1" => Object::Reference(font_id) },
3933        });
3934        let pages_id = doc.new_object_id();
3935        let page_id = doc.add_object(dictionary! {
3936            "Type" => "Page",
3937            "Parent" => Object::Reference(pages_id),
3938            "MediaBox" => vec![0.into(), 0.into(), 300.into(), 400.into()],
3939            "Contents" => streams,
3940            "Resources" => Object::Reference(resources_id),
3941        });
3942        doc.objects.insert(
3943            pages_id,
3944            Object::Dictionary(dictionary! {
3945                "Type" => "Pages",
3946                "Kids" => vec![Object::Reference(page_id)],
3947                "Count" => 1,
3948            }),
3949        );
3950        let catalog_id = doc.add_object(dictionary! {
3951            "Type" => "Catalog",
3952            "Pages" => Object::Reference(pages_id),
3953        });
3954        doc.trailer.set("Root", Object::Reference(catalog_id));
3955        let mut out = Vec::new();
3956        doc.save_to(&mut out).expect("save synthetic PDF");
3957        out
3958    }
3959
3960    fn only_page(bytes: &[u8]) -> (PageBox, Vec<Glyph>) {
3961        let doc = load_document(bytes).expect("loads");
3962        let pid = *doc.get_pages().values().next().expect("one page");
3963        (page_box(&doc, pid), page_glyphs(&doc, pid))
3964    }
3965
3966    /// The trimmed-book-page shape: MediaBox and CropBox both away from the
3967    /// origin (inherited from `/Pages`). The display box is the CropBox, and a
3968    /// glyph's coordinates count from its lower-left corner — the same numbers
3969    /// the same text gets on a page whose CropBox *is* `[0 0 w h]`.
3970    #[test]
3971    fn glyphs_count_from_the_cropbox_corner_like_pdfium() {
3972        let (pb, shifted) = only_page(&pdf(
3973            "/MediaBox[-56.505 -58.25 576.303 723.31]/CropBox[1.095 -0.65 518.703 665.71]",
3974            37.0 + 1.095,
3975            58.0 - 0.65,
3976        ));
3977        assert!((pb.l - 1.095).abs() < 1e-3 && (pb.b + 0.65).abs() < 1e-3);
3978        assert!(
3979            (pb.w - 517.608).abs() < 1e-3 && (pb.h - 666.36).abs() < 1e-3,
3980            "{pb:?}"
3981        );
3982        let (pb0, plain) = only_page(&pdf("/MediaBox[0 0 517.608 666.36]", 37.0, 58.0));
3983        assert!((pb0.w - pb.w).abs() < 1e-3 && (pb0.h - pb.h).abs() < 1e-3);
3984        assert_eq!(shifted.len(), plain.len());
3985        assert!(!plain.is_empty());
3986        for (a, b) in shifted.iter().zip(&plain) {
3987            assert!(
3988                (a.l - b.l).abs() < 1e-3 && (a.b - b.b).abs() < 1e-3,
3989                "{:?} vs {:?}",
3990                (a.l, a.b),
3991                (b.l, b.b)
3992            );
3993        }
3994        assert!((plain[0].l - 37.0).abs() < 1e-3, "{}", plain[0].l);
3995    }
3996
3997    fn text(glyphs: &[Glyph]) -> String {
3998        glyphs.iter().map(|g| g.ch).collect()
3999    }
4000
4001    /// #529: text drawn outside the display box — a print slug beside the
4002    /// CropBox, a tiled page's neighbour beyond the MediaBox — is dropped,
4003    /// glyph by glyph like docling-parse, instead of being clamped onto the
4004    /// page edge; a line crossing the edge keeps the glyphs that fit.
4005    #[test]
4006    fn glyphs_outside_the_display_box_are_dropped() {
4007        let (_, g) = only_page(&pdf_content(
4008            "/MediaBox[0 0 400 400]/CropBox[100 100 400 400]",
4009            "BT /F1 14 Tf 120 300 Td (Visible.) Tj ET\nBT /F1 8 Tf 5 40 Td (Slug) Tj ET\n",
4010        ));
4011        assert_eq!(text(&g), "Visible.");
4012        let (_, g) = only_page(&pdf_content(
4013            "/MediaBox[0 0 300 400]",
4014            "BT /F1 14 Tf 40 300 Td (On page) Tj ET\nBT /F1 14 Tf 400 250 Td (Beyond) Tj ET\n",
4015        ));
4016        assert_eq!(text(&g), "On page");
4017        // docling-parse: "Crossing the right edge" on a 300 pt page → "Crossing the rig".
4018        let (_, g) = only_page(&pdf_content(
4019            "/MediaBox[0 0 300 400]",
4020            "BT /F1 14 Tf 200 250 Td (Crossing the right edge) Tj ET\n",
4021        ));
4022        assert_eq!(text(&g), "Crossing the rig");
4023    }
4024
4025    #[test]
4026    fn leading_comment_content_stream_does_not_hide_later_text() {
4027        // The comment has its own EOL; lopdf inserts another while assembling
4028        // the streams, which is the boundary its 0.44 parser stopped at.
4029        let (_, glyphs) = only_page(&pdf_content_streams(&[
4030            b"% CANON_PFINF_TYPE2_TEXTOFF\n",
4031            b"BT /F1 14 Tf 40 300 Td (Second stream text) Tj ET",
4032        ]));
4033        assert_eq!(text(&glyphs), "Second stream text");
4034    }
4035
4036    /// The containment test is docling-parse's: the whole char box (advance
4037    /// × the font's ascent / descent) inside the display box, edges
4038    /// included — a Helvetica `W` at 50 pt (47.2 wide) ending exactly on the
4039    /// right edge stays, 0.01 pt further it goes; likewise on the left.
4040    #[test]
4041    fn a_glyph_on_the_edge_stays_one_over_it_goes() {
4042        let w_at = |x: f32, y: f32| {
4043            let (_, g) = only_page(&pdf_content(
4044                "/MediaBox[0 0 300 400]",
4045                &format!("BT /F1 50 Tf {x} {y} Td (W) Tj ET\n"),
4046            ));
4047            text(&g)
4048        };
4049        assert_eq!(w_at(252.8, 200.0), "W");
4050        assert_eq!(w_at(252.81, 200.0), "");
4051        assert_eq!(w_at(0.0, 200.0), "W");
4052        assert_eq!(w_at(-0.01, 200.0), "");
4053        // Vertically the box is the font's ascent / descent: a baseline far
4054        // enough up keeps the descender on the page, one at 0 does not.
4055        assert_eq!(w_at(100.0, 20.0), "W");
4056        assert_eq!(w_at(100.0, 0.0), "");
4057        assert_eq!(w_at(100.0, 380.0), "");
4058    }
4059
4060    /// pdfium's fallbacks: no MediaBox → Letter; a CropBox is clipped to the
4061    /// MediaBox, and one that misses it entirely is ignored.
4062    #[test]
4063    fn page_box_follows_pdfium_fallbacks() {
4064        let (pb, _) = only_page(&pdf("", 10.0, 10.0));
4065        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 612.0, 792.0));
4066        let (pb, _) = only_page(&pdf(
4067            "/MediaBox[0 0 500 700]/CropBox[-100 100 600 900]",
4068            10.0,
4069            10.0,
4070        ));
4071        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 100.0, 500.0, 600.0));
4072        let (pb, _) = only_page(&pdf(
4073            "/MediaBox[0 0 500 700]/CropBox[800 800 900 900]",
4074            10.0,
4075            10.0,
4076        ));
4077        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
4078        // Reversed corners normalize.
4079        let (pb, _) = only_page(&pdf("/MediaBox[500 700 0 0]", 10.0, 10.0));
4080        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
4081    }
4082}
4083
4084#[cfg(test)]
4085mod xref_repair {
4086    /// Build a tiny one-page PDF whose cross-reference entries are either the
4087    /// spec's 20 bytes (`two_byte_eol`) or the 19-byte form some generators
4088    /// emit — everything else about the two files is identical.
4089    fn pdf_with_xref(two_byte_eol: bool) -> Vec<u8> {
4090        let content = b"BT /F1 12 Tf 72 700 Td (Invoice 922769430725) Tj ET\n";
4091        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
4092        let objs: Vec<Vec<u8>> = vec![
4093            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
4094            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
4095            b"<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
4096               /Resources<</Font<</F1 5 0 R>>>>>>"
4097                .to_vec(),
4098            [stream.as_slice(), content.as_slice(), b"endstream"].concat(),
4099            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".to_vec(),
4100        ];
4101
4102        let mut out = b"%PDF-1.4\n".to_vec();
4103        let mut offsets = Vec::new();
4104        for (i, body) in objs.iter().enumerate() {
4105            offsets.push(out.len());
4106            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
4107            out.extend_from_slice(body);
4108            out.extend_from_slice(b"endobj\n");
4109        }
4110        let xref_at = out.len();
4111        let eol: &[u8] = if two_byte_eol { b" \n" } else { b"\n" };
4112        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
4113        out.extend_from_slice(b"0000000000 65535 f");
4114        out.extend_from_slice(eol);
4115        for off in &offsets {
4116            out.extend_from_slice(format!("{off:010} 00000 n").as_bytes());
4117            out.extend_from_slice(eol);
4118        }
4119        out.extend_from_slice(
4120            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
4121        );
4122        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
4123        out
4124    }
4125
4126    /// A 19-byte cross-reference entry (a bare LF where the spec wants a
4127    /// two-byte EOL) used to make lopdf reject the whole file. Lopdf 0.45
4128    /// accepts it directly, so exercise the fallback itself and prove it still
4129    /// produces the exact 20-byte entries lopdf 0.44 needed.
4130    #[test]
4131    fn short_xref_entries_are_padded_for_fallback() {
4132        let good = pdf_with_xref(true);
4133        let broken = pdf_with_xref(false);
4134        assert!(
4135            broken.len() < good.len(),
4136            "the broken file is the shorter one"
4137        );
4138        assert!(
4139            lopdf::Document::load_mem(&good).is_ok(),
4140            "the control file must load unaided"
4141        );
4142        assert!(
4143            lopdf::Document::load_mem(&broken).is_ok(),
4144            "lopdf 0.45 should accept 19-byte entries"
4145        );
4146
4147        let repaired = super::pad_short_xref_entries(&broken).expect("repair short entries");
4148        assert_eq!(repaired, good, "repair should recreate the valid xref");
4149        assert!(
4150            lopdf::Document::load_mem(&repaired).is_ok(),
4151            "the bytes produced for lopdf 0.44 must load"
4152        );
4153
4154        let cells = |b: &[u8]| -> Vec<String> {
4155            super::pdf_textlines(b)
4156                .into_iter()
4157                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
4158                .collect()
4159        };
4160        let from_good = cells(&good);
4161        assert!(
4162            from_good.iter().any(|t| t.contains("922769430725")),
4163            "control text: {from_good:?}"
4164        );
4165        assert_eq!(
4166            cells(&repaired),
4167            from_good,
4168            "repair must match the good parse"
4169        );
4170    }
4171
4172    /// The same generator overstates `/Length`. Lopdf 0.45 recovers at
4173    /// `endstream`, so exercise our fallback directly and prove it still writes
4174    /// the exact length lopdf 0.44 needed without moving any bytes.
4175    #[test]
4176    fn overstated_stream_length_is_fixed_for_fallback() {
4177        let good = pdf_with_xref(true);
4178        // Inflate the content stream's /Length by one, exactly as the invoice
4179        // that prompted this does.
4180        let broken = {
4181            let at = good
4182                .windows(8)
4183                .position(|w| w == b"/Length ")
4184                .expect("a /Length")
4185                + 8;
4186            let digits = good[at..].iter().take_while(|c| c.is_ascii_digit()).count();
4187            let n: usize = std::str::from_utf8(&good[at..at + digits])
4188                .unwrap()
4189                .parse()
4190                .unwrap();
4191            let inflated = (n + 1).to_string();
4192            assert_eq!(inflated.len(), digits, "keep the digit count");
4193            let mut b = good.clone();
4194            b[at..at + digits].copy_from_slice(inflated.as_bytes());
4195            b
4196        };
4197        assert_eq!(broken.len(), good.len(), "the defect must not move bytes");
4198        // Lopdf 0.45 keeps the stream despite the overstated length.
4199        let raw = lopdf::Document::load_mem(&broken).expect("still loads");
4200        assert!(
4201            raw.get_pages()
4202                .into_values()
4203                .any(|p| !raw.get_page_content(p).is_empty()),
4204            "lopdf 0.45 should recover content at endstream"
4205        );
4206
4207        let repaired = super::fix_stream_lengths(&broken);
4208        assert_eq!(repaired.len(), broken.len(), "repair must preserve offsets");
4209        assert_eq!(repaired, good, "repair should restore the true length");
4210        let repaired_doc = lopdf::Document::load_mem(&repaired).expect("repaired PDF loads");
4211        assert!(
4212            repaired_doc
4213                .get_pages()
4214                .into_values()
4215                .any(|p| !repaired_doc.get_page_content(p).is_empty()),
4216            "the bytes produced for lopdf 0.44 must retain page content"
4217        );
4218
4219        // Ours recovers the same text the well-formed file gives.
4220        let text = |b: &[u8]| -> Vec<String> {
4221            super::pdf_textlines(b)
4222                .into_iter()
4223                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
4224                .collect()
4225        };
4226        let expected = text(&good);
4227        assert!(!expected.is_empty(), "control must produce text");
4228        assert_eq!(text(&repaired), expected);
4229    }
4230
4231    /// The repair only fires where padding cannot move an object: it declines a
4232    /// file whose xref precedes an object (an incremental update), rather than
4233    /// shifting every offset the table records.
4234    #[test]
4235    fn repair_declines_when_padding_would_move_objects() {
4236        let mut incremental = pdf_with_xref(false);
4237        incremental.extend_from_slice(b"6 0 obj<</Type/Whatever>>endobj\n");
4238        let declined = super::pad_short_xref_entries(&incremental).unwrap_err();
4239        assert!(
4240            declined.contains("object follows the xref"),
4241            "reason: {declined}"
4242        );
4243    }
4244}
4245
4246/// #187: standard-14 fonts referenced without an embedded program (and thus
4247/// usually without `/Widths` or a `/FontDescriptor`) must decode with the
4248/// built-in Adobe Core 14 metrics instead of collapsing every cell to zero
4249/// width — the failure mode where a valid text layer was silently dropped
4250/// while pdfium read the same file fine.
4251#[cfg(test)]
4252mod base14_fonts {
4253    /// A one-page PDF whose single `Tj` uses `fontdict` (no embedded program).
4254    fn pdf_with_font(fontdict: &[u8], text: &[u8]) -> Vec<u8> {
4255        let content = [b"BT /F1 12 Tf 72 700 Td (".as_slice(), text, b") Tj ET\n"].concat();
4256        pdf_with_content(fontdict, &content)
4257    }
4258
4259    /// A one-page A4 PDF drawing `content` with `fontdict` as `/F1`.
4260    pub(super) fn pdf_with_content(fontdict: &[u8], content: &[u8]) -> Vec<u8> {
4261        pdf_with_fonts(&[fontdict], content)
4262    }
4263
4264    /// A one-page A4 PDF drawing `content` with `fontdicts[i]` bound as
4265    /// `/F{i+1}` (objects 5, 6, …).
4266    fn pdf_with_fonts(fontdicts: &[&[u8]], content: &[u8]) -> Vec<u8> {
4267        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
4268        let font_res: String = fontdicts
4269            .iter()
4270            .enumerate()
4271            .map(|(i, _)| format!("/F{} {} 0 R", i + 1, i + 5))
4272            .collect();
4273        let mut objs: Vec<Vec<u8>> = vec![
4274            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
4275            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
4276            format!(
4277                "<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
4278                 /Resources<</Font<<{font_res}>>>>>>"
4279            )
4280            .into_bytes(),
4281            [stream.as_slice(), content, b"endstream"].concat(),
4282        ];
4283        objs.extend(fontdicts.iter().map(|d| d.to_vec()));
4284        let mut out = b"%PDF-1.4\n".to_vec();
4285        let mut offsets = Vec::new();
4286        for (i, body) in objs.iter().enumerate() {
4287            offsets.push(out.len());
4288            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
4289            out.extend_from_slice(body);
4290            out.extend_from_slice(b"endobj\n");
4291        }
4292        let xref_at = out.len();
4293        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
4294        out.extend_from_slice(b"0000000000 65535 f \n");
4295        for off in &offsets {
4296            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
4297        }
4298        out.extend_from_slice(
4299            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
4300        );
4301        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
4302        out
4303    }
4304
4305    /// The parsed cells of the only page.
4306    pub(super) fn cells(pdf: &[u8]) -> Vec<crate::pdfium_backend::TextCell> {
4307        super::pdf_textlines(pdf)
4308            .into_iter()
4309            .flat_map(|(_, _, c)| c)
4310            .collect()
4311    }
4312
4313    /// Text drawn with a rotated text matrix reads as one line in its own
4314    /// reading order, boxed by the run's axis-aligned extent (#528). The
4315    /// landscape case is `0 s -s 0 tx ty Tm` on a `/Rotate 90` page; every
4316    /// such run used to collapse to zero width and vanish (90°/270°/tilted)
4317    /// or read backwards (180°). Expected boxes are docling-parse 7.22's line
4318    /// cells for the same runs (y-up, PDF points).
4319    #[test]
4320    fn rotated_text_matrix_reads_in_order() {
4321        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
4322        const TEXT: &str = "Upright text on a rotated page.";
4323        for (tm, [l, b, r, t]) in [
4324            ("0 14 -14 0 200 100", [189.9, 100.0, 202.9, 289.1]), // 90°
4325            ("-14 0 0 -14 350 300", [160.9, 289.9, 350.0, 302.9]), // 180°
4326            ("0 -14 14 0 200 500", [197.1, 310.9, 210.1, 500.0]), // 270°
4327            (
4328                "9.8995 9.8995 -9.8995 9.8995 100 100",
4329                [92.9, 98.0, 235.8, 240.8],
4330            ), // 45°
4331        ] {
4332            let content = format!("BT /F1 1 Tf {tm} Tm ({TEXT}) Tj ET\n");
4333            let pdf = pdf_with_content(HELV, content.as_bytes());
4334            let cs = cells(&pdf);
4335            assert_eq!(cs.len(), 1, "{tm}: {cs:?}");
4336            let c = &cs[0];
4337            assert_eq!(c.text, TEXT, "{tm}");
4338            // `cells` is top-left-origin on the 842 pt page. The descriptor-
4339            // less base-14 face gets the parser's 1-em box where docling-parse
4340            // reads Helvetica's AFM (718/−207) — the same ≤ 0.6 pt
4341            // across-the-baseline offset upright text has — so allow 1 pt.
4342            let got = [c.l, 842.0 - c.b, c.r, 842.0 - c.t];
4343            for (g, w) in got.iter().zip([l, b, r, t]) {
4344                assert!(
4345                    (g - w).abs() < 1.0,
4346                    "{tm}: box {got:?} vs docling-parse {:?}",
4347                    [l, b, r, t]
4348                );
4349            }
4350            let words: Vec<String> = super::pdf_words(&pdf)
4351                .into_iter()
4352                .flat_map(|(_, _, c)| c)
4353                .map(|c| c.text)
4354                .collect();
4355            assert_eq!(words, TEXT.split(' ').collect::<Vec<_>>(), "{tm}");
4356        }
4357    }
4358
4359    /// A rotated run past the page edge (a cover's spine title) is dropped
4360    /// by the on-page test (#529) — measured on the quad's extent, not the
4361    /// zero-width box rotated glyphs used to get — instead of clamping onto
4362    /// the edge as a zero-width cell. One inside the page is kept.
4363    #[test]
4364    fn rotated_text_off_the_page_is_dropped() {
4365        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
4366        let off = pdf_with_content(
4367            HELV,
4368            b"BT /F1 1 Tf 0 14 -14 0 -4 200 Tm (Spine title) Tj ET\n",
4369        );
4370        assert!(cells(&off).is_empty(), "{:?}", cells(&off));
4371        let on = pdf_with_content(
4372            HELV,
4373            b"BT /F1 1 Tf 0 14 -14 0 30 200 Tm (Margin note) Tj ET\n",
4374        );
4375        assert_eq!(cells(&on).len(), 1);
4376    }
4377
4378    /// Upright text keeps the plain loose rectangle — no quad, and the
4379    /// sanitizer path is the one every pinned PDF baseline was made with.
4380    #[test]
4381    fn upright_glyphs_carry_no_quad() {
4382        let only_page = |pdf: &[u8]| {
4383            let doc = super::load_document(pdf).expect("loads");
4384            let pid = *doc.get_pages().values().next().expect("one page");
4385            super::page_glyphs(&doc, pid)
4386        };
4387        let pdf = pdf_with_font(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", b"Plain");
4388        let glyphs = only_page(&pdf);
4389        assert!(!glyphs.is_empty());
4390        assert!(glyphs.iter().all(|g| g.quad.is_none() && g.lr > g.ll));
4391        // A 90° glyph's style height is its font size, not its advance.
4392        let pdf = pdf_with_content(
4393            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>",
4394            b"BT /F1 1 Tf 0 14 -14 0 200 100 Tm (W) Tj ET\n",
4395        );
4396        let glyphs = only_page(&pdf);
4397        let g = &glyphs[0];
4398        assert!(g.quad.is_some());
4399        // 14 pt × the face's 1-em box (no descriptor: ascent − descent = 1).
4400        assert!((g.height() - 14.0).abs() < 0.05, "{}", g.height());
4401    }
4402
4403    /// #609: ReportLab's chart axis sets each tick label with the *same* `Tm`
4404    /// inside its own `q … cm … Q`, so only the CTM stacks `3`…`8` one under
4405    /// another at one x. Each tick is its own line cell with its own box (as
4406    /// docling-parse reads it), not one `345678` cell boxed like the first.
4407    #[test]
4408    fn ticks_stacked_by_cm_are_separate_line_cells() {
4409        let mut content = String::new();
4410        for (k, tick) in "345678".chars().enumerate() {
4411            let y = 300.0 + 25.92 * k as f64;
4412            content.push_str(&format!(
4413                "q 1 0 0 1 125 {y} cm BT /F1 10 Tf 1 0 0 1 -5 -4 Tm ({tick}) Tj ET Q\n"
4414            ));
4415        }
4416        let pdf = pdf_with_content(
4417            b"<</Type/Font/Subtype/Type1/BaseFont/Times-Roman>>",
4418            content.as_bytes(),
4419        );
4420        let cells = cells(&pdf);
4421        let texts: Vec<&str> = cells.iter().map(|c| c.text.as_str()).collect();
4422        assert_eq!(texts, ["3", "4", "5", "6", "7", "8"]);
4423        for (k, c) in cells.iter().enumerate() {
4424            // Top-left y of a tick whose baseline is 4 pt under its cm origin.
4425            let base = 842.0 - (300.0 + 25.92 * k as f32 - 4.0);
4426            assert!((c.l - 120.0).abs() < 0.1, "{k}: l {}", c.l);
4427            assert!(
4428                c.t < base && base - c.t < 10.0,
4429                "{k}: t {} base {base}",
4430                c.t
4431            );
4432        }
4433    }
4434
4435    /// #609: the path walk finds drawn checkbox squares — ReportLab's four
4436    /// `m … l S` edges under a `cm`, and a stroked `re` with a tick inside
4437    /// (checked) — in the text cells' top-left frame, and a stroked box
4438    /// filled white; a filled bar, a big frame and a colour-filled legend
4439    /// swatch are not checkboxes.
4440    #[test]
4441    fn drawn_checkbox_squares_are_found() {
4442        let content = b"q 1 0 0 1 119.52 449.04 cm \
4443            n 0 12.96 m 12.96 12.96 l S n 0 0 m 12.96 0 l S \
4444            n 0 0 m 0 12.96 l S n 12.96 0 m 12.96 12.96 l S Q\n\
4445            200 400 10 10 re S 202 405 m 204.5 402 l 208.5 408.5 l S\n\
4446            300 400 12 12 re f 50 50 100 100 re S\n\
4447            q 0.2 0.4 0.8 rg 400 400 12 12 re B Q q 1 g 500 400 12 12 re B Q\n";
4448        let pdf = pdf_with_content(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", content);
4449        let mut parser = super::PageTextParser::open(&pdf).expect("parses");
4450        let mut boxes = parser.cells(0).checkboxes;
4451        boxes.sort_by(|a, b| a.l.total_cmp(&b.l));
4452        let got: Vec<([i32; 4], bool)> = boxes
4453            .iter()
4454            .map(|c| {
4455                let r = |v: f32| (v * 100.0).round() as i32;
4456                ([r(c.l), r(c.t), r(c.r), r(c.b)], c.checked)
4457            })
4458            .collect();
4459        // Page height 842: y-up 449.04..462.0 → top-left 380.0..392.96.
4460        assert_eq!(
4461            got,
4462            [
4463                ([11952, 38000, 13248, 39296], false),
4464                ([20000, 43200, 21000, 44200], true),
4465                ([50000, 43000, 51200, 44200], false),
4466            ]
4467        );
4468    }
4469
4470    /// Every standard-14 alias/style decodes with real (positive-width) boxes.
4471    #[test]
4472    fn standard14_faces_get_builtin_widths() {
4473        for fontdict in [
4474            // ReportLab's default: base-14 Helvetica, WinAnsi, nothing else.
4475            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>".as_slice(),
4476            // No /Encoding at all (StandardEncoding-ish default).
4477            b"<</Type/Font/Subtype/Type1/BaseFont/Times-BoldItalic>>",
4478            // Substitution aliases + a subset prefix.
4479            b"<</Type/Font/Subtype/TrueType/BaseFont/Arial,Bold>>",
4480            b"<</Type/Font/Subtype/Type1/BaseFont/ABCDEF+Courier-Oblique>>",
4481        ] {
4482            let pdf = pdf_with_font(fontdict, b"Words have width now");
4483            let cs = cells(&pdf);
4484            let text: String = cs
4485                .iter()
4486                .map(|c| c.text.as_str())
4487                .collect::<Vec<_>>()
4488                .join(" ");
4489            assert!(
4490                text.contains("Words have width now"),
4491                "{}: text lost: {text:?}",
4492                String::from_utf8_lossy(fontdict)
4493            );
4494            assert!(
4495                cs.iter().all(|c| c.r > c.l),
4496                "{}: zero-width cells: {cs:?}",
4497                String::from_utf8_lossy(fontdict)
4498            );
4499        }
4500    }
4501
4502    /// MacRomanEncoding without a ToUnicode CMap (#744) — Word for Mac's
4503    /// Quartz output: the whole high range decodes, Danish æ ø å, é, ü, ß
4504    /// and § included, through the plain encoding name and as the base of a
4505    /// `/Differences` dictionary alike.
4506    #[test]
4507    fn macroman_fonts_keep_accented_letters() {
4508        let sentence = b"Bl\x8cb\xbergr\xbfd p\x8c \xbfen \xd0 caf\x8e M\x9fller, Stra\xa7e \xa4 3";
4509        for (fontdict, want) in [
4510            (
4511                b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/MacRomanEncoding>>"
4512                    .as_slice(),
4513                "Blåbærgrød på øen – café Müller, Straße § 3",
4514            ),
4515            (
4516                b"<</Type/Font/Subtype/TrueType/BaseFont/ABCDEF+Arial\
4517                   /Encoding<</Type/Encoding/BaseEncoding/MacRomanEncoding/Differences[51/four]>>>>",
4518                "Blåbærgrød på øen – café Müller, Straße § 4",
4519            ),
4520        ] {
4521            let text: String = cells(&pdf_with_font(fontdict, sentence))
4522                .iter()
4523                .map(|c| c.text.as_str())
4524                .collect::<Vec<_>>()
4525                .join(" ");
4526            assert_eq!(text, want, "{}", String::from_utf8_lossy(fontdict));
4527        }
4528
4529        // The table itself: all of 0x80–0xFF but the Apple logo, ¤ at 0xDB.
4530        let table = super::macroman_table();
4531        assert_eq!(
4532            (0x80u8..=0xFF).filter(|b| table.contains_key(b)).count(),
4533            127
4534        );
4535        assert_eq!(table.get(&0xF0), None);
4536        assert_eq!(table[&0xDB], '¤');
4537        assert_eq!(
4538            [0x80, 0x8C, 0xAE, 0xAF, 0xBE, 0xBF, 0xFF].map(|b| table[&b]),
4539            ['Ä', 'å', 'Æ', 'Ø', 'æ', 'ø', 'ˇ']
4540        );
4541    }
4542
4543    /// Unmapped glyphs read U+FFFD (#1003, docling#5070): a ToUnicode NUL —
4544    /// docling's `test_threaded_page_backend_replaces_nul_with_replacement_char`
4545    /// case, `mg.mL \x00 1` — and a bare glyph-index `/Differences` name,
4546    /// while an unknown code-like name keeps its text without the slash
4547    /// (docling-parse 7.24's `SM590000`).
4548    #[test]
4549    fn unmapped_glyphs_read_as_the_replacement_char() {
4550        let cmap = b"/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n\
4551            1 begincodespacerange <00> <FF> endcodespacerange\n\
4552            1 beginbfchar <7e> <0000> endbfchar\n\
4553            endcmap CMapName currentdict /CMap defineresource pop end end";
4554        let stream = [
4555            format!("<</Length {}>>stream\n", cmap.len()).as_bytes(),
4556            cmap.as_slice(),
4557            b"\nendstream",
4558        ]
4559        .concat();
4560        let pdf = pdf_with_fonts(
4561            &[
4562                b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/ToUnicode 6 0 R\
4563                   /Encoding<</Type/Encoding/Differences[1/g115 2/SM590000]>>>>",
4564                &stream,
4565            ],
4566            b"BT /F1 12 Tf 72 700 Td (mg.mL ~ 1 \x01 x \x02 y) Tj ET\n",
4567        );
4568        let text: String = cells(&pdf)
4569            .iter()
4570            .map(|c| c.text.as_str())
4571            .collect::<Vec<_>>()
4572            .join(" ");
4573        assert_eq!(text, "mg.mL \u{FFFD} 1 \u{FFFD} x SM590000 y");
4574    }
4575
4576    /// redp5110's subset font names every glyph by index (`/g115/g3 /g40…`):
4577    /// the painted ones read U+FFFD, the blank `/g3` program is a space, so
4578    /// the line keeps its words (docling-parse 7.24's line cell).
4579    #[cfg(feature = "render")]
4580    #[test]
4581    fn a_blank_glyph_program_is_a_space() {
4582        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
4583            .join("../../tests/data/pdf/sources/redp5110_sampled.pdf");
4584        let Ok(bytes) = std::fs::read(path) else {
4585            return;
4586        };
4587        let lines: Vec<String> = super::pdf_textlines(&bytes)
4588            .into_iter()
4589            .flat_map(|(_, _, c)| c)
4590            .map(|c| c.text)
4591            .collect();
4592        let want = "\u{FFFD} \u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD} \
4593                    \u{FFFD}\u{FFFD}\u{FFFD} ";
4594        assert!(
4595            lines.iter().any(|l| l.starts_with(want)),
4596            "no `{want}` line in {:?}",
4597            lines
4598                .iter()
4599                .filter(|l| l.contains('\u{FFFD}'))
4600                .collect::<Vec<_>>()
4601        );
4602    }
4603
4604    /// The page's image placements reach the cells (#999): the scan under
4605    /// the #999 fixture's one native word is the whole page.
4606    #[test]
4607    fn image_placements_are_the_pages_bitmaps() {
4608        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
4609            .join("../docling/tests/fixtures/partial_text_layer/scan_with_native_word.pdf");
4610        let bytes = std::fs::read(path).expect("fixture");
4611        let mut parser = super::PageTextParser::open(&bytes).expect("parser");
4612        let page = parser.cells(0);
4613        assert_eq!(page.bitmaps, vec![[0.0, 0.0, 612.0, 792.0]]);
4614        assert_eq!(page.words.len(), 1);
4615        assert!(page.checkboxes.is_empty());
4616    }
4617
4618    /// A `bfrange` whose destination is a surrogate pair steps its last code
4619    /// unit (#1003: equations.pdf's `H𝛂`, `<1d59> <1d5a> <d835dec2>`); a
4620    /// single-unit base keeps counting in code points.
4621    #[test]
4622    fn surrogate_pair_bfrange_maps_each_code() {
4623        let map = super::parse_tounicode(
4624            b"1 beginbfrange <1d59> <1d5a> <d835dec2> endbfrange\n\
4625              1 beginbfrange <0010> <0011> <0041> endbfrange",
4626        );
4627        assert_eq!(map.get(&0x1d59).map(String::as_str), Some("\u{1D6C2}"));
4628        assert_eq!(map.get(&0x1d5a).map(String::as_str), Some("\u{1D6C3}"));
4629        assert_eq!(map.get(&0x11).map(String::as_str), Some("B"));
4630    }
4631
4632    /// Which unknown glyph names are text and which are bare indexes
4633    /// (docling-parse 7.24's `resolve_unknown_name`).
4634    #[test]
4635    fn unknown_glyph_names_follow_docling_parse() {
4636        for name in ["g115", "gid00043", "Glyph7", "cid42", "index9", "g3.alt"] {
4637            assert_eq!(super::unknown_name_text(name), None, "{name}");
4638        }
4639        assert_eq!(
4640            super::unknown_name_text("SM590000").as_deref(),
4641            Some("SM590000")
4642        );
4643        assert_eq!(
4644            super::unknown_name_text("SM590000.sc").as_deref(),
4645            Some("SM590000")
4646        );
4647    }
4648
4649    /// An explicit `/Widths` array always wins over the built-in metrics, and a
4650    /// non-standard face without `/Widths` stays as before (no invented boxes).
4651    #[test]
4652    fn explicit_widths_win_and_unknown_faces_are_untouched() {
4653        // Helvetica with explicit 100/1000-em widths: the word's box must be
4654        // ~4×100 units at 12pt = 4.8pt wide — far narrower than the ~2.7×
4655        // wider built-in Helvetica advances would make it.
4656        let explicit = pdf_with_font(
4657            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/FirstChar 65\
4658               /Widths[100 100 100 100]/Encoding/WinAnsiEncoding>>",
4659            b"ABBA",
4660        );
4661        let builtin = pdf_with_font(
4662            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>",
4663            b"ABBA",
4664        );
4665        let w = |pdf: &[u8]| {
4666            let cs = cells(pdf);
4667            assert_eq!(cs.len(), 1, "one word cell: {cs:?}");
4668            cs[0].r - cs[0].l
4669        };
4670        let (we, wb) = (w(&explicit), w(&builtin));
4671        assert!(
4672            (we - 4.8).abs() < 0.1,
4673            "explicit widths must win: got {we}, want 4×100×12/1000"
4674        );
4675        assert!(
4676            wb > 2.0 * we,
4677            "built-in Helvetica is much wider: {wb} vs {we}"
4678        );
4679
4680        // An unknown face with no /Widths: still parses (text kept), but no
4681        // built-in table applies — the old zero-width behavior is preserved
4682        // rather than inventing Helvetica metrics for an arbitrary font.
4683        let unknown = pdf_with_font(
4684            b"<</Type/Font/Subtype/Type1/BaseFont/FancyCorp-Display>>",
4685            b"Mystery",
4686        );
4687        let cs = cells(&unknown);
4688        let text: String = cs.iter().map(|c| c.text.as_str()).collect();
4689        assert!(text.contains("Mystery"), "text still decodes: {cs:?}");
4690    }
4691    /// #728: one face bound under two resource keys — `/F1` Helvetica in
4692    /// WinAnsi, `/F2` the same Helvetica re-encoded so code 0x81 is the `fi`
4693    /// glyph — with every fi-word drawn in `/F2` right after an `/F1` run.
4694    /// docling-parse names a font by `/FontDescriptor/FontName` → `/Name` →
4695    /// `/BaseFont`, never by the resource key, so both resources are the same
4696    /// font and the line reads `financial`. Keyed by resource key the `fi`
4697    /// ligature (font-neutral in `applicable_for_merge`) joined the `/F1`
4698    /// run and the font barrier then split it from `nancial`.
4699    #[test]
4700    fn same_face_under_two_resource_keys_is_one_font_for_the_sanitizer() {
4701        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4702        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding<</Type/Encoding\
4703                   /BaseEncoding/WinAnsiEncoding/Differences[129/fi]>>>>";
4704        let content = b"BT 12 TL 72 720 Td \
4705            /F1 12 Tf (The ) Tj /F2 12 Tf (\\201nancial position) Tj \
4706            /F1 12 Tf ( of the company is shown in ) Tj /F2 12 Tf (\\201gure 3) Tj \
4707            /F1 12 Tf ( with the ) Tj /F2 12 Tf (\\201rst results.) Tj \
4708            T* T* /F2 12 Tf (Same words in /F2 only: \\201nancial, \\201gure, \\201rst.) Tj ET\n";
4709        let cs = cells(&pdf_with_fonts(&[f1, f2], content));
4710        let lines: Vec<&str> = cs.iter().map(|c| c.text.as_str()).collect();
4711        assert_eq!(
4712            lines,
4713            [
4714                "The financial position of the company is shown in figure 3 with the first results.",
4715                "Same words in /F2 only: financial, figure, first.",
4716            ],
4717            "{cs:?}"
4718        );
4719    }
4720
4721    /// Under `legacy` the font barrier itself is untouched: two *different*
4722    /// faces (regular and bold Helvetica, which docling-parse names apart) set
4723    /// flush against each other stay separate line cells, as
4724    /// `enforce_same_font` demands. docling-parse 7 dropped the barrier — its
4725    /// line runs follow geometry alone — so under the default `adaptive` the
4726    /// flush pair is one line and one word.
4727    #[test]
4728    fn different_faces_keep_the_font_barrier_only_under_legacy() {
4729        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4730        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica-Bold/Encoding/WinAnsiEncoding>>";
4731        let content = b"BT 72 720 Td /F2 12 Tf (Total:) Tj /F1 12 Tf (42) Tj ET\n";
4732        let pdf = pdf_with_fonts(&[f1, f2], content);
4733        let lines = |gap| {
4734            let mut p = super::PageTextParser::open(&pdf).expect("opens");
4735            p.set_word_gap(Some(gap));
4736            p.cells(0)
4737                .prose
4738                .into_iter()
4739                .map(|c| c.text)
4740                .collect::<Vec<_>>()
4741        };
4742        assert_eq!(lines(crate::dp_lines::WordGap::Legacy), ["Total:", "42"]);
4743        assert_eq!(lines(crate::dp_lines::WordGap::Adaptive), ["Total:42"]);
4744    }
4745
4746    /// #992: a space glyph whose advance the next `TJ` offset takes back —
4747    /// EUR-Lex's superscript footnote marker `( 1 )`, set with spaces the
4748    /// digit is then drawn over — is not a character of the text: the page
4749    /// shows `(1)`, pdftotext and pdfium read `(1)`, and so does the line
4750    /// cell here (docling-parse keeps the spaces, `( 1 )`). A space the next
4751    /// glyph only kerns into (−100/1000 em, well past the space's start) is
4752    /// a real gap and stays, as does one before a jump to the next line.
4753    #[test]
4754    fn a_space_the_next_glyph_is_drawn_over_is_not_a_character() {
4755        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4756        let lines = |content: &[u8]| {
4757            let pdf = pdf_with_content(f1, content);
4758            super::PageTextParser::open(&pdf)
4759                .expect("opens")
4760                .cells(0)
4761                .prose
4762                .into_iter()
4763                .map(|c| c.text)
4764                .collect::<Vec<_>>()
4765        };
4766        // Helvetica's space is 278/1000 em: an offset of 300 moves the pen
4767        // back past where the space began, 100 leaves two thirds of it.
4768        assert_eq!(
4769            lines(b"BT /F1 12 Tf 72 700 Td [(EG \\() ( ) 300 (1) ( ) 300 (\\) erf)] TJ ET\n"),
4770            ["EG (1) erf"]
4771        );
4772        assert_eq!(
4773            lines(b"BT /F1 12 Tf 72 700 Td [(EG \\() ( ) 100 (1) ( ) 100 (\\) erf)] TJ ET\n"),
4774            ["EG ( 1 ) erf"]
4775        );
4776        // A trailing space, then the next line starts at the left margin.
4777        assert_eq!(
4778            lines(b"BT /F1 12 Tf 72 700 Td (first line ) Tj 0 -14 Td (second) Tj ET\n"),
4779            ["first line", "second"]
4780        );
4781        // A line drawn rotated 90° (a landscape slide on a portrait page,
4782        // #528): its glyphs' loose boxes stack on one left edge, which is
4783        // not an overprint — the space stays.
4784        assert_eq!(
4785            lines(b"BT 0 1 -1 0 300 100 Tm /F1 12 Tf (Nullam mollis) Tj ET\n"),
4786            ["Nullam mollis"]
4787        );
4788    }
4789
4790    /// A text shadow: a word processor exports a heading with a shadow
4791    /// effect as the same run drawn twice — the shadow in black, then the
4792    /// text in colour over it, shifted up and left by a fraction of a point.
4793    /// pdfium's text page reads the heading once and so does the line cell
4794    /// here (docling-parse reads `Safety instructions Safety instructions`).
4795    /// #1092: docling-parse's view of a page's shapes — a stroked straight
4796    /// rule is a shape line and a shape box, a thin filled box only a box, a
4797    /// curve or a diagonal no line, a path outside the clip nothing, and a
4798    /// fill in an unresolved Pattern space nothing at all. In the cells'
4799    /// frame: top-left, on the 595 × 842 test page.
4800    #[test]
4801    fn the_page_shapes_follow_docling_parse() {
4802        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4803        let shapes = |content: &[u8]| {
4804            let pdf = pdf_with_content(f1, content);
4805            super::PageTextParser::open(&pdf)
4806                .expect("opens")
4807                .cells(0)
4808                .shapes
4809        };
4810        let s = shapes(b"2 w 72 400 m 540 400 l S\n");
4811        assert_eq!(s.lines, vec![[72.0, 442.0, 540.0, 442.0]]);
4812        // The stroke's half width pads the box.
4813        assert_eq!(s.boxes, vec![[71.0, 441.0, 541.0, 443.0]]);
4814
4815        let s = shapes(b"72 300 468 1 re f\n");
4816        assert!(s.lines.is_empty(), "a fill strokes no line");
4817        assert_eq!(s.boxes, vec![[72.0, 541.0, 540.0, 542.0]]);
4818
4819        // A stroked rectangle: four edges, `h`'s closing one included.
4820        let s = shapes(b"100 100 200 50 re S\n");
4821        assert_eq!(s.lines.len(), 4);
4822
4823        let s = shapes(b"72 400 m 100 450 l 200 300 300 500 400 400 c S\n");
4824        assert!(s.lines.is_empty(), "neither a diagonal nor a curve");
4825        assert_eq!(s.boxes.len(), 1);
4826
4827        let s = shapes(b"q 0 0 300 842 re W n 72 400 m 540 400 l S 400 100 m 400 700 l S Q\n");
4828        assert_eq!(
4829            s.lines,
4830            vec![[72.0, 442.0, 300.0, 442.0]],
4831            "cut to the clip"
4832        );
4833        assert_eq!(s.boxes.len(), 1, "the vertical rule is clipped away");
4834
4835        let s = shapes(b"/Pattern cs /P0 scn 72 300 468 1 re f\n");
4836        assert!(s.lines.is_empty() && s.boxes.is_empty());
4837
4838        // Two touching boxes merge into one component.
4839        let s = shapes(b"72 300 100 1 re f 172 300 100 1 re f 400 600 10 10 re f\n");
4840        assert_eq!(
4841            s.boxes,
4842            vec![[72.0, 541.0, 272.0, 542.0], [400.0, 232.0, 410.0, 242.0]]
4843        );
4844    }
4845
4846    /// A transparent stroke or fill is no shape (`CA` / `ca` 0).
4847    #[test]
4848    fn transparent_paths_are_no_shapes() {
4849        let rule = |closed| super::ShapeSub {
4850            start: (0.0, 0.0),
4851            cur: (100.0, 0.0),
4852            pts: vec![(0.0, 0.0), (100.0, 0.0)],
4853            lines: vec![((0.0, 0.0), (100.0, 0.0))],
4854            closed,
4855        };
4856        let gs = |stroke_alpha, fill_alpha| super::ShapeGs {
4857            stroke_alpha,
4858            fill_alpha,
4859            ..super::ShapeGs::INIT
4860        };
4861        let mut shapes = super::Shapes::default();
4862        super::record_shape(&[rule(false)], true, false, &gs(0.0, 1.0), 1.0, &mut shapes);
4863        assert_eq!(shapes, super::Shapes::default());
4864        super::record_shape(&[rule(true)], false, true, &gs(1.0, 0.0), 1.0, &mut shapes);
4865        assert_eq!(shapes, super::Shapes::default());
4866        super::record_shape(&[rule(false)], true, false, &gs(0.5, 0.0), 1.0, &mut shapes);
4867        assert_eq!(shapes.lines.len(), 1);
4868    }
4869
4870    /// Text that really repeats sits an advance apart or more and stays: a
4871    /// word set twice, a doubled letter, the same words on the next line.
4872    #[test]
4873    fn a_run_drawn_again_over_itself_is_read_once() {
4874        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica-Bold/Encoding/WinAnsiEncoding>>";
4875        let lines = |content: &[u8]| {
4876            let pdf = pdf_with_content(f1, content);
4877            super::PageTextParser::open(&pdf)
4878                .expect("opens")
4879                .cells(0)
4880                .prose
4881                .into_iter()
4882                .map(|c| c.text)
4883                .collect::<Vec<_>>()
4884        };
4885        // The shadow, then the text 0.5 pt up and left of it, at 18 pt.
4886        assert_eq!(
4887            lines(
4888                b"BT 0 g /F1 18 Tf 1 0 0 1 72 720 Tm (Safety instructions) Tj ET\n\
4889                  BT 0.8 0 0 rg /F1 18 Tf 1 0 0 1 71.5 720.5 Tm (Safety instructions) Tj ET\n"
4890            ),
4891            ["Safety instructions"]
4892        );
4893        // 0.8 pt at 14 pt, and the line under it untouched.
4894        assert_eq!(
4895            lines(
4896                b"BT /F1 14 Tf 1 0 0 1 72 656 Tm (Maintenance) Tj ET\n\
4897                  BT /F1 14 Tf 1 0 0 1 71.2 656.8 Tm (Maintenance) Tj ET\n\
4898                  BT /F1 11 Tf 1 0 0 1 72 632 Tm (Clean the filter) Tj ET\n"
4899            ),
4900            ["Maintenance", "Clean the filter"]
4901        );
4902        // A word set twice, a doubled letter, the same words a line apart.
4903        assert_eq!(
4904            lines(b"BT /F1 12 Tf 72 700 Td (very very small) Tj ET\n"),
4905            ["very very small"]
4906        );
4907        assert_eq!(
4908            lines(b"BT /F1 12 Tf 72 700 Td (Hello, all) Tj ET\n"),
4909            ["Hello, all"]
4910        );
4911        assert_eq!(
4912            lines(b"BT /F1 12 Tf 72 700 Td (Total) Tj 0 -14 Td (Total) Tj ET\n"),
4913            ["Total", "Total"]
4914        );
4915        // Dot leaders squeezed by `Tc` to 0.05, 0.07 and 0.1 em: two dots on,
4916        // a dot sits within a shadow's shift and the dots after it repeat by
4917        // the same offset, but the next dot is nearer still.
4918        for tc in ["-2.28", "-2.08", "-1.78"] {
4919            let content = format!(
4920                "BT /F1 10 Tf 72 700 Td (Contents) Tj {tc} Tc (..........) Tj 0 Tc ( 5) Tj ET\n"
4921            );
4922            assert_eq!(
4923                lines(content.as_bytes()),
4924                ["Contents.......... 5"],
4925                "Tc {tc}"
4926            );
4927        }
4928    }
4929
4930    /// docling-parse's `test_unit_actual_text.py` (docling-parse#300): a
4931    /// marked-content span's `/ActualText` replaces the glyphs drawn in it.
4932    #[test]
4933    fn actual_text_replaces_the_glyphs_of_its_span() {
4934        let helvetica = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4935        // A non-embedded TrueType whose only code, 0x4D (`M` in WinAnsi), is a
4936        // zero-advance caron: drawn after `s` it makes `š`.
4937        let caron = b"<</Type/Font/Subtype/TrueType/BaseFont/SynthCaron/FirstChar 77/LastChar 77\
4938            /Widths[0]/Encoding/WinAnsiEncoding/FontDescriptor<</Type/FontDescriptor\
4939            /FontName/SynthCaron/Flags 32/FontBBox[-540 -200 0 800]/ItalicAngle 0/Ascent 800\
4940            /Descent -200/CapHeight 700/StemV 80>>>>";
4941        // Code 1 is the `fi` ligature: two glyphs on one pen, one cell.
4942        let ligature = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica\
4943            /Encoding<</Type/Encoding/BaseEncoding/WinAnsiEncoding/Differences[1/fi]>>>>";
4944        let text = |content: &str| {
4945            let pdf = pdf_with_fonts(&[helvetica, caron, ligature], content.as_bytes());
4946            super::PageTextParser::open(&pdf)
4947                .expect("opens")
4948                .cells(0)
4949                .prose
4950                .into_iter()
4951                .map(|c| c.text)
4952                .collect::<String>()
4953        };
4954        // Unmarked, the caron reads as its WinAnsi letter.
4955        assert_eq!(
4956            text("BT /F1 24 Tf 40 50 Td (Ore) Tj (s) Tj /F2 24 Tf <4D> Tj /F1 24 Tf (i) Tj ET"),
4957            "OresMi"
4958        );
4959        // Two cells, one character: the accent composes onto its base letter.
4960        assert_eq!(
4961            text(
4962                "BT /F1 24 Tf 40 50 Td (Ore) Tj /Span <</ActualText <FEFF0161>>> BDC \
4963                  (s) Tj /F2 24 Tf <4D> Tj EMC /F1 24 Tf (i) Tj ET"
4964            ),
4965            "Oreši"
4966        );
4967        // As many characters as cells: one for one.
4968        assert_eq!(
4969            text("BT /F1 24 Tf 40 50 Td /Span <</ActualText (ab)>> BDC (xy) Tj EMC (z) Tj ET"),
4970            "abz"
4971        );
4972        // A cell of two glyphs takes one character.
4973        assert_eq!(
4974            text("BT /F3 24 Tf 40 50 Td <01> Tj /F1 24 Tf (a) Tj ET"),
4975            "fia"
4976        );
4977        assert_eq!(
4978            text(
4979                "BT /F3 24 Tf 40 50 Td /Span <</ActualText (XY)>> BDC <01> Tj \
4980                  /F1 24 Tf (a) Tj EMC (z) Tj ET"
4981            ),
4982            "XYz"
4983        );
4984        // An empty text declares the glyphs not text (a visual line-break
4985        // hyphen); its advance may still read as a gap.
4986        let hyphen = text(
4987            "BT /F1 24 Tf 40 50 Td (exam) Tj /Span <</ActualText <FEFF>>> BDC (-) Tj EMC \
4988             (ple) Tj ET",
4989        );
4990        assert!(!hyphen.contains('-'), "{hyphen}");
4991        assert_eq!(hyphen.replace(' ', ""), "example");
4992        // `/Alt` written into the wrong key: implausibly long, ignored.
4993        assert_eq!(
4994            text(
4995                "BT /F1 24 Tf 40 50 Td /Span <</ActualText (A long description that belongs \
4996                  in Alt not here)>> BDC (ab) Tj EMC ET"
4997            ),
4998            "ab"
4999        );
5000        // Over graphics only: nothing to replace.
5001        assert_eq!(
5002            text(
5003                "BT /F1 24 Tf 40 50 Td (x) Tj ET /Span <</ActualText (ghost)>> BDC \
5004                  10 10 50 50 re f EMC"
5005            ),
5006            "x"
5007        );
5008        // A named property list is not resolved.
5009        assert_eq!(
5010            text("BT /F1 24 Tf 40 50 Td /OC /MC0 BDC (ok) Tj EMC ET"),
5011            "ok"
5012        );
5013        // Nested spans: the outer text describes the whole run.
5014        assert_eq!(
5015            text(
5016                "BT /F1 24 Tf 40 50 Td /Span <</ActualText (AB)>> BDC (a) Tj \
5017                  /Span <</ActualText (Z)>> BDC (b) Tj EMC EMC ET"
5018            ),
5019            "AB"
5020        );
5021        // A span nested past the depth limit keeps its glyphs.
5022        let nested = |depth: usize| {
5023            text(&format!(
5024                "BT /F1 24 Tf 40 50 Td {}/Span <</ActualText (z)>> BDC (a) Tj EMC{} ET",
5025                "/X BMC ".repeat(depth),
5026                " EMC".repeat(depth)
5027            ))
5028        };
5029        assert_eq!(nested(super::MAX_ACTUAL_TEXT_DEPTH - 1), "z");
5030        assert_eq!(nested(super::MAX_ACTUAL_TEXT_DEPTH), "a");
5031    }
5032
5033    /// One for one, each character keeps its cell's box, and a cell of
5034    /// several glyphs on one pen (a ligature's) becomes its first glyph.
5035    #[test]
5036    fn actual_text_one_for_one_keeps_each_cells_box() {
5037        let glyph = |ch, x: f32, pen: f32| super::Glyph {
5038            ch,
5039            l: x,
5040            b: 0.0,
5041            r: x + 5.0,
5042            t: 10.0,
5043            ll: x,
5044            lb: -2.0,
5045            lr: x + 5.0,
5046            lt: 10.0,
5047            font: 0,
5048            quad: None,
5049            pen: Some([pen, 0.0, 10.0]),
5050            math_font: false,
5051            invisible: false,
5052        };
5053        let mut out = vec![
5054            glyph('o', 0.0, 0.0),
5055            glyph('f', 10.0, 10.0),
5056            glyph('i', 15.0, 10.0),
5057            glyph('a', 20.0, 20.0),
5058        ];
5059        super::apply_actual_text(&mut out, 1, "XY");
5060        let got: Vec<(char, f32)> = out.iter().map(|g| (g.ch, g.l)).collect();
5061        assert_eq!(got, [('o', 0.0), ('X', 10.0), ('Y', 20.0)]);
5062    }
5063
5064    /// The identity follows docling-parse's precedence: a `/FontDescriptor/FontName`
5065    /// wins over `/BaseFont`, so two dictionaries that share a `/BaseFont` but
5066    /// describe different programs are two fonts, and two that share the
5067    /// descriptor name are one whatever their `/BaseFont` says.
5068    #[test]
5069    fn math_faces_are_named_like_docling_parse_names_them() {
5070        for name in [
5071            "ABCDEF+CMR7",
5072            "CMMI10",
5073            "MSBM10",
5074            "Cambria-Math",
5075            "Symbol",
5076            "MTExtra",
5077        ] {
5078            assert!(super::is_math_font_name(name.as_bytes()), "{name}");
5079        }
5080        for name in ["Helvetica", "ABCDEF+TimesNewRomanPSMT", "Arial-BoldMT"] {
5081            assert!(!super::is_math_font_name(name.as_bytes()), "{name}");
5082        }
5083    }
5084
5085    #[test]
5086    fn sanitizer_font_name_follows_docling_parse_precedence() {
5087        use lopdf::{dictionary, Document, Object};
5088        let mut doc = Document::with_version("1.4");
5089        let fd_a =
5090            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "AAAAAA+Face" });
5091        let fd_b =
5092            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "BBBBBB+Face" });
5093        let with = |fd: Option<lopdf::ObjectId>, base: &str| {
5094            let mut d = dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => base };
5095            if let Some(id) = fd {
5096                d.set("FontDescriptor", Object::Reference(id));
5097            }
5098            d
5099        };
5100        let name = |d: &lopdf::Dictionary| {
5101            String::from_utf8(super::sanitizer_font_name(&doc, d, false)).unwrap()
5102        };
5103        assert_eq!(name(&with(Some(fd_a), "Face")), "AAAAAA+Face");
5104        assert_eq!(name(&with(Some(fd_b), "Face")), "BBBBBB+Face");
5105        assert_eq!(name(&with(None, "ABCDEF+Face")), "ABCDEF+Face");
5106        assert_eq!(
5107            name(&dictionary! { "Type" => "Font", "Name" => "F7" }),
5108            "F7"
5109        );
5110        assert_eq!(name(&dictionary! { "Type" => "Font" }), "null");
5111    }
5112}
5113
5114#[cfg(test)]
5115mod overpainted {
5116    use crate::pdfium_backend::TextCell;
5117
5118    fn cell(text: &str, l: f32, t: f32, r: f32, b: f32) -> TextCell {
5119        TextCell {
5120            text: text.into(),
5121            l,
5122            t,
5123            r,
5124            b,
5125        }
5126    }
5127
5128    /// The reporting invoice's logo: a `"` painted inside a `==` on one band —
5129    /// artwork drawn with glyphs. Both cells go; the real text on the next
5130    /// band stays.
5131    #[test]
5132    fn stacked_logo_glyphs_are_dropped() {
5133        let mut cells = vec![
5134            cell("\"", 72.7, 21.5, 86.4, 31.5),
5135            cell("==", 59.4, 21.5, 99.6, 31.5),
5136            cell("Herr", 65.2, 151.3, 81.7, 161.3),
5137        ];
5138        super::drop_overpainted_cells(&mut cells);
5139        assert_eq!(cells.len(), 1, "cells: {cells:?}");
5140        assert_eq!(cells[0].text, "Herr");
5141    }
5142
5143    /// Adjacent words on a line touch but never contain each other — prose is
5144    /// untouched, and so is a same-text near-duplicate (double-drawn faux
5145    /// bold), which is not evidence of artwork.
5146    #[test]
5147    fn prose_and_double_draw_are_kept() {
5148        let mut cells = vec![
5149            cell("Telefon", 354.3, 133.2, 381.5, 143.2),
5150            cell("0676/2000", 387.3, 133.2, 428.7, 143.2),
5151            cell("Bold", 100.0, 50.0, 130.0, 60.0),
5152            cell("Bold", 100.3, 50.0, 130.3, 60.0),
5153        ];
5154        super::drop_overpainted_cells(&mut cells);
5155        assert_eq!(cells.len(), 4);
5156    }
5157}
5158
5159#[cfg(test)]
5160mod vestigial_layer {
5161    use crate::pdfium_backend::{PdfPage, TextCell};
5162
5163    fn page_with(texts: &[&str]) -> PdfPage {
5164        let cells = texts
5165            .iter()
5166            .enumerate()
5167            .map(|(i, t)| TextCell {
5168                text: t.to_string(),
5169                l: 10.0,
5170                t: 10.0 + 12.0 * i as f32,
5171                r: 90.0,
5172                b: 20.0 + 12.0 * i as f32,
5173            })
5174            .collect();
5175        PdfPage::from_cells(595.0, 842.0, 1.0, cells)
5176    }
5177
5178    /// The reported scanned form: three typed-in field values ("03", "05",
5179    /// "2025") over three image pages. That must read as *no usable layer*,
5180    /// so the browser routes the document to OCR instead of extracting
5181    /// thirteen characters and skipping the letter entirely.
5182    #[test]
5183    fn typed_in_form_fields_are_not_a_text_layer() {
5184        let pages = vec![
5185            page_with(&["03", "05", "2025"]),
5186            page_with(&[]),
5187            page_with(&[]),
5188        ];
5189        assert!(super::text_layer_is_vestigial(&pages));
5190        assert!(super::text_layer_is_vestigial(&[page_with(&[])]));
5191    }
5192
5193    /// A short but genuine digital document — one page, a few real lines —
5194    /// keeps the fast text path.
5195    #[test]
5196    fn sparse_but_real_documents_pass() {
5197        let one_pager = vec![page_with(&[
5198            "Confidential briefing",
5199            "Prepared for the board meeting",
5200            "Do not distribute",
5201        ])];
5202        assert!(!super::text_layer_is_vestigial(&one_pager));
5203    }
5204}
5205
5206/// #994: decoded Form XObject content is cached per document only from a
5207/// form's second sighting, so one-form-per-page documents do not hold every
5208/// page's decoded form until the document ends.
5209#[cfg(test)]
5210mod form_cache {
5211    use lopdf::content::{Content, Operation};
5212    use lopdf::{dictionary, Document, Object, Stream};
5213
5214    fn form(doc: &mut Document, text: &str) -> lopdf::ObjectId {
5215        let ops = vec![
5216            Operation::new("BT", vec![]),
5217            Operation::new("Tf", vec!["F1".into(), 12.into()]),
5218            Operation::new("Td", vec![40.into(), 500.into()]),
5219            Operation::new("Tj", vec![Object::string_literal(text)]),
5220            Operation::new("ET", vec![]),
5221        ];
5222        let data = Content { operations: ops }.encode().unwrap();
5223        doc.add_object(Stream::new(
5224            dictionary! { "Type" => "XObject", "Subtype" => "Form",
5225            "BBox" => vec![0.into(), 0.into(), 595.into(), 842.into()] },
5226            data,
5227        ))
5228    }
5229
5230    #[test]
5231    fn single_use_forms_are_not_retained_shared_ones_are() {
5232        let mut doc = Document::with_version("1.5");
5233        let pages_id = doc.new_object_id();
5234        let font = doc.add_object(
5235            dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => "Helvetica" },
5236        );
5237        let header = form(&mut doc, "Header");
5238        let mut kids = Vec::new();
5239        let mut own_forms = Vec::new();
5240        for p in 0..3 {
5241            let own = form(&mut doc, &format!("Body{p}"));
5242            own_forms.push(own);
5243            let content = doc.add_object(Stream::new(
5244                dictionary! {},
5245                b"q /Hd Do Q q /Fx Do Q".to_vec(),
5246            ));
5247            kids.push(Object::Reference(doc.add_object(dictionary! {
5248                "Type" => "Page", "Parent" => pages_id,
5249                "MediaBox" => vec![0.into(), 0.into(), 595.into(), 842.into()],
5250                "Contents" => content,
5251                "Resources" => dictionary! {
5252                    "Font" => dictionary! { "F1" => font },
5253                    "XObject" => dictionary! { "Hd" => header, "Fx" => own },
5254                },
5255            })));
5256        }
5257        doc.objects.insert(
5258            pages_id,
5259            Object::Dictionary(dictionary! { "Type" => "Pages", "Kids" => kids, "Count" => 3 }),
5260        );
5261        let catalog = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages_id });
5262        doc.trailer.set("Root", catalog);
5263
5264        let mut caches = super::DocCaches::default();
5265        let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
5266        pages.sort_by_key(|(n, _)| *n);
5267        for (i, (_, pid)) in pages.iter().enumerate() {
5268            let text: String = super::page_glyphs_cached(&doc, *pid, &mut caches)
5269                .iter()
5270                .map(|g| g.ch)
5271                .collect();
5272            // Both forms' text is still read, cached or not.
5273            assert!(text.contains("Header"), "page {i}: {text:?}");
5274            assert!(text.contains(&format!("Body{i}")), "page {i}: {text:?}");
5275        }
5276        // The shared header form is cached from its second page on; the
5277        // per-page forms, each used once, are not held.
5278        assert!(caches.forms.contains_key(&header));
5279        for own in &own_forms {
5280            assert!(!caches.forms.contains_key(own));
5281        }
5282        assert_eq!(caches.forms.len(), 1);
5283    }
5284}
5285
5286#[cfg(test)]
5287mod cid_collections {
5288    use super::base14_fonts::{cells, pdf_with_content};
5289
5290    fn type0(encoding: &str) -> Vec<u8> {
5291        format!(
5292            "<</Type/Font/Subtype/Type0/BaseFont/Test-Font-Light/Encoding/{encoding}\
5293             /DescendantFonts[<</Type/Font/Subtype/CIDFontType0/BaseFont/Test-Font-Light\
5294             /CIDSystemInfo<</Registry(Adobe)/Ordering(Japan1)/Supplement 7>>/DW 1000>>]>>"
5295        )
5296        .into_bytes()
5297    }
5298
5299    fn text(fontdict: &[u8], hex: &str) -> String {
5300        let content = format!("BT /F1 12 Tf 72 700 Td <{hex}> Tj ET\n");
5301        cells(&pdf_with_content(fontdict, content.as_bytes()))
5302            .iter()
5303            .map(|c| c.text.as_str())
5304            .collect()
5305    }
5306
5307    #[test]
5308    #[cfg(feature = "cid-collections")]
5309    fn identity_codes_read_through_the_collection() {
5310        assert_eq!(text(&type0("Identity-H"), "0CD40E8A07A0"), "日本語");
5311    }
5312
5313    #[test]
5314    fn unicode_keyed_cmaps_read_the_code() {
5315        assert_eq!(text(&type0("UniJIS-UCS2-H"), "65E5672C8A9E"), "日本語");
5316    }
5317
5318    #[test]
5319    #[cfg(feature = "cid-collections")]
5320    fn collection_ranges_keep_supplementary_characters() {
5321        assert_eq!(text(&type0("Identity-H"), "0CD44507450807A0"), "日𢭏𢭐語");
5322        let text = |o: &[u8], cid| super::collection_text(super::collection_table(o).unwrap(), cid);
5323        assert_eq!(text(b"Japan1", 0x2714).as_deref(), Some("🄐"));
5324        assert_eq!(text(b"CNS1", 0x3a32).as_deref(), Some("𡤃"));
5325    }
5326
5327    #[test]
5328    #[cfg(feature = "cid-collections")]
5329    fn collection_entries_are_plain_characters() {
5330        assert_eq!(text(&type0("Identity-H"), "046D"), "逢");
5331        let text = |o: &[u8], cid| super::collection_text(super::collection_table(o).unwrap(), cid);
5332        assert_eq!(text(b"Korea1", 8752), None);
5333        assert_eq!(text(b"KR", 11704), None);
5334        assert_eq!(text(b"Japan1", 0x1ffd).as_deref(), Some("sec"));
5335    }
5336
5337    #[test]
5338    fn tounicode_ranges_increment_the_utf16_string() {
5339        let map = super::parse_tounicode(
5340            b"3 beginbfrange\n\
5341              <01> <02> <d842dfb7>\n\
5342              <03> <04> <00660069>\n\
5343              <05> <06> <0061>\n\
5344              endbfrange",
5345        );
5346        assert_eq!(map[&1], "𠮷");
5347        assert_eq!(map[&2], "𠮸");
5348        assert_eq!(map[&3], "fi");
5349        assert_eq!(map[&4], "fj");
5350        assert_eq!(map[&5], "a");
5351        assert_eq!(map[&6], "b");
5352    }
5353
5354    #[test]
5355    fn tounicode_skips_lone_surrogates_overflows_and_huge_ranges() {
5356        let map = super::parse_tounicode(
5357            b"2 beginbfchar\n<01> <d842>\n<02> <0041>\nendbfchar\n\
5358              4 beginbfrange\n\
5359              <10> <11> [<dc00> <0042>]\n\
5360              <20> <22> <0041fffe>\n\
5361              <30> <31> <d842dfff>\n\
5362              <00010000> <ffffffff> <>\n\
5363              endbfrange",
5364        );
5365        assert!(!map.contains_key(&0x01));
5366        assert_eq!(map[&0x02], "A");
5367        assert!(!map.contains_key(&0x10));
5368        assert_eq!(map[&0x11], "B");
5369        assert_eq!(map[&0x20], "A\u{fffe}");
5370        assert_eq!(map[&0x21], "A\u{ffff}");
5371        assert!(!map.contains_key(&0x22));
5372        assert_eq!(map[&0x30], "\u{20bff}");
5373        assert!(!map.contains_key(&0x31));
5374        assert_eq!(map[&0x1ffff], "");
5375        assert_eq!(map.len(), 5 + 0x10000);
5376    }
5377
5378    #[test]
5379    fn utf16_pairs_are_one_glyph_and_one_advance() {
5380        let font = type0("UniJIS-UTF16-H");
5381        assert_eq!(text(&font, "65E5D842DFB78A9E"), "日𠮷語");
5382        let content = b"BT /F1 12 Tf 72 700 Td <65E5D842DFB78A9E> Tj ET\n";
5383        let pdf = pdf_with_content(&font, content);
5384        let doc = lopdf::Document::load_mem(&pdf).unwrap();
5385        let glyphs = super::page_glyphs(&doc, doc.get_pages()[&1]);
5386        assert_eq!(glyphs.len(), 3);
5387        assert!((glyphs[2].l - glyphs[0].l - 24.0).abs() < 0.001);
5388    }
5389
5390    #[test]
5391    fn utf16_codes_preserve_tounicode_keys_and_identity_cids() {
5392        use lopdf::{dictionary, Document};
5393        let doc = Document::with_version("1.4");
5394        let dict = dictionary! {"Subtype" => "Type0", "Encoding" => "UniJIS-UTF16-H"};
5395        let mut font = super::parse_font(&doc, b"F1", &dict);
5396        assert_eq!(
5397            super::codes(&font, &[0xd8, 0x42, 0xdf, 0xb7]),
5398            vec![0xd842dfb7]
5399        );
5400        font.to_unicode.insert(0xd842dfb7, "override".into());
5401        assert_eq!(font.decode_code(0xd842dfb7).0.as_deref(), Some("override"));
5402        assert_eq!(
5403            super::codes(&font, &[0xd8, 0x42, 0x65, 0xe5]),
5404            vec![0xd842, 0x65e5]
5405        );
5406        let dict = dictionary! {"Subtype" => "Type0", "Encoding" => "Identity-H"};
5407        let font = super::parse_font(&doc, b"F1", &dict);
5408        assert_eq!(
5409            super::codes(&font, &[0xd8, 0x42, 0xdf, 0xb7]),
5410            vec![0xd842, 0xdfb7]
5411        );
5412    }
5413}