Skip to main content

docling_pdf/
textparse.rs

1//! Pure-Rust PDF text extraction (replacing pdfium's glyph layer).
2//!
3//! pdfium reports *rendered* glyph boxes, which diverge from docling's
4//! `docling-parse` C++ parser at exactly the points that drive conformance:
5//! generated spaces get a zero-width box, combining diacritics get a real-width
6//! box, and ligature/fraction glyphs land at different x. This module instead
7//! reconstructs each glyph's box from the **font's own advance widths** and the
8//! PDF text/graphics matrices — the same information docling-parse uses — so a
9//! space is as wide as the font says and a combining mark has zero advance.
10//!
11//! The output is the same [`Glyph`] stream pdfium produces (native PDF
12//! coordinates, y-up), fed straight into the existing docling-parse line
13//! sanitizer ([`crate::dp_lines`]). Only the digital text layer is handled here;
14//! pages without one still fall back to OCR upstream.
15
16use std::collections::{HashMap, HashSet};
17use std::sync::Arc;
18
19use lopdf::{Dictionary, Document, Object};
20
21use crate::pdfium_backend::Glyph;
22use crate::tounicode::parse_tounicode;
23
24/// Per-document caches for the content-stream interpreter. Fonts are indirect
25/// objects shared by many pages, but were fully re-parsed — ToUnicode CMap
26/// decompression + tokenization, embedded Type1 program scan, width tables —
27/// for **every page and every Form XObject invocation**; decoded form content
28/// streams were likewise re-inflated on every `Do`. Cached per document,
29/// keyed by the referenced object id (fonts also by resource name — the
30/// `/Font` entry a page binds the object to). Inline (non-reference) dicts
31/// are rare and stay uncached.
32#[derive(Default)]
33struct DocCaches {
34    fonts: HashMap<(lopdf::ObjectId, Vec<u8>), Arc<Font>>,
35    forms: HashMap<lopdf::ObjectId, Arc<lopdf::content::Content>>,
36    /// Forms decoded once and not (yet) cached (#994). A form enters
37    /// [`Self::forms`] on its *second* sighting: a shared header/footer form
38    /// is decoded twice and then hits the cache on every later page, while
39    /// a generator's one-form-per-page body (each page's chart in its own
40    /// form, used once) is dropped with its page instead of every decoded
41    /// `Content` — many times its stream's size — living until the document
42    /// ends. Caching on first use made peak RSS grow ~45 MB per page on such
43    /// files (7 GiB at 60 pages of the issue's repro, flat ~0.6 GiB now).
44    /// Only ids are kept here, so the set itself stays tiny.
45    forms_seen: HashSet<lopdf::ObjectId>,
46}
47
48/// A 2×3 affine matrix `[a b c d e f]`: maps `(x,y)` → `(a·x+c·y+e, b·x+d·y+f)`.
49#[derive(Clone, Copy)]
50struct Mat {
51    a: f64,
52    b: f64,
53    c: f64,
54    d: f64,
55    e: f64,
56    f: f64,
57}
58
59impl Mat {
60    const ID: Mat = Mat {
61        a: 1.0,
62        b: 0.0,
63        c: 0.0,
64        d: 1.0,
65        e: 0.0,
66        f: 0.0,
67    };
68
69    /// `self ∘ m`: the matrix that applies `self` first, then `m`.
70    fn then(self, m: Mat) -> Mat {
71        Mat {
72            a: self.a * m.a + self.b * m.c,
73            b: self.a * m.b + self.b * m.d,
74            c: self.c * m.a + self.d * m.c,
75            d: self.c * m.b + self.d * m.d,
76            e: self.e * m.a + self.f * m.c + m.e,
77            f: self.e * m.b + self.f * m.d + m.f,
78        }
79    }
80
81    fn apply(self, x: f64, y: f64) -> (f64, f64) {
82        (
83            self.a * x + self.c * y + self.e,
84            self.b * x + self.d * y + self.f,
85        )
86    }
87}
88
89/// A parsed font: how to turn raw string bytes into (unicode, advance) pairs.
90struct Font {
91    /// 2-byte codes (Type0 / Identity-H) vs 1-byte (simple fonts).
92    two_byte: bool,
93    /// code → Unicode string (from ToUnicode; may be multi-char, e.g. ligatures).
94    to_unicode: HashMap<u32, String>,
95    /// For codes ToUnicode lacks (ISO 32000-1, 9.10.2).
96    cid_fallback: CidFallback,
97    /// code → glyph advance, in 1000-unit glyph space.
98    widths: HashMap<u32, f64>,
99    default_width: f64,
100    /// 1-byte fallback decoding when ToUnicode lacks a code (WinAnsi-ish).
101    simple_encoding: Option<HashMap<u8, char>>,
102    /// code → raw `/Differences` glyph name, for placeholder names (`g115`,
103    /// `SM590000`) that have no Unicode mapping; they override the base
104    /// encoding (see [`unknown_name_text`]).
105    fallback_names: HashMap<u8, String>,
106    /// The unmapped codes whose embedded glyph paints nothing — semantic
107    /// spaces (docling-parse 7.24's blank glyph programs: the redp5110
108    /// list's `/g3`). Every other unmapped code reads U+FFFD.
109    blank_codes: HashSet<u32>,
110    /// code → char from the embedded Type1 font program's own `/Encoding` vector,
111    /// used only as a last resort for glyphs the base encoding leaves unmapped
112    /// (standard TeX math fonts: `λ`, `≤`, …).
113    program_encoding: HashMap<u8, char>,
114    ascent: f64,
115    descent: f64,
116    hash: u64,
117    /// Whether the font's docling-parse name marks a math face
118    /// ([`is_math_font_name`]) — for the word contraction's local
119    /// attachments (#921).
120    math: bool,
121    /// Weight and slant read from the `/BaseFont` name (the heading
122    /// hierarchy's style signal, #302).
123    style: crate::font_style::FontStyle,
124}
125
126impl Font {
127    /// The text of `code` and its advance; the flag marks a blank unmapped
128    /// glyph, whose box has no height.
129    ///
130    /// A code no source maps to Unicode is visibly painted but carries no
131    /// text: docling-parse 7.24 emits U+FFFD for it, and docling turns a NUL
132    /// into U+FFFD too (docling#5070), so the loss stays visible and counted
133    /// by the text-quality check instead of silently dropping the glyph
134    /// (`mL⁻¹` reading `mL1`). A glyph whose embedded program paints nothing
135    /// is a space — a word boundary — instead.
136    fn decode_code(&self, code: u32) -> (Option<String>, f64, bool) {
137        let w = self
138            .widths
139            .get(&code)
140            .copied()
141            .unwrap_or(self.default_width);
142        if let Some(s) = self.to_unicode.get(&code) {
143            return (Some(decompose_ligatures(s)), w, false);
144        }
145        let unmapped = |font: &Font| {
146            let blank = font.blank_codes.contains(&code);
147            (
148                Some(if blank { " " } else { "\u{FFFD}" }.to_string()),
149                w,
150                blank,
151            )
152        };
153        if !self.two_byte {
154            // A placeholder `/Differences` name (no Unicode) overrides the
155            // base encoding. docling-parse 7.24 reads a bare glyph index
156            // (`g115`, `gid00043`) as unmapped and any other unknown name as
157            // its text (`SM590000`, the redp5110 bullets).
158            if let Some(name) = self.fallback_names.get(&(code as u8)) {
159                return match unknown_name_text(name) {
160                    Some(text) => (Some(text), w, false),
161                    None => unmapped(self),
162                };
163            }
164            if let Some(enc) = &self.simple_encoding {
165                if let Some(&ch) = enc.get(&(code as u8)) {
166                    return (Some(decompose_ligatures(&ch.to_string())), w, false);
167                }
168            }
169            // Last resort: the embedded Type1 font program's own `/Encoding`
170            // vector (`dup N /glyphname put`). Standard TeX math fonts (CMMI, CMSY,
171            // …) ship no PDF `/Encoding` and no ToUnicode, so a glyph like `λ`
172            // (CMMI code 21 → `/lambda`) or `≤` (CMSY code 20 → `/lessequal`) has
173            // no other mapping and would otherwise be silently dropped. docling
174            // recovers these from the same font program. This only fills codes the
175            // base encoding left unmapped, so it never changes an existing decode.
176            if let Some(&ch) = self.program_encoding.get(&(code as u8)) {
177                return (Some(decompose_ligatures(&ch.to_string())), w, false);
178            }
179        } else {
180            match self.cid_fallback {
181                CidFallback::None => {}
182                CidFallback::UnicodeCodes { .. } => {
183                    if let Some(ch) = unicode_cmap_char(code).filter(|c| !c.is_control()) {
184                        return (Some(ch.to_string()), w, false);
185                    }
186                }
187                CidFallback::Collection(table) => {
188                    if let Some(s) = collection_text(table, code) {
189                        return (Some(decompose_ligatures(&s)), w, false);
190                    }
191                }
192            }
193        }
194        unmapped(self)
195    }
196}
197
198/// Spell out Latin presentation-form ligatures (`fi`→`fi`, `ffi`→`ffi`, …) the way
199/// docling does, so `configuration`/`difficult` don't keep the ligature glyph.
200/// The chars share the ligature's box, so the line sanitizer recomposes them.
201fn decompose_ligatures(s: &str) -> String {
202    if !s.chars().any(|c| ('\u{FB00}'..='\u{FB06}').contains(&c)) {
203        return s.to_string();
204    }
205    s.chars()
206        .map(|c| {
207            match c {
208                '\u{FB00}' => "ff",
209                '\u{FB01}' => "fi",
210                '\u{FB02}' => "fl",
211                '\u{FB03}' => "ffi",
212                '\u{FB04}' => "ffl",
213                // U+FB05 is LATIN SMALL LIGATURE LONG S T (`ſt`): `st`, as
214                // docling's ligature map and Unicode's decomposition (#1001).
215                '\u{FB05}' => "st",
216                '\u{FB06}' => "st",
217                _ => return c.to_string(),
218            }
219            .to_string()
220        })
221        .collect()
222}
223
224fn hash_name(name: &[u8]) -> u64 {
225    use std::hash::{Hash, Hasher};
226    let mut h = std::collections::hash_map::DefaultHasher::new();
227    name.hash(&mut h);
228    h.finish()
229}
230
231/// Resolve a possibly-indirect object to a dictionary.
232fn as_dict<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Dictionary> {
233    match obj {
234        Object::Dictionary(d) => Some(d),
235        Object::Reference(id) => doc.get_object(*id).ok().and_then(|o| o.as_dict().ok()),
236        _ => None,
237    }
238}
239
240fn deref<'a>(doc: &'a Document, obj: &'a Object) -> Option<&'a Object> {
241    match obj {
242        Object::Reference(id) => doc.get_object(*id).ok(),
243        other => Some(other),
244    }
245}
246
247/// The font identity the line sanitizer's `enforce_same_font` compares —
248/// docling-parse's `page_font::init_font_name`: `/FontDescriptor/FontName`
249/// (the descendant CIDFont's for a Type0), else `/Name`, else `/BaseFont`
250/// (its `base_font`), else the literal `"null"`. The resource key (`/F1`,
251/// `/F2`) is deliberately **not** part of it: a producer that references one
252/// face twice — the same Helvetica under two keys, one re-encoded so a code
253/// maps to the `fi` glyph — draws one word across the two resources, and
254/// docling-parse reads both as the same font. Keyed by resource name (as
255/// this port was until #728) the `fi` ligature, which bridges fonts, glued
256/// itself to the preceding run while the font barrier then cut it off from
257/// its own word (`fi nancial`); docling's `financial` needs the two
258/// resources to compare equal.
259/// docling-parse's `has_math_positioning_semantics`, the font-name half: a
260/// TeX math or symbol face (`cmmi`, `cmsy`, `cmex`, `cmr`, `cmbx`, `msam`,
261/// `msbm`) or any name saying `math`, `symbol` or `mtextra`.
262fn is_math_font_name(name: &[u8]) -> bool {
263    let lower = name.to_ascii_lowercase();
264    [
265        "cmmi", "cmsy", "cmex", "cmr", "cmbx", "msam", "msbm", "math", "symbol", "mtextra",
266    ]
267    .iter()
268    .any(|t| lower.windows(t.len()).any(|w| w == t.as_bytes()))
269}
270
271fn sanitizer_font_name(doc: &Document, fdict: &Dictionary, two_byte: bool) -> Vec<u8> {
272    let descriptor_font_name = |owner: &Dictionary| {
273        owner
274            .get(b"FontDescriptor")
275            .ok()
276            .and_then(|o| as_dict(doc, o))
277            .and_then(|fd| fd.get(b"FontName").ok())
278            .and_then(|o| deref(doc, o))
279            .and_then(|o| o.as_name().ok().map(<[u8]>::to_vec))
280    };
281    if let Some(n) = descriptor_font_name(fdict) {
282        return n;
283    }
284    if two_byte {
285        let descendant = fdict
286            .get(b"DescendantFonts")
287            .ok()
288            .and_then(|o| deref(doc, o))
289            .and_then(|o| match o {
290                Object::Array(a) => a.first(),
291                _ => None,
292            })
293            .and_then(|o| as_dict(doc, o));
294        if let Some(n) = descendant.and_then(descriptor_font_name) {
295            return n;
296        }
297    }
298    if let Some(n) = fdict
299        .get(b"Name")
300        .ok()
301        .and_then(|o| deref(doc, o))
302        .and_then(|o| o.as_name().ok())
303    {
304        return n.to_vec();
305    }
306    // docling-parse's `base_font` keeps the subset prefix (`ABCDEF+Helvetica`):
307    // two subsets of one face are two fonts there, and so here.
308    if let Some(n) = fdict.get(b"BaseFont").ok().and_then(|o| o.as_name().ok()) {
309        return n.to_vec();
310    }
311    b"null".to_vec()
312}
313
314/// Parse one font dictionary into a [`Font`].
315fn parse_font(doc: &Document, name: &[u8], fdict: &Dictionary) -> Font {
316    let subtype: &[u8] = fdict
317        .get(b"Subtype")
318        .ok()
319        .and_then(|o| o.as_name().ok())
320        .unwrap_or(&[]);
321    let two_byte = subtype == b"Type0".as_slice();
322
323    let mut to_unicode = fdict
324        .get(b"ToUnicode")
325        .ok()
326        .and_then(|o| deref(doc, o))
327        .and_then(|o| o.as_stream().ok())
328        .and_then(|s| s.decompressed_content().ok())
329        .map(|data| parse_tounicode(&data))
330        .unwrap_or_default();
331    if !two_byte {
332        if let Some(diffs) = differences_array(doc, fdict) {
333            reconcile_stale_tounicode(&mut to_unicode, &diffs);
334        }
335    }
336
337    let cid_fallback = if two_byte {
338        cid_fallback(doc, fdict)
339    } else {
340        CidFallback::None
341    };
342
343    let (mut widths, mut default_width) = if two_byte {
344        cid_widths(doc, fdict)
345    } else {
346        simple_widths(doc, fdict)
347    };
348
349    let simple_encoding = if two_byte {
350        None
351    } else {
352        Some(simple_encoding_table(doc, fdict))
353    };
354
355    // A standard-14 font referenced without an embedded program usually ships
356    // no `/Widths` and no `/FontDescriptor` either (ReportLab's default, #187).
357    // Every advance then resolved to 0, the cells collapsed to zero width, and
358    // the page's whole text layer was silently dropped — while pdfium, with
359    // its built-in metrics, reads the same file fine. Fill the widths from the
360    // built-in Adobe Core 14 AFM tables via the font's own code→char decode
361    // (base encoding + `/Differences`), so an explicit `/Widths` always wins.
362    if !two_byte && widths.is_empty() && default_width == 0.0 {
363        if let Some(std14) = base_font_name(fdict).and_then(|n| crate::std14::widths_for(&n)) {
364            if let Some(enc) = &simple_encoding {
365                for (&code, &ch) in enc {
366                    if let Some(w) = std14.width(ch) {
367                        widths.insert(u32::from(code), w);
368                    }
369                }
370            }
371            // Codes the table misses still advance a typical width instead of
372            // stacking at x=0 (the failure mode this whole branch fixes).
373            default_width = 500.0;
374        }
375    }
376    let fallback_names = if two_byte {
377        HashMap::new()
378    } else {
379        differences_gid_names(doc, fdict)
380    };
381    let program_encoding = if two_byte {
382        HashMap::new()
383    } else {
384        type1_program_encoding(doc, fdict)
385    };
386
387    let (ascent, descent) = font_ascent_descent(doc, fdict, two_byte);
388    let sanitizer_name = sanitizer_font_name(doc, fdict, two_byte);
389    let mut font = Font {
390        two_byte,
391        to_unicode,
392        cid_fallback,
393        widths,
394        default_width,
395        simple_encoding,
396        fallback_names,
397        blank_codes: HashSet::new(),
398        program_encoding,
399        ascent,
400        descent,
401        hash: hash_name(&sanitizer_name),
402        math: is_math_font_name(&sanitizer_name),
403        style: crate::font_style::parse_font_style(&String::from_utf8_lossy(
404            &base_font_name(fdict).unwrap_or_else(|| name.to_vec()),
405        )),
406    };
407    font.blank_codes = blank_codes(doc, fdict, &font);
408    font
409}
410
411/// The codes `font` leaves unmapped whose embedded glyph paints nothing
412/// ([`Font::blank_codes`]). The program is loaded only when the font has
413/// unmapped codes among those it declares widths for; without the
414/// renderer (the wasm text-layer build) there is no program to ask, and
415/// every unmapped glyph reads U+FFFD.
416#[cfg(feature = "render")]
417fn blank_codes(doc: &Document, fdict: &Dictionary, font: &Font) -> HashSet<u32> {
418    // A CID font can declare tens of thousands of widths: past this many
419    // unmapped codes nothing is classified (the glyphs read U+FFFD).
420    const MAX_CANDIDATES: usize = 4096;
421    let unmapped: Vec<u32> = font
422        .widths
423        .keys()
424        .copied()
425        .filter(|&c| font.decode_code(c).0.as_deref() == Some("\u{FFFD}"))
426        .take(MAX_CANDIDATES + 1)
427        .collect();
428    if unmapped.is_empty() || unmapped.len() > MAX_CANDIDATES {
429        return HashSet::new();
430    }
431    let loaded = crate::render::font::LoadedFont::load(doc, fdict);
432    unmapped
433        .into_iter()
434        .filter(|&c| loaded.glyph_has_ink(c) == Some(false))
435        .collect()
436}
437
438#[cfg(not(feature = "render"))]
439fn blank_codes(_doc: &Document, _fdict: &Dictionary, _font: &Font) -> HashSet<u32> {
440    HashSet::new()
441}
442
443#[derive(Clone, Copy)]
444enum CidFallback {
445    None,
446    UnicodeCodes { utf16: bool },
447    Collection(&'static [u8]),
448}
449
450/// A Unicode CMap code retains its original bytes for ToUnicode lookups;
451/// a four-byte UTF-16 code is a packed surrogate pair, not a scalar value.
452pub(crate) fn unicode_cmap_char(code: u32) -> Option<char> {
453    if code <= 0xffff {
454        char::from_u32(code)
455    } else {
456        let units = [(code >> 16) as u16, code as u16];
457        let mut chars = char::decode_utf16(units);
458        let ch = chars.next()?.ok()?;
459        chars.next().is_none().then_some(ch)
460    }
461}
462
463#[cfg(not(feature = "cid-collections"))]
464fn collection_table(_ordering: &[u8]) -> Option<&'static [u8]> {
465    None
466}
467
468#[cfg(feature = "cid-collections")]
469fn collection_table(ordering: &[u8]) -> Option<&'static [u8]> {
470    macro_rules! table {
471        ($file:literal) => {
472            include_bytes!(concat!(env!("OUT_DIR"), "/", $file))
473        };
474    }
475    Some(match ordering {
476        b"Japan1" => table!("Adobe-Japan1-UCS2"),
477        b"GB1" => table!("Adobe-GB1-UCS2"),
478        b"CNS1" => table!("Adobe-CNS1-UCS2"),
479        b"Korea1" => table!("Adobe-Korea1-UCS2"),
480        b"KR" => table!("Adobe-KR-UCS2"),
481        _ => return None,
482    })
483}
484
485/// The text of `cid` in a table build.rs wrote: the CID count, then three
486/// bytes per CID (0 for none, a code point, or 0x110000 + the offset of a
487/// length-prefixed UTF-8 string after the entries).
488pub(crate) fn collection_text(table: &[u8], cid: u32) -> Option<String> {
489    let n = u32::from_le_bytes(table.get(..4)?.try_into().ok()?) as usize;
490    let at = 4 + 3 * usize::try_from(cid).ok().filter(|&c| c < n)?;
491    let &[a, b, c] = table.get(at..at + 3)? else {
492        return None;
493    };
494    match u32::from_be_bytes([0, a, b, c]) {
495        0 => None,
496        v @ ..0x110000 => char::from_u32(v).map(String::from),
497        v => {
498            let at = 4 + 3 * n + (v - 0x110000) as usize;
499            let len = *table.get(at)? as usize;
500            std::str::from_utf8(table.get(at + 1..at + 1 + len)?)
501                .ok()
502                .map(str::to_owned)
503        }
504    }
505}
506
507fn cid_fallback(doc: &Document, fdict: &Dictionary) -> CidFallback {
508    let encoding = fdict
509        .get(b"Encoding")
510        .ok()
511        .and_then(|o| deref(doc, o))
512        .and_then(|o| o.as_name().ok())
513        .unwrap_or(&[]);
514    if encoding.starts_with(b"Uni")
515        && (encoding.windows(4).any(|w| w == b"UCS2") || encoding.windows(5).any(|w| w == b"UTF16"))
516    {
517        return CidFallback::UnicodeCodes {
518            utf16: encoding.windows(5).any(|w| w == b"UTF16"),
519        };
520    }
521    if encoding != b"Identity-H" && encoding != b"Identity-V" {
522        // Other CMaps need code → CID tables that are not bundled.
523        return CidFallback::None;
524    }
525    cid_collection(doc, fdict).map_or(CidFallback::None, CidFallback::Collection)
526}
527
528/// The CID → Unicode table of the character collection a Type0 font uses.
529pub(crate) fn cid_collection(doc: &Document, fdict: &Dictionary) -> Option<&'static [u8]> {
530    let Object::Array(descendants) = deref(doc, fdict.get(b"DescendantFonts").ok()?)? else {
531        return None;
532    };
533    let descendant = as_dict(doc, descendants.first()?)?;
534    let info = as_dict(doc, descendant.get(b"CIDSystemInfo").ok()?)?;
535    let registry = deref(doc, info.get(b"Registry").ok()?)?.as_str().ok()?;
536    let ordering = deref(doc, info.get(b"Ordering").ok()?)?.as_str().ok()?;
537    (registry == b"Adobe").then(|| collection_table(ordering))?
538}
539
540/// A simple font's `/Encoding` `/Differences` array, dereferenced.
541fn differences_array(doc: &Document, fdict: &Dictionary) -> Option<Vec<Object>> {
542    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))?;
543    let diffs = enc.as_dict().ok()?.get(b"Differences").ok()?;
544    match deref(doc, diffs)? {
545        Object::Array(a) => Some(a.clone()),
546        _ => None,
547    }
548}
549
550/// A ToUnicode CMap copied from the *original* font while the glyphs were
551/// re-encoded: the encrypted `2206.01062_pg3` fixture (a Quartz re-save of
552/// pdfTeX output) keeps the T1-encoding CMap, keyed on the old codes, next
553/// to a `/Differences [33 /f_f_i /f_i /f_f /f_l]` that names the ligatures at
554/// new ones, so the CMap maps code 33 to `!` — its own identity — and
555/// `difficulties` read `di ! culties`, in docling's pdfium chain too. A
556/// ToUnicode entry that is the code's identity, for a code whose
557/// `/Differences` name resolves to something else, is that stale copy: the
558/// name wins. A CMap that disagrees any other way is left alone — ToUnicode
559/// is the producer's word on what the glyph means.
560fn reconcile_stale_tounicode(to_unicode: &mut HashMap<u32, String>, diffs: &[Object]) {
561    let mut code = 0u8;
562    for el in diffs {
563        match el {
564            Object::Integer(i) => code = *i as u8,
565            Object::Name(name) => {
566                if let Some(ch) = glyph_name_to_char(name) {
567                    let identity = (code as char).to_string();
568                    if ch.to_string() != identity
569                        && to_unicode.get(&u32::from(code)) == Some(&identity)
570                    {
571                        to_unicode.insert(u32::from(code), ch.to_string());
572                    }
573                }
574                code = code.wrapping_add(1);
575            }
576            _ => {}
577        }
578    }
579}
580
581/// Collect `/Differences` entries whose glyph name is a GID placeholder
582/// (`g115`, `cid42`, `glyph7`, `index9`, `SM590000`) with no Unicode mapping
583/// (subsetted fonts with no ToUnicode); [`unknown_name_text`] decides what
584/// each reads as. The placeholder-name restriction keeps real Adobe glyph
585/// names on the normal path so this never invents garbage on the clean files.
586fn differences_gid_names(doc: &Document, fdict: &Dictionary) -> HashMap<u8, String> {
587    let mut map = HashMap::new();
588    let Some(Object::Dictionary(enc)) = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o))
589    else {
590        return map;
591    };
592    let Some(Object::Array(diffs)) = enc.get(b"Differences").ok().and_then(|o| deref(doc, o))
593    else {
594        return map;
595    };
596    let mut code = 0u8;
597    for el in diffs {
598        match el {
599            Object::Integer(i) => code = *i as u8,
600            Object::Name(name) => {
601                if glyph_name_to_char(name).is_none() && is_gid_name(name) {
602                    map.insert(code, String::from_utf8_lossy(name).into_owned());
603                }
604                code = code.wrapping_add(1);
605            }
606            _ => {}
607        }
608    }
609    map
610}
611
612/// Parse the embedded Type1 font program's built-in `/Encoding` vector
613/// (`dup <code> /<glyphname> put` entries in the clear-text header before
614/// `eexec`) into `code → char`. This is how docling recovers glyphs from
615/// standard TeX math fonts (CMMI/CMSY/…) that carry no PDF `/Encoding` and no
616/// ToUnicode — e.g. CMMI's `dup 21 /lambda` or CMSY's `dup 20 /lessequal`.
617/// Only `FontFile` (Type1) is parsed; CFF (`FontFile3`) and TrueType
618/// (`FontFile2`) store their encoding in a binary table and are left alone.
619fn type1_program_encoding(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
620    let mut map = HashMap::new();
621    let Some(desc) = fdict
622        .get(b"FontDescriptor")
623        .ok()
624        .and_then(|o| deref(doc, o))
625        .and_then(|o| o.as_dict().ok())
626    else {
627        return map;
628    };
629    let Some(data) = desc
630        .get(b"FontFile")
631        .ok()
632        .and_then(|o| deref(doc, o))
633        .and_then(|o| o.as_stream().ok())
634        .and_then(|s| s.decompressed_content().ok())
635    else {
636        return map;
637    };
638    // The clear-text header (PostScript) ends at `eexec`; the rest is encrypted.
639    let head_end = data
640        .windows(5)
641        .position(|w| w == b"eexec")
642        .unwrap_or(data.len());
643    let head = String::from_utf8_lossy(&data[..head_end]);
644    // Scan for `dup <code> /<name> put` tokens.
645    let toks: Vec<&str> = head.split_whitespace().collect();
646    for w in toks.windows(4) {
647        if w[0] == "dup" && w[3] == "put" {
648            if let (Ok(code), Some(name)) = (w[1].parse::<u32>(), w[2].strip_prefix('/')) {
649                if code <= 255 {
650                    if let Some(ch) = glyph_name_to_char(name.as_bytes()) {
651                        map.insert(code as u8, ch);
652                    }
653                }
654            }
655        }
656    }
657    map
658}
659
660/// The reading text docling-parse 7.24 keeps for a `/Differences` glyph name
661/// no table resolves (`resolve_unknown_name`): the name without any
662/// `.suffix` — `None` when that is a bare glyph index
663/// (`(gid|glyph|g|cid|index)\d+`, any case), which names a glyph of the
664/// font program but carries no text (docling-parse#238), so it is unmapped.
665fn unknown_name_text(name: &str) -> Option<String> {
666    let stem = name.rsplit_once('.').map_or(name, |(s, _)| s);
667    let lower = stem.to_ascii_lowercase();
668    let index = ["gid", "glyph", "g", "cid", "index"].iter().any(|p| {
669        lower
670            .strip_prefix(p)
671            .is_some_and(|d| !d.is_empty() && d.bytes().all(|b| b.is_ascii_digit()))
672    });
673    (!index).then(|| stem.to_string())
674}
675
676/// A glyph name that is a synthetic placeholder, not a real Adobe name:
677/// `g115`, `cid42`, `glyph7`, `index9`, `G12`, or a short-prefix code name like
678/// `SM590000` (IBM BookMaster). These carry no Unicode meaning (docling-parse
679/// keeps the code-like names as text, `SM590000`, and reads the glyph indexes
680/// as unmapped, see [`unknown_name_text`]). `afii####` / `uni####` are real Adobe names
681/// and excluded. The restriction keeps genuine glyph names on the Unicode path.
682fn is_gid_name(name: &[u8]) -> bool {
683    let Ok(s) = std::str::from_utf8(name) else {
684        return false;
685    };
686    if s.starts_with("afii") || s.starts_with("uni") {
687        return false;
688    }
689    for prefix in ["g", "G", "cid", "CID", "glyph", "index"] {
690        if let Some(rest) = s.strip_prefix(prefix) {
691            if !rest.is_empty() && rest.bytes().all(|b| b.is_ascii_digit()) {
692                return true;
693            }
694        }
695    }
696    // Short alpha prefix (≤3 letters) followed by a run of ≥3 digits — synthetic
697    // code names like `SM590000`, distinct from real Adobe names (whole words or
698    // letter+`.suffix` variants).
699    let alpha = s.bytes().take_while(|b| b.is_ascii_alphabetic()).count();
700    let digits = s.len() - alpha;
701    (1..=3).contains(&alpha)
702        && digits >= 3
703        && s.as_bytes()[alpha..].iter().all(|b| b.is_ascii_digit())
704}
705
706fn font_ascent_descent(doc: &Document, fdict: &Dictionary, two_byte: bool) -> (f64, f64) {
707    // For Type0, the descriptor lives on the descendant CIDFont.
708    let descr_owner = if two_byte {
709        fdict
710            .get(b"DescendantFonts")
711            .ok()
712            .and_then(|o| deref(doc, o))
713            .and_then(|o| match o {
714                Object::Array(a) => a.first(),
715                _ => None,
716            })
717            .and_then(|o| as_dict(doc, o))
718    } else {
719        Some(fdict)
720    };
721    let fd = descr_owner
722        .and_then(|d| d.get(b"FontDescriptor").ok())
723        .and_then(|o| as_dict(doc, o));
724    let asc = fd
725        .and_then(|d| d.get(b"Ascent").ok())
726        .and_then(|o| {
727            o.as_float()
728                .ok()
729                .or_else(|| o.as_i64().ok().map(|i| i as f32))
730        })
731        .unwrap_or(750.0) as f64;
732    let desc = fd
733        .and_then(|d| d.get(b"Descent").ok())
734        .and_then(|o| {
735            o.as_float()
736                .ok()
737                .or_else(|| o.as_i64().ok().map(|i| i as f32))
738        })
739        .unwrap_or(-250.0) as f64;
740    // Some subsetted fonts carry a degenerate FontDescriptor (`/Ascent 0
741    // /Descent 0`) — the real metrics live in the font program. That collapses
742    // the loose box to zero height, so the line cells get zero area and the
743    // layout's region/text assignment drops them (2305's References list lost
744    // every prose line, keeping only the URLs). Fall back to typical text metrics
745    // so the box has height.
746    if asc - desc <= 1.0 {
747        return (750.0, -250.0);
748    }
749    (asc, desc)
750}
751
752/// The `/BaseFont` name with any `ABCDEF+` subset prefix stripped (#187).
753fn base_font_name(fdict: &Dictionary) -> Option<Vec<u8>> {
754    let name = fdict.get(b"BaseFont").ok()?.as_name().ok()?;
755    let stripped = match name.iter().position(|&b| b == b'+') {
756        Some(i) if i == 6 => &name[i + 1..],
757        _ => name,
758    };
759    Some(stripped.to_vec())
760}
761
762/// Simple-font widths: `/FirstChar` + `/Widths` array, `/MissingWidth` default.
763fn simple_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
764    let mut map = HashMap::new();
765    let first = fdict
766        .get(b"FirstChar")
767        .ok()
768        .and_then(|o| o.as_i64().ok())
769        .unwrap_or(0) as u32;
770    if let Some(Object::Array(arr)) = fdict.get(b"Widths").ok().and_then(|o| deref(doc, o)) {
771        for (i, w) in arr.iter().enumerate() {
772            if let Some(w) = num(w) {
773                map.insert(first + i as u32, w);
774            }
775        }
776    }
777    let dw = fdict
778        .get(b"FontDescriptor")
779        .ok()
780        .and_then(|o| as_dict(doc, o))
781        .and_then(|d| d.get(b"MissingWidth").ok())
782        .and_then(num)
783        .unwrap_or(0.0);
784    (map, dw)
785}
786
787/// CIDFont widths: the `/W` array on the descendant font (`/DW` default = 1000).
788fn cid_widths(doc: &Document, fdict: &Dictionary) -> (HashMap<u32, f64>, f64) {
789    let mut map = HashMap::new();
790    let Some(desc) = fdict
791        .get(b"DescendantFonts")
792        .ok()
793        .and_then(|o| deref(doc, o))
794        .and_then(|o| match o {
795            Object::Array(a) => a.first(),
796            _ => None,
797        })
798        .and_then(|o| as_dict(doc, o))
799    else {
800        return (map, 1000.0);
801    };
802    let dw = desc.get(b"DW").ok().and_then(num).unwrap_or(1000.0);
803    if let Some(Object::Array(w)) = desc.get(b"W").ok().and_then(|o| deref(doc, o)) {
804        let mut i = 0;
805        while i < w.len() {
806            let c = w.get(i).and_then(num);
807            match (c, w.get(i + 1)) {
808                // `c [w1 w2 ...]`: consecutive CIDs starting at c.
809                (Some(c), Some(Object::Array(list))) => {
810                    for (k, wv) in list.iter().enumerate() {
811                        if let Some(wv) = num(wv) {
812                            map.insert(c as u32 + k as u32, wv);
813                        }
814                    }
815                    i += 2;
816                }
817                // `c_first c_last w`: a run all of width w.
818                (Some(c1), Some(o2)) => {
819                    if let (Some(c2), Some(wv)) = (num(o2), w.get(i + 2).and_then(num)) {
820                        for cid in c1 as u32..=c2 as u32 {
821                            map.insert(cid, wv);
822                        }
823                    }
824                    i += 3;
825                }
826                _ => break,
827            }
828        }
829    }
830    (map, dw)
831}
832
833fn num(o: &Object) -> Option<f64> {
834    match o {
835        Object::Integer(i) => Some(*i as f64),
836        Object::Real(r) => Some(*r as f64),
837        _ => None,
838    }
839}
840
841/// Decode a PDF string literal into codes, retaining packed UTF-16 pairs
842/// as the original four-byte keys used by ToUnicode.
843fn codes(font: &Font, bytes: &[u8]) -> Vec<u32> {
844    if font.two_byte {
845        let mut units = bytes
846            .chunks(2)
847            .map(|c| {
848                if c.len() == 2 {
849                    ((c[0] as u32) << 8) | c[1] as u32
850                } else {
851                    c[0] as u32
852                }
853            })
854            .peekable();
855        let utf16 = matches!(font.cid_fallback, CidFallback::UnicodeCodes { utf16: true });
856        let mut out = Vec::with_capacity(bytes.len() / 2 + 1);
857        while let Some(mut code) = units.next() {
858            if utf16 && (0xd800..=0xdbff).contains(&code) {
859                if let Some(low) = units.next_if(|u| (0xdc00..=0xdfff).contains(u)) {
860                    code = (code << 16) | low;
861                }
862            }
863            out.push(code);
864        }
865        out
866    } else {
867        bytes.iter().map(|&b| b as u32).collect()
868    }
869}
870
871/// A page's display box, in PDF user space: the `/CropBox` clipped to the
872/// `/MediaBox`, both inherited through the page tree and normalized — a
873/// missing or empty MediaBox is US Letter, an empty CropBox is the MediaBox
874/// (pdfium's `CPDF_Page::UpdateDimensions`). pdfium reports the page size from
875/// this box, renders exactly it, and translates every content coordinate so
876/// its lower-left corner is the origin (`m_PageMatrix`); docling's backends
877/// inherit that frame, so text cells, `prov` boxes and destinations all count
878/// from the CropBox corner, not the MediaBox one. The parser used to flip
879/// glyphs with the MediaBox *height* and no translation at all, so a page
880/// whose boxes do not start at (0, 0) — a trimmed book page with
881/// `MediaBox [-56 -58 576 723]` / `CropBox [1 -0.6 519 666]`, or a LaTeX
882/// figure cropped to `[156 147 637 391]` — had its text displaced against the
883/// rendered bitmap by the box offset, the bottom lines pushed past the page
884/// edge and clamped to `t = b`.
885#[derive(Debug, Clone, Copy, PartialEq)]
886pub(crate) struct PageBox {
887    /// Left edge, user space.
888    pub l: f32,
889    /// Bottom edge, user space.
890    pub b: f32,
891    pub w: f32,
892    pub h: f32,
893}
894
895impl PageBox {
896    /// Top edge, user space — the y that becomes `0` in the y-down frame.
897    pub fn top(&self) -> f32 {
898        self.b + self.h
899    }
900}
901
902/// A page-tree rect attribute (`/MediaBox`, `/CropBox`), inherited from the
903/// nearest ancestor that sets it, as normalized `(l, b, r, t)`.
904fn inherited_rect(
905    doc: &Document,
906    page_id: lopdf::ObjectId,
907    key: &[u8],
908) -> Option<(f32, f32, f32, f32)> {
909    let mut id = page_id;
910    for _ in 0..32 {
911        let dict = doc.get_object(id).ok()?.as_dict().ok()?;
912        if let Some(Object::Array(a)) = dict.get(key).ok().and_then(|o| deref(doc, o)) {
913            let v: Vec<f32> = a.iter().filter_map(|o| num(o).map(|x| x as f32)).collect();
914            if v.len() == 4 && v.iter().all(|x| x.is_finite()) {
915                return Some((
916                    v[0].min(v[2]),
917                    v[1].min(v[3]),
918                    v[0].max(v[2]),
919                    v[1].max(v[3]),
920                ));
921            }
922            return None;
923        }
924        id = dict.get(b"Parent").ok()?.as_reference().ok()?;
925    }
926    None
927}
928
929pub(crate) fn page_box(doc: &Document, page_id: lopdf::ObjectId) -> PageBox {
930    let nonempty = |r: &(f32, f32, f32, f32)| r.2 > r.0 && r.3 > r.1;
931    let media = inherited_rect(doc, page_id, b"MediaBox")
932        .filter(nonempty)
933        .unwrap_or((0.0, 0.0, 612.0, 792.0));
934    let crop = inherited_rect(doc, page_id, b"CropBox")
935        .map(|c| {
936            (
937                c.0.max(media.0),
938                c.1.max(media.1),
939                c.2.min(media.2),
940                c.3.min(media.3),
941            )
942        })
943        .filter(nonempty)
944        .unwrap_or(media);
945    PageBox {
946        l: crop.0,
947        b: crop.1,
948        w: crop.2 - crop.0,
949        h: crop.3 - crop.1,
950    }
951}
952
953/// Page size (width, height) in PDF points — the display box's, like pdfium's
954/// `FPDF_GetPageWidthF/HeightF`.
955fn page_size(doc: &Document, page_id: lopdf::ObjectId) -> (f32, f32) {
956    let pb = page_box(doc, page_id);
957    (pb.w, pb.h)
958}
959
960/// Localize where a page's text is lost, for the `text_layer` diagnostic.
961/// Extraction can come up empty at three different points — no content stream
962/// reached the parser, the stream did not decode into operators, or it ran but
963/// produced no glyphs (fonts/encodings) — and from the outside all three look
964/// the same. Report them per page.
965pub fn content_diagnosis(bytes: &[u8]) -> String {
966    let Some(doc) = load_document(bytes) else {
967        return "document does not load".into();
968    };
969    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
970    pages.sort_by_key(|(n, _)| *n);
971    let mut out = String::new();
972    let mut caches = DocCaches::default();
973    for (n, pid) in pages.into_iter().take(4) {
974        let content_bytes = doc.get_page_content(pid);
975        let ops = lopdf::content::Content::decode(&content_bytes)
976            .map(|c| c.operations.len())
977            .ok();
978        let res = page_res(&doc, pid);
979        let fonts = res.map(|r| fonts_from_res(&doc, r, &mut caches).len());
980        let glyphs = page_glyphs_cached(&doc, pid, &mut caches).len();
981        out.push_str(&format!(
982            "\n   page {n}: content {} B, ops {}, resources {}, fonts {}, glyphs {}",
983            content_bytes.len(),
984            ops.map_or("UNDECODABLE".to_string(), |n| n.to_string()),
985            if res.is_some() { "ok" } else { "MISSING" },
986            fonts.map_or("-".to_string(), |n| n.to_string()),
987            glyphs,
988        ));
989    }
990    out
991}
992
993/// Is this "text layer" a vestige rather than the document's text?
994///
995/// Scanned forms often carry a handful of typed-in strings — a date filled
996/// into three form fields, say — on top of pages that are otherwise images.
997/// Treating that as a real text layer is the worst of both worlds: the text
998/// path proudly extracts thirteen characters, and no OCR ever runs on the
999/// letter the pages actually show. The reported form did exactly this (3
1000/// lines, 13 chars, 3 pages).
1001///
1002/// The rule is deliberately tight so genuinely sparse *digital* documents are
1003/// not misrouted into OCR: only a document averaging at most one line per page
1004/// **and** totalling fewer than 32 characters is called vestigial.
1005pub fn text_layer_is_vestigial(pages: &[crate::pdfium_backend::PdfPage]) -> bool {
1006    let mut tally = TextLayerTally::default();
1007    pages.iter().for_each(|p| tally.add(p));
1008    tally.is_vestigial()
1009}
1010
1011/// [`text_layer_is_vestigial`]'s counts, accumulated a page at a time so a
1012/// streaming caller need not keep the pages around to decide.
1013#[derive(Debug, Default, Clone, Copy)]
1014pub struct TextLayerTally {
1015    pages: usize,
1016    lines: usize,
1017    chars: usize,
1018}
1019
1020impl TextLayerTally {
1021    pub fn add(&mut self, page: &crate::pdfium_backend::PdfPage) {
1022        self.pages += 1;
1023        self.lines += page.cells.len();
1024        self.chars += page
1025            .cells
1026            .iter()
1027            .map(|c| c.text.chars().count())
1028            .sum::<usize>();
1029    }
1030
1031    /// The verdict over the pages added so far.
1032    pub fn is_vestigial(&self) -> bool {
1033        self.lines == 0 || (self.lines <= self.pages && self.chars < 32)
1034    }
1035
1036    /// Whether the pages added so far already rule vestigial out for a
1037    /// document of `total_pages`, whatever the remaining pages hold — both
1038    /// counts only grow, so a caller may stop parsing pages it does not need.
1039    pub fn proves_text(&self, total_pages: usize) -> bool {
1040        self.lines > 0 && (self.lines > total_pages || self.chars >= 32)
1041    }
1042}
1043
1044/// Why the cross-reference repair did or did not fire, for the `text_layer`
1045/// diagnostic. A PDF that will not load is indistinguishable from a scan in
1046/// production (both convert to nothing), so the reason has to be askable.
1047pub fn xref_repair_status(bytes: &[u8]) -> String {
1048    if Document::load_mem(bytes).is_ok() {
1049        return "loads unaided; no repair needed".into();
1050    }
1051    match pad_short_xref_entries(bytes) {
1052        Ok(fixed) => match Document::load_mem(&fixed) {
1053            Ok(_) => "repaired: cross-reference entries padded to 20 bytes".into(),
1054            Err(e) => format!("padded the entries, but it still will not load: {e}"),
1055        },
1056        Err(why) => format!("repair declined — {why}"),
1057    }
1058}
1059
1060/// Load a PDF, repairing the one malformation that otherwise costs us the whole
1061/// document: **19-byte cross-reference entries**.
1062///
1063/// The spec fixes an xref entry at 20 bytes — `nnnnnnnnnn ggggg n` plus a
1064/// *two*-byte EOL. Some generators (an Austrian telecom's invoices, for one)
1065/// emit a bare LF instead, making each entry 19 bytes. lopdf rejects the file
1066/// outright (`invalid file trailer`) where pdfium reads it happily, so a
1067/// perfectly good text layer looked to the browser exactly like a scan and cost
1068/// ten seconds of OCR.
1069///
1070/// Padding is only attempted when it cannot move anything the xref points at:
1071/// a single `xref` section that begins after the last object. The repair then
1072/// has to prove itself — the padded bytes are used only if they load — so a
1073/// mis-repair degrades to today's behaviour rather than to silent garbage.
1074pub(crate) fn load_document(bytes: &[u8]) -> Option<Document> {
1075    open_document(bytes, None).ok()
1076}
1077
1078/// Why [`open_document`] could not hand back a readable document.
1079#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1080pub(crate) enum OpenError {
1081    /// lopdf cannot read the file even after the repairs.
1082    Unreadable,
1083    /// The file is encrypted and the password given (or none) does not open it.
1084    Password,
1085}
1086
1087/// Load `bytes` with the document's password. lopdf decrypts while reading —
1088/// with `password`, or the empty user password most "protected" PDFs carry
1089/// (what every viewer opens silently) — so every reader downstream sees plain
1090/// streams; a file the password does not open loads with its streams still
1091/// encrypted, which is reported as [`OpenError::Password`] rather than handed
1092/// on as a document whose every stream decodes to nothing.
1093pub(crate) fn open_document(bytes: &[u8], password: Option<&str>) -> Result<Document, OpenError> {
1094    let Some(doc) = load_document_raw(bytes, password) else {
1095        // lopdf refuses to load at all under a *wrong* password (a missing
1096        // one loads the file with its streams still encrypted); tell the two
1097        // apart by loading without it.
1098        if password.is_some() && load_document_raw(bytes, None).is_some_and(|d| d.is_encrypted()) {
1099            return Err(OpenError::Password);
1100        }
1101        return Err(OpenError::Unreadable);
1102    };
1103    if doc.is_encrypted() {
1104        return Err(OpenError::Password);
1105    }
1106    Ok(doc)
1107}
1108
1109fn load_options(password: Option<&str>) -> lopdf::LoadOptions {
1110    lopdf::LoadOptions {
1111        password: password.map(str::to_string),
1112        ..lopdf::LoadOptions::default()
1113    }
1114}
1115
1116fn load_document_raw(bytes: &[u8], password: Option<&str>) -> Option<Document> {
1117    // Try progressively more repair, and accept a candidate only once the pages
1118    // actually carry content — a document whose streams were dropped still
1119    // "loads", so loading alone is not evidence the repair helped. A
1120    // well-formed file returns on the first attempt and pays for nothing.
1121    let mut fallback = None;
1122    if let Some(doc) = best_effort_load(bytes, password, &mut fallback) {
1123        return Some(doc);
1124    }
1125    let xref_fixed = pad_short_xref_entries(bytes).ok();
1126    if let Some(fixed) = &xref_fixed {
1127        if let Some(doc) = best_effort_load(fixed, password, &mut fallback) {
1128            return Some(doc);
1129        }
1130    }
1131    // Both defects can coexist, and the second only becomes visible once the
1132    // first is repaired, so build on whatever the previous step produced.
1133    let lengths_fixed = fix_stream_lengths(xref_fixed.as_deref().unwrap_or(bytes));
1134    if let Some(doc) = best_effort_load(&lengths_fixed, password, &mut fallback) {
1135        return Some(doc);
1136    }
1137    fallback
1138}
1139
1140/// Load `data`, returning it only when its pages carry content; a document that
1141/// merely parses is remembered as the fallback for when nothing does better.
1142fn best_effort_load(
1143    data: &[u8],
1144    password: Option<&str>,
1145    fallback: &mut Option<Document>,
1146) -> Option<Document> {
1147    match Document::load_mem_with_options(data, load_options(password)) {
1148        Ok(doc) if has_page_content(&doc) => Some(doc),
1149        Ok(doc) => {
1150            fallback.get_or_insert(doc);
1151            None
1152        }
1153        Err(_) => None,
1154    }
1155}
1156
1157/// Does any page actually hand us a content stream? A document whose streams
1158/// were dropped still parses — it simply has nothing to read — so this is what
1159/// tells a successful repair from a pointless one.
1160fn has_page_content(doc: &Document) -> bool {
1161    doc.get_pages()
1162        .into_values()
1163        .take(4)
1164        .any(|pid| !doc.get_page_content(pid).is_empty())
1165}
1166
1167/// Correct `/Length` values that disagree with where `endstream` actually is.
1168///
1169/// The same generator that writes short xref entries also overstates its
1170/// content-stream lengths by a byte or two. lopdf trusts `/Length`, reads past
1171/// the data, fails to find `endstream` there and drops the stream — the object
1172/// comes back as a bare dictionary, so the page has no content at all and the
1173/// document looks like a scan. pdfium instead trusts `endstream`, which is what
1174/// this does.
1175///
1176/// The rewrite is length-preserving: the corrected number is written over the
1177/// old digits and padded with spaces, so every byte offset in the file — and
1178/// therefore the whole cross-reference table — stays valid.
1179fn fix_stream_lengths(bytes: &[u8]) -> Vec<u8> {
1180    let mut out = bytes.to_vec();
1181    let mut i = 0;
1182    while let Some(rel) = find(&out[i..], b"stream") {
1183        let kw = i + rel;
1184        i = kw + 6;
1185        // Skip `endstream` (the keyword we are measuring *to*).
1186        if kw >= 3 && &out[kw - 3..kw] == b"end" {
1187            continue;
1188        }
1189        // The stream data starts after the EOL that follows the keyword.
1190        let mut data = kw + 6;
1191        if out.get(data..data + 2) == Some(b"\r\n".as_slice()) {
1192            data += 2;
1193        } else if matches!(out.get(data), Some(b'\n' | b'\r')) {
1194            data += 1;
1195        }
1196        let Some(end) = find(&out[data..], b"endstream").map(|r| data + r) else {
1197            continue;
1198        };
1199        // `/Length <digits>` in the dictionary just before the keyword.
1200        let dict_start = out[..kw].iter().rposition(|&c| c == b'<').unwrap_or(0);
1201        let Some(lrel) = find(&out[dict_start..kw], b"/Length") else {
1202            continue;
1203        };
1204        let mut d = dict_start + lrel + 7;
1205        while matches!(out.get(d), Some(b' ')) {
1206            d += 1;
1207        }
1208        let digits = out[d..].iter().take_while(|c| c.is_ascii_digit()).count();
1209        if digits == 0 {
1210            continue;
1211        }
1212        let declared: usize = match std::str::from_utf8(&out[d..d + digits])
1213            .ok()
1214            .and_then(|s| s.parse().ok())
1215        {
1216            Some(v) => v,
1217            None => continue,
1218        };
1219        let actual = end - data;
1220        // Only shrink, and only when the new value fits the space the old one
1221        // occupied — growing the number would move every following byte.
1222        let replacement = actual.to_string();
1223        if actual == declared || replacement.len() > digits {
1224            continue;
1225        }
1226        out[d..d + digits].fill(b' ');
1227        out[d..d + replacement.len()].copy_from_slice(replacement.as_bytes());
1228    }
1229    out
1230}
1231
1232fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
1233    haystack.windows(needle.len()).position(|w| w == needle)
1234}
1235
1236/// Rewrite a classic cross-reference table's entries to the spec's 20 bytes,
1237/// or `None` when the file's shape makes that unsafe (see [`load_document`]).
1238pub(crate) fn pad_short_xref_entries(bytes: &[u8]) -> Result<Vec<u8>, &'static str> {
1239    // Exactly one xref section, and it must start after every object, so that
1240    // growing it shifts nothing the table's offsets refer to.
1241    let is_boundary = |i: usize| i == 0 || matches!(bytes[i - 1], b'\n' | b'\r');
1242    let mut starts = (0..bytes.len().saturating_sub(4))
1243        .filter(|&i| &bytes[i..i + 4] == b"xref" && is_boundary(i));
1244    let xref_at = starts
1245        .next()
1246        .ok_or("no classic `xref` section (an xref stream?)")?;
1247    if starts.next().is_some() {
1248        return Err("more than one xref section (incremental update)");
1249    }
1250    let last_obj = bytes
1251        .windows(3)
1252        .rposition(|w| w == b"obj")
1253        .ok_or("no objects found")?;
1254    if last_obj > xref_at {
1255        return Err("an object follows the xref — padding would move it");
1256    }
1257
1258    let mut out = bytes[..xref_at].to_vec();
1259    out.extend_from_slice(b"xref\n");
1260    let mut i = xref_at + 4;
1261    let skip_ws = |i: &mut usize| {
1262        while matches!(bytes.get(*i), Some(b'\r' | b'\n' | b' ')) {
1263            *i += 1;
1264        }
1265    };
1266    loop {
1267        skip_ws(&mut i);
1268        // Either the next subsection header ("first count") or the trailer.
1269        if bytes[i..].starts_with(b"trailer") {
1270            out.extend_from_slice(&bytes[i..]);
1271            return Ok(out);
1272        }
1273        let header_end = i + bytes[i..]
1274            .iter()
1275            .position(|c| matches!(c, b'\n' | b'\r'))
1276            .ok_or("subsection header runs off the end")?;
1277        let header = std::str::from_utf8(&bytes[i..header_end])
1278            .map_err(|_| "subsection header is not text")?
1279            .trim();
1280        let mut parts = header.split_whitespace();
1281        let count: usize = parts
1282            .nth(1)
1283            .and_then(|c| c.parse().ok())
1284            .ok_or("unparseable subsection header")?;
1285        if parts.next().is_some() || count == 0 {
1286            return Err("unexpected subsection header shape");
1287        }
1288        out.extend_from_slice(header.as_bytes());
1289        out.push(b'\n');
1290        i = header_end;
1291        for _ in 0..count {
1292            skip_ws(&mut i);
1293            // `nnnnnnnnnn ggggg n` — the 18 bytes before whatever EOL follows.
1294            let entry = bytes.get(i..i + 18).ok_or("xref entry runs off the end")?;
1295            let well_formed = entry[..10].iter().all(u8::is_ascii_digit)
1296                && entry[10] == b' '
1297                && entry[11..16].iter().all(u8::is_ascii_digit)
1298                && entry[16] == b' '
1299                && matches!(entry[17], b'n' | b'f');
1300            if !well_formed {
1301                return Err("xref entry is not `nnnnnnnnnn ggggg n`");
1302            }
1303            out.extend_from_slice(entry);
1304            out.extend_from_slice(b" \n"); // the spec's 2-byte EOL -> 20 bytes
1305            i += 18;
1306        }
1307    }
1308}
1309
1310/// Debug: raw glyph stream `(ch, ll, lr, lb, lt)` (native coords) for page
1311/// `index`, before the sanitizer. For comparing char cells to docling-parse.
1312pub fn debug_glyphs(bytes: &[u8], index: usize) -> Vec<(char, f32, f32, f32, f32)> {
1313    let Some(doc) = load_document(bytes) else {
1314        return Vec::new();
1315    };
1316    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1317    pages.sort_by_key(|(n, _)| *n);
1318    let Some((_, pid)) = pages.get(index) else {
1319        return Vec::new();
1320    };
1321    page_glyphs(&doc, *pid)
1322        .into_iter()
1323        .map(|g| (g.ch, g.ll, g.lr, g.lb, g.lt))
1324        .collect()
1325}
1326
1327/// Public entry: per-page (width, height, line cells) for a PDF, via the Rust
1328/// text parser + the docling-parse line sanitizer. Used by the pipeline and the
1329/// `textparse_dump` example.
1330pub fn pdf_textlines(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1331    let Some(doc) = load_document(bytes) else {
1332        return Vec::new();
1333    };
1334    let mut caches = DocCaches::default();
1335    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1336    pages.sort_by_key(|(n, _)| *n);
1337    pages
1338        .into_iter()
1339        .map(|(_, pid)| {
1340            let (w, h) = page_size(&doc, pid);
1341            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1342            let cells = crate::dp_lines::line_cells(&glyphs, h, true);
1343            (w, h, cells)
1344        })
1345        .collect()
1346}
1347
1348/// Debug/diagnostic entry: per-page (width, height, word cells) for a PDF, via
1349/// the Rust parser glyphs run through the docling-parse word grouping. Used to
1350/// compare parser word cells against docling-parse's `word_cells` oracle (roadmap
1351/// item 6).
1352pub fn pdf_words(bytes: &[u8]) -> Vec<(f32, f32, Vec<crate::pdfium_backend::TextCell>)> {
1353    let Some(doc) = load_document(bytes) else {
1354        return Vec::new();
1355    };
1356    let mut caches = DocCaches::default();
1357    let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1358    pages.sort_by_key(|(n, _)| *n);
1359    pages
1360        .into_iter()
1361        .map(|(_, pid)| {
1362            let (w, h) = page_size(&doc, pid);
1363            let glyphs = page_glyphs_cached(&doc, pid, &mut caches);
1364            let cells = crate::dp_lines::word_cells(&glyphs, h, true);
1365            (w, h, cells)
1366        })
1367        .collect()
1368}
1369
1370/// One page's text cells from the pure-Rust parser: prose line cells, per-word
1371/// cells, and code line cells — all from a single glyph parse. Replaces the
1372/// pdfium text path (roadmap item 6) when the parser drop is enabled.
1373#[derive(Default)]
1374pub struct PageParserCells {
1375    pub prose: Vec<crate::pdfium_backend::TextCell>,
1376    pub words: Vec<crate::pdfium_backend::TextCell>,
1377    pub code: Vec<crate::pdfium_backend::TextCell>,
1378    /// Drawn checkbox squares (#609), in the cells' frame.
1379    pub checkboxes: Vec<crate::checkbox::CheckBox>,
1380    /// Image placements (#999), in the cells' frame — see
1381    /// [`crate::pdfium_backend::PdfPage::bitmaps`].
1382    pub bitmaps: Vec<[f32; 4]>,
1383    /// The page's shapes for the reading-order separators (#1092).
1384    pub shapes: PageShapes,
1385}
1386
1387/// The parser text layer, driven one page at a time: the document is loaded
1388/// (and repaired, see [`load_document`]) once, the font/form caches persist
1389/// across pages, and each page's glyphs are parsed only when asked for.
1390///
1391/// The eager whole-document walk this replaces ran *before* the first page
1392/// was rendered, so on a long PDF it was a serial prefix the page-worker pool
1393/// sat idle through — 6.2 s on the 1913-page .NET reference, in front of a
1394/// pipeline that otherwise overlaps parsing with inference — and a `--pages`
1395/// window still paid for every page in the file. Pulling pages on demand
1396/// keeps the parse on the producer thread but interleaved with rendering,
1397/// and skips unselected pages entirely. Output per page is unchanged: same
1398/// glyph walk, same shared caches, same contraction.
1399pub struct PageTextParser {
1400    doc: Document,
1401    caches: DocCaches,
1402    /// Page object ids in document order (page 1 first).
1403    pages: Vec<lopdf::ObjectId>,
1404    /// How words split where no space glyph does (#921); from
1405    /// `DOCLING_RS_PDF_WORD_GAP` unless [`Self::set_word_gap`] says.
1406    word_gap: crate::dp_lines::WordGap,
1407    /// Whether [`Self::text_page`]'s page keeps typographic punctuation as
1408    /// drawn (#921); `DOCLING_RS_PDF_KEEP_TYPOGRAPHY` unless
1409    /// [`Self::set_keep_typography`] says.
1410    keep_typography: bool,
1411}
1412
1413impl PageTextParser {
1414    /// Load the document; `None` when lopdf cannot read it at all.
1415    pub fn open(bytes: &[u8]) -> Option<Self> {
1416        Self::open_with_password(bytes, None)
1417    }
1418
1419    /// [`open`](Self::open) with the document's password (an encrypted file
1420    /// the password does not open reads as `None` here; the pipeline reports
1421    /// it through [`crate::pdf_meta::PdfMeta::open_with_password`] first).
1422    pub fn open_with_password(bytes: &[u8], password: Option<&str>) -> Option<Self> {
1423        let doc = open_document(bytes, password).ok()?;
1424        let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
1425        pages.sort_by_key(|(n, _)| *n);
1426        Some(Self {
1427            doc,
1428            caches: DocCaches::default(),
1429            pages: pages.into_iter().map(|(_, pid)| pid).collect(),
1430            word_gap: crate::dp_lines::WordGap::from_env(),
1431            keep_typography: crate::assemble::keep_typography_from_env(),
1432        })
1433    }
1434
1435    /// The word-gap rule the line and word cells are built with (#921);
1436    /// `None` keeps the environment's (`DOCLING_RS_PDF_WORD_GAP`, else
1437    /// `adaptive`).
1438    pub fn set_word_gap(&mut self, gap: Option<crate::dp_lines::WordGap>) {
1439        if let Some(g) = gap {
1440            self.word_gap = g;
1441        }
1442    }
1443
1444    /// Whether the pages [`Self::text_page`] builds keep typographic
1445    /// punctuation as drawn (#921); `None` keeps the environment's
1446    /// (`DOCLING_RS_PDF_KEEP_TYPOGRAPHY`, else folding).
1447    pub fn set_keep_typography(&mut self, keep: Option<bool>) {
1448        if let Some(k) = keep {
1449            self.keep_typography = k;
1450        }
1451    }
1452
1453    /// The glyph boxes and font styles of the 0-based page `index` — the
1454    /// heading-hierarchy stage's style signal (#302): every non-space glyph's
1455    /// box (font ascent + descent at its size — the font-size proxy pdfium's
1456    /// loose char box also gave) in top-left coordinates, with the weight
1457    /// class and slant its `/BaseFont` name declares. Empty for a page
1458    /// without a text layer (a scan), and the stage falls back to its other
1459    /// signals.
1460    pub(crate) fn glyph_styles(
1461        &mut self,
1462        index: usize,
1463    ) -> Vec<crate::heading_hierarchy::GlyphStyle> {
1464        let Some(&pid) = self.pages.get(index) else {
1465            return Vec::new();
1466        };
1467        let (_w, h) = page_size(&self.doc, pid);
1468        let glyphs = page_glyphs_cached(&self.doc, pid, &mut self.caches);
1469        // Font hash → style, over the fonts the walk just parsed (an inline,
1470        // uncached font dictionary reads as unstyled).
1471        let styles: HashMap<u64, crate::font_style::FontStyle> = self
1472            .caches
1473            .fonts
1474            .values()
1475            .map(|f| (f.hash, f.style))
1476            .collect();
1477        glyphs
1478            .iter()
1479            .filter(|g| !g.ch.is_whitespace() && g.ll.is_finite())
1480            .map(|g| {
1481                let st = styles.get(&g.font).copied().unwrap_or_default();
1482                crate::heading_hierarchy::GlyphStyle {
1483                    l: g.ll,
1484                    t: h - g.lt,
1485                    r: g.lr,
1486                    b: h - g.lb,
1487                    height: g.height(),
1488                    weight_cls: crate::font_style::weight_class(st.weight),
1489                    italic: st.italic,
1490                    styled: st.known,
1491                }
1492            })
1493            .collect()
1494    }
1495
1496    /// The characters of the 0-based page `index` as the script-position
1497    /// stage measures them (#1005, docling's `read_page_chars`): every
1498    /// visible, non-blank glyph with its font-metric box, baseline and drawn
1499    /// size in the unrotated page frame (y up), and beside it the same box in
1500    /// the display frame (`/Rotate` applied, top-left origin: the frame the
1501    /// items' boxes live in). Invisible text (`Tr` 3 / 7, an OCR layer over a
1502    /// scan) is left out, as docling leaves it: it follows the scanned lines,
1503    /// not a typeset baseline.
1504    pub(crate) fn script_chars(
1505        &mut self,
1506        index: usize,
1507    ) -> Vec<(crate::script_position::ScriptChar, [f32; 4])> {
1508        let Some(&pid) = self.pages.get(index) else {
1509            return Vec::new();
1510        };
1511        let (w, h) = page_size(&self.doc, pid);
1512        let rotation = page_rotation(&self.doc, pid);
1513        page_glyphs_cached(&self.doc, pid, &mut self.caches)
1514            .iter()
1515            .filter(|g| !g.invisible && !g.ch.is_whitespace() && g.ll.is_finite())
1516            .filter_map(|g| {
1517                let [_, y, size] = g.pen?;
1518                let c = crate::script_position::ScriptChar {
1519                    text: g.ch,
1520                    left: g.ll,
1521                    bottom: g.lb,
1522                    right: g.lr,
1523                    top: g.lt,
1524                    baseline: y,
1525                    size,
1526                    horizontal: g.quad.is_none(),
1527                };
1528                let (l, t, r, b) = crate::pdfium_backend::to_display_frame(
1529                    (g.ll, h - g.lt, g.lr, h - g.lb),
1530                    rotation,
1531                    w,
1532                    h,
1533                );
1534                Some((c, [l, t, r, b]))
1535            })
1536            .collect()
1537    }
1538
1539    /// Prose, word and code cells of the 0-based page `index` — empty for an
1540    /// index the parser's page tree doesn't have (a damaged file whose page
1541    /// tree disagrees with the object model's count).
1542    pub fn cells(&mut self, index: usize) -> PageParserCells {
1543        let Some(&pid) = self.pages.get(index) else {
1544            return PageParserCells::default();
1545        };
1546        let (_w, h) = page_size(&self.doc, pid);
1547        let (glyphs, inks, bitmaps, shapes) =
1548            page_glyphs_and_inks(&self.doc, pid, &mut self.caches);
1549        let (prose, words) =
1550            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1551        PageParserCells {
1552            prose,
1553            words,
1554            code: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1555            checkboxes: crate::checkbox::find(&inks, h),
1556            bitmaps,
1557            shapes,
1558        }
1559    }
1560
1561    /// Pages in the document's page tree.
1562    pub fn page_count(&self) -> usize {
1563        self.pages.len()
1564    }
1565
1566    /// The 0-based page `index` as the text-layer-only conversion sees it —
1567    /// one element of [`pdf_text_pages`], parsed on demand so a caller can
1568    /// drop each page once it is assembled instead of holding the whole
1569    /// document's cells. A blank page (no cells) for an index the page tree
1570    /// doesn't have.
1571    pub fn text_page(&mut self, index: usize) -> crate::pdfium_backend::PdfPage {
1572        let (w, h, (glyphs, inks, _, shapes)) = match self.pages.get(index) {
1573            Some(&pid) => {
1574                let (w, h) = page_size(&self.doc, pid);
1575                (w, h, page_glyphs_and_inks(&self.doc, pid, &mut self.caches))
1576            }
1577            None => (0.0, 0.0, Default::default()),
1578        };
1579        let (mut prose, mut words) =
1580            crate::dp_lines::line_and_word_cells_with(&glyphs, h, true, self.word_gap);
1581        drop_overpainted_cells(&mut prose);
1582        drop_overpainted_cells(&mut words);
1583        crate::pdfium_backend::PdfPage {
1584            #[cfg(feature = "ocr-prep")]
1585            image_layout: None,
1586            width: w,
1587            height: h,
1588            // Cells are native PDF points; there is no rendered bitmap.
1589            scale: 1.0,
1590            cells: prose,
1591            code_cells: crate::pdfium_backend::code_cells_from_glyphs(&glyphs, h),
1592            word_cells: words,
1593            checkboxes: crate::checkbox::find(&inks, h),
1594            bitmaps: Vec::new(),
1595            shapes,
1596            #[cfg(feature = "ocr-prep")]
1597            image: image::RgbImage::new(1, 1),
1598            links: Vec::new(),
1599            rotation: 0,
1600            keep_typography: self.keep_typography,
1601            #[cfg(feature = "render")]
1602            embedded_images: Vec::new(),
1603            #[cfg(feature = "render")]
1604            pending_image: None,
1605        }
1606    }
1607}
1608
1609/// Full parser text layer: prose + word + code cells per page, glyphs parsed once.
1610/// `prose`/`words` come from the docling-parse contraction ([`crate::dp_lines`]);
1611/// `code` splits only at the parser's own space glyphs (monospace keeps its
1612/// source spacing). The eager form of [`PageTextParser`].
1613pub fn pdf_all_cells(bytes: &[u8]) -> Vec<PageParserCells> {
1614    let Some(mut parser) = PageTextParser::open(bytes) else {
1615        return Vec::new();
1616    };
1617    (0..parser.pages.len()).map(|i| parser.cells(i)).collect()
1618}
1619
1620/// Whole pages for the text-layer-only conversion ([`crate::convert_text_layer`]):
1621/// the parser's prose/word/code cells plus page geometry, assembled into
1622/// [`PdfPage`]s with no rendered image and no link annotations. Everything here
1623/// is pure Rust (lopdf), so it compiles without the `ml` feature — including on
1624/// wasm32. A page the parser can't read (no text layer) comes back with empty
1625/// cells; there is no pdfium fallback on this path.
1626pub fn pdf_text_pages(bytes: &[u8]) -> Vec<crate::pdfium_backend::PdfPage> {
1627    let Some(mut parser) = PageTextParser::open(bytes) else {
1628        return Vec::new();
1629    };
1630    (0..parser.page_count())
1631        .map(|i| parser.text_page(i))
1632        .collect()
1633}
1634
1635/// Drop line cells that are *painted over each other* — glyphs used as artwork.
1636///
1637/// Some generators draw their logo with a symbol font: on the reporting
1638/// invoice, a `TeleLogo` Type1 paints the T-Mobile mark by stacking the glyphs
1639/// encoded as `"` and `==` on top of one another, and the flat text-layer
1640/// output opened with that garbage. Nothing in the font metadata gives it away
1641/// (the *text* fonts in the same file are also flagged symbolic, and the logo
1642/// font names its glyphs `quotedbl` &c.), but the geometry does: two cells with
1643/// different text where one lies inside the other on the same line is
1644/// physically impossible for prose — ink from two words never occupies the
1645/// same box. Both cells of such a pair are paint, not text.
1646///
1647/// Containment (not mere overlap) keeps this narrow: adjacent words touch but
1648/// never contain each other, and a same-text near-duplicate (double-draw faux
1649/// bold) is left alone for the sanitizer's usual handling. Applied on the
1650/// flat/browser path only — the ML pipeline's text layer is byte-pinned by the
1651/// PDF corpus, and there the layout model already sinks logo marks into
1652/// `picture` regions.
1653fn drop_overpainted_cells(cells: &mut Vec<crate::pdfium_backend::TextCell>) {
1654    let mut paint = vec![false; cells.len()];
1655    for i in 0..cells.len() {
1656        for j in 0..cells.len() {
1657            if i == j || cells[i].text == cells[j].text {
1658                continue;
1659            }
1660            let (a, b) = (&cells[i], &cells[j]);
1661            // Same line band: the vertical overlap covers most of the shorter.
1662            let vo = (a.b.min(b.b) - a.t.max(b.t)).max(0.0);
1663            if vo < 0.6 * (a.b - a.t).min(b.b - b.t) {
1664                continue;
1665            }
1666            // `a` horizontally inside `b` (with a small tolerance).
1667            let ho = (a.r.min(b.r) - a.l.max(b.l)).max(0.0);
1668            if ho >= 0.8 * (a.r - a.l) && (a.r - a.l) <= (b.r - b.l) {
1669                paint[i] = true;
1670                paint[j] = true;
1671            }
1672        }
1673    }
1674    let mut keep = paint.iter().map(|p| !p);
1675    cells.retain(|_| keep.next().unwrap());
1676}
1677
1678/// The text-state scalars inherited by a Form XObject when it is invoked via
1679/// `Do` (the PDF graphics state includes the text parameters, but not the text
1680/// matrices, which a form re-establishes inside its own `BT`/`ET`).
1681#[derive(Clone, Copy)]
1682struct TextState {
1683    tc: f64,
1684    tw: f64,
1685    th: f64,
1686    tl: f64,
1687    trise: f64,
1688    fsize: f64,
1689    /// The text rendering mode (`Tr`).
1690    tr: i64,
1691}
1692
1693impl TextState {
1694    const INIT: TextState = TextState {
1695        tc: 0.0,
1696        tw: 0.0,
1697        th: 1.0,
1698        tl: 0.0,
1699        trise: 0.0,
1700        fsize: 0.0,
1701        tr: 0,
1702    };
1703}
1704
1705/// pdfium's `CPDF_Page::GetPageRotation` — the inherited `/Rotate`, in
1706/// quarter turns, as degrees (`pdf_meta`'s rule).
1707fn page_rotation(doc: &Document, pid: lopdf::ObjectId) -> u16 {
1708    let mut id = pid;
1709    for _ in 0..32 {
1710        let Ok(dict) = doc.get_dictionary(id) else {
1711            return 0;
1712        };
1713        if let Some(obj) = dict.get(b"Rotate").ok().and_then(|o| deref(doc, o)) {
1714            let quarter = (num(obj).unwrap_or(0.0) as i64 / 90).rem_euclid(4);
1715            return (quarter * 90) as u16;
1716        }
1717        match dict.get(b"Parent").ok().and_then(|o| o.as_reference().ok()) {
1718            Some(parent) => id = parent,
1719            None => return 0,
1720        }
1721    }
1722    0
1723}
1724
1725/// The effective `/Resources` dictionary for a page (inline or via reference,
1726/// falling back to an inherited one from a `/Parent`).
1727fn page_res(doc: &Document, page_id: lopdf::ObjectId) -> Option<&Dictionary> {
1728    let (inline, ids) = doc.get_page_resources(page_id).ok()?;
1729    if let Some(d) = inline {
1730        return Some(d);
1731    }
1732    ids.into_iter().find_map(|id| doc.get_dictionary(id).ok())
1733}
1734
1735/// Build the code→[`Font`] map for a resources dictionary's `/Font` sub-dict,
1736/// reusing the per-document cache for fonts referenced indirectly (the common
1737/// case — the same font objects recur on every page).
1738fn fonts_from_res(
1739    doc: &Document,
1740    res: &Dictionary,
1741    caches: &mut DocCaches,
1742) -> HashMap<Vec<u8>, Arc<Font>> {
1743    let mut map = HashMap::new();
1744    let font_dict = res
1745        .get(b"Font")
1746        .ok()
1747        .and_then(|o| deref(doc, o))
1748        .and_then(|o| o.as_dict().ok());
1749    if let Some(fd) = font_dict {
1750        for (name, value) in fd.iter() {
1751            let font = match value {
1752                Object::Reference(id) => {
1753                    let key = (*id, name.clone());
1754                    if let Some(f) = caches.fonts.get(&key) {
1755                        Arc::clone(f)
1756                    } else if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
1757                        let f = Arc::new(parse_font(doc, name, fdict));
1758                        caches.fonts.insert(key, Arc::clone(&f));
1759                        f
1760                    } else {
1761                        continue;
1762                    }
1763                }
1764                _ => {
1765                    if let Some(fdict) = deref(doc, value).and_then(|o| o.as_dict().ok()) {
1766                        Arc::new(parse_font(doc, name, fdict))
1767                    } else {
1768                        continue;
1769                    }
1770                }
1771            };
1772            map.insert(name.clone(), font);
1773        }
1774    }
1775    map
1776}
1777
1778/// Extract every glyph on a page as a native-coordinate [`Glyph`].
1779/// [`PageTextParser::glyph_styles`] for the given **1-based** pages of a
1780/// document, keyed by page number — a separate, on-demand pass (no
1781/// rendering), so the extraction pipeline stays byte-identical whether or not
1782/// the heading-hierarchy stage runs. Empty when lopdf cannot open the file.
1783pub(crate) fn glyph_styles(
1784    bytes: &[u8],
1785    pages: &[usize],
1786) -> HashMap<usize, Vec<crate::heading_hierarchy::GlyphStyle>> {
1787    let mut out = HashMap::new();
1788    let Some(mut parser) = PageTextParser::open(bytes) else {
1789        return out;
1790    };
1791    for &page_no in pages {
1792        if page_no == 0 {
1793            continue;
1794        }
1795        let styles = parser.glyph_styles(page_no - 1);
1796        if !styles.is_empty() {
1797            out.insert(page_no, styles);
1798        }
1799    }
1800    out
1801}
1802
1803pub(crate) fn page_glyphs(doc: &Document, page_id: lopdf::ObjectId) -> Vec<Glyph> {
1804    page_glyphs_cached(doc, page_id, &mut DocCaches::default())
1805}
1806
1807/// [`page_glyphs`] with an explicit per-document cache, so a multi-page walk
1808/// parses each font / decodes each form once instead of once per page.
1809fn page_glyphs_cached(
1810    doc: &Document,
1811    page_id: lopdf::ObjectId,
1812    caches: &mut DocCaches,
1813) -> Vec<Glyph> {
1814    page_glyphs_and_inks(doc, page_id, caches).0
1815}
1816
1817/// [`page_glyphs_cached`] plus the small painted path pieces the same walk
1818/// passed — what [`crate::checkbox::find`] looks for checkbox squares in
1819/// (#609) — the page's image placements (#999, `[l, t, r, b]` top-left
1820/// page points clipped to the page, the cells' frame), and its shapes for the
1821/// reading-order separators (#1092, [`PageShapes`]).
1822fn page_glyphs_and_inks(
1823    doc: &Document,
1824    page_id: lopdf::ObjectId,
1825    caches: &mut DocCaches,
1826) -> (
1827    Vec<Glyph>,
1828    Vec<crate::checkbox::Ink>,
1829    Vec<[f32; 4]>,
1830    PageShapes,
1831) {
1832    let mut out = Vec::new();
1833    let mut inks = Vec::new();
1834    // get_page_content returns the assembled content-stream bytes directly
1835    // (an empty Vec when the page has none).
1836    let content_bytes = doc.get_page_content(page_id);
1837    let Ok(content) = lopdf::content::Content::decode(&content_bytes) else {
1838        return (out, inks, Vec::new(), PageShapes::default());
1839    };
1840    let mut bitmaps = Vec::new();
1841    let mut page_shapes = PageShapes::default();
1842    if let Some(res) = page_res(doc, page_id) {
1843        // pdfium's page matrix: user space translated so the display box's
1844        // lower-left corner is the origin (see [`PageBox`]).
1845        let pb = page_box(doc, page_id);
1846        let base = Mat {
1847            e: -(pb.l as f64),
1848            f: -(pb.b as f64),
1849            ..Mat::ID
1850        };
1851        let mut shapes = Shapes::default();
1852        run_content(
1853            doc,
1854            res,
1855            &content,
1856            base,
1857            TextState::INIT,
1858            0,
1859            caches,
1860            &mut out,
1861            &mut inks,
1862            ShapeGs::INIT,
1863            &mut shapes,
1864        );
1865        out.retain(|g| on_page(g, pb.w, pb.h));
1866        drop_overprinted_spaces(&mut out);
1867        drop_shadow_runs(&mut out);
1868        let (w, h) = (f64::from(pb.w), f64::from(pb.h));
1869        inks.retain(|ink| match *ink {
1870            crate::checkbox::Ink::Bitmap { l, b, r, t } => {
1871                let (l, r, b, t) = (l.max(0.0), r.min(w), b.max(0.0), t.min(h));
1872                if r > l && t > b {
1873                    bitmaps.push([l as f32, (h - t) as f32, r as f32, (h - b) as f32]);
1874                }
1875                false
1876            }
1877            _ => true,
1878        });
1879        let top_left =
1880            |[l, b, r, t]: [f64; 4]| [l as f32, (h - t) as f32, r as f32, (h - b) as f32];
1881        page_shapes.lines = shapes.lines.into_iter().map(top_left).collect();
1882        if shapes.boxes.len() <= MAX_SHAPE_BOXES {
1883            page_shapes.boxes = connected_boxes(&shapes.boxes)
1884                .into_iter()
1885                .map(top_left)
1886                .collect();
1887        } else {
1888            docling_core::debug_log!(
1889                "textparse: {} painted paths on the page; their boxes are not merged",
1890                shapes.boxes.len()
1891            );
1892        }
1893    }
1894    (out, inks, bitmaps, page_shapes)
1895}
1896
1897/// A page's shapes for the reading-order separators (#1092), in the cells'
1898/// frame (`[l, t, r, b]`, top-left page points): docling-parse's
1899/// `get_shape_lines` and `get_connected_shape_bounding_boxes`.
1900#[derive(Default, Clone, Debug, PartialEq)]
1901pub struct PageShapes {
1902    pub lines: Vec<[f32; 4]>,
1903    pub boxes: Vec<[f32; 4]>,
1904}
1905
1906/// Drop a run of glyphs painted again over itself — a text shadow. A word
1907/// processor exports a heading with a shadow effect as the same run drawn
1908/// twice: the shadow, then the text over it, shifted by a fraction of a point
1909/// (0.5–0.8 pt up and left, black under red, in the manuals that showed it).
1910/// docling-parse keeps both copies and reads `Safety instructions Safety
1911/// instructions`; pdfium's text page, behind docling's pypdfium2 backend,
1912/// drops the second, and the page shows the heading once. A deliberate
1913/// divergence (`docs/MIGRATION.md` §4 item 25): a glyph that repeats an
1914/// earlier one — same character, font and size, its pen shifted from the
1915/// earlier pen by at most 0.15 em on each axis, and by less than the earlier
1916/// glyph's step to its own successor — starts a copy when the glyphs after it
1917/// repeat the earlier one's successors by the same offset and size, and the
1918/// whole stretch that keeps the offset is dropped. Two glyphs at least: a lone
1919/// mark stamped twice is left alone, and so is the per-glyph re-stamp the
1920/// sanitizer already folds (`dp_lines`, the overprint duplicate). Text that
1921/// really repeats sits an advance apart or more — a doubled letter (`ll`,
1922/// 0.22 em in Helvetica), a word set twice (`very very`), the same text on
1923/// the next line or in the next table cell — and is kept, and so is a run of
1924/// one character set tighter than a shadow's shift (dot leaders squeezed by
1925/// `Tc`): its glyph two on is farther than the next one. A line stamped two
1926/// or three times at a small offset to fake bold reads once the same way. The
1927/// original must start within the 1024 glyphs before its copy, so a shadowed
1928/// block longer than that is not caught; a shadowed heading is drawn right
1929/// after itself. A rotated run, whose pen positions are not compared here, is
1930/// left alone.
1931fn drop_shadow_runs(glyphs: &mut Vec<Glyph>) {
1932    // How far back a copy's original may start.
1933    const LOOKBACK: usize = 1024;
1934    // The largest shift of a copy, per axis, in ems of the glyph's size.
1935    const NEAR_EM: f32 = 0.15;
1936    // How much the shift and the size may vary along a copy, in ems.
1937    const SAME_EM: f32 = 0.01;
1938    let pen = |g: &Glyph| if g.quad.is_none() { g.pen } else { None };
1939    let n = glyphs.len();
1940    let mut keep = vec![true; n];
1941    // The glyphs a copy may start on, by character and font, in stream
1942    // order: kept (not part of a copy), upright, sized and not whitespace. A
1943    // copy repeats both, so a glyph walks only its own list back through the
1944    // window; scanning the whole window nearly doubled a `--text-layer-only`
1945    // read of pages without a shadow.
1946    let mut earlier: HashMap<(char, u64), Vec<usize>> = HashMap::new();
1947    let mut j = 0;
1948    while j < n {
1949        let b = &glyphs[j];
1950        let start = pen(b).filter(|p| p[2] > 0.0 && !b.ch.is_whitespace());
1951        let mut copy = 0;
1952        if let Some([xj, yj, sj]) = start {
1953            let lo = j.saturating_sub(LOOKBACK);
1954            let candidates = earlier.entry((b.ch, b.font)).or_default();
1955            for &i in candidates.iter().rev().take_while(|&&i| i >= lo) {
1956                let Some([xi, yi, si]) = pen(&glyphs[i]) else {
1957                    continue;
1958                };
1959                if (si - sj).abs() > SAME_EM * sj {
1960                    continue;
1961                }
1962                let (dx, dy) = (xj - xi, yj - yi);
1963                if dx.abs() > NEAR_EM * sj || dy.abs() > NEAR_EM * sj {
1964                    continue;
1965                }
1966                // A shadow is shifted: the characters one glyph expands to
1967                // (a code-like name read as `SM590000`, a decomposed
1968                // ligature) share the glyph's pen and are not a copy.
1969                if dx.abs() + dy.abs() <= SAME_EM * sj {
1970                    continue;
1971                }
1972                // And it lies over its original, nearer to it than the
1973                // original's next glyph is: one character set tighter than
1974                // half the largest shift (dot leaders squeezed by `Tc`) meets
1975                // its own successors at one offset two glyphs on.
1976                if let Some([xn, yn, _]) = glyphs.get(i + 1).and_then(pen) {
1977                    let step = (xn - xi).abs().max((yn - yi).abs());
1978                    if step > SAME_EM * sj && dx.abs().max(dy.abs()) >= step {
1979                        continue;
1980                    }
1981                }
1982                // The copy goes on while each next glyph repeats the
1983                // original's next one by the same shift; the original
1984                // ends before the copy starts.
1985                let mut m = 1;
1986                while j + m < n && i + m < j {
1987                    let (a, b) = (&glyphs[i + m], &glyphs[j + m]);
1988                    let (Some([xa, ya, sa]), Some([xb, yb, sb])) = (pen(a), pen(b)) else {
1989                        break;
1990                    };
1991                    if a.ch != b.ch
1992                        || a.font != b.font
1993                        || (sa - sb).abs() > SAME_EM * sj
1994                        || ((xb - xa) - dx).abs() > SAME_EM * sj
1995                        || ((yb - ya) - dy).abs() > SAME_EM * sj
1996                    {
1997                        break;
1998                    }
1999                    m += 1;
2000                }
2001                if m >= 2 {
2002                    copy = m;
2003                    break;
2004                }
2005            }
2006            if copy == 0 {
2007                candidates.push(j);
2008            }
2009        }
2010        if copy > 0 {
2011            keep[j..j + copy].iter_mut().for_each(|k| *k = false);
2012            j += copy;
2013        } else {
2014            j += 1;
2015        }
2016    }
2017    let mut it = keep.into_iter();
2018    glyphs.retain(|_| it.next().unwrap_or(true));
2019}
2020
2021/// How many marked-content spans may be open for one's `/ActualText` to be
2022/// read (see [`apply_actual_text`]); real documents nest a few.
2023const MAX_ACTUAL_TEXT_DEPTH: usize = 64;
2024
2025/// Put a marked-content span's `/ActualText` in place of the glyphs drawn
2026/// inside it, as docling-parse does since 7.9 (`apply_actual_text`,
2027/// docling-parse#300, on by default; PDF 32000-1 §14.9.4). A writer draws a
2028/// glyph cluster whose characters the font cannot spell this way: a base
2029/// letter and a zero-advance accent whose code collides with a Latin letter
2030/// (`Oreši` read as `OresMi`, docling#3783), or the Arabic marks of a
2031/// LibreOffice font subset whose `/ToUnicode` skips them, so the byte reads as
2032/// WinAnsi (`منعًا` as `منع8ا`). docling-parse's rules, cell for cell: a span
2033/// with no glyphs (over graphics) is left alone, and so is a text implausibly
2034/// long for it (more than 8 characters and 4 per cell: `/Alt` written into
2035/// the wrong key); an empty text removes the glyphs; as many characters as
2036/// cells replace them one by one, keeping each cell's geometry; any other
2037/// count puts the whole text on the span's first cell, over the union of the
2038/// span's boxes, its baseline running to the cell that reaches furthest along
2039/// it (an accent drawn after its base letter trails behind it). Nested spans
2040/// apply innermost first, so the outer text wins. docling-parse has a cell per
2041/// character code; a code whose `/ToUnicode` spells several characters is
2042/// several glyphs here, all on the code's pen, and counts as one cell.
2043///
2044/// Each `EMC` rewrites the glyphs under its span, so a crafted nest of spans
2045/// over the same glyphs would cost glyphs × depth: past
2046/// [`MAX_ACTUAL_TEXT_DEPTH`] open spans, a span still pairs with its `EMC` but
2047/// its `/ActualText` is not read.
2048fn apply_actual_text(out: &mut Vec<Glyph>, begin: usize, text: &str) {
2049    let begin = begin.min(out.len());
2050    let mut cells: Vec<std::ops::Range<usize>> = Vec::new();
2051    for i in begin..out.len() {
2052        match cells.last_mut() {
2053            Some(cell) if out[i].pen.is_some() && out[i].pen == out[cell.end - 1].pen => {
2054                cell.end = i + 1;
2055            }
2056            _ => cells.push(i..i + 1),
2057        }
2058    }
2059    if cells.is_empty() {
2060        return;
2061    }
2062    let chars: Vec<char> = text.chars().collect();
2063    if chars.len() > (4 * cells.len()).max(8) {
2064        return;
2065    }
2066    if chars.is_empty() {
2067        out.truncate(begin);
2068        return;
2069    }
2070    if chars.len() == cells.len() {
2071        // Each cell's first glyph takes its character, the rest of the cell
2072        // goes: one pass, writing behind the cells read.
2073        for (w, (cell, &ch)) in cells.iter().zip(&chars).enumerate() {
2074            out[begin + w] = Glyph {
2075                ch,
2076                ..out[cell.start]
2077            };
2078        }
2079        out.truncate(begin + cells.len());
2080        return;
2081    }
2082    // The leading edge (quad corners 0 and 3) stays the first cell's; the
2083    // trailing one (1 and 2) is the cell's that reaches furthest along the
2084    // baseline, projected on it so a rotated run keeps its direction.
2085    let quad_of = |g: &Glyph| {
2086        g.quad
2087            .unwrap_or([g.ll, g.lb, g.lr, g.lb, g.lr, g.lt, g.ll, g.lt])
2088    };
2089    let mut merged = Glyph { ..out[begin] };
2090    let mut quad = quad_of(&merged);
2091    let dir = (quad[2] - quad[0], quad[3] - quad[1]);
2092    let along = |q: &[f32; 8]| q[2] * dir.0 + q[3] * dir.1;
2093    let mut reach = along(&quad);
2094    for g in &out[begin + 1..] {
2095        merged.l = merged.l.min(g.l);
2096        merged.b = merged.b.min(g.b);
2097        merged.r = merged.r.max(g.r);
2098        merged.t = merged.t.max(g.t);
2099        merged.ll = merged.ll.min(g.ll);
2100        merged.lb = merged.lb.min(g.lb);
2101        merged.lr = merged.lr.max(g.lr);
2102        merged.lt = merged.lt.max(g.lt);
2103        let q = quad_of(g);
2104        if along(&q) > reach {
2105            reach = along(&q);
2106            quad[2..6].copy_from_slice(&q[2..6]);
2107        }
2108    }
2109    if merged.quad.is_some() {
2110        merged.quad = Some(quad);
2111    }
2112    out.truncate(begin);
2113    // The characters of one cell share its box, as a ligature's do.
2114    out.extend(chars.iter().map(|&ch| Glyph { ch, ..merged }));
2115}
2116
2117/// Drop a space glyph the next glyph is drawn over (#992). A typesetter
2118/// that sets a superscript footnote marker as `( 1 )` — space glyphs around
2119/// the raised digit — and then cancels each space's advance with a negative
2120/// `TJ` offset (Arbortext on EUR-Lex: the `1` starts 0.05 pt *before* its
2121/// space begins) renders `(1)`; the spaces exist in the content stream
2122/// only. docling-parse keeps them as characters and its line cell reads
2123/// `( 1 )`; pdftotext and pdfium, which break words by geometry alone,
2124/// read `(1)`, and so does the page. A deliberate divergence
2125/// (`docs/MIGRATION.md` §4 item 22): a space whose successor in stream
2126/// order sits on the same line and starts within the first 30 % of the
2127/// space's advance — the pen went back to (or before) where the space
2128/// began, so it left no gap — is not a character of the text. A space
2129/// followed by a glyph on another line (a trailing space before the
2130/// cursor jumps to the next line's left margin, far to the left of it) or
2131/// by one that starts past that mark (a kerned pair, a justified word gap
2132/// pulled in a little) is kept; the word-gap rules then see the geometry
2133/// as before.
2134fn drop_overprinted_spaces(glyphs: &mut Vec<Glyph>) {
2135    if glyphs.len() < 2 {
2136        return;
2137    }
2138    let overprinted = |sp: &Glyph, next: &Glyph| {
2139        if sp.ch != ' ' || next.ch == ' ' {
2140            return false;
2141        }
2142        // Upright glyphs only: a rotated run's loose boxes are the extents
2143        // of its quads, every glyph of a 90° line shares one left edge and
2144        // the advance runs along y, so the test below would read every
2145        // space of a landscape slide drawn through a 90° `cm` as
2146        // overprinted (`Nullammollis`, odf_presentation_02).
2147        if sp.quad.is_some() || next.quad.is_some() {
2148            return false;
2149        }
2150        let width = sp.lr - sp.ll;
2151        if !width.is_finite() || width <= 0.0 || !next.ll.is_finite() {
2152            return false;
2153        }
2154        // Same line: the loose boxes overlap vertically (a raised digit's box
2155        // still overlaps its neighbours' em boxes).
2156        let same_line = next.lb < sp.lt && next.lt > sp.lb;
2157        same_line && next.ll >= sp.ll - 0.1 * width && next.ll <= sp.ll + 0.3 * width
2158    };
2159    let mut keep = vec![true; glyphs.len()];
2160    for i in 0..glyphs.len() - 1 {
2161        if overprinted(&glyphs[i], &glyphs[i + 1]) {
2162            keep[i] = false;
2163        }
2164    }
2165    let mut it = keep.into_iter();
2166    glyphs.retain(|_| it.next().unwrap_or(true));
2167}
2168
2169/// Whether a glyph is on the page (#529): docling-parse keeps a character
2170/// only when its whole box lies inside the display box — the CropBox, or
2171/// the MediaBox without one — edges included, so a FrameMaker print slug
2172/// drawn beside the CropBox or a tiled page's neighbouring text beyond the
2173/// MediaBox never becomes a cell, and a line crossing the edge is cut at the
2174/// last glyph that fits (`Crossing the rig`). The box is docling-parse's
2175/// char box — the advance by the font's ascent/descent, the loose box here
2176/// (its axis-aligned extent for rotated text) — in the frame `page_glyphs`
2177/// already moved to the display box's corner, so the page is `[0, w] × [0,
2178/// h]`. A glyph without a finite box is kept, as before. (A standard-14 font
2179/// without a FontDescriptor gets the 750 / −250 default ascent / descent
2180/// here where docling-parse reads the AFM's — Helvetica's 718 / −207 — so
2181/// for those a glyph within ~0.04 em of the top or bottom edge can fall the
2182/// other way; horizontally the advance is the same.)
2183fn on_page(g: &Glyph, w: f32, h: f32) -> bool {
2184    // f32 noise only: docling-parse already drops a glyph 0.01 pt over.
2185    const EPS: f32 = 1e-3;
2186    let (l, b, r, t) = if [g.ll, g.lb, g.lr, g.lt].iter().all(|v| v.is_finite()) {
2187        (g.ll, g.lb, g.lr, g.lt)
2188    } else {
2189        (g.l, g.b, g.r, g.t)
2190    };
2191    if ![l, b, r, t].iter().all(|v| v.is_finite()) {
2192        return true;
2193    }
2194    l >= -EPS && b >= -EPS && r <= w + EPS && t <= h + EPS
2195}
2196
2197/// Run a content stream's operators, emitting glyphs into `out`. Recurses into
2198/// Form XObjects on `Do` (bulk body text in heavy PDFs lives inside a form, not
2199/// the page content stream). `res` is the resources dict in scope (the page's,
2200/// or the form's own); `base_ctm` is the CTM at the point of invocation.
2201#[allow(clippy::too_many_arguments)]
2202fn run_content(
2203    doc: &Document,
2204    res: &Dictionary,
2205    content: &lopdf::content::Content,
2206    base_ctm: Mat,
2207    init: TextState,
2208    depth: u32,
2209    caches: &mut DocCaches,
2210    out: &mut Vec<Glyph>,
2211    inks: &mut Vec<crate::checkbox::Ink>,
2212    shape_init: ShapeGs,
2213    shapes: &mut Shapes,
2214) {
2215    let fonts = fonts_from_res(doc, res, caches);
2216    let xobjects = res
2217        .get(b"XObject")
2218        .ok()
2219        .and_then(|o| deref(doc, o))
2220        .and_then(|o| o.as_dict().ok());
2221
2222    // Graphics + text state. `q`/`Q` save and restore the whole graphics state,
2223    // which includes the text parameters (Tc, Tw, Tz, TL, Tfs, Trise, font) —
2224    // *not* the text matrix (that is reset by BT). Saving only the CTM let a Tc
2225    // set inside a `q…Q` block leak out and drift every later glyph.
2226    #[allow(clippy::type_complexity)]
2227    let mut gstate_stack: Vec<(Mat, f64, f64, f64, f64, f64, f64, i64, Option<&Arc<Font>>)> =
2228        Vec::new();
2229    let mut ctm = base_ctm;
2230    let mut tm = Mat::ID;
2231    let mut tlm = Mat::ID;
2232    let mut font: Option<&Arc<Font>> = None;
2233    let mut fsize = init.fsize;
2234    let mut tc = init.tc; // char spacing
2235    let mut tw = init.tw; // word spacing
2236    let mut th = init.th; // horizontal scale (Tz/100)
2237    let mut tl = init.tl; // leading
2238    let mut trise = init.trise;
2239    let mut tr = init.tr; // rendering mode
2240
2241    let op_f = |operands: &[Object], i: usize| operands.get(i).and_then(num).unwrap_or(0.0);
2242    // The path under construction, already in page space (a path is built
2243    // under one CTM — `cm` is not allowed inside it), for the checkbox
2244    // squares (#609). Text extraction never reads it.
2245    let mut path: Vec<PathPiece> = Vec::new();
2246    let (mut cur, mut start) = ((0.0, 0.0), (0.0, 0.0));
2247    // Whether the fill colour is (near) white, saved/restored with `q`/`Q`:
2248    // a white-filled stroked box is a checkbox outline, a coloured one a
2249    // chart legend's swatch. The initial fill colour is black.
2250    let mut fill_light = false;
2251    let mut fill_stack: Vec<bool> = Vec::new();
2252    // Open marked-content sequences (`BMC`/`BDC` … `EMC`): where each one's
2253    // glyphs start in `out`, and its inline `/ActualText`, if any.
2254    let mut marked: Vec<(usize, Option<String>)> = Vec::new();
2255    // The reading-order shapes (#1092): docling-parse's view of the path
2256    // under construction, its graphics state and the pending clip.
2257    let mut shape_gs = shape_init;
2258    let mut shape_stack: Vec<ShapeGs> = Vec::new();
2259    let mut shape_path: Vec<ShapeSub> = Vec::new();
2260    let mut clip_pending = false;
2261    let ext_gstates = res
2262        .get(b"ExtGState")
2263        .ok()
2264        .and_then(|o| deref(doc, o))
2265        .and_then(|o| o.as_dict().ok());
2266    let colorspaces = res
2267        .get(b"ColorSpace")
2268        .ok()
2269        .and_then(|o| deref(doc, o))
2270        .and_then(|o| o.as_dict().ok());
2271
2272    for op in &content.operations {
2273        let operands = &op.operands;
2274        let shown = out.len();
2275        match op.operator.as_str() {
2276            "q" => {
2277                gstate_stack.push((ctm, tc, tw, th, tl, trise, fsize, tr, font));
2278                fill_stack.push(fill_light);
2279                shape_stack.push(shape_gs.clone());
2280            }
2281            "BMC" => marked.push((out.len(), None)),
2282            // Only an inline property list is read: a named one (`/OC /MC0`)
2283            // would need the `/Properties` resource, as in docling-parse.
2284            "BDC" => {
2285                let actual = operands
2286                    .get(1)
2287                    .filter(|_| marked.len() < MAX_ACTUAL_TEXT_DEPTH)
2288                    .and_then(|o| o.as_dict().ok())
2289                    .and_then(|d| d.get(b"ActualText").ok())
2290                    .and_then(|o| match o {
2291                        Object::String(bytes, _) => Some(crate::info::decode_text(bytes)),
2292                        _ => None,
2293                    });
2294                marked.push((out.len(), actual));
2295            }
2296            "EMC" => {
2297                if let Some((begin, Some(text))) = marked.pop() {
2298                    apply_actual_text(out, begin, &text);
2299                }
2300                // Shows no text: the invisible-mode stamp below has nothing
2301                // new to mark, and the span may have shortened `out`.
2302                continue;
2303            }
2304            "Q" => {
2305                fill_light = fill_stack.pop().unwrap_or(false);
2306                if let Some(gs) = shape_stack.pop() {
2307                    shape_gs = gs;
2308                }
2309                if let Some((c, a, b, h, l, r, fs, mode, f)) = gstate_stack.pop() {
2310                    ctm = c;
2311                    tc = a;
2312                    tw = b;
2313                    th = h;
2314                    tl = l;
2315                    trise = r;
2316                    fsize = fs;
2317                    tr = mode;
2318                    font = f;
2319                }
2320            }
2321            "cm" => {
2322                let m = Mat {
2323                    a: op_f(operands, 0),
2324                    b: op_f(operands, 1),
2325                    c: op_f(operands, 2),
2326                    d: op_f(operands, 3),
2327                    e: op_f(operands, 4),
2328                    f: op_f(operands, 5),
2329                };
2330                ctm = m.then(ctm);
2331            }
2332            "BT" => {
2333                tm = Mat::ID;
2334                tlm = Mat::ID;
2335            }
2336            "ET" => {}
2337            "Tf" => {
2338                if let Some(Object::Name(n)) = operands.first() {
2339                    font = fonts.get(n.as_slice());
2340                }
2341                fsize = op_f(operands, 1);
2342            }
2343            "Td" => {
2344                tlm = Mat {
2345                    a: 1.0,
2346                    b: 0.0,
2347                    c: 0.0,
2348                    d: 1.0,
2349                    e: op_f(operands, 0),
2350                    f: op_f(operands, 1),
2351                }
2352                .then(tlm);
2353                tm = tlm;
2354            }
2355            "TD" => {
2356                tl = -op_f(operands, 1);
2357                tlm = Mat {
2358                    a: 1.0,
2359                    b: 0.0,
2360                    c: 0.0,
2361                    d: 1.0,
2362                    e: op_f(operands, 0),
2363                    f: op_f(operands, 1),
2364                }
2365                .then(tlm);
2366                tm = tlm;
2367            }
2368            "Tm" => {
2369                tlm = Mat {
2370                    a: op_f(operands, 0),
2371                    b: op_f(operands, 1),
2372                    c: op_f(operands, 2),
2373                    d: op_f(operands, 3),
2374                    e: op_f(operands, 4),
2375                    f: op_f(operands, 5),
2376                };
2377                tm = tlm;
2378            }
2379            "T*" => {
2380                tlm = Mat {
2381                    a: 1.0,
2382                    b: 0.0,
2383                    c: 0.0,
2384                    d: 1.0,
2385                    e: 0.0,
2386                    f: -tl,
2387                }
2388                .then(tlm);
2389                tm = tlm;
2390            }
2391            "Tc" => tc = op_f(operands, 0),
2392            "Tw" => tw = op_f(operands, 0),
2393            "Tz" => th = op_f(operands, 0) / 100.0,
2394            "TL" => tl = op_f(operands, 0),
2395            "Ts" => trise = op_f(operands, 0),
2396            "Tr" => tr = op_f(operands, 0) as i64,
2397            "Tj" | "'" | "\"" => {
2398                if op.operator == "'" || op.operator == "\"" {
2399                    // move to next line first
2400                    tlm = Mat {
2401                        a: 1.0,
2402                        b: 0.0,
2403                        c: 0.0,
2404                        d: 1.0,
2405                        e: 0.0,
2406                        f: -tl,
2407                    }
2408                    .then(tlm);
2409                    tm = tlm;
2410                }
2411                if op.operator == "\"" {
2412                    // `aw ac string "` sets word- and char-spacing before
2413                    // showing the string (PDF 32000-1 §9.4.3), persisting after.
2414                    tw = op_f(operands, 0);
2415                    tc = op_f(operands, 1);
2416                }
2417                if let (Some(f), Some(Object::String(s, _))) = (font, operands.last()) {
2418                    show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out);
2419                }
2420            }
2421            "TJ" => {
2422                if let (Some(f), Some(Object::Array(arr))) = (font, operands.first()) {
2423                    for el in arr {
2424                        match el {
2425                            Object::String(s, _) => {
2426                                show_text(f, s, fsize, tc, tw, th, trise, &mut tm, ctm, out)
2427                            }
2428                            other => {
2429                                if let Some(adj) = num(other) {
2430                                    // negative number moves text right (PDF: subtract)
2431                                    let tx = -adj / 1000.0 * fsize * th;
2432                                    tm = Mat {
2433                                        a: 1.0,
2434                                        b: 0.0,
2435                                        c: 0.0,
2436                                        d: 1.0,
2437                                        e: tx,
2438                                        f: 0.0,
2439                                    }
2440                                    .then(tm);
2441                                }
2442                            }
2443                        }
2444                    }
2445                }
2446            }
2447            "m" => {
2448                cur = ctm.apply(op_f(operands, 0), op_f(operands, 1));
2449                start = cur;
2450                shape_path.push(ShapeSub::at(cur));
2451            }
2452            "l" => {
2453                let p = ctm.apply(op_f(operands, 0), op_f(operands, 1));
2454                path.push(PathPiece::Line(cur, p));
2455                cur = p;
2456                if let Some(sub) = shape_path.last_mut() {
2457                    sub.lines.push((sub.cur, p));
2458                    sub.pts.push(p);
2459                    sub.cur = p;
2460                }
2461            }
2462            "c" | "v" | "y" => {
2463                let pts: Vec<(f64, f64)> = (0..operands.len() / 2)
2464                    .map(|k| ctm.apply(op_f(operands, 2 * k), op_f(operands, 2 * k + 1)))
2465                    .collect();
2466                if let Some(&end) = pts.last() {
2467                    path.push(PathPiece::Curve(
2468                        std::iter::once(cur).chain(pts.iter().copied()).collect(),
2469                    ));
2470                    cur = end;
2471                    if let Some(sub) = shape_path.last_mut() {
2472                        sub.pts.extend(pts.iter().copied());
2473                        sub.cur = end;
2474                    }
2475                }
2476            }
2477            "h" => {
2478                if cur != start {
2479                    path.push(PathPiece::Line(cur, start));
2480                }
2481                cur = start;
2482                // docling-parse closes the subpath and opens the next one at
2483                // its start, where a following `l` continues.
2484                if let Some(sub) = shape_path.last_mut() {
2485                    sub.closed = true;
2486                    sub.pts.push(sub.start);
2487                    let at = sub.start;
2488                    shape_path.push(ShapeSub::at(at));
2489                }
2490            }
2491            "re" => {
2492                let (x, y, w, h) = (
2493                    op_f(operands, 0),
2494                    op_f(operands, 1),
2495                    op_f(operands, 2),
2496                    op_f(operands, 3),
2497                );
2498                path.push(PathPiece::Rect([
2499                    ctm.apply(x, y),
2500                    ctm.apply(x + w, y),
2501                    ctm.apply(x + w, y + h),
2502                    ctm.apply(x, y + h),
2503                ]));
2504                cur = ctm.apply(x, y);
2505                start = cur;
2506                // `m` + three `l` + `h`, as docling-parse builds it.
2507                let c = [
2508                    ctm.apply(x, y),
2509                    ctm.apply(x + w, y),
2510                    ctm.apply(x + w, y + h),
2511                    ctm.apply(x, y + h),
2512                ];
2513                shape_path.push(ShapeSub {
2514                    start: c[0],
2515                    cur: c[3],
2516                    pts: vec![c[0], c[1], c[2], c[3], c[0]],
2517                    lines: vec![(c[0], c[1]), (c[1], c[2]), (c[2], c[3])],
2518                    closed: true,
2519                });
2520                shape_path.push(ShapeSub::at(c[0]));
2521            }
2522            "S" | "s" | "f" | "F" | "f*" | "B" | "B*" | "b" | "b*" => {
2523                let op = op.operator.as_str();
2524                if matches!(op, "s" | "b" | "b*") && cur != start {
2525                    path.push(PathPiece::Line(cur, start));
2526                }
2527                let stroke = matches!(op, "S" | "s" | "B" | "B*" | "b" | "b*");
2528                let fill = !matches!(op, "S" | "s");
2529                paint_path(&path, stroke, fill, fill_light, inks);
2530                path.clear();
2531                // `s`, `f`, `f*`, `b`, `b*` close the last subpath first.
2532                if !matches!(op, "S" | "B" | "B*") {
2533                    if let Some(sub) = shape_path.last_mut() {
2534                        sub.closed = true;
2535                    }
2536                }
2537                let scale = (ctm.a * ctm.d - ctm.b * ctm.c).abs().sqrt();
2538                record_shape(
2539                    &shape_path,
2540                    stroke,
2541                    fill,
2542                    &shape_gs,
2543                    if scale > 0.0 { scale } else { 1.0 },
2544                    shapes,
2545                );
2546                // A pending `W` takes effect after the painting operator.
2547                if std::mem::take(&mut clip_pending) {
2548                    capture_clip(&shape_path, &mut shape_gs.clips);
2549                }
2550                shape_path.clear();
2551            }
2552            "n" => {
2553                path.clear();
2554                if std::mem::take(&mut clip_pending) {
2555                    capture_clip(&shape_path, &mut shape_gs.clips);
2556                }
2557                shape_path.clear();
2558            }
2559            "W" | "W*" => clip_pending = true,
2560            "w" => shape_gs.line_width = op_f(operands, 0),
2561            "gs" => {
2562                let params = operands
2563                    .first()
2564                    .and_then(|o| o.as_name().ok())
2565                    .and_then(|n| ext_gstates?.get(n).ok())
2566                    .and_then(|o| deref(doc, o))
2567                    .and_then(|o| o.as_dict().ok());
2568                if let Some(params) = params {
2569                    let get = |k: &[u8]| params.get(k).ok().and_then(num);
2570                    if let Some(lw) = get(b"LW") {
2571                        shape_gs.line_width = lw;
2572                    }
2573                    if let Some(a) = get(b"CA") {
2574                        shape_gs.stroke_alpha = a;
2575                    }
2576                    if let Some(a) = get(b"ca") {
2577                        shape_gs.fill_alpha = a;
2578                    }
2579                }
2580            }
2581            // The fill colour, by its components: gray `g`, `rg`, CMYK `k`,
2582            // and `sc`/`scn` by operand count (a pattern name is not light).
2583            "g" | "rg" | "k" | "sc" | "scn" => {
2584                // docling-parse: a colour resolves a pattern fill; a pattern
2585                // name in `scn` makes it one.
2586                if matches!(op.operator.as_str(), "scn")
2587                    && matches!(operands.last(), Some(Object::Name(_)))
2588                {
2589                    shape_gs.fill_pattern = true;
2590                } else if !operands.is_empty() {
2591                    shape_gs.fill_pattern = false;
2592                }
2593                let v: Vec<f64> = operands
2594                    .iter()
2595                    .map(num)
2596                    .collect::<Option<_>>()
2597                    .unwrap_or_default();
2598                fill_light = match v.as_slice() {
2599                    [g] => *g >= 0.9,
2600                    [r, g, b] => r.min(*g).min(*b) >= 0.9,
2601                    [c, m, y, k] => c.max(*m).max(*y).max(*k) <= 0.1,
2602                    _ => false,
2603                };
2604            }
2605            "cs" => {
2606                fill_light = false;
2607                // A Pattern space carries no colour until `scn` names one.
2608                let pattern = match operands.first() {
2609                    Some(Object::Name(n)) if n.as_slice() == b"Pattern" => true,
2610                    Some(Object::Name(n)) => colorspaces
2611                        .and_then(|cs| cs.get(n).ok())
2612                        .and_then(|o| deref(doc, o))
2613                        .is_some_and(|o| match o {
2614                            Object::Name(f) => f.as_slice() == b"Pattern",
2615                            Object::Array(a) => a
2616                                .first()
2617                                .and_then(|f| f.as_name().ok())
2618                                .is_some_and(|f| f == b"Pattern"),
2619                            _ => false,
2620                        }),
2621                    _ => false,
2622                };
2623                if pattern {
2624                    shape_gs.fill_pattern = true;
2625                }
2626            }
2627            // An inline image: the unit square under the CTM (#999).
2628            "BI" => inks.push(image_ink(ctm)),
2629            "Do" => {
2630                // Invoke a Form XObject: bulk body text in many PDFs lives inside
2631                // a form, reached only here. An image XObject carries no text,
2632                // only its placement (#999).
2633                if depth >= 8 {
2634                    continue;
2635                }
2636                let Some(Object::Name(n)) = operands.first() else {
2637                    continue;
2638                };
2639                let obj = xobjects.and_then(|d| d.get(n.as_slice()).ok());
2640                let form_id = match obj {
2641                    Some(Object::Reference(id)) => Some(*id),
2642                    _ => None,
2643                };
2644                let stream = obj
2645                    .and_then(|o| deref(doc, o))
2646                    .and_then(|o| o.as_stream().ok());
2647                let Some(stream) = stream else { continue };
2648                let is_form = stream
2649                    .dict
2650                    .get(b"Subtype")
2651                    .ok()
2652                    .and_then(|o| o.as_name().ok())
2653                    == Some(b"Form".as_slice());
2654                if !is_form {
2655                    let is_image = stream
2656                        .dict
2657                        .get(b"Subtype")
2658                        .ok()
2659                        .and_then(|o| o.as_name().ok())
2660                        == Some(b"Image".as_slice());
2661                    if is_image {
2662                        inks.push(image_ink(ctm));
2663                    }
2664                    continue;
2665                }
2666                // Decode a reused form's content once per document
2667                // (headers/footers and bulk body text invoke the same form on
2668                // every page); see `DocCaches::forms_seen` for single-use ones.
2669                let cached = form_id.and_then(|id| caches.forms.get(&id).cloned());
2670                let form_content = match cached {
2671                    Some(c) => c,
2672                    None => {
2673                        let Ok(data) = stream.decompressed_content() else {
2674                            continue;
2675                        };
2676                        let Ok(c) = lopdf::content::Content::decode(&data) else {
2677                            continue;
2678                        };
2679                        let c = Arc::new(c);
2680                        if let Some(id) = form_id {
2681                            if !caches.forms_seen.insert(id) {
2682                                caches.forms.insert(id, Arc::clone(&c));
2683                            }
2684                        }
2685                        c
2686                    }
2687                };
2688                // The form's /Matrix maps form space into the CTM at invocation.
2689                let form_mat = match stream.dict.get(b"Matrix").ok() {
2690                    Some(Object::Array(a)) if a.len() == 6 => {
2691                        let v: Vec<f64> = a.iter().filter_map(num).collect();
2692                        if v.len() == 6 {
2693                            Mat {
2694                                a: v[0],
2695                                b: v[1],
2696                                c: v[2],
2697                                d: v[3],
2698                                e: v[4],
2699                                f: v[5],
2700                            }
2701                        } else {
2702                            Mat::ID
2703                        }
2704                    }
2705                    _ => Mat::ID,
2706                };
2707                // The form's own /Resources, falling back to the inherited ones.
2708                let form_res = stream
2709                    .dict
2710                    .get(b"Resources")
2711                    .ok()
2712                    .and_then(|o| deref(doc, o))
2713                    .and_then(|o| o.as_dict().ok())
2714                    .unwrap_or(res);
2715                let state = TextState {
2716                    tc,
2717                    tw,
2718                    th,
2719                    tl,
2720                    trise,
2721                    fsize,
2722                    tr,
2723                };
2724                run_content(
2725                    doc,
2726                    form_res,
2727                    &form_content,
2728                    form_mat.then(ctm),
2729                    state,
2730                    depth + 1,
2731                    caches,
2732                    out,
2733                    inks,
2734                    shape_gs.clone(),
2735                    shapes,
2736                );
2737            }
2738            _ => {}
2739        }
2740        // Invisible text (`Tr` 3, or 7: clip only) — an OCR layer over a scan
2741        // follows the scanned lines, not a typeset baseline (#1005).
2742        if matches!(tr, 3 | 7) {
2743            for g in &mut out[shown..] {
2744                g.invisible = true;
2745            }
2746        }
2747    }
2748}
2749
2750/// An image placement's box — the unit square under `ctm`, y-up page
2751/// points — as an [`Ink::Bitmap`](crate::checkbox::Ink::Bitmap).
2752fn image_ink(ctm: Mat) -> crate::checkbox::Ink {
2753    let pts = [(0.0, 0.0), (1.0, 0.0), (0.0, 1.0), (1.0, 1.0)].map(|(x, y)| ctm.apply(x, y));
2754    let (mut l, mut b, mut r, mut t) = (
2755        f64::INFINITY,
2756        f64::INFINITY,
2757        f64::NEG_INFINITY,
2758        f64::NEG_INFINITY,
2759    );
2760    for (x, y) in pts {
2761        (l, b, r, t) = (l.min(x), b.min(y), r.max(x), t.max(y));
2762    }
2763    crate::checkbox::Ink::Bitmap { l, b, r, t }
2764}
2765
2766/// The page's vector shapes as docling-parse's page decoder reports them to
2767/// docling's reading order (#1092): `get_shape_lines` — every stroked,
2768/// axis-aligned straight segment, clipped — and the boxes
2769/// `get_connected_shape_bounding_boxes` merges — every visible painted path,
2770/// its stroke's half width included, clipped. `[l, b, r, t]`, page points,
2771/// bottom-left origin, in the frame `run_content` walks (the display box's
2772/// lower-left corner at the origin, before `/Rotate`).
2773#[derive(Default, Clone, Debug, PartialEq)]
2774pub(crate) struct Shapes {
2775    pub lines: Vec<[f64; 4]>,
2776    pub boxes: Vec<[f64; 4]>,
2777}
2778
2779/// More painted paths than this on a page and their boxes are not merged —
2780/// docling-parse's merge is quadratic in them, and a page that dense (a map,
2781/// a seat plan, text drawn as outlines) yields no trustworthy rule anyway;
2782/// the stroked lines still count.
2783const MAX_SHAPE_BOXES: usize = 10_000;
2784
2785/// One subpath of the shape path under construction, page space: what
2786/// docling-parse's `page_item<PAGE_SHAPE>` keeps of it.
2787struct ShapeSub {
2788    start: (f64, f64),
2789    cur: (f64, f64),
2790    /// Every point the path passes or is controlled by — a curve's control
2791    /// points included, as docling-parse's segment data has them.
2792    pts: Vec<(f64, f64)>,
2793    /// Its straight segments, in order.
2794    lines: Vec<((f64, f64), (f64, f64))>,
2795    closed: bool,
2796}
2797
2798impl ShapeSub {
2799    fn at(p: (f64, f64)) -> Self {
2800        ShapeSub {
2801            start: p,
2802            cur: p,
2803            pts: vec![p],
2804            lines: Vec::new(),
2805            closed: false,
2806        }
2807    }
2808
2809    /// `keep_shape`: two points at least, spanning more than a speck (the
2810    /// box diagonal, squared, over 1e-3).
2811    fn kept(&self) -> bool {
2812        let Some((l, b, r, t)) = bbox_of(&self.pts) else {
2813            return false;
2814        };
2815        self.pts.len() >= 2 && (r - l).powi(2) + (t - b).powi(2) >= 1e-3
2816    }
2817}
2818
2819fn bbox_of(pts: &[(f64, f64)]) -> Option<(f64, f64, f64, f64)> {
2820    let (&first, rest) = pts.split_first()?;
2821    Some(rest.iter().fold(
2822        (first.0, first.1, first.0, first.1),
2823        |(l, b, r, t), &(x, y)| (l.min(x), b.min(y), r.max(x), t.max(y)),
2824    ))
2825}
2826
2827/// The graphics state docling-parse's shapes read: line width (user space),
2828/// stroke and fill alpha (`CA` / `ca`), whether the fill colour is a pattern
2829/// not yet resolved to a paint, and the clip paths' boxes. Saved and
2830/// restored with `q` / `Q`.
2831#[derive(Clone)]
2832struct ShapeGs {
2833    line_width: f64,
2834    stroke_alpha: f64,
2835    fill_alpha: f64,
2836    fill_pattern: bool,
2837    clips: Vec<[f64; 4]>,
2838}
2839
2840impl ShapeGs {
2841    const INIT: ShapeGs = ShapeGs {
2842        line_width: 1.0,
2843        stroke_alpha: 1.0,
2844        fill_alpha: 1.0,
2845        fill_pattern: false,
2846        clips: Vec::new(),
2847    };
2848}
2849
2850/// docling-parse's `bbox_intersects`: the boxes share area (a degenerate box
2851/// lying inside the other counts).
2852fn boxes_intersect(a: [f64; 4], b: [f64; 4]) -> bool {
2853    a[0] < b[2] && a[2] > b[0] && a[1] < b[3] && a[3] > b[1]
2854}
2855
2856/// `clipped_bbox`: a box cut to every clip path's box in turn; `None` once
2857/// one of them misses it.
2858fn clipped(mut bbox: [f64; 4], clips: &[[f64; 4]]) -> Option<[f64; 4]> {
2859    for c in clips {
2860        if !boxes_intersect(bbox, *c) {
2861            return None;
2862        }
2863        bbox = [
2864            bbox[0].max(c[0]),
2865            bbox[1].max(c[1]),
2866            bbox[2].min(c[2]),
2867            bbox[3].min(c[3]),
2868        ];
2869    }
2870    Some(bbox)
2871}
2872
2873/// `get_shape_lines`' `maybe_add_line` + `clip_axis_aligned_segment`: a
2874/// straight segment that is horizontal or vertical within 1e-3 (not both: a
2875/// point), cut to every clip path's box along its axis and dropped when one
2876/// misses it.
2877fn shape_line(a: (f64, f64), z: (f64, f64), clips: &[[f64; 4]], out: &mut Vec<[f64; 4]>) {
2878    const TOL: f64 = 1e-3;
2879    let horizontal = (z.1 - a.1).abs() <= TOL;
2880    let vertical = (z.0 - a.0).abs() <= TOL;
2881    if horizontal == vertical {
2882        return;
2883    }
2884    let mut bbox = [a.0.min(z.0), a.1.min(z.1), a.0.max(z.0), a.1.max(z.1)];
2885    for c in clips {
2886        if horizontal {
2887            if a.1 < c[1] - TOL || a.1 > c[3] + TOL {
2888                return;
2889            }
2890            bbox[0] = bbox[0].max(c[0]);
2891            bbox[2] = bbox[2].min(c[2]);
2892            if bbox[0] > bbox[2] + TOL {
2893                return;
2894            }
2895        } else {
2896            if a.0 < c[0] - TOL || a.0 > c[2] + TOL {
2897                return;
2898            }
2899            bbox[1] = bbox[1].max(c[1]);
2900            bbox[3] = bbox[3].min(c[3]);
2901            if bbox[1] > bbox[3] + TOL {
2902                return;
2903            }
2904        }
2905    }
2906    out.push(bbox);
2907}
2908
2909/// `capture_pending_clip`: each subpath of the clipping path with a segment
2910/// (two points or more) bounds what later operators paint — appended, which
2911/// approximates the intersection the spec asks for, as docling-parse does.
2912fn capture_clip(subs: &[ShapeSub], clips: &mut Vec<[f64; 4]>) {
2913    for sub in subs.iter().filter(|s| s.pts.len() >= 2) {
2914        if let Some((l, b, r, t)) = bbox_of(&sub.pts) {
2915            clips.push([l, b, r, t]);
2916        }
2917    }
2918}
2919
2920/// docling-parse's `register_paths` for the reading-order geometry: a painted
2921/// path's kept subpaths become shape lines (when its stroke shows) and a
2922/// shape box (when anything of it shows). A fill whose colour is an
2923/// unresolved pattern paints nothing, so a fill-only path is no shape and a
2924/// fill-and-stroke one keeps its stroke. `scale` maps the user-space line
2925/// width to page space (the CTM's `sqrt|det|`; a hairline stays 0).
2926fn record_shape(
2927    subs: &[ShapeSub],
2928    stroke: bool,
2929    fill: bool,
2930    gs: &ShapeGs,
2931    scale: f64,
2932    shapes: &mut Shapes,
2933) {
2934    let kept: Vec<&ShapeSub> = subs.iter().filter(|s| s.kept()).collect();
2935    if kept.is_empty() {
2936        return;
2937    }
2938    let fill = fill && !gs.fill_pattern;
2939    if gs.fill_pattern && !stroke {
2940        return;
2941    }
2942    let strokes_visible = stroke && gs.stroke_alpha > 0.0;
2943    let visible = strokes_visible || (fill && gs.fill_alpha > 0.0);
2944    if strokes_visible {
2945        for sub in &kept {
2946            for &(a, z) in &sub.lines {
2947                shape_line(a, z, &gs.clips, &mut shapes.lines);
2948            }
2949            if sub.closed {
2950                shape_line(sub.cur, sub.start, &gs.clips, &mut shapes.lines);
2951            }
2952        }
2953    }
2954    if visible {
2955        let pts: Vec<(f64, f64)> = kept.iter().flat_map(|s| s.pts.iter().copied()).collect();
2956        if let Some((l, b, r, t)) = bbox_of(&pts) {
2957            let pad = if strokes_visible {
2958                (gs.line_width * scale).max(0.0) * 0.5
2959            } else {
2960                0.0
2961            };
2962            if let Some(bbox) = clipped([l - pad, b - pad, r + pad, t + pad], &gs.clips) {
2963                shapes.boxes.push(bbox);
2964            }
2965        }
2966    }
2967}
2968
2969/// `get_connected_shape_bounding_boxes`: each box in turn (unless already
2970/// taken) grows by every untaken box its growing union touches, until none
2971/// does — docling-parse's greedy pass, so a component finished early keeps a
2972/// box a later one would also have reached.
2973fn connected_boxes(boxes: &[[f64; 4]]) -> Vec<[f64; 4]> {
2974    let touches =
2975        |a: [f64; 4], b: [f64; 4]| a[0] <= b[2] && a[2] >= b[0] && a[1] <= b[3] && a[3] >= b[1];
2976    let mut taken = vec![false; boxes.len()];
2977    let mut out = Vec::new();
2978    for i in 0..boxes.len() {
2979        if taken[i] {
2980            continue;
2981        }
2982        taken[i] = true;
2983        let mut component = boxes[i];
2984        let mut changed = true;
2985        while changed {
2986            changed = false;
2987            for j in 0..boxes.len() {
2988                if !taken[j] && touches(component, boxes[j]) {
2989                    let b = boxes[j];
2990                    component = [
2991                        component[0].min(b[0]),
2992                        component[1].min(b[1]),
2993                        component[2].max(b[2]),
2994                        component[3].max(b[3]),
2995                    ];
2996                    taken[j] = true;
2997                    changed = true;
2998                }
2999            }
3000        }
3001        out.push(component);
3002    }
3003    out
3004}
3005
3006/// One piece of a path under construction, page space (see `run_content`).
3007enum PathPiece {
3008    Line((f64, f64), (f64, f64)),
3009    /// An `re`: its four corners, counter-clockwise from `(x, y)`.
3010    Rect([(f64, f64); 4]),
3011    /// A Bézier segment: its start and control/end points (their hull).
3012    Curve(Vec<(f64, f64)>),
3013}
3014
3015/// Record a painted path's pieces for the checkbox finder (#609): a stroked
3016/// line is a [`Seg`](crate::checkbox::Ink::Seg), a stroked rectangle that
3017/// stays axis-aligned a [`Rect`](crate::checkbox::Ink::Rect) (one filled
3018/// white too — still an outline), and a coloured fill or a curve only its
3019/// bounding box, a [`Blot`](crate::checkbox::Ink::Blot): a possible tick
3020/// mark, or a swatch. White fills draw nothing and are dropped. Only pieces small
3021/// enough to be a checkbox edge or a mark inside one are kept.
3022fn paint_path(
3023    path: &[PathPiece],
3024    stroke: bool,
3025    fill: bool,
3026    fill_light: bool,
3027    inks: &mut Vec<crate::checkbox::Ink>,
3028) {
3029    use crate::checkbox::Ink;
3030    let bbox = |pts: &mut dyn Iterator<Item = (f64, f64)>| {
3031        pts.fold(
3032            (
3033                f64::INFINITY,
3034                f64::INFINITY,
3035                f64::NEG_INFINITY,
3036                f64::NEG_INFINITY,
3037            ),
3038            |(l, b, r, t), (x, y)| (l.min(x), b.min(y), r.max(x), t.max(y)),
3039        )
3040    };
3041    let mut keep = |ink: Ink| {
3042        if ink.worth_keeping() {
3043            inks.push(ink);
3044        }
3045    };
3046    if fill && !fill_light {
3047        // A coloured fill is a blot — a tick mark, or (the size of a square)
3048        // a legend swatch, which `checkbox::find` then rules out. A white
3049        // fill is no ink on paper: it only leaves the stroke, if any, below.
3050        let (l, b, r, t) = bbox(&mut path.iter().flat_map(|p| match p {
3051            PathPiece::Line(a, z) => vec![*a, *z],
3052            PathPiece::Rect(c) => c.to_vec(),
3053            PathPiece::Curve(c) => c.clone(),
3054        }));
3055        if l.is_finite() {
3056            keep(Ink::Blot { l, b, r, t });
3057        }
3058        return;
3059    }
3060    if !stroke {
3061        return;
3062    }
3063    for piece in path {
3064        match piece {
3065            PathPiece::Line(a, z) => keep(Ink::Seg {
3066                x0: a.0,
3067                y0: a.1,
3068                x1: z.0,
3069                y1: z.1,
3070            }),
3071            PathPiece::Rect(c) => {
3072                let axis = ((c[0].1 - c[1].1).abs() < 1e-6 && (c[1].0 - c[2].0).abs() < 1e-6)
3073                    || ((c[0].0 - c[1].0).abs() < 1e-6 && (c[1].1 - c[2].1).abs() < 1e-6);
3074                if axis {
3075                    let (l, b, r, t) = bbox(&mut c.iter().copied());
3076                    keep(Ink::Rect { l, b, r, t });
3077                } else {
3078                    for k in 0..4 {
3079                        let (a, z) = (c[k], c[(k + 1) % 4]);
3080                        keep(Ink::Seg {
3081                            x0: a.0,
3082                            y0: a.1,
3083                            x1: z.0,
3084                            y1: z.1,
3085                        });
3086                    }
3087                }
3088            }
3089            PathPiece::Curve(c) => {
3090                let (l, b, r, t) = bbox(&mut c.iter().copied());
3091                keep(Ink::Blot { l, b, r, t });
3092            }
3093        }
3094    }
3095}
3096
3097#[allow(clippy::too_many_arguments)]
3098fn show_text(
3099    font: &Font,
3100    bytes: &[u8],
3101    fsize: f64,
3102    tc: f64,
3103    tw: f64,
3104    th: f64,
3105    trise: f64,
3106    tm: &mut Mat,
3107    ctm: Mat,
3108    out: &mut Vec<Glyph>,
3109) {
3110    for code in codes(font, bytes) {
3111        let (text, w, blank) = font.decode_code(code);
3112        let w0 = w / 1000.0; // advance in text-space (em) units
3113                             // The glyph→user transform: scale glyph space by font size, then Tm, CTM.
3114        let scale = Mat {
3115            a: fsize * th,
3116            b: 0.0,
3117            c: 0.0,
3118            d: fsize,
3119            e: 0.0,
3120            f: trise,
3121        };
3122        let trm = scale.then(*tm).then(ctm);
3123        // Box in glyph space (1000-unit em): x 0..w, y descent..ascent.
3124        // A blank unmapped glyph keeps its advance but no ink height
3125        // (docling-parse collapses its ascent and descent).
3126        let (desc, asc) = if blank {
3127            (0.0, 0.0)
3128        } else {
3129            (font.descent / 1000.0, font.ascent / 1000.0)
3130        };
3131        let (x0, y0) = trm.apply(0.0, desc);
3132        let (x1, y1) = trm.apply(w0, desc);
3133        let (x2, y2) = trm.apply(w0, asc);
3134        let (x3, y3) = trm.apply(0.0, asc);
3135        // Upright = the baseline runs left-to-right along +x. Only then is the
3136        // loose box the rectangle spanned by the baseline's x and the em's y
3137        // (a synthetic oblique's skew is ignored, as it always was). A rotated
3138        // matrix — the `0 s -s 0 tx ty Tm` landscape pages are built with
3139        // (#528) — collapsed that rectangle to zero width, and every such run
3140        // vanished; those glyphs keep their real quad (docling-parse's char
3141        // rect) and its extent instead.
3142        let upright = trm.a > 0.0 && trm.b.abs() <= 1e-6 * trm.a;
3143        let (left, bot, right, top, quad) = if upright {
3144            (x0.min(x1), y0.min(y3), x0.max(x1), y0.max(y3), None)
3145        } else {
3146            let (xs, ys) = ([x0, x1, x2, x3], [y0, y1, y2, y3]);
3147            let fold = |v: [f64; 4], f: fn(f64, f64) -> f64| v.into_iter().reduce(f).unwrap();
3148            let q = [x0, y0, x1, y1, x2, y2, x3, y3].map(|v| v as f32);
3149            (
3150                fold(xs, f64::min),
3151                fold(ys, f64::min),
3152                fold(xs, f64::max),
3153                fold(ys, f64::max),
3154                Some(q),
3155            )
3156        };
3157        // The pen's origin (rise included) and the font size's extent from
3158        // it, as docling-parse maps them (`map_text_point(0, rise)` and
3159        // `(0, rise + font_size)`).
3160        let (ox, oy) = trm.apply(0.0, 0.0);
3161        let (sx, sy) = trm.apply(0.0, 1.0);
3162        let pen = Some([ox as f32, oy as f32, (sx - ox).hypot(sy - oy) as f32]);
3163        if let Some(s) = text {
3164            // A run may map one code to multiple chars (ligature/fraction); share box.
3165            for ch in s.chars() {
3166                // A NUL from the `/ToUnicode` is an unmapped glyph too
3167                // (docling#5070): U+FFFD.
3168                let ch = if ch == '\u{0}' { '\u{FFFD}' } else { ch };
3169                out.push(Glyph {
3170                    ch,
3171                    l: left as f32,
3172                    b: bot as f32,
3173                    r: right as f32,
3174                    t: top as f32,
3175                    ll: left as f32,
3176                    lb: bot as f32,
3177                    lr: right as f32,
3178                    lt: top as f32,
3179                    font: font.hash,
3180                    quad,
3181                    pen,
3182                    math_font: font.math,
3183                    invisible: false,
3184                });
3185            }
3186        }
3187        // Advance the text matrix. Word spacing applies to single-byte code 32.
3188        let is_space = !font.two_byte && code == 32;
3189        let tx = (w0 * fsize + tc + if is_space { tw } else { 0.0 }) * th;
3190        *tm = Mat {
3191            a: 1.0,
3192            b: 0.0,
3193            c: 0.0,
3194            d: 1.0,
3195            e: tx,
3196            f: 0.0,
3197        }
3198        .then(*tm);
3199    }
3200}
3201
3202/// Build a simple font's code→char table from its `/Encoding`: the base
3203/// encoding (WinAnsi / MacRoman) plus any `/Differences` overrides (glyph names
3204/// resolved through a small Adobe-glyph-name subset).
3205fn simple_encoding_table(doc: &Document, fdict: &Dictionary) -> HashMap<u8, char> {
3206    let enc = fdict.get(b"Encoding").ok().and_then(|o| deref(doc, o));
3207    let base_name = match enc {
3208        Some(Object::Name(n)) => n.clone(),
3209        Some(Object::Dictionary(d)) => d
3210            .get(b"BaseEncoding")
3211            .ok()
3212            .and_then(|o| o.as_name().ok())
3213            .map(|n| n.to_vec())
3214            .unwrap_or_default(),
3215        _ => Vec::new(),
3216    };
3217    let mut m = if base_name == b"MacRomanEncoding" {
3218        macroman_table()
3219    } else if base_name.is_empty() {
3220        // No PDF /Encoding at all: the font's *built-in* encoding applies. For
3221        // the standard TeX math fonts that is their fixed TeX layout — falling
3222        // back to StandardEncoding read CMSY's braces as `f`/`g`, `→` as `!`,
3223        // `∈` as `2` (2203's `{ahn,…}` author line). The font program (often
3224        // CFF, which this parser does not read) carries the same mapping;
3225        // docling-parse decodes it from there.
3226        tex_math_builtin(fdict).unwrap_or_else(winansi_table)
3227    } else if base_name == b"WinAnsiEncoding" {
3228        let mut m = winansi_table();
3229        // PDF 32000-1:2008 Annex D.2, note 3 to the Latin set: "In
3230        // WinAnsiEncoding, all unused codes greater than 40 [octal] map to the
3231        // bullet character" — pdfium and pdf.js read them so. ReportLab draws
3232        // its default list bullet as `\177` in a non-embedded WinAnsi
3233        // Helvetica, which read U+FFFD before (#1102). Only a font that names
3234        // WinAnsiEncoding gets them: the no-/Encoding fallback above stands
3235        // in for a built-in encoding, where these codes stay undefined.
3236        for b in [0x7Fu8, 0x81, 0x8D, 0x8F, 0x90, 0x9D] {
3237            m.insert(b, '\u{2022}');
3238        }
3239        m
3240    } else {
3241        winansi_table()
3242    };
3243    // Apply /Differences: `code /glyphname /glyphname ... code ...`.
3244    if let Some(Object::Dictionary(d)) = enc {
3245        if let Some(Object::Array(diffs)) = d.get(b"Differences").ok().and_then(|o| deref(doc, o)) {
3246            let mut code = 0u8;
3247            for el in diffs {
3248                match el {
3249                    Object::Integer(i) => code = *i as u8,
3250                    Object::Name(name) => {
3251                        if let Some(ch) = glyph_name_to_char(name) {
3252                            m.insert(code, ch);
3253                        }
3254                        code = code.wrapping_add(1);
3255                    }
3256                    _ => {}
3257                }
3258            }
3259        }
3260    }
3261    m
3262}
3263
3264/// The fixed built-in encodings of the standard TeX math fonts (TeXbook
3265/// Appendix F), keyed off the base font name: `CMSY*` (symbols; `CMBSY` is its
3266/// bold) and `CMMI*` (math italic). These fonts ship no PDF `/Encoding` and no
3267/// ToUnicode, and their program is usually CFF — without this table the codes
3268/// fell through to StandardEncoding and rendered as the wrong ASCII.
3269fn tex_math_builtin(fdict: &Dictionary) -> Option<HashMap<u8, char>> {
3270    const CMSY: [char; 128] = [
3271        '−', '·', '×', '∗', '÷', '⋄', '±', '∓', '⊕', '⊖', '⊗', '⊘', '⊙', '◯', '∘', '•', '≍', '≡',
3272        '⊆', '⊇', '≤', '≥', '≼', '≽', '∼', '≈', '⊂', '⊃', '≪', '≫', '≺', '≻', '←', '→', '↑', '↓',
3273        '↔', '↗', '↘', '≃', '⇐', '⇒', '⇑', '⇓', '⇔', '↖', '↙', '∝', '′', '∞', '∈', '∋', '△', '▽',
3274        '\u{338}', '↦', '∀', '∃', '¬', '∅', 'ℜ', 'ℑ', '⊤', '⊥', 'ℵ', 'A', 'B', 'C', 'D', 'E', 'F',
3275        'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
3276        'Y', 'Z', '∪', '∩', '⊎', '∧', '∨', '⊢', '⊣', '⌊', '⌋', '⌈', '⌉', '{', '}', '⟨', '⟩', '|',
3277        '∥', '↕', '⇕', '\\', '≀', '√', '∐', '∇', '∫', '⊔', '⊓', '⊑', '⊒', '§', '†', '‡', '¶', '♣',
3278        '♢', '♡', '♠',
3279    ];
3280    const CMMI: [char; 128] = [
3281        'Γ', 'Δ', 'Θ', 'Λ', 'Ξ', 'Π', 'Σ', 'Υ', 'Φ', 'Ψ', 'Ω', 'α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η',
3282        'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'π', 'ρ', 'σ', 'τ', 'υ', 'φ', 'χ', 'ψ', 'ω', 'ϵ', 'ϑ',
3283        'ϖ', 'ϱ', 'ς', 'ϕ', '↼', '↽', '⇀', '⇁', '↩', '↪', '▷', '◁', '0', '1', '2', '3', '4', '5',
3284        '6', '7', '8', '9', '.', ',', '<', '/', '>', '⋆', '∂', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
3285        'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y',
3286        'Z', '♭', '♮', '♯', '⌣', '⌢', 'ℓ', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k',
3287        'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'ı', 'ȷ', '℘',
3288        '\u{20d7}', '⁀',
3289    ];
3290    let name = base_font_name(fdict)?;
3291    let up = name.to_ascii_uppercase();
3292    let table: &[char; 128] = if up.starts_with(b"CMSY") || up.starts_with(b"CMBSY") {
3293        &CMSY
3294    } else if up.starts_with(b"CMMI") {
3295        &CMMI
3296    } else {
3297        return None;
3298    };
3299    Some(
3300        table
3301            .iter()
3302            .enumerate()
3303            .map(|(i, &c)| (i as u8, c))
3304            .collect(),
3305    )
3306}
3307
3308/// The Adobe Glyph List's names for Latin-1 Supplement and Latin Extended-A
3309/// (plus `Euro`, `florin`, the comma-accent pairs): what a Type1 font's
3310/// `/Differences` or builtin encoding calls an accented or stroked letter.
3311/// 2203.01017v2 names `Ł` as `/Lslash` in a NimbusRomNo9L subset with no
3312/// ToUnicode, and the resolver, which knew only the ASCII, ligature and symbol
3313/// names, dropped the glyph outright (`Ł ukasz Kaiser` read `ukasz Kaiser`).
3314/// Sorted by name for the binary search in [`glyph_name_to_char`].
3315const AGL_LATIN: &[(&str, char)] = &[
3316    ("AE", '\u{00C6}'),
3317    ("Aacute", '\u{00C1}'),
3318    ("Abreve", '\u{0102}'),
3319    ("Acircumflex", '\u{00C2}'),
3320    ("Adieresis", '\u{00C4}'),
3321    ("Agrave", '\u{00C0}'),
3322    ("Amacron", '\u{0100}'),
3323    ("Aogonek", '\u{0104}'),
3324    ("Aring", '\u{00C5}'),
3325    ("Atilde", '\u{00C3}'),
3326    ("Cacute", '\u{0106}'),
3327    ("Ccaron", '\u{010C}'),
3328    ("Ccedilla", '\u{00C7}'),
3329    ("Ccircumflex", '\u{0108}'),
3330    ("Cdotaccent", '\u{010A}'),
3331    ("Dcaron", '\u{010E}'),
3332    ("Dcroat", '\u{0110}'),
3333    ("Eacute", '\u{00C9}'),
3334    ("Ebreve", '\u{0114}'),
3335    ("Ecaron", '\u{011A}'),
3336    ("Ecircumflex", '\u{00CA}'),
3337    ("Edieresis", '\u{00CB}'),
3338    ("Edotaccent", '\u{0116}'),
3339    ("Egrave", '\u{00C8}'),
3340    ("Emacron", '\u{0112}'),
3341    ("Eng", '\u{014A}'),
3342    ("Eogonek", '\u{0118}'),
3343    ("Eth", '\u{00D0}'),
3344    ("Euro", '\u{20AC}'),
3345    ("Gbreve", '\u{011E}'),
3346    ("Gcircumflex", '\u{011C}'),
3347    ("Gcommaaccent", '\u{0122}'),
3348    ("Gdotaccent", '\u{0120}'),
3349    ("Hbar", '\u{0126}'),
3350    ("Hcircumflex", '\u{0124}'),
3351    ("IJ", '\u{0132}'),
3352    ("Iacute", '\u{00CD}'),
3353    ("Ibreve", '\u{012C}'),
3354    ("Icircumflex", '\u{00CE}'),
3355    ("Idieresis", '\u{00CF}'),
3356    ("Idotaccent", '\u{0130}'),
3357    ("Igrave", '\u{00CC}'),
3358    ("Imacron", '\u{012A}'),
3359    ("Iogonek", '\u{012E}'),
3360    ("Itilde", '\u{0128}'),
3361    ("Jcircumflex", '\u{0134}'),
3362    ("Kcommaaccent", '\u{0136}'),
3363    ("Lacute", '\u{0139}'),
3364    ("Lcaron", '\u{013D}'),
3365    ("Lcommaaccent", '\u{013B}'),
3366    ("Ldot", '\u{013F}'),
3367    ("Lslash", '\u{0141}'),
3368    ("Nacute", '\u{0143}'),
3369    ("Ncaron", '\u{0147}'),
3370    ("Ncommaaccent", '\u{0145}'),
3371    ("Ntilde", '\u{00D1}'),
3372    ("OE", '\u{0152}'),
3373    ("Oacute", '\u{00D3}'),
3374    ("Obreve", '\u{014E}'),
3375    ("Ocircumflex", '\u{00D4}'),
3376    ("Odieresis", '\u{00D6}'),
3377    ("Ograve", '\u{00D2}'),
3378    ("Ohungarumlaut", '\u{0150}'),
3379    ("Omacron", '\u{014C}'),
3380    ("Oslash", '\u{00D8}'),
3381    ("Otilde", '\u{00D5}'),
3382    ("Racute", '\u{0154}'),
3383    ("Rcaron", '\u{0158}'),
3384    ("Rcommaaccent", '\u{0156}'),
3385    ("Sacute", '\u{015A}'),
3386    ("Scaron", '\u{0160}'),
3387    ("Scedilla", '\u{015E}'),
3388    ("Scircumflex", '\u{015C}'),
3389    ("Scommaaccent", '\u{0218}'),
3390    ("Tbar", '\u{0166}'),
3391    ("Tcaron", '\u{0164}'),
3392    ("Tcommaaccent", '\u{0162}'),
3393    ("Thorn", '\u{00DE}'),
3394    ("Uacute", '\u{00DA}'),
3395    ("Ubreve", '\u{016C}'),
3396    ("Ucircumflex", '\u{00DB}'),
3397    ("Udieresis", '\u{00DC}'),
3398    ("Ugrave", '\u{00D9}'),
3399    ("Uhungarumlaut", '\u{0170}'),
3400    ("Umacron", '\u{016A}'),
3401    ("Uogonek", '\u{0172}'),
3402    ("Uring", '\u{016E}'),
3403    ("Utilde", '\u{0168}'),
3404    ("Wcircumflex", '\u{0174}'),
3405    ("Yacute", '\u{00DD}'),
3406    ("Ycircumflex", '\u{0176}'),
3407    ("Ydieresis", '\u{0178}'),
3408    ("Zacute", '\u{0179}'),
3409    ("Zcaron", '\u{017D}'),
3410    ("Zdotaccent", '\u{017B}'),
3411    ("aacute", '\u{00E1}'),
3412    ("abreve", '\u{0103}'),
3413    ("acircumflex", '\u{00E2}'),
3414    ("acute", '\u{00B4}'),
3415    ("adieresis", '\u{00E4}'),
3416    ("ae", '\u{00E6}'),
3417    ("agrave", '\u{00E0}'),
3418    ("amacron", '\u{0101}'),
3419    ("aogonek", '\u{0105}'),
3420    ("aring", '\u{00E5}'),
3421    ("atilde", '\u{00E3}'),
3422    ("brokenbar", '\u{00A6}'),
3423    ("cacute", '\u{0107}'),
3424    ("ccaron", '\u{010D}'),
3425    ("ccedilla", '\u{00E7}'),
3426    ("ccircumflex", '\u{0109}'),
3427    ("cdotaccent", '\u{010B}'),
3428    ("cedilla", '\u{00B8}'),
3429    ("cent", '\u{00A2}'),
3430    ("currency", '\u{00A4}'),
3431    ("dcaron", '\u{010F}'),
3432    ("dcroat", '\u{0111}'),
3433    ("dieresis", '\u{00A8}'),
3434    ("dotlessi", '\u{0131}'),
3435    ("eacute", '\u{00E9}'),
3436    ("ebreve", '\u{0115}'),
3437    ("ecaron", '\u{011B}'),
3438    ("ecircumflex", '\u{00EA}'),
3439    ("edieresis", '\u{00EB}'),
3440    ("edotaccent", '\u{0117}'),
3441    ("egrave", '\u{00E8}'),
3442    ("emacron", '\u{0113}'),
3443    ("eng", '\u{014B}'),
3444    ("eogonek", '\u{0119}'),
3445    ("eth", '\u{00F0}'),
3446    ("exclamdown", '\u{00A1}'),
3447    ("florin", '\u{0192}'),
3448    ("gbreve", '\u{011F}'),
3449    ("gcircumflex", '\u{011D}'),
3450    ("gcommaaccent", '\u{0123}'),
3451    ("gdotaccent", '\u{0121}'),
3452    ("germandbls", '\u{00DF}'),
3453    ("guillemotleft", '\u{00AB}'),
3454    ("guillemotright", '\u{00BB}'),
3455    ("hbar", '\u{0127}'),
3456    ("hcircumflex", '\u{0125}'),
3457    ("iacute", '\u{00ED}'),
3458    ("ibreve", '\u{012D}'),
3459    ("icircumflex", '\u{00EE}'),
3460    ("idieresis", '\u{00EF}'),
3461    ("igrave", '\u{00EC}'),
3462    ("ij", '\u{0133}'),
3463    ("imacron", '\u{012B}'),
3464    ("iogonek", '\u{012F}'),
3465    ("itilde", '\u{0129}'),
3466    ("jcircumflex", '\u{0135}'),
3467    ("kcommaaccent", '\u{0137}'),
3468    ("kgreenlandic", '\u{0138}'),
3469    ("lacute", '\u{013A}'),
3470    ("lcaron", '\u{013E}'),
3471    ("lcommaaccent", '\u{013C}'),
3472    ("ldot", '\u{0140}'),
3473    ("logicalnot", '\u{00AC}'),
3474    ("longs", '\u{017F}'),
3475    ("lslash", '\u{0142}'),
3476    ("macron", '\u{00AF}'),
3477    ("nacute", '\u{0144}'),
3478    ("napostrophe", '\u{0149}'),
3479    ("ncaron", '\u{0148}'),
3480    ("ncommaaccent", '\u{0146}'),
3481    ("ntilde", '\u{00F1}'),
3482    ("oacute", '\u{00F3}'),
3483    ("obreve", '\u{014F}'),
3484    ("ocircumflex", '\u{00F4}'),
3485    ("odieresis", '\u{00F6}'),
3486    ("oe", '\u{0153}'),
3487    ("ograve", '\u{00F2}'),
3488    ("ohungarumlaut", '\u{0151}'),
3489    ("omacron", '\u{014D}'),
3490    ("onehalf", '\u{00BD}'),
3491    ("onequarter", '\u{00BC}'),
3492    ("onesuperior", '\u{00B9}'),
3493    ("ordfeminine", '\u{00AA}'),
3494    ("ordmasculine", '\u{00BA}'),
3495    ("oslash", '\u{00F8}'),
3496    ("otilde", '\u{00F5}'),
3497    ("paragraph", '\u{00B6}'),
3498    ("questiondown", '\u{00BF}'),
3499    ("racute", '\u{0155}'),
3500    ("rcaron", '\u{0159}'),
3501    ("rcommaaccent", '\u{0157}'),
3502    ("sacute", '\u{015B}'),
3503    ("scaron", '\u{0161}'),
3504    ("scedilla", '\u{015F}'),
3505    ("scircumflex", '\u{015D}'),
3506    ("scommaaccent", '\u{0219}'),
3507    ("section", '\u{00A7}'),
3508    ("sfthyphen", '\u{00AD}'),
3509    ("sterling", '\u{00A3}'),
3510    ("tbar", '\u{0167}'),
3511    ("tcaron", '\u{0165}'),
3512    ("tcommaaccent", '\u{0163}'),
3513    ("thorn", '\u{00FE}'),
3514    ("threequarters", '\u{00BE}'),
3515    ("threesuperior", '\u{00B3}'),
3516    ("twosuperior", '\u{00B2}'),
3517    ("uacute", '\u{00FA}'),
3518    ("ubreve", '\u{016D}'),
3519    ("ucircumflex", '\u{00FB}'),
3520    ("udieresis", '\u{00FC}'),
3521    ("ugrave", '\u{00F9}'),
3522    ("uhungarumlaut", '\u{0171}'),
3523    ("umacron", '\u{016B}'),
3524    ("uogonek", '\u{0173}'),
3525    ("uring", '\u{016F}'),
3526    ("utilde", '\u{0169}'),
3527    ("wcircumflex", '\u{0175}'),
3528    ("yacute", '\u{00FD}'),
3529    ("ycircumflex", '\u{0177}'),
3530    ("ydieresis", '\u{00FF}'),
3531    ("yen", '\u{00A5}'),
3532    ("zacute", '\u{017A}'),
3533    ("zcaron", '\u{017E}'),
3534    ("zdotaccent", '\u{017C}'),
3535];
3536
3537/// Resolve an Adobe glyph name to Unicode: `uniXXXX`, single ASCII letters, the
3538/// digit/punctuation names from the Adobe Glyph List, and common typographic
3539/// names. A `.suffix` (`one.taboldstyle`, `a.sc`) is stripped and the base name
3540/// retried — docling renders these as the base character.
3541pub(crate) fn glyph_name_to_char(name: &[u8]) -> Option<char> {
3542    let s = std::str::from_utf8(name).ok()?;
3543    if let Some(hex) = s.strip_prefix("uni") {
3544        if let Ok(cp) = u32::from_str_radix(hex.get(0..4)?, 16) {
3545            return char::from_u32(cp);
3546        }
3547    }
3548    // Single ASCII letter names (`A`, `m`) map to themselves.
3549    if s.len() == 1 {
3550        let b = s.as_bytes()[0];
3551        if b.is_ascii_alphabetic() {
3552            return Some(b as char);
3553        }
3554    }
3555    let resolved = match s {
3556        "space" => ' ',
3557        "exclam" => '!',
3558        "quotedbl" => '"',
3559        "numbersign" => '#',
3560        "dollar" => '$',
3561        "percent" => '%',
3562        "ampersand" => '&',
3563        "quotesingle" => '\'',
3564        "parenleft" => '(',
3565        "parenright" => ')',
3566        "asterisk" => '*',
3567        "plus" => '+',
3568        "comma" => ',',
3569        "hyphen" => '-',
3570        "period" => '.',
3571        "slash" => '/',
3572        "zero" => '0',
3573        "one" => '1',
3574        "two" => '2',
3575        "three" => '3',
3576        "four" => '4',
3577        "five" => '5',
3578        "six" => '6',
3579        "seven" => '7',
3580        "eight" => '8',
3581        "nine" => '9',
3582        "colon" => ':',
3583        "semicolon" => ';',
3584        "less" => '<',
3585        "equal" => '=',
3586        "greater" => '>',
3587        "question" => '?',
3588        "at" => '@',
3589        "bracketleft" => '[',
3590        "backslash" => '\\',
3591        "bracketright" => ']',
3592        "asciicircum" => '^',
3593        "underscore" => '_',
3594        "grave" => '`',
3595        "braceleft" => '{',
3596        "bar" => '|',
3597        "braceright" => '}',
3598        "asciitilde" => '~',
3599        "bullet" => '\u{2022}',
3600        "periodcentered" => '\u{00B7}',
3601        "endash" => '\u{2013}',
3602        "emdash" => '\u{2014}',
3603        "quoteright" => '\u{2019}',
3604        "quoteleft" => '\u{2018}',
3605        "quotedblleft" => '\u{201C}',
3606        "quotedblright" => '\u{201D}',
3607        "quotedblbase" => '\u{201E}',
3608        "quotesinglbase" => '\u{201A}',
3609        // Latin f-ligatures named in `/Differences` (e.g. 2305's body font). These
3610        // map to the presentation-form code points, which `decompose_ligatures`
3611        // then spells back out (`ff`→"ff") — without them the glyph decodes to
3612        // nothing and the sanitizer fills the gap with a space (`di erences`).
3613        "ff" => '\u{FB00}',
3614        "fi" => '\u{FB01}',
3615        "fl" => '\u{FB02}',
3616        "ffi" => '\u{FB03}',
3617        "ffl" => '\u{FB04}',
3618        "ft" => '\u{FB05}',
3619        "st" => '\u{FB06}',
3620        "degree" => '\u{00B0}',
3621        "trademark" => '\u{2122}',
3622        "registered" => '\u{00AE}',
3623        "copyright" => '\u{00A9}',
3624        "ellipsis" => '\u{2026}',
3625        "minus" => '\u{2212}',
3626        "fraction" => '\u{2044}',
3627        "nbspace" => '\u{00A0}',
3628        // Greek + math glyph names (standard Adobe Glyph List). Standard TeX math
3629        // fonts (CMMI/CMSY/…) name their glyphs this way in the embedded font
3630        // program's `/Encoding`; without these a `λ`/`≤` decodes to nothing and is
3631        // dropped from body text (`and λ set to 0.5` → `and set to 0.5`).
3632        "alpha" => '\u{03B1}',
3633        "beta" => '\u{03B2}',
3634        "gamma" => '\u{03B3}',
3635        "delta" => '\u{03B4}',
3636        "epsilon" | "epsilon1" => '\u{03B5}',
3637        "zeta" => '\u{03B6}',
3638        "eta" => '\u{03B7}',
3639        "theta" | "theta1" => '\u{03B8}',
3640        "iota" => '\u{03B9}',
3641        "kappa" => '\u{03BA}',
3642        "lambda" => '\u{03BB}',
3643        "mu" => '\u{03BC}',
3644        "nu" => '\u{03BD}',
3645        "xi" => '\u{03BE}',
3646        "omicron" => '\u{03BF}',
3647        "pi" | "pi1" => '\u{03C0}',
3648        "rho" | "rho1" => '\u{03C1}',
3649        "sigma" => '\u{03C3}',
3650        "sigma1" => '\u{03C2}',
3651        "tau" => '\u{03C4}',
3652        "upsilon" => '\u{03C5}',
3653        "phi" | "phi1" => '\u{03C6}',
3654        "chi" => '\u{03C7}',
3655        "psi" => '\u{03C8}',
3656        "omega" | "omega1" => '\u{03C9}',
3657        "Gamma" => '\u{0393}',
3658        "Delta" => '\u{0394}',
3659        "Theta" => '\u{0398}',
3660        "Lambda" => '\u{039B}',
3661        "Xi" => '\u{039E}',
3662        "Pi" => '\u{03A0}',
3663        "Sigma" => '\u{03A3}',
3664        "Upsilon" => '\u{03A5}',
3665        "Phi" => '\u{03A6}',
3666        "Psi" => '\u{03A8}',
3667        "Omega" => '\u{03A9}',
3668        "lessequal" => '\u{2264}',
3669        "greaterequal" => '\u{2265}',
3670        "notequal" => '\u{2260}',
3671        "approxequal" => '\u{2248}',
3672        "equivalence" => '\u{2261}',
3673        "element" => '\u{2208}',
3674        "plusminus" => '\u{00B1}',
3675        "multiply" => '\u{00D7}',
3676        "divide" => '\u{00F7}',
3677        "infinity" => '\u{221E}',
3678        "partialdiff" => '\u{2202}',
3679        "gradient" => '\u{2207}',
3680        "summation" => '\u{2211}',
3681        "product" => '\u{220F}',
3682        "integral" => '\u{222B}',
3683        "radical" => '\u{221A}',
3684        "proportional" => '\u{221D}',
3685        "arrowright" => '\u{2192}',
3686        "arrowleft" => '\u{2190}',
3687        "arrowup" => '\u{2191}',
3688        "arrowdown" => '\u{2193}',
3689        "arrowboth" => '\u{2194}',
3690        "arrowdblright" => '\u{21D2}',
3691        "logicaland" => '\u{2227}',
3692        "logicalor" => '\u{2228}',
3693        "intersection" => '\u{2229}',
3694        "union" => '\u{222A}',
3695        "similar" => '\u{223C}',
3696        "congruent" => '\u{2245}',
3697        "dotmath" => '\u{22C5}',
3698        "asteriskmath" => '\u{2217}',
3699        _ => {
3700            if let Ok(i) = AGL_LATIN.binary_search_by_key(&s, |(n, _)| n) {
3701                return Some(AGL_LATIN[i].1);
3702            }
3703            // Strip an AGL `.suffix` (oldstyle/small-cap variant) and retry.
3704            if let Some((base, _)) = s.split_once('.') {
3705                if !base.is_empty() {
3706                    return glyph_name_to_char(base.as_bytes());
3707                }
3708            }
3709            // An AGL ligature name joins its components with `_` (`f_f_i`,
3710            // `f_l` — what a re-encoded Libertine subset calls them); joined,
3711            // the presentation-form arms above resolve them.
3712            if s.contains('_') {
3713                let joined: String = s.chars().filter(|&c| c != '_').collect();
3714                if !joined.is_empty() && joined.len() < s.len() {
3715                    return glyph_name_to_char(joined.as_bytes());
3716                }
3717            }
3718            return None;
3719        }
3720    };
3721    Some(resolved)
3722}
3723
3724/// Minimal WinAnsiEncoding (Latin-1-ish) for simple fonts lacking ToUnicode.
3725fn winansi_table() -> HashMap<u8, char> {
3726    let mut m = HashMap::new();
3727    for b in 0x20u8..=0x7e {
3728        m.insert(b, b as char);
3729    }
3730    // High range: Windows-1252 printable points that differ from Latin-1.
3731    let extra: &[(u8, char)] = &[
3732        (0x91, '\u{2018}'),
3733        (0x92, '\u{2019}'),
3734        (0x93, '\u{201C}'),
3735        (0x94, '\u{201D}'),
3736        (0x95, '\u{2022}'),
3737        (0x96, '\u{2013}'),
3738        (0x97, '\u{2014}'),
3739        (0x85, '\u{2026}'),
3740        (0xA0, '\u{00A0}'),
3741        // The rest of Windows-1252's 0x80–0x9F row. Left out, the German
3742        // opening quote `„` (0x84) vanished from the text layer while its
3743        // closing `“` (0x93) stayed (#922); `€ ‚ ƒ † ‡ ˆ ‰ Š ‹ Œ Ž ˜ ™ š › œ ž Ÿ`
3744        // were lost the same way. 0x81/0x8D/0x8F/0x90/0x9D (and 0x7F) are
3745        // unused; a font naming WinAnsiEncoding reads them as bullets
3746        // (`simple_encoding_table`, #1102).
3747        (0x80, '\u{20AC}'),
3748        (0x82, '\u{201A}'),
3749        (0x83, '\u{0192}'),
3750        (0x84, '\u{201E}'),
3751        (0x86, '\u{2020}'),
3752        (0x87, '\u{2021}'),
3753        (0x88, '\u{02C6}'),
3754        (0x89, '\u{2030}'),
3755        (0x8A, '\u{0160}'),
3756        (0x8B, '\u{2039}'),
3757        (0x8C, '\u{0152}'),
3758        (0x8E, '\u{017D}'),
3759        (0x98, '\u{02DC}'),
3760        (0x99, '\u{2122}'),
3761        (0x9A, '\u{0161}'),
3762        (0x9B, '\u{203A}'),
3763        (0x9C, '\u{0153}'),
3764        (0x9E, '\u{017E}'),
3765        (0x9F, '\u{0178}'),
3766    ];
3767    for &(b, c) in extra {
3768        m.insert(b, c);
3769    }
3770    for b in 0xA1u8..=0xFF {
3771        m.entry(b).or_insert(b as char);
3772    }
3773    m
3774}
3775
3776/// MacRomanEncoding: ASCII plus the whole high range, 0x80–0xFF, as Mac OS
3777/// Roman maps it (#744) — the table pdfium, pdftotext and pdf.js decode with.
3778/// Two codes follow the PDF reference instead of Apple's later revisions:
3779/// 0xDB is `currency` (¤, not €), and 0xF0 (`apple`, a private-use glyph) is
3780/// left unmapped. 0xCA, the PDF table's second `space`, reads as a no-break
3781/// space.
3782fn macroman_table() -> HashMap<u8, char> {
3783    #[rustfmt::skip]
3784    const HIGH: [char; 128] = [
3785        'Ä', 'Å', 'Ç', 'É', 'Ñ', 'Ö', 'Ü', 'á',  // 0x80
3786        'à', 'â', 'ä', 'ã', 'å', 'ç', 'é', 'è',  // 0x88
3787        'ê', 'ë', 'í', 'ì', 'î', 'ï', 'ñ', 'ó',  // 0x90
3788        'ò', 'ô', 'ö', 'õ', 'ú', 'ù', 'û', 'ü',  // 0x98
3789        '†', '°', '¢', '£', '§', '•', '¶', 'ß',  // 0xA0
3790        '®', '©', '™', '´', '¨', '≠', 'Æ', 'Ø',  // 0xA8
3791        '∞', '±', '≤', '≥', '¥', 'µ', '∂', '∑',  // 0xB0
3792        '∏', 'π', '∫', 'ª', 'º', 'Ω', 'æ', 'ø',  // 0xB8
3793        '¿', '¡', '¬', '√', 'ƒ', '≈', '∆', '«',  // 0xC0
3794        '»', '…', '\u{00A0}', 'À', 'Ã', 'Õ', 'Œ', 'œ',  // 0xC8
3795        '–', '—', '“', '”', '‘', '’', '÷', '◊',  // 0xD0
3796        'ÿ', 'Ÿ', '⁄', '¤', '‹', '›', 'fi', 'fl',  // 0xD8
3797        '‡', '·', '‚', '„', '‰', 'Â', 'Ê', 'Á',  // 0xE0
3798        'Ë', 'È', 'Í', 'Î', 'Ï', 'Ì', 'Ó', 'Ô',  // 0xE8
3799        '\0', 'Ò', 'Ú', 'Û', 'Ù', 'ı', 'ˆ', '˜',  // 0xF0
3800        '¯', '˘', '˙', '˚', '¸', '˝', '˛', 'ˇ',  // 0xF8
3801    ];
3802    let mut m = HashMap::new();
3803    for b in 0x20u8..=0x7e {
3804        m.insert(b, b as char);
3805    }
3806    for (b, &c) in (0x80u8..=0xFF).zip(HIGH.iter()) {
3807        if c != '\0' {
3808            m.insert(b, c);
3809        }
3810    }
3811    m
3812}
3813
3814#[cfg(test)]
3815mod glyph_names {
3816    /// `/Lslash` and the other Latin-1 / Latin Extended-A glyph names a Type1
3817    /// `/Differences` uses resolve (2203.01017v2's `Łukasz`); the table the
3818    /// lookup bisects is sorted and the ligature / symbol arms still win.
3819    #[test]
3820    fn latin_glyph_names_resolve() {
3821        for w in super::AGL_LATIN.windows(2) {
3822            assert!(
3823                w[0].0 < w[1].0,
3824                "AGL_LATIN not sorted at {:?} / {:?}",
3825                w[0].0,
3826                w[1].0
3827            );
3828        }
3829        for (name, ch) in [
3830            ("Lslash", 'Ł'),
3831            ("lslash", 'ł'),
3832            ("Aacute", 'Á'),
3833            ("aacute", 'á'),
3834            ("Scaron", 'Š'),
3835            ("germandbls", 'ß'),
3836            ("oe", 'œ'),
3837            ("Ydieresis", 'Ÿ'),
3838            ("Euro", '€'),
3839            ("Lslash.sc", 'Ł'),
3840            ("fi", '\u{FB01}'),
3841            ("Omega", '\u{03A9}'),
3842        ] {
3843            assert_eq!(
3844                super::glyph_name_to_char(name.as_bytes()),
3845                Some(ch),
3846                "{name}"
3847            );
3848        }
3849        assert_eq!(super::glyph_name_to_char(b"g123"), None);
3850        // AGL ligature names: components joined with `_`.
3851        assert_eq!(super::glyph_name_to_char(b"f_f_i"), Some('\u{FB03}'));
3852        assert_eq!(super::glyph_name_to_char(b"f_l"), Some('\u{FB02}'));
3853        assert_eq!(super::glyph_name_to_char(b"a_b"), None);
3854    }
3855
3856    /// The stale-CMap case: ToUnicode maps a code to its own identity while
3857    /// `/Differences` names a ligature there — the name wins; a CMap that
3858    /// maps the code anywhere else, or a code with no entry, is untouched.
3859    #[test]
3860    fn a_stale_tounicode_yields_to_differences() {
3861        use lopdf::Object;
3862        use std::collections::HashMap;
3863        let mut tu: HashMap<u32, String> = HashMap::new();
3864        tu.insert(33, "!".into()); // identity — stale
3865        tu.insert(34, "x".into()); // the producer meant it
3866        let diffs = vec![
3867            Object::Integer(33),
3868            Object::Name(b"f_f_i".to_vec()),
3869            Object::Name(b"f_i".to_vec()),
3870            Object::Name(b"f_f".to_vec()),
3871            Object::Integer(65),
3872            Object::Name(b"A".to_vec()), // identity on both sides: nothing to do
3873        ];
3874        super::reconcile_stale_tounicode(&mut tu, &diffs);
3875        assert_eq!(tu.get(&33).map(String::as_str), Some("\u{FB03}"));
3876        assert_eq!(tu.get(&34).map(String::as_str), Some("x"));
3877        assert_eq!(tu.get(&35), None);
3878        assert_eq!(tu.get(&65), None);
3879    }
3880}
3881
3882#[cfg(test)]
3883mod page_box_frame {
3884    use super::*;
3885    use lopdf::dictionary;
3886
3887    /// One page, `boxes` spliced into the page dictionary verbatim, one text
3888    /// run at user-space `(x, y)`.
3889    fn pdf(boxes: &str, x: f32, y: f32) -> Vec<u8> {
3890        pdf_content(
3891            boxes,
3892            &format!("BT /F1 12 Tf {x} {y} Td (First printing) Tj ET\n"),
3893        )
3894    }
3895
3896    /// One page, `boxes` spliced into the page dictionary, `content` as its
3897    /// content stream (font `/F1` = Helvetica).
3898    fn pdf_content(boxes: &str, content: &str) -> Vec<u8> {
3899        let objs: Vec<String> = vec![
3900            "<</Type/Catalog/Pages 2 0 R>>".into(),
3901            format!("<</Type/Pages/Kids[3 0 R]/Count 1{boxes}>>"),
3902            "<</Type/Page/Parent 2 0 R/Contents 4 0 R/Resources<</Font<</F1 5 0 R>>>>>>".into(),
3903            format!("<</Length {}>>stream\n{content}endstream", content.len()),
3904            "<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".into(),
3905        ];
3906        let mut out = b"%PDF-1.4\n".to_vec();
3907        let mut offsets = Vec::new();
3908        for (i, body) in objs.iter().enumerate() {
3909            offsets.push(out.len());
3910            out.extend_from_slice(format!("{} 0 obj{body}endobj\n", i + 1).as_bytes());
3911        }
3912        let xref_at = out.len();
3913        out.extend_from_slice(
3914            format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes(),
3915        );
3916        for off in &offsets {
3917            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
3918        }
3919        out.extend_from_slice(
3920            format!(
3921                "trailer<</Size {}/Root 1 0 R>>\nstartxref\n{xref_at}\n%%EOF\n",
3922                objs.len() + 1
3923            )
3924            .as_bytes(),
3925        );
3926        out
3927    }
3928
3929    /// One page whose `/Contents` is an array of separate streams.
3930    fn pdf_content_streams(contents: &[&[u8]]) -> Vec<u8> {
3931        let mut doc = Document::with_version("1.5");
3932        let streams = contents
3933            .iter()
3934            .map(|content| {
3935                Object::Reference(
3936                    doc.add_object(lopdf::Stream::new(Dictionary::new(), content.to_vec())),
3937                )
3938            })
3939            .collect::<Vec<_>>();
3940        let font_id = doc.add_object(dictionary! {
3941            "Type" => "Font",
3942            "Subtype" => "Type1",
3943            "BaseFont" => "Helvetica",
3944        });
3945        let resources_id = doc.add_object(dictionary! {
3946            "Font" => dictionary! { "F1" => Object::Reference(font_id) },
3947        });
3948        let pages_id = doc.new_object_id();
3949        let page_id = doc.add_object(dictionary! {
3950            "Type" => "Page",
3951            "Parent" => Object::Reference(pages_id),
3952            "MediaBox" => vec![0.into(), 0.into(), 300.into(), 400.into()],
3953            "Contents" => streams,
3954            "Resources" => Object::Reference(resources_id),
3955        });
3956        doc.objects.insert(
3957            pages_id,
3958            Object::Dictionary(dictionary! {
3959                "Type" => "Pages",
3960                "Kids" => vec![Object::Reference(page_id)],
3961                "Count" => 1,
3962            }),
3963        );
3964        let catalog_id = doc.add_object(dictionary! {
3965            "Type" => "Catalog",
3966            "Pages" => Object::Reference(pages_id),
3967        });
3968        doc.trailer.set("Root", Object::Reference(catalog_id));
3969        let mut out = Vec::new();
3970        doc.save_to(&mut out).expect("save synthetic PDF");
3971        out
3972    }
3973
3974    fn only_page(bytes: &[u8]) -> (PageBox, Vec<Glyph>) {
3975        let doc = load_document(bytes).expect("loads");
3976        let pid = *doc.get_pages().values().next().expect("one page");
3977        (page_box(&doc, pid), page_glyphs(&doc, pid))
3978    }
3979
3980    /// The trimmed-book-page shape: MediaBox and CropBox both away from the
3981    /// origin (inherited from `/Pages`). The display box is the CropBox, and a
3982    /// glyph's coordinates count from its lower-left corner — the same numbers
3983    /// the same text gets on a page whose CropBox *is* `[0 0 w h]`.
3984    #[test]
3985    fn glyphs_count_from_the_cropbox_corner_like_pdfium() {
3986        let (pb, shifted) = only_page(&pdf(
3987            "/MediaBox[-56.505 -58.25 576.303 723.31]/CropBox[1.095 -0.65 518.703 665.71]",
3988            37.0 + 1.095,
3989            58.0 - 0.65,
3990        ));
3991        assert!((pb.l - 1.095).abs() < 1e-3 && (pb.b + 0.65).abs() < 1e-3);
3992        assert!(
3993            (pb.w - 517.608).abs() < 1e-3 && (pb.h - 666.36).abs() < 1e-3,
3994            "{pb:?}"
3995        );
3996        let (pb0, plain) = only_page(&pdf("/MediaBox[0 0 517.608 666.36]", 37.0, 58.0));
3997        assert!((pb0.w - pb.w).abs() < 1e-3 && (pb0.h - pb.h).abs() < 1e-3);
3998        assert_eq!(shifted.len(), plain.len());
3999        assert!(!plain.is_empty());
4000        for (a, b) in shifted.iter().zip(&plain) {
4001            assert!(
4002                (a.l - b.l).abs() < 1e-3 && (a.b - b.b).abs() < 1e-3,
4003                "{:?} vs {:?}",
4004                (a.l, a.b),
4005                (b.l, b.b)
4006            );
4007        }
4008        assert!((plain[0].l - 37.0).abs() < 1e-3, "{}", plain[0].l);
4009    }
4010
4011    fn text(glyphs: &[Glyph]) -> String {
4012        glyphs.iter().map(|g| g.ch).collect()
4013    }
4014
4015    /// #529: text drawn outside the display box — a print slug beside the
4016    /// CropBox, a tiled page's neighbour beyond the MediaBox — is dropped,
4017    /// glyph by glyph like docling-parse, instead of being clamped onto the
4018    /// page edge; a line crossing the edge keeps the glyphs that fit.
4019    #[test]
4020    fn glyphs_outside_the_display_box_are_dropped() {
4021        let (_, g) = only_page(&pdf_content(
4022            "/MediaBox[0 0 400 400]/CropBox[100 100 400 400]",
4023            "BT /F1 14 Tf 120 300 Td (Visible.) Tj ET\nBT /F1 8 Tf 5 40 Td (Slug) Tj ET\n",
4024        ));
4025        assert_eq!(text(&g), "Visible.");
4026        let (_, g) = only_page(&pdf_content(
4027            "/MediaBox[0 0 300 400]",
4028            "BT /F1 14 Tf 40 300 Td (On page) Tj ET\nBT /F1 14 Tf 400 250 Td (Beyond) Tj ET\n",
4029        ));
4030        assert_eq!(text(&g), "On page");
4031        // docling-parse: "Crossing the right edge" on a 300 pt page → "Crossing the rig".
4032        let (_, g) = only_page(&pdf_content(
4033            "/MediaBox[0 0 300 400]",
4034            "BT /F1 14 Tf 200 250 Td (Crossing the right edge) Tj ET\n",
4035        ));
4036        assert_eq!(text(&g), "Crossing the rig");
4037    }
4038
4039    #[test]
4040    fn leading_comment_content_stream_does_not_hide_later_text() {
4041        // The comment has its own EOL; lopdf inserts another while assembling
4042        // the streams, which is the boundary its 0.44 parser stopped at.
4043        let (_, glyphs) = only_page(&pdf_content_streams(&[
4044            b"% CANON_PFINF_TYPE2_TEXTOFF\n",
4045            b"BT /F1 14 Tf 40 300 Td (Second stream text) Tj ET",
4046        ]));
4047        assert_eq!(text(&glyphs), "Second stream text");
4048    }
4049
4050    /// The containment test is docling-parse's: the whole char box (advance
4051    /// × the font's ascent / descent) inside the display box, edges
4052    /// included — a Helvetica `W` at 50 pt (47.2 wide) ending exactly on the
4053    /// right edge stays, 0.01 pt further it goes; likewise on the left.
4054    #[test]
4055    fn a_glyph_on_the_edge_stays_one_over_it_goes() {
4056        let w_at = |x: f32, y: f32| {
4057            let (_, g) = only_page(&pdf_content(
4058                "/MediaBox[0 0 300 400]",
4059                &format!("BT /F1 50 Tf {x} {y} Td (W) Tj ET\n"),
4060            ));
4061            text(&g)
4062        };
4063        assert_eq!(w_at(252.8, 200.0), "W");
4064        assert_eq!(w_at(252.81, 200.0), "");
4065        assert_eq!(w_at(0.0, 200.0), "W");
4066        assert_eq!(w_at(-0.01, 200.0), "");
4067        // Vertically the box is the font's ascent / descent: a baseline far
4068        // enough up keeps the descender on the page, one at 0 does not.
4069        assert_eq!(w_at(100.0, 20.0), "W");
4070        assert_eq!(w_at(100.0, 0.0), "");
4071        assert_eq!(w_at(100.0, 380.0), "");
4072    }
4073
4074    /// pdfium's fallbacks: no MediaBox → Letter; a CropBox is clipped to the
4075    /// MediaBox, and one that misses it entirely is ignored.
4076    #[test]
4077    fn page_box_follows_pdfium_fallbacks() {
4078        let (pb, _) = only_page(&pdf("", 10.0, 10.0));
4079        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 612.0, 792.0));
4080        let (pb, _) = only_page(&pdf(
4081            "/MediaBox[0 0 500 700]/CropBox[-100 100 600 900]",
4082            10.0,
4083            10.0,
4084        ));
4085        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 100.0, 500.0, 600.0));
4086        let (pb, _) = only_page(&pdf(
4087            "/MediaBox[0 0 500 700]/CropBox[800 800 900 900]",
4088            10.0,
4089            10.0,
4090        ));
4091        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
4092        // Reversed corners normalize.
4093        let (pb, _) = only_page(&pdf("/MediaBox[500 700 0 0]", 10.0, 10.0));
4094        assert_eq!((pb.l, pb.b, pb.w, pb.h), (0.0, 0.0, 500.0, 700.0));
4095    }
4096}
4097
4098#[cfg(test)]
4099mod xref_repair {
4100    /// Build a tiny one-page PDF whose cross-reference entries are either the
4101    /// spec's 20 bytes (`two_byte_eol`) or the 19-byte form some generators
4102    /// emit — everything else about the two files is identical.
4103    fn pdf_with_xref(two_byte_eol: bool) -> Vec<u8> {
4104        let content = b"BT /F1 12 Tf 72 700 Td (Invoice 922769430725) Tj ET\n";
4105        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
4106        let objs: Vec<Vec<u8>> = vec![
4107            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
4108            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
4109            b"<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
4110               /Resources<</Font<</F1 5 0 R>>>>>>"
4111                .to_vec(),
4112            [stream.as_slice(), content.as_slice(), b"endstream"].concat(),
4113            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>".to_vec(),
4114        ];
4115
4116        let mut out = b"%PDF-1.4\n".to_vec();
4117        let mut offsets = Vec::new();
4118        for (i, body) in objs.iter().enumerate() {
4119            offsets.push(out.len());
4120            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
4121            out.extend_from_slice(body);
4122            out.extend_from_slice(b"endobj\n");
4123        }
4124        let xref_at = out.len();
4125        let eol: &[u8] = if two_byte_eol { b" \n" } else { b"\n" };
4126        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
4127        out.extend_from_slice(b"0000000000 65535 f");
4128        out.extend_from_slice(eol);
4129        for off in &offsets {
4130            out.extend_from_slice(format!("{off:010} 00000 n").as_bytes());
4131            out.extend_from_slice(eol);
4132        }
4133        out.extend_from_slice(
4134            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
4135        );
4136        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
4137        out
4138    }
4139
4140    /// A 19-byte cross-reference entry (a bare LF where the spec wants a
4141    /// two-byte EOL) used to make lopdf reject the whole file. Lopdf 0.45
4142    /// accepts it directly, so exercise the fallback itself and prove it still
4143    /// produces the exact 20-byte entries lopdf 0.44 needed.
4144    #[test]
4145    fn short_xref_entries_are_padded_for_fallback() {
4146        let good = pdf_with_xref(true);
4147        let broken = pdf_with_xref(false);
4148        assert!(
4149            broken.len() < good.len(),
4150            "the broken file is the shorter one"
4151        );
4152        assert!(
4153            lopdf::Document::load_mem(&good).is_ok(),
4154            "the control file must load unaided"
4155        );
4156        assert!(
4157            lopdf::Document::load_mem(&broken).is_ok(),
4158            "lopdf 0.45 should accept 19-byte entries"
4159        );
4160
4161        let repaired = super::pad_short_xref_entries(&broken).expect("repair short entries");
4162        assert_eq!(repaired, good, "repair should recreate the valid xref");
4163        assert!(
4164            lopdf::Document::load_mem(&repaired).is_ok(),
4165            "the bytes produced for lopdf 0.44 must load"
4166        );
4167
4168        let cells = |b: &[u8]| -> Vec<String> {
4169            super::pdf_textlines(b)
4170                .into_iter()
4171                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
4172                .collect()
4173        };
4174        let from_good = cells(&good);
4175        assert!(
4176            from_good.iter().any(|t| t.contains("922769430725")),
4177            "control text: {from_good:?}"
4178        );
4179        assert_eq!(
4180            cells(&repaired),
4181            from_good,
4182            "repair must match the good parse"
4183        );
4184    }
4185
4186    /// The same generator overstates `/Length`. Lopdf 0.45 recovers at
4187    /// `endstream`, so exercise our fallback directly and prove it still writes
4188    /// the exact length lopdf 0.44 needed without moving any bytes.
4189    #[test]
4190    fn overstated_stream_length_is_fixed_for_fallback() {
4191        let good = pdf_with_xref(true);
4192        // Inflate the content stream's /Length by one, exactly as the invoice
4193        // that prompted this does.
4194        let broken = {
4195            let at = good
4196                .windows(8)
4197                .position(|w| w == b"/Length ")
4198                .expect("a /Length")
4199                + 8;
4200            let digits = good[at..].iter().take_while(|c| c.is_ascii_digit()).count();
4201            let n: usize = std::str::from_utf8(&good[at..at + digits])
4202                .unwrap()
4203                .parse()
4204                .unwrap();
4205            let inflated = (n + 1).to_string();
4206            assert_eq!(inflated.len(), digits, "keep the digit count");
4207            let mut b = good.clone();
4208            b[at..at + digits].copy_from_slice(inflated.as_bytes());
4209            b
4210        };
4211        assert_eq!(broken.len(), good.len(), "the defect must not move bytes");
4212        // Lopdf 0.45 keeps the stream despite the overstated length.
4213        let raw = lopdf::Document::load_mem(&broken).expect("still loads");
4214        assert!(
4215            raw.get_pages()
4216                .into_values()
4217                .any(|p| !raw.get_page_content(p).is_empty()),
4218            "lopdf 0.45 should recover content at endstream"
4219        );
4220
4221        let repaired = super::fix_stream_lengths(&broken);
4222        assert_eq!(repaired.len(), broken.len(), "repair must preserve offsets");
4223        assert_eq!(repaired, good, "repair should restore the true length");
4224        let repaired_doc = lopdf::Document::load_mem(&repaired).expect("repaired PDF loads");
4225        assert!(
4226            repaired_doc
4227                .get_pages()
4228                .into_values()
4229                .any(|p| !repaired_doc.get_page_content(p).is_empty()),
4230            "the bytes produced for lopdf 0.44 must retain page content"
4231        );
4232
4233        // Ours recovers the same text the well-formed file gives.
4234        let text = |b: &[u8]| -> Vec<String> {
4235            super::pdf_textlines(b)
4236                .into_iter()
4237                .flat_map(|(_, _, c)| c.into_iter().map(|c| c.text))
4238                .collect()
4239        };
4240        let expected = text(&good);
4241        assert!(!expected.is_empty(), "control must produce text");
4242        assert_eq!(text(&repaired), expected);
4243    }
4244
4245    /// The repair only fires where padding cannot move an object: it declines a
4246    /// file whose xref precedes an object (an incremental update), rather than
4247    /// shifting every offset the table records.
4248    #[test]
4249    fn repair_declines_when_padding_would_move_objects() {
4250        let mut incremental = pdf_with_xref(false);
4251        incremental.extend_from_slice(b"6 0 obj<</Type/Whatever>>endobj\n");
4252        let declined = super::pad_short_xref_entries(&incremental).unwrap_err();
4253        assert!(
4254            declined.contains("object follows the xref"),
4255            "reason: {declined}"
4256        );
4257    }
4258}
4259
4260/// #187: standard-14 fonts referenced without an embedded program (and thus
4261/// usually without `/Widths` or a `/FontDescriptor`) must decode with the
4262/// built-in Adobe Core 14 metrics instead of collapsing every cell to zero
4263/// width — the failure mode where a valid text layer was silently dropped
4264/// while pdfium read the same file fine.
4265#[cfg(test)]
4266mod base14_fonts {
4267    /// A one-page PDF whose single `Tj` uses `fontdict` (no embedded program).
4268    fn pdf_with_font(fontdict: &[u8], text: &[u8]) -> Vec<u8> {
4269        let content = [b"BT /F1 12 Tf 72 700 Td (".as_slice(), text, b") Tj ET\n"].concat();
4270        pdf_with_content(fontdict, &content)
4271    }
4272
4273    /// A one-page A4 PDF drawing `content` with `fontdict` as `/F1`.
4274    pub(super) fn pdf_with_content(fontdict: &[u8], content: &[u8]) -> Vec<u8> {
4275        pdf_with_fonts(&[fontdict], content)
4276    }
4277
4278    /// A one-page A4 PDF drawing `content` with `fontdicts[i]` bound as
4279    /// `/F{i+1}` (objects 5, 6, …).
4280    fn pdf_with_fonts(fontdicts: &[&[u8]], content: &[u8]) -> Vec<u8> {
4281        let stream = format!("<</Length {}>>stream\n", content.len()).into_bytes();
4282        let font_res: String = fontdicts
4283            .iter()
4284            .enumerate()
4285            .map(|(i, _)| format!("/F{} {} 0 R", i + 1, i + 5))
4286            .collect();
4287        let mut objs: Vec<Vec<u8>> = vec![
4288            b"<</Type/Catalog/Pages 2 0 R>>".to_vec(),
4289            b"<</Type/Pages/Kids[3 0 R]/Count 1>>".to_vec(),
4290            format!(
4291                "<</Type/Page/Parent 2 0 R/MediaBox[0 0 595 842]/Contents 4 0 R\
4292                 /Resources<</Font<<{font_res}>>>>>>"
4293            )
4294            .into_bytes(),
4295            [stream.as_slice(), content, b"endstream"].concat(),
4296        ];
4297        objs.extend(fontdicts.iter().map(|d| d.to_vec()));
4298        let mut out = b"%PDF-1.4\n".to_vec();
4299        let mut offsets = Vec::new();
4300        for (i, body) in objs.iter().enumerate() {
4301            offsets.push(out.len());
4302            out.extend_from_slice(format!("{} 0 obj", i + 1).as_bytes());
4303            out.extend_from_slice(body);
4304            out.extend_from_slice(b"endobj\n");
4305        }
4306        let xref_at = out.len();
4307        out.extend_from_slice(format!("xref\n0 {}\n", objs.len() + 1).as_bytes());
4308        out.extend_from_slice(b"0000000000 65535 f \n");
4309        for off in &offsets {
4310            out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes());
4311        }
4312        out.extend_from_slice(
4313            format!("trailer<</Size {}/Root 1 0 R>>\n", objs.len() + 1).as_bytes(),
4314        );
4315        out.extend_from_slice(format!("startxref\n{xref_at}\n%%EOF\n").as_bytes());
4316        out
4317    }
4318
4319    /// The parsed cells of the only page.
4320    pub(super) fn cells(pdf: &[u8]) -> Vec<crate::pdfium_backend::TextCell> {
4321        super::pdf_textlines(pdf)
4322            .into_iter()
4323            .flat_map(|(_, _, c)| c)
4324            .collect()
4325    }
4326
4327    /// Text drawn with a rotated text matrix reads as one line in its own
4328    /// reading order, boxed by the run's axis-aligned extent (#528). The
4329    /// landscape case is `0 s -s 0 tx ty Tm` on a `/Rotate 90` page; every
4330    /// such run used to collapse to zero width and vanish (90°/270°/tilted)
4331    /// or read backwards (180°). Expected boxes are docling-parse 7.22's line
4332    /// cells for the same runs (y-up, PDF points).
4333    #[test]
4334    fn rotated_text_matrix_reads_in_order() {
4335        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
4336        const TEXT: &str = "Upright text on a rotated page.";
4337        for (tm, [l, b, r, t]) in [
4338            ("0 14 -14 0 200 100", [189.9, 100.0, 202.9, 289.1]), // 90°
4339            ("-14 0 0 -14 350 300", [160.9, 289.9, 350.0, 302.9]), // 180°
4340            ("0 -14 14 0 200 500", [197.1, 310.9, 210.1, 500.0]), // 270°
4341            (
4342                "9.8995 9.8995 -9.8995 9.8995 100 100",
4343                [92.9, 98.0, 235.8, 240.8],
4344            ), // 45°
4345        ] {
4346            let content = format!("BT /F1 1 Tf {tm} Tm ({TEXT}) Tj ET\n");
4347            let pdf = pdf_with_content(HELV, content.as_bytes());
4348            let cs = cells(&pdf);
4349            assert_eq!(cs.len(), 1, "{tm}: {cs:?}");
4350            let c = &cs[0];
4351            assert_eq!(c.text, TEXT, "{tm}");
4352            // `cells` is top-left-origin on the 842 pt page. The descriptor-
4353            // less base-14 face gets the parser's 1-em box where docling-parse
4354            // reads Helvetica's AFM (718/−207) — the same ≤ 0.6 pt
4355            // across-the-baseline offset upright text has — so allow 1 pt.
4356            let got = [c.l, 842.0 - c.b, c.r, 842.0 - c.t];
4357            for (g, w) in got.iter().zip([l, b, r, t]) {
4358                assert!(
4359                    (g - w).abs() < 1.0,
4360                    "{tm}: box {got:?} vs docling-parse {:?}",
4361                    [l, b, r, t]
4362                );
4363            }
4364            let words: Vec<String> = super::pdf_words(&pdf)
4365                .into_iter()
4366                .flat_map(|(_, _, c)| c)
4367                .map(|c| c.text)
4368                .collect();
4369            assert_eq!(words, TEXT.split(' ').collect::<Vec<_>>(), "{tm}");
4370        }
4371    }
4372
4373    /// A rotated run past the page edge (a cover's spine title) is dropped
4374    /// by the on-page test (#529) — measured on the quad's extent, not the
4375    /// zero-width box rotated glyphs used to get — instead of clamping onto
4376    /// the edge as a zero-width cell. One inside the page is kept.
4377    #[test]
4378    fn rotated_text_off_the_page_is_dropped() {
4379        const HELV: &[u8] = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
4380        let off = pdf_with_content(
4381            HELV,
4382            b"BT /F1 1 Tf 0 14 -14 0 -4 200 Tm (Spine title) Tj ET\n",
4383        );
4384        assert!(cells(&off).is_empty(), "{:?}", cells(&off));
4385        let on = pdf_with_content(
4386            HELV,
4387            b"BT /F1 1 Tf 0 14 -14 0 30 200 Tm (Margin note) Tj ET\n",
4388        );
4389        assert_eq!(cells(&on).len(), 1);
4390    }
4391
4392    /// Upright text keeps the plain loose rectangle — no quad, and the
4393    /// sanitizer path is the one every pinned PDF baseline was made with.
4394    #[test]
4395    fn upright_glyphs_carry_no_quad() {
4396        let only_page = |pdf: &[u8]| {
4397            let doc = super::load_document(pdf).expect("loads");
4398            let pid = *doc.get_pages().values().next().expect("one page");
4399            super::page_glyphs(&doc, pid)
4400        };
4401        let pdf = pdf_with_font(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", b"Plain");
4402        let glyphs = only_page(&pdf);
4403        assert!(!glyphs.is_empty());
4404        assert!(glyphs.iter().all(|g| g.quad.is_none() && g.lr > g.ll));
4405        // A 90° glyph's style height is its font size, not its advance.
4406        let pdf = pdf_with_content(
4407            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>",
4408            b"BT /F1 1 Tf 0 14 -14 0 200 100 Tm (W) Tj ET\n",
4409        );
4410        let glyphs = only_page(&pdf);
4411        let g = &glyphs[0];
4412        assert!(g.quad.is_some());
4413        // 14 pt × the face's 1-em box (no descriptor: ascent − descent = 1).
4414        assert!((g.height() - 14.0).abs() < 0.05, "{}", g.height());
4415    }
4416
4417    /// #609: ReportLab's chart axis sets each tick label with the *same* `Tm`
4418    /// inside its own `q … cm … Q`, so only the CTM stacks `3`…`8` one under
4419    /// another at one x. Each tick is its own line cell with its own box (as
4420    /// docling-parse reads it), not one `345678` cell boxed like the first.
4421    #[test]
4422    fn ticks_stacked_by_cm_are_separate_line_cells() {
4423        let mut content = String::new();
4424        for (k, tick) in "345678".chars().enumerate() {
4425            let y = 300.0 + 25.92 * k as f64;
4426            content.push_str(&format!(
4427                "q 1 0 0 1 125 {y} cm BT /F1 10 Tf 1 0 0 1 -5 -4 Tm ({tick}) Tj ET Q\n"
4428            ));
4429        }
4430        let pdf = pdf_with_content(
4431            b"<</Type/Font/Subtype/Type1/BaseFont/Times-Roman>>",
4432            content.as_bytes(),
4433        );
4434        let cells = cells(&pdf);
4435        let texts: Vec<&str> = cells.iter().map(|c| c.text.as_str()).collect();
4436        assert_eq!(texts, ["3", "4", "5", "6", "7", "8"]);
4437        for (k, c) in cells.iter().enumerate() {
4438            // Top-left y of a tick whose baseline is 4 pt under its cm origin.
4439            let base = 842.0 - (300.0 + 25.92 * k as f32 - 4.0);
4440            assert!((c.l - 120.0).abs() < 0.1, "{k}: l {}", c.l);
4441            assert!(
4442                c.t < base && base - c.t < 10.0,
4443                "{k}: t {} base {base}",
4444                c.t
4445            );
4446        }
4447    }
4448
4449    /// #609: the path walk finds drawn checkbox squares — ReportLab's four
4450    /// `m … l S` edges under a `cm`, and a stroked `re` with a tick inside
4451    /// (checked) — in the text cells' top-left frame, and a stroked box
4452    /// filled white; a filled bar, a big frame and a colour-filled legend
4453    /// swatch are not checkboxes.
4454    #[test]
4455    fn drawn_checkbox_squares_are_found() {
4456        let content = b"q 1 0 0 1 119.52 449.04 cm \
4457            n 0 12.96 m 12.96 12.96 l S n 0 0 m 12.96 0 l S \
4458            n 0 0 m 0 12.96 l S n 12.96 0 m 12.96 12.96 l S Q\n\
4459            200 400 10 10 re S 202 405 m 204.5 402 l 208.5 408.5 l S\n\
4460            300 400 12 12 re f 50 50 100 100 re S\n\
4461            q 0.2 0.4 0.8 rg 400 400 12 12 re B Q q 1 g 500 400 12 12 re B Q\n";
4462        let pdf = pdf_with_content(b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>", content);
4463        let mut parser = super::PageTextParser::open(&pdf).expect("parses");
4464        let mut boxes = parser.cells(0).checkboxes;
4465        boxes.sort_by(|a, b| a.l.total_cmp(&b.l));
4466        let got: Vec<([i32; 4], bool)> = boxes
4467            .iter()
4468            .map(|c| {
4469                let r = |v: f32| (v * 100.0).round() as i32;
4470                ([r(c.l), r(c.t), r(c.r), r(c.b)], c.checked)
4471            })
4472            .collect();
4473        // Page height 842: y-up 449.04..462.0 → top-left 380.0..392.96.
4474        assert_eq!(
4475            got,
4476            [
4477                ([11952, 38000, 13248, 39296], false),
4478                ([20000, 43200, 21000, 44200], true),
4479                ([50000, 43000, 51200, 44200], false),
4480            ]
4481        );
4482    }
4483
4484    /// Every standard-14 alias/style decodes with real (positive-width) boxes.
4485    #[test]
4486    fn standard14_faces_get_builtin_widths() {
4487        for fontdict in [
4488            // ReportLab's default: base-14 Helvetica, WinAnsi, nothing else.
4489            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>".as_slice(),
4490            // No /Encoding at all (StandardEncoding-ish default).
4491            b"<</Type/Font/Subtype/Type1/BaseFont/Times-BoldItalic>>",
4492            // Substitution aliases + a subset prefix.
4493            b"<</Type/Font/Subtype/TrueType/BaseFont/Arial,Bold>>",
4494            b"<</Type/Font/Subtype/Type1/BaseFont/ABCDEF+Courier-Oblique>>",
4495        ] {
4496            let pdf = pdf_with_font(fontdict, b"Words have width now");
4497            let cs = cells(&pdf);
4498            let text: String = cs
4499                .iter()
4500                .map(|c| c.text.as_str())
4501                .collect::<Vec<_>>()
4502                .join(" ");
4503            assert!(
4504                text.contains("Words have width now"),
4505                "{}: text lost: {text:?}",
4506                String::from_utf8_lossy(fontdict)
4507            );
4508            assert!(
4509                cs.iter().all(|c| c.r > c.l),
4510                "{}: zero-width cells: {cs:?}",
4511                String::from_utf8_lossy(fontdict)
4512            );
4513        }
4514    }
4515
4516    /// MacRomanEncoding without a ToUnicode CMap (#744) — Word for Mac's
4517    /// Quartz output: the whole high range decodes, Danish æ ø å, é, ü, ß
4518    /// and § included, through the plain encoding name and as the base of a
4519    /// `/Differences` dictionary alike.
4520    #[test]
4521    fn macroman_fonts_keep_accented_letters() {
4522        let sentence = b"Bl\x8cb\xbergr\xbfd p\x8c \xbfen \xd0 caf\x8e M\x9fller, Stra\xa7e \xa4 3";
4523        for (fontdict, want) in [
4524            (
4525                b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/MacRomanEncoding>>"
4526                    .as_slice(),
4527                "Blåbærgrød på øen – café Müller, Straße § 3",
4528            ),
4529            (
4530                b"<</Type/Font/Subtype/TrueType/BaseFont/ABCDEF+Arial\
4531                   /Encoding<</Type/Encoding/BaseEncoding/MacRomanEncoding/Differences[51/four]>>>>",
4532                "Blåbærgrød på øen – café Müller, Straße § 4",
4533            ),
4534        ] {
4535            let text: String = cells(&pdf_with_font(fontdict, sentence))
4536                .iter()
4537                .map(|c| c.text.as_str())
4538                .collect::<Vec<_>>()
4539                .join(" ");
4540            assert_eq!(text, want, "{}", String::from_utf8_lossy(fontdict));
4541        }
4542
4543        // The table itself: all of 0x80–0xFF but the Apple logo, ¤ at 0xDB.
4544        let table = super::macroman_table();
4545        assert_eq!(
4546            (0x80u8..=0xFF).filter(|b| table.contains_key(b)).count(),
4547            127
4548        );
4549        assert_eq!(table.get(&0xF0), None);
4550        assert_eq!(table[&0xDB], '¤');
4551        assert_eq!(
4552            [0x80, 0x8C, 0xAE, 0xAF, 0xBE, 0xBF, 0xFF].map(|b| table[&b]),
4553            ['Ä', 'å', 'Æ', 'Ø', 'æ', 'ø', 'ˇ']
4554        );
4555    }
4556
4557    /// WinAnsiEncoding's unused codes above octal 40 are bullets (PDF
4558    /// 32000-1 Annex D.2) — ReportLab's default list bullet is `\177` in a
4559    /// non-embedded WinAnsi Helvetica and read U+FFFD (#1102). A font
4560    /// without an `/Encoding` keeps them undefined.
4561    #[test]
4562    fn winansi_unused_codes_are_bullets() {
4563        let read = |fontdict: &[u8], text: &[u8]| -> String {
4564            cells(&pdf_with_font(fontdict, text))
4565                .iter()
4566                .map(|c| c.text.as_str())
4567                .collect::<Vec<_>>()
4568                .join(" ")
4569        };
4570        let winansi = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4571        assert_eq!(read(winansi, b"\\177 Cut costs"), "\u{2022} Cut costs");
4572        for code in [0x7Fu8, 0x81, 0x8D, 0x8F, 0x90, 0x9D] {
4573            assert_eq!(
4574                read(winansi, &[b'x', code, b'x']),
4575                "x\u{2022}x",
4576                "{code:#x}"
4577            );
4578        }
4579        let dict = b"<</Type/Font/Subtype/TrueType/BaseFont/Arial\
4580                     /Encoding<</Type/Encoding/BaseEncoding/WinAnsiEncoding>>>>";
4581        assert_eq!(read(dict, b"\\177 Cut costs"), "\u{2022} Cut costs");
4582        let bare = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>";
4583        assert!(!read(bare, b"x\\177x").contains('\u{2022}'));
4584    }
4585
4586    /// Unmapped glyphs read U+FFFD (#1003, docling#5070): a ToUnicode NUL —
4587    /// docling's `test_threaded_page_backend_replaces_nul_with_replacement_char`
4588    /// case, `mg.mL \x00 1` — and a bare glyph-index `/Differences` name,
4589    /// while an unknown code-like name keeps its text without the slash
4590    /// (docling-parse 7.24's `SM590000`).
4591    #[test]
4592    fn unmapped_glyphs_read_as_the_replacement_char() {
4593        let cmap = b"/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n\
4594            1 begincodespacerange <00> <FF> endcodespacerange\n\
4595            1 beginbfchar <7e> <0000> endbfchar\n\
4596            endcmap CMapName currentdict /CMap defineresource pop end end";
4597        let stream = [
4598            format!("<</Length {}>>stream\n", cmap.len()).as_bytes(),
4599            cmap.as_slice(),
4600            b"\nendstream",
4601        ]
4602        .concat();
4603        let pdf = pdf_with_fonts(
4604            &[
4605                b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/ToUnicode 6 0 R\
4606                   /Encoding<</Type/Encoding/Differences[1/g115 2/SM590000]>>>>",
4607                &stream,
4608            ],
4609            b"BT /F1 12 Tf 72 700 Td (mg.mL ~ 1 \x01 x \x02 y) Tj ET\n",
4610        );
4611        let text: String = cells(&pdf)
4612            .iter()
4613            .map(|c| c.text.as_str())
4614            .collect::<Vec<_>>()
4615            .join(" ");
4616        assert_eq!(text, "mg.mL \u{FFFD} 1 \u{FFFD} x SM590000 y");
4617    }
4618
4619    /// redp5110's subset font names every glyph by index (`/g115/g3 /g40…`):
4620    /// the painted ones read U+FFFD, the blank `/g3` program is a space, so
4621    /// the line keeps its words (docling-parse 7.24's line cell).
4622    #[cfg(feature = "render")]
4623    #[test]
4624    fn a_blank_glyph_program_is_a_space() {
4625        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
4626            .join("../../tests/data/pdf/sources/redp5110_sampled.pdf");
4627        let Ok(bytes) = std::fs::read(path) else {
4628            return;
4629        };
4630        let lines: Vec<String> = super::pdf_textlines(&bytes)
4631            .into_iter()
4632            .flat_map(|(_, _, c)| c)
4633            .map(|c| c.text)
4634            .collect();
4635        let want = "\u{FFFD} \u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD} \
4636                    \u{FFFD}\u{FFFD}\u{FFFD} ";
4637        assert!(
4638            lines.iter().any(|l| l.starts_with(want)),
4639            "no `{want}` line in {:?}",
4640            lines
4641                .iter()
4642                .filter(|l| l.contains('\u{FFFD}'))
4643                .collect::<Vec<_>>()
4644        );
4645    }
4646
4647    /// The page's image placements reach the cells (#999): the scan under
4648    /// the #999 fixture's one native word is the whole page.
4649    #[test]
4650    fn image_placements_are_the_pages_bitmaps() {
4651        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
4652            .join("../docling/tests/fixtures/partial_text_layer/scan_with_native_word.pdf");
4653        let bytes = std::fs::read(path).expect("fixture");
4654        let mut parser = super::PageTextParser::open(&bytes).expect("parser");
4655        let page = parser.cells(0);
4656        assert_eq!(page.bitmaps, vec![[0.0, 0.0, 612.0, 792.0]]);
4657        assert_eq!(page.words.len(), 1);
4658        assert!(page.checkboxes.is_empty());
4659    }
4660
4661    /// A `bfrange` whose destination is a surrogate pair steps its last code
4662    /// unit (#1003: equations.pdf's `H𝛂`, `<1d59> <1d5a> <d835dec2>`); a
4663    /// single-unit base keeps counting in code points.
4664    #[test]
4665    fn surrogate_pair_bfrange_maps_each_code() {
4666        let map = super::parse_tounicode(
4667            b"1 beginbfrange <1d59> <1d5a> <d835dec2> endbfrange\n\
4668              1 beginbfrange <0010> <0011> <0041> endbfrange",
4669        );
4670        assert_eq!(map.get(&0x1d59).map(String::as_str), Some("\u{1D6C2}"));
4671        assert_eq!(map.get(&0x1d5a).map(String::as_str), Some("\u{1D6C3}"));
4672        assert_eq!(map.get(&0x11).map(String::as_str), Some("B"));
4673    }
4674
4675    /// Which unknown glyph names are text and which are bare indexes
4676    /// (docling-parse 7.24's `resolve_unknown_name`).
4677    #[test]
4678    fn unknown_glyph_names_follow_docling_parse() {
4679        for name in ["g115", "gid00043", "Glyph7", "cid42", "index9", "g3.alt"] {
4680            assert_eq!(super::unknown_name_text(name), None, "{name}");
4681        }
4682        assert_eq!(
4683            super::unknown_name_text("SM590000").as_deref(),
4684            Some("SM590000")
4685        );
4686        assert_eq!(
4687            super::unknown_name_text("SM590000.sc").as_deref(),
4688            Some("SM590000")
4689        );
4690    }
4691
4692    /// An explicit `/Widths` array always wins over the built-in metrics, and a
4693    /// non-standard face without `/Widths` stays as before (no invented boxes).
4694    #[test]
4695    fn explicit_widths_win_and_unknown_faces_are_untouched() {
4696        // Helvetica with explicit 100/1000-em widths: the word's box must be
4697        // ~4×100 units at 12pt = 4.8pt wide — far narrower than the ~2.7×
4698        // wider built-in Helvetica advances would make it.
4699        let explicit = pdf_with_font(
4700            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/FirstChar 65\
4701               /Widths[100 100 100 100]/Encoding/WinAnsiEncoding>>",
4702            b"ABBA",
4703        );
4704        let builtin = pdf_with_font(
4705            b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>",
4706            b"ABBA",
4707        );
4708        let w = |pdf: &[u8]| {
4709            let cs = cells(pdf);
4710            assert_eq!(cs.len(), 1, "one word cell: {cs:?}");
4711            cs[0].r - cs[0].l
4712        };
4713        let (we, wb) = (w(&explicit), w(&builtin));
4714        assert!(
4715            (we - 4.8).abs() < 0.1,
4716            "explicit widths must win: got {we}, want 4×100×12/1000"
4717        );
4718        assert!(
4719            wb > 2.0 * we,
4720            "built-in Helvetica is much wider: {wb} vs {we}"
4721        );
4722
4723        // An unknown face with no /Widths: still parses (text kept), but no
4724        // built-in table applies — the old zero-width behavior is preserved
4725        // rather than inventing Helvetica metrics for an arbitrary font.
4726        let unknown = pdf_with_font(
4727            b"<</Type/Font/Subtype/Type1/BaseFont/FancyCorp-Display>>",
4728            b"Mystery",
4729        );
4730        let cs = cells(&unknown);
4731        let text: String = cs.iter().map(|c| c.text.as_str()).collect();
4732        assert!(text.contains("Mystery"), "text still decodes: {cs:?}");
4733    }
4734    /// #728: one face bound under two resource keys — `/F1` Helvetica in
4735    /// WinAnsi, `/F2` the same Helvetica re-encoded so code 0x81 is the `fi`
4736    /// glyph — with every fi-word drawn in `/F2` right after an `/F1` run.
4737    /// docling-parse names a font by `/FontDescriptor/FontName` → `/Name` →
4738    /// `/BaseFont`, never by the resource key, so both resources are the same
4739    /// font and the line reads `financial`. Keyed by resource key the `fi`
4740    /// ligature (font-neutral in `applicable_for_merge`) joined the `/F1`
4741    /// run and the font barrier then split it from `nancial`.
4742    #[test]
4743    fn same_face_under_two_resource_keys_is_one_font_for_the_sanitizer() {
4744        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4745        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding<</Type/Encoding\
4746                   /BaseEncoding/WinAnsiEncoding/Differences[129/fi]>>>>";
4747        let content = b"BT 12 TL 72 720 Td \
4748            /F1 12 Tf (The ) Tj /F2 12 Tf (\\201nancial position) Tj \
4749            /F1 12 Tf ( of the company is shown in ) Tj /F2 12 Tf (\\201gure 3) Tj \
4750            /F1 12 Tf ( with the ) Tj /F2 12 Tf (\\201rst results.) Tj \
4751            T* T* /F2 12 Tf (Same words in /F2 only: \\201nancial, \\201gure, \\201rst.) Tj ET\n";
4752        let cs = cells(&pdf_with_fonts(&[f1, f2], content));
4753        let lines: Vec<&str> = cs.iter().map(|c| c.text.as_str()).collect();
4754        assert_eq!(
4755            lines,
4756            [
4757                "The financial position of the company is shown in figure 3 with the first results.",
4758                "Same words in /F2 only: financial, figure, first.",
4759            ],
4760            "{cs:?}"
4761        );
4762    }
4763
4764    /// Under `legacy` the font barrier itself is untouched: two *different*
4765    /// faces (regular and bold Helvetica, which docling-parse names apart) set
4766    /// flush against each other stay separate line cells, as
4767    /// `enforce_same_font` demands. docling-parse 7 dropped the barrier — its
4768    /// line runs follow geometry alone — so under the default `adaptive` the
4769    /// flush pair is one line and one word.
4770    #[test]
4771    fn different_faces_keep_the_font_barrier_only_under_legacy() {
4772        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4773        let f2 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica-Bold/Encoding/WinAnsiEncoding>>";
4774        let content = b"BT 72 720 Td /F2 12 Tf (Total:) Tj /F1 12 Tf (42) Tj ET\n";
4775        let pdf = pdf_with_fonts(&[f1, f2], content);
4776        let lines = |gap| {
4777            let mut p = super::PageTextParser::open(&pdf).expect("opens");
4778            p.set_word_gap(Some(gap));
4779            p.cells(0)
4780                .prose
4781                .into_iter()
4782                .map(|c| c.text)
4783                .collect::<Vec<_>>()
4784        };
4785        assert_eq!(lines(crate::dp_lines::WordGap::Legacy), ["Total:", "42"]);
4786        assert_eq!(lines(crate::dp_lines::WordGap::Adaptive), ["Total:42"]);
4787    }
4788
4789    /// #992: a space glyph whose advance the next `TJ` offset takes back —
4790    /// EUR-Lex's superscript footnote marker `( 1 )`, set with spaces the
4791    /// digit is then drawn over — is not a character of the text: the page
4792    /// shows `(1)`, pdftotext and pdfium read `(1)`, and so does the line
4793    /// cell here (docling-parse keeps the spaces, `( 1 )`). A space the next
4794    /// glyph only kerns into (−100/1000 em, well past the space's start) is
4795    /// a real gap and stays, as does one before a jump to the next line.
4796    #[test]
4797    fn a_space_the_next_glyph_is_drawn_over_is_not_a_character() {
4798        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4799        let lines = |content: &[u8]| {
4800            let pdf = pdf_with_content(f1, content);
4801            super::PageTextParser::open(&pdf)
4802                .expect("opens")
4803                .cells(0)
4804                .prose
4805                .into_iter()
4806                .map(|c| c.text)
4807                .collect::<Vec<_>>()
4808        };
4809        // Helvetica's space is 278/1000 em: an offset of 300 moves the pen
4810        // back past where the space began, 100 leaves two thirds of it.
4811        assert_eq!(
4812            lines(b"BT /F1 12 Tf 72 700 Td [(EG \\() ( ) 300 (1) ( ) 300 (\\) erf)] TJ ET\n"),
4813            ["EG (1) erf"]
4814        );
4815        assert_eq!(
4816            lines(b"BT /F1 12 Tf 72 700 Td [(EG \\() ( ) 100 (1) ( ) 100 (\\) erf)] TJ ET\n"),
4817            ["EG ( 1 ) erf"]
4818        );
4819        // A trailing space, then the next line starts at the left margin.
4820        assert_eq!(
4821            lines(b"BT /F1 12 Tf 72 700 Td (first line ) Tj 0 -14 Td (second) Tj ET\n"),
4822            ["first line", "second"]
4823        );
4824        // A line drawn rotated 90° (a landscape slide on a portrait page,
4825        // #528): its glyphs' loose boxes stack on one left edge, which is
4826        // not an overprint — the space stays.
4827        assert_eq!(
4828            lines(b"BT 0 1 -1 0 300 100 Tm /F1 12 Tf (Nullam mollis) Tj ET\n"),
4829            ["Nullam mollis"]
4830        );
4831    }
4832
4833    /// A text shadow: a word processor exports a heading with a shadow
4834    /// effect as the same run drawn twice — the shadow in black, then the
4835    /// text in colour over it, shifted up and left by a fraction of a point.
4836    /// pdfium's text page reads the heading once and so does the line cell
4837    /// here (docling-parse reads `Safety instructions Safety instructions`).
4838    /// #1092: docling-parse's view of a page's shapes — a stroked straight
4839    /// rule is a shape line and a shape box, a thin filled box only a box, a
4840    /// curve or a diagonal no line, a path outside the clip nothing, and a
4841    /// fill in an unresolved Pattern space nothing at all. In the cells'
4842    /// frame: top-left, on the 595 × 842 test page.
4843    #[test]
4844    fn the_page_shapes_follow_docling_parse() {
4845        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4846        let shapes = |content: &[u8]| {
4847            let pdf = pdf_with_content(f1, content);
4848            super::PageTextParser::open(&pdf)
4849                .expect("opens")
4850                .cells(0)
4851                .shapes
4852        };
4853        let s = shapes(b"2 w 72 400 m 540 400 l S\n");
4854        assert_eq!(s.lines, vec![[72.0, 442.0, 540.0, 442.0]]);
4855        // The stroke's half width pads the box.
4856        assert_eq!(s.boxes, vec![[71.0, 441.0, 541.0, 443.0]]);
4857
4858        let s = shapes(b"72 300 468 1 re f\n");
4859        assert!(s.lines.is_empty(), "a fill strokes no line");
4860        assert_eq!(s.boxes, vec![[72.0, 541.0, 540.0, 542.0]]);
4861
4862        // A stroked rectangle: four edges, `h`'s closing one included.
4863        let s = shapes(b"100 100 200 50 re S\n");
4864        assert_eq!(s.lines.len(), 4);
4865
4866        let s = shapes(b"72 400 m 100 450 l 200 300 300 500 400 400 c S\n");
4867        assert!(s.lines.is_empty(), "neither a diagonal nor a curve");
4868        assert_eq!(s.boxes.len(), 1);
4869
4870        let s = shapes(b"q 0 0 300 842 re W n 72 400 m 540 400 l S 400 100 m 400 700 l S Q\n");
4871        assert_eq!(
4872            s.lines,
4873            vec![[72.0, 442.0, 300.0, 442.0]],
4874            "cut to the clip"
4875        );
4876        assert_eq!(s.boxes.len(), 1, "the vertical rule is clipped away");
4877
4878        let s = shapes(b"/Pattern cs /P0 scn 72 300 468 1 re f\n");
4879        assert!(s.lines.is_empty() && s.boxes.is_empty());
4880
4881        // Two touching boxes merge into one component.
4882        let s = shapes(b"72 300 100 1 re f 172 300 100 1 re f 400 600 10 10 re f\n");
4883        assert_eq!(
4884            s.boxes,
4885            vec![[72.0, 541.0, 272.0, 542.0], [400.0, 232.0, 410.0, 242.0]]
4886        );
4887    }
4888
4889    /// A transparent stroke or fill is no shape (`CA` / `ca` 0).
4890    #[test]
4891    fn transparent_paths_are_no_shapes() {
4892        let rule = |closed| super::ShapeSub {
4893            start: (0.0, 0.0),
4894            cur: (100.0, 0.0),
4895            pts: vec![(0.0, 0.0), (100.0, 0.0)],
4896            lines: vec![((0.0, 0.0), (100.0, 0.0))],
4897            closed,
4898        };
4899        let gs = |stroke_alpha, fill_alpha| super::ShapeGs {
4900            stroke_alpha,
4901            fill_alpha,
4902            ..super::ShapeGs::INIT
4903        };
4904        let mut shapes = super::Shapes::default();
4905        super::record_shape(&[rule(false)], true, false, &gs(0.0, 1.0), 1.0, &mut shapes);
4906        assert_eq!(shapes, super::Shapes::default());
4907        super::record_shape(&[rule(true)], false, true, &gs(1.0, 0.0), 1.0, &mut shapes);
4908        assert_eq!(shapes, super::Shapes::default());
4909        super::record_shape(&[rule(false)], true, false, &gs(0.5, 0.0), 1.0, &mut shapes);
4910        assert_eq!(shapes.lines.len(), 1);
4911    }
4912
4913    /// Text that really repeats sits an advance apart or more and stays: a
4914    /// word set twice, a doubled letter, the same words on the next line.
4915    #[test]
4916    fn a_run_drawn_again_over_itself_is_read_once() {
4917        let f1 = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica-Bold/Encoding/WinAnsiEncoding>>";
4918        let lines = |content: &[u8]| {
4919            let pdf = pdf_with_content(f1, content);
4920            super::PageTextParser::open(&pdf)
4921                .expect("opens")
4922                .cells(0)
4923                .prose
4924                .into_iter()
4925                .map(|c| c.text)
4926                .collect::<Vec<_>>()
4927        };
4928        // The shadow, then the text 0.5 pt up and left of it, at 18 pt.
4929        assert_eq!(
4930            lines(
4931                b"BT 0 g /F1 18 Tf 1 0 0 1 72 720 Tm (Safety instructions) Tj ET\n\
4932                  BT 0.8 0 0 rg /F1 18 Tf 1 0 0 1 71.5 720.5 Tm (Safety instructions) Tj ET\n"
4933            ),
4934            ["Safety instructions"]
4935        );
4936        // 0.8 pt at 14 pt, and the line under it untouched.
4937        assert_eq!(
4938            lines(
4939                b"BT /F1 14 Tf 1 0 0 1 72 656 Tm (Maintenance) Tj ET\n\
4940                  BT /F1 14 Tf 1 0 0 1 71.2 656.8 Tm (Maintenance) Tj ET\n\
4941                  BT /F1 11 Tf 1 0 0 1 72 632 Tm (Clean the filter) Tj ET\n"
4942            ),
4943            ["Maintenance", "Clean the filter"]
4944        );
4945        // A word set twice, a doubled letter, the same words a line apart.
4946        assert_eq!(
4947            lines(b"BT /F1 12 Tf 72 700 Td (very very small) Tj ET\n"),
4948            ["very very small"]
4949        );
4950        assert_eq!(
4951            lines(b"BT /F1 12 Tf 72 700 Td (Hello, all) Tj ET\n"),
4952            ["Hello, all"]
4953        );
4954        assert_eq!(
4955            lines(b"BT /F1 12 Tf 72 700 Td (Total) Tj 0 -14 Td (Total) Tj ET\n"),
4956            ["Total", "Total"]
4957        );
4958        // Dot leaders squeezed by `Tc` to 0.05, 0.07 and 0.1 em: two dots on,
4959        // a dot sits within a shadow's shift and the dots after it repeat by
4960        // the same offset, but the next dot is nearer still.
4961        for tc in ["-2.28", "-2.08", "-1.78"] {
4962            let content = format!(
4963                "BT /F1 10 Tf 72 700 Td (Contents) Tj {tc} Tc (..........) Tj 0 Tc ( 5) Tj ET\n"
4964            );
4965            assert_eq!(
4966                lines(content.as_bytes()),
4967                ["Contents.......... 5"],
4968                "Tc {tc}"
4969            );
4970        }
4971    }
4972
4973    /// docling-parse's `test_unit_actual_text.py` (docling-parse#300): a
4974    /// marked-content span's `/ActualText` replaces the glyphs drawn in it.
4975    #[test]
4976    fn actual_text_replaces_the_glyphs_of_its_span() {
4977        let helvetica = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>";
4978        // A non-embedded TrueType whose only code, 0x4D (`M` in WinAnsi), is a
4979        // zero-advance caron: drawn after `s` it makes `š`.
4980        let caron = b"<</Type/Font/Subtype/TrueType/BaseFont/SynthCaron/FirstChar 77/LastChar 77\
4981            /Widths[0]/Encoding/WinAnsiEncoding/FontDescriptor<</Type/FontDescriptor\
4982            /FontName/SynthCaron/Flags 32/FontBBox[-540 -200 0 800]/ItalicAngle 0/Ascent 800\
4983            /Descent -200/CapHeight 700/StemV 80>>>>";
4984        // Code 1 is the `fi` ligature: two glyphs on one pen, one cell.
4985        let ligature = b"<</Type/Font/Subtype/Type1/BaseFont/Helvetica\
4986            /Encoding<</Type/Encoding/BaseEncoding/WinAnsiEncoding/Differences[1/fi]>>>>";
4987        let text = |content: &str| {
4988            let pdf = pdf_with_fonts(&[helvetica, caron, ligature], content.as_bytes());
4989            super::PageTextParser::open(&pdf)
4990                .expect("opens")
4991                .cells(0)
4992                .prose
4993                .into_iter()
4994                .map(|c| c.text)
4995                .collect::<String>()
4996        };
4997        // Unmarked, the caron reads as its WinAnsi letter.
4998        assert_eq!(
4999            text("BT /F1 24 Tf 40 50 Td (Ore) Tj (s) Tj /F2 24 Tf <4D> Tj /F1 24 Tf (i) Tj ET"),
5000            "OresMi"
5001        );
5002        // Two cells, one character: the accent composes onto its base letter.
5003        assert_eq!(
5004            text(
5005                "BT /F1 24 Tf 40 50 Td (Ore) Tj /Span <</ActualText <FEFF0161>>> BDC \
5006                  (s) Tj /F2 24 Tf <4D> Tj EMC /F1 24 Tf (i) Tj ET"
5007            ),
5008            "Oreši"
5009        );
5010        // As many characters as cells: one for one.
5011        assert_eq!(
5012            text("BT /F1 24 Tf 40 50 Td /Span <</ActualText (ab)>> BDC (xy) Tj EMC (z) Tj ET"),
5013            "abz"
5014        );
5015        // A cell of two glyphs takes one character.
5016        assert_eq!(
5017            text("BT /F3 24 Tf 40 50 Td <01> Tj /F1 24 Tf (a) Tj ET"),
5018            "fia"
5019        );
5020        assert_eq!(
5021            text(
5022                "BT /F3 24 Tf 40 50 Td /Span <</ActualText (XY)>> BDC <01> Tj \
5023                  /F1 24 Tf (a) Tj EMC (z) Tj ET"
5024            ),
5025            "XYz"
5026        );
5027        // An empty text declares the glyphs not text (a visual line-break
5028        // hyphen); its advance may still read as a gap.
5029        let hyphen = text(
5030            "BT /F1 24 Tf 40 50 Td (exam) Tj /Span <</ActualText <FEFF>>> BDC (-) Tj EMC \
5031             (ple) Tj ET",
5032        );
5033        assert!(!hyphen.contains('-'), "{hyphen}");
5034        assert_eq!(hyphen.replace(' ', ""), "example");
5035        // `/Alt` written into the wrong key: implausibly long, ignored.
5036        assert_eq!(
5037            text(
5038                "BT /F1 24 Tf 40 50 Td /Span <</ActualText (A long description that belongs \
5039                  in Alt not here)>> BDC (ab) Tj EMC ET"
5040            ),
5041            "ab"
5042        );
5043        // Over graphics only: nothing to replace.
5044        assert_eq!(
5045            text(
5046                "BT /F1 24 Tf 40 50 Td (x) Tj ET /Span <</ActualText (ghost)>> BDC \
5047                  10 10 50 50 re f EMC"
5048            ),
5049            "x"
5050        );
5051        // A named property list is not resolved.
5052        assert_eq!(
5053            text("BT /F1 24 Tf 40 50 Td /OC /MC0 BDC (ok) Tj EMC ET"),
5054            "ok"
5055        );
5056        // Nested spans: the outer text describes the whole run.
5057        assert_eq!(
5058            text(
5059                "BT /F1 24 Tf 40 50 Td /Span <</ActualText (AB)>> BDC (a) Tj \
5060                  /Span <</ActualText (Z)>> BDC (b) Tj EMC EMC ET"
5061            ),
5062            "AB"
5063        );
5064        // A span nested past the depth limit keeps its glyphs.
5065        let nested = |depth: usize| {
5066            text(&format!(
5067                "BT /F1 24 Tf 40 50 Td {}/Span <</ActualText (z)>> BDC (a) Tj EMC{} ET",
5068                "/X BMC ".repeat(depth),
5069                " EMC".repeat(depth)
5070            ))
5071        };
5072        assert_eq!(nested(super::MAX_ACTUAL_TEXT_DEPTH - 1), "z");
5073        assert_eq!(nested(super::MAX_ACTUAL_TEXT_DEPTH), "a");
5074    }
5075
5076    /// One for one, each character keeps its cell's box, and a cell of
5077    /// several glyphs on one pen (a ligature's) becomes its first glyph.
5078    #[test]
5079    fn actual_text_one_for_one_keeps_each_cells_box() {
5080        let glyph = |ch, x: f32, pen: f32| super::Glyph {
5081            ch,
5082            l: x,
5083            b: 0.0,
5084            r: x + 5.0,
5085            t: 10.0,
5086            ll: x,
5087            lb: -2.0,
5088            lr: x + 5.0,
5089            lt: 10.0,
5090            font: 0,
5091            quad: None,
5092            pen: Some([pen, 0.0, 10.0]),
5093            math_font: false,
5094            invisible: false,
5095        };
5096        let mut out = vec![
5097            glyph('o', 0.0, 0.0),
5098            glyph('f', 10.0, 10.0),
5099            glyph('i', 15.0, 10.0),
5100            glyph('a', 20.0, 20.0),
5101        ];
5102        super::apply_actual_text(&mut out, 1, "XY");
5103        let got: Vec<(char, f32)> = out.iter().map(|g| (g.ch, g.l)).collect();
5104        assert_eq!(got, [('o', 0.0), ('X', 10.0), ('Y', 20.0)]);
5105    }
5106
5107    /// The identity follows docling-parse's precedence: a `/FontDescriptor/FontName`
5108    /// wins over `/BaseFont`, so two dictionaries that share a `/BaseFont` but
5109    /// describe different programs are two fonts, and two that share the
5110    /// descriptor name are one whatever their `/BaseFont` says.
5111    #[test]
5112    fn math_faces_are_named_like_docling_parse_names_them() {
5113        for name in [
5114            "ABCDEF+CMR7",
5115            "CMMI10",
5116            "MSBM10",
5117            "Cambria-Math",
5118            "Symbol",
5119            "MTExtra",
5120        ] {
5121            assert!(super::is_math_font_name(name.as_bytes()), "{name}");
5122        }
5123        for name in ["Helvetica", "ABCDEF+TimesNewRomanPSMT", "Arial-BoldMT"] {
5124            assert!(!super::is_math_font_name(name.as_bytes()), "{name}");
5125        }
5126    }
5127
5128    #[test]
5129    fn sanitizer_font_name_follows_docling_parse_precedence() {
5130        use lopdf::{dictionary, Document, Object};
5131        let mut doc = Document::with_version("1.4");
5132        let fd_a =
5133            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "AAAAAA+Face" });
5134        let fd_b =
5135            doc.add_object(dictionary! { "Type" => "FontDescriptor", "FontName" => "BBBBBB+Face" });
5136        let with = |fd: Option<lopdf::ObjectId>, base: &str| {
5137            let mut d = dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => base };
5138            if let Some(id) = fd {
5139                d.set("FontDescriptor", Object::Reference(id));
5140            }
5141            d
5142        };
5143        let name = |d: &lopdf::Dictionary| {
5144            String::from_utf8(super::sanitizer_font_name(&doc, d, false)).unwrap()
5145        };
5146        assert_eq!(name(&with(Some(fd_a), "Face")), "AAAAAA+Face");
5147        assert_eq!(name(&with(Some(fd_b), "Face")), "BBBBBB+Face");
5148        assert_eq!(name(&with(None, "ABCDEF+Face")), "ABCDEF+Face");
5149        assert_eq!(
5150            name(&dictionary! { "Type" => "Font", "Name" => "F7" }),
5151            "F7"
5152        );
5153        assert_eq!(name(&dictionary! { "Type" => "Font" }), "null");
5154    }
5155}
5156
5157#[cfg(test)]
5158mod overpainted {
5159    use crate::pdfium_backend::TextCell;
5160
5161    fn cell(text: &str, l: f32, t: f32, r: f32, b: f32) -> TextCell {
5162        TextCell {
5163            text: text.into(),
5164            l,
5165            t,
5166            r,
5167            b,
5168        }
5169    }
5170
5171    /// The reporting invoice's logo: a `"` painted inside a `==` on one band —
5172    /// artwork drawn with glyphs. Both cells go; the real text on the next
5173    /// band stays.
5174    #[test]
5175    fn stacked_logo_glyphs_are_dropped() {
5176        let mut cells = vec![
5177            cell("\"", 72.7, 21.5, 86.4, 31.5),
5178            cell("==", 59.4, 21.5, 99.6, 31.5),
5179            cell("Herr", 65.2, 151.3, 81.7, 161.3),
5180        ];
5181        super::drop_overpainted_cells(&mut cells);
5182        assert_eq!(cells.len(), 1, "cells: {cells:?}");
5183        assert_eq!(cells[0].text, "Herr");
5184    }
5185
5186    /// Adjacent words on a line touch but never contain each other — prose is
5187    /// untouched, and so is a same-text near-duplicate (double-drawn faux
5188    /// bold), which is not evidence of artwork.
5189    #[test]
5190    fn prose_and_double_draw_are_kept() {
5191        let mut cells = vec![
5192            cell("Telefon", 354.3, 133.2, 381.5, 143.2),
5193            cell("0676/2000", 387.3, 133.2, 428.7, 143.2),
5194            cell("Bold", 100.0, 50.0, 130.0, 60.0),
5195            cell("Bold", 100.3, 50.0, 130.3, 60.0),
5196        ];
5197        super::drop_overpainted_cells(&mut cells);
5198        assert_eq!(cells.len(), 4);
5199    }
5200}
5201
5202#[cfg(test)]
5203mod vestigial_layer {
5204    use crate::pdfium_backend::{PdfPage, TextCell};
5205
5206    fn page_with(texts: &[&str]) -> PdfPage {
5207        let cells = texts
5208            .iter()
5209            .enumerate()
5210            .map(|(i, t)| TextCell {
5211                text: t.to_string(),
5212                l: 10.0,
5213                t: 10.0 + 12.0 * i as f32,
5214                r: 90.0,
5215                b: 20.0 + 12.0 * i as f32,
5216            })
5217            .collect();
5218        PdfPage::from_cells(595.0, 842.0, 1.0, cells)
5219    }
5220
5221    /// The reported scanned form: three typed-in field values ("03", "05",
5222    /// "2025") over three image pages. That must read as *no usable layer*,
5223    /// so the browser routes the document to OCR instead of extracting
5224    /// thirteen characters and skipping the letter entirely.
5225    #[test]
5226    fn typed_in_form_fields_are_not_a_text_layer() {
5227        let pages = vec![
5228            page_with(&["03", "05", "2025"]),
5229            page_with(&[]),
5230            page_with(&[]),
5231        ];
5232        assert!(super::text_layer_is_vestigial(&pages));
5233        assert!(super::text_layer_is_vestigial(&[page_with(&[])]));
5234    }
5235
5236    /// A short but genuine digital document — one page, a few real lines —
5237    /// keeps the fast text path.
5238    #[test]
5239    fn sparse_but_real_documents_pass() {
5240        let one_pager = vec![page_with(&[
5241            "Confidential briefing",
5242            "Prepared for the board meeting",
5243            "Do not distribute",
5244        ])];
5245        assert!(!super::text_layer_is_vestigial(&one_pager));
5246    }
5247}
5248
5249/// #994: decoded Form XObject content is cached per document only from a
5250/// form's second sighting, so one-form-per-page documents do not hold every
5251/// page's decoded form until the document ends.
5252#[cfg(test)]
5253mod form_cache {
5254    use lopdf::content::{Content, Operation};
5255    use lopdf::{dictionary, Document, Object, Stream};
5256
5257    fn form(doc: &mut Document, text: &str) -> lopdf::ObjectId {
5258        let ops = vec![
5259            Operation::new("BT", vec![]),
5260            Operation::new("Tf", vec!["F1".into(), 12.into()]),
5261            Operation::new("Td", vec![40.into(), 500.into()]),
5262            Operation::new("Tj", vec![Object::string_literal(text)]),
5263            Operation::new("ET", vec![]),
5264        ];
5265        let data = Content { operations: ops }.encode().unwrap();
5266        doc.add_object(Stream::new(
5267            dictionary! { "Type" => "XObject", "Subtype" => "Form",
5268            "BBox" => vec![0.into(), 0.into(), 595.into(), 842.into()] },
5269            data,
5270        ))
5271    }
5272
5273    #[test]
5274    fn single_use_forms_are_not_retained_shared_ones_are() {
5275        let mut doc = Document::with_version("1.5");
5276        let pages_id = doc.new_object_id();
5277        let font = doc.add_object(
5278            dictionary! { "Type" => "Font", "Subtype" => "Type1", "BaseFont" => "Helvetica" },
5279        );
5280        let header = form(&mut doc, "Header");
5281        let mut kids = Vec::new();
5282        let mut own_forms = Vec::new();
5283        for p in 0..3 {
5284            let own = form(&mut doc, &format!("Body{p}"));
5285            own_forms.push(own);
5286            let content = doc.add_object(Stream::new(
5287                dictionary! {},
5288                b"q /Hd Do Q q /Fx Do Q".to_vec(),
5289            ));
5290            kids.push(Object::Reference(doc.add_object(dictionary! {
5291                "Type" => "Page", "Parent" => pages_id,
5292                "MediaBox" => vec![0.into(), 0.into(), 595.into(), 842.into()],
5293                "Contents" => content,
5294                "Resources" => dictionary! {
5295                    "Font" => dictionary! { "F1" => font },
5296                    "XObject" => dictionary! { "Hd" => header, "Fx" => own },
5297                },
5298            })));
5299        }
5300        doc.objects.insert(
5301            pages_id,
5302            Object::Dictionary(dictionary! { "Type" => "Pages", "Kids" => kids, "Count" => 3 }),
5303        );
5304        let catalog = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages_id });
5305        doc.trailer.set("Root", catalog);
5306
5307        let mut caches = super::DocCaches::default();
5308        let mut pages: Vec<_> = doc.get_pages().into_iter().collect();
5309        pages.sort_by_key(|(n, _)| *n);
5310        for (i, (_, pid)) in pages.iter().enumerate() {
5311            let text: String = super::page_glyphs_cached(&doc, *pid, &mut caches)
5312                .iter()
5313                .map(|g| g.ch)
5314                .collect();
5315            // Both forms' text is still read, cached or not.
5316            assert!(text.contains("Header"), "page {i}: {text:?}");
5317            assert!(text.contains(&format!("Body{i}")), "page {i}: {text:?}");
5318        }
5319        // The shared header form is cached from its second page on; the
5320        // per-page forms, each used once, are not held.
5321        assert!(caches.forms.contains_key(&header));
5322        for own in &own_forms {
5323            assert!(!caches.forms.contains_key(own));
5324        }
5325        assert_eq!(caches.forms.len(), 1);
5326    }
5327}
5328
5329#[cfg(test)]
5330mod cid_collections {
5331    use super::base14_fonts::{cells, pdf_with_content};
5332
5333    fn type0(encoding: &str) -> Vec<u8> {
5334        format!(
5335            "<</Type/Font/Subtype/Type0/BaseFont/Test-Font-Light/Encoding/{encoding}\
5336             /DescendantFonts[<</Type/Font/Subtype/CIDFontType0/BaseFont/Test-Font-Light\
5337             /CIDSystemInfo<</Registry(Adobe)/Ordering(Japan1)/Supplement 7>>/DW 1000>>]>>"
5338        )
5339        .into_bytes()
5340    }
5341
5342    fn text(fontdict: &[u8], hex: &str) -> String {
5343        let content = format!("BT /F1 12 Tf 72 700 Td <{hex}> Tj ET\n");
5344        cells(&pdf_with_content(fontdict, content.as_bytes()))
5345            .iter()
5346            .map(|c| c.text.as_str())
5347            .collect()
5348    }
5349
5350    #[test]
5351    #[cfg(feature = "cid-collections")]
5352    fn identity_codes_read_through_the_collection() {
5353        assert_eq!(text(&type0("Identity-H"), "0CD40E8A07A0"), "日本語");
5354    }
5355
5356    #[test]
5357    fn unicode_keyed_cmaps_read_the_code() {
5358        assert_eq!(text(&type0("UniJIS-UCS2-H"), "65E5672C8A9E"), "日本語");
5359    }
5360
5361    #[test]
5362    #[cfg(feature = "cid-collections")]
5363    fn collection_ranges_keep_supplementary_characters() {
5364        assert_eq!(text(&type0("Identity-H"), "0CD44507450807A0"), "日𢭏𢭐語");
5365        let text = |o: &[u8], cid| super::collection_text(super::collection_table(o).unwrap(), cid);
5366        assert_eq!(text(b"Japan1", 0x2714).as_deref(), Some("🄐"));
5367        assert_eq!(text(b"CNS1", 0x3a32).as_deref(), Some("𡤃"));
5368    }
5369
5370    #[test]
5371    #[cfg(feature = "cid-collections")]
5372    fn collection_entries_are_plain_characters() {
5373        assert_eq!(text(&type0("Identity-H"), "046D"), "逢");
5374        let text = |o: &[u8], cid| super::collection_text(super::collection_table(o).unwrap(), cid);
5375        assert_eq!(text(b"Korea1", 8752), None);
5376        assert_eq!(text(b"KR", 11704), None);
5377        assert_eq!(text(b"Japan1", 0x1ffd).as_deref(), Some("sec"));
5378    }
5379
5380    #[test]
5381    fn tounicode_ranges_increment_the_utf16_string() {
5382        let map = super::parse_tounicode(
5383            b"3 beginbfrange\n\
5384              <01> <02> <d842dfb7>\n\
5385              <03> <04> <00660069>\n\
5386              <05> <06> <0061>\n\
5387              endbfrange",
5388        );
5389        assert_eq!(map[&1], "𠮷");
5390        assert_eq!(map[&2], "𠮸");
5391        assert_eq!(map[&3], "fi");
5392        assert_eq!(map[&4], "fj");
5393        assert_eq!(map[&5], "a");
5394        assert_eq!(map[&6], "b");
5395    }
5396
5397    #[test]
5398    fn tounicode_skips_lone_surrogates_overflows_and_huge_ranges() {
5399        let map = super::parse_tounicode(
5400            b"2 beginbfchar\n<01> <d842>\n<02> <0041>\nendbfchar\n\
5401              4 beginbfrange\n\
5402              <10> <11> [<dc00> <0042>]\n\
5403              <20> <22> <0041fffe>\n\
5404              <30> <31> <d842dfff>\n\
5405              <00010000> <ffffffff> <>\n\
5406              endbfrange",
5407        );
5408        assert!(!map.contains_key(&0x01));
5409        assert_eq!(map[&0x02], "A");
5410        assert!(!map.contains_key(&0x10));
5411        assert_eq!(map[&0x11], "B");
5412        assert_eq!(map[&0x20], "A\u{fffe}");
5413        assert_eq!(map[&0x21], "A\u{ffff}");
5414        assert!(!map.contains_key(&0x22));
5415        assert_eq!(map[&0x30], "\u{20bff}");
5416        assert!(!map.contains_key(&0x31));
5417        assert_eq!(map[&0x1ffff], "");
5418        assert_eq!(map.len(), 5 + 0x10000);
5419    }
5420
5421    #[test]
5422    fn utf16_pairs_are_one_glyph_and_one_advance() {
5423        let font = type0("UniJIS-UTF16-H");
5424        assert_eq!(text(&font, "65E5D842DFB78A9E"), "日𠮷語");
5425        let content = b"BT /F1 12 Tf 72 700 Td <65E5D842DFB78A9E> Tj ET\n";
5426        let pdf = pdf_with_content(&font, content);
5427        let doc = lopdf::Document::load_mem(&pdf).unwrap();
5428        let glyphs = super::page_glyphs(&doc, doc.get_pages()[&1]);
5429        assert_eq!(glyphs.len(), 3);
5430        assert!((glyphs[2].l - glyphs[0].l - 24.0).abs() < 0.001);
5431    }
5432
5433    #[test]
5434    fn utf16_codes_preserve_tounicode_keys_and_identity_cids() {
5435        use lopdf::{dictionary, Document};
5436        let doc = Document::with_version("1.4");
5437        let dict = dictionary! {"Subtype" => "Type0", "Encoding" => "UniJIS-UTF16-H"};
5438        let mut font = super::parse_font(&doc, b"F1", &dict);
5439        assert_eq!(
5440            super::codes(&font, &[0xd8, 0x42, 0xdf, 0xb7]),
5441            vec![0xd842dfb7]
5442        );
5443        font.to_unicode.insert(0xd842dfb7, "override".into());
5444        assert_eq!(font.decode_code(0xd842dfb7).0.as_deref(), Some("override"));
5445        assert_eq!(
5446            super::codes(&font, &[0xd8, 0x42, 0x65, 0xe5]),
5447            vec![0xd842, 0x65e5]
5448        );
5449        let dict = dictionary! {"Subtype" => "Type0", "Encoding" => "Identity-H"};
5450        let font = super::parse_font(&doc, b"F1", &dict);
5451        assert_eq!(
5452            super::codes(&font, &[0xd8, 0x42, 0xdf, 0xb7]),
5453            vec![0xd842, 0xdfb7]
5454        );
5455    }
5456}