paperforge_extract/
extractor.rs

1use std::collections::HashMap;
2
3use paperforge_pdf::{ObjectId, PdfObject, StreamDecoder, StreamFilter};
4
5use crate::error::{ExtractError, ExtractResult};
6
7/// Decoding information for one font referenced by a page's `/Resources`: the
8/// `/ToUnicode` CMap mapping character codes to Unicode, and whether codes are
9/// two bytes each (`/Type0` composite fonts, e.g. `/Identity-H`).
10struct FontInfo {
11    two_byte: bool,
12    to_unicode: HashMap<u16, String>,
13}
14
15pub struct Extractor;
16
17impl Extractor {
18    /// Extracts the text of every page, walking the catalog's page tree and
19    /// reading only `/Contents` streams (never metadata or other objects).
20    /// Flate-compressed streams are decoded; `Tj`/`TJ` operators are parsed,
21    /// including literal strings with escapes, hex strings, and UTF-16BE text.
22    /// Text shown with an embedded font is mapped back to Unicode through the
23    /// font's `/ToUnicode` CMap (`bfchar` and `bfrange`).
24    pub fn extract_text(&self, data: &[u8]) -> ExtractResult<String> {
25        let doc = paperforge_pdf::Parser::new()
26            .parse(data)
27            .map_err(|e| ExtractError::Parse(e.to_string()))?;
28
29        let contents = self.page_content_streams(&doc);
30        let mut text = String::new();
31        for (page_index, (content_ids, page_dict)) in contents.iter().enumerate() {
32            if page_index > 0 {
33                text.push('\n');
34            }
35            let fonts = self.page_fonts(&doc, page_dict);
36            for id in content_ids {
37                let Some(obj) = doc.get_object(*id) else {
38                    continue;
39                };
40                let Some(stream) = obj.as_stream() else {
41                    continue;
42                };
43                if let Ok(bytes) = self.decode(stream) {
44                    text.push_str(&extract_text_operators(&bytes, &fonts));
45                }
46            }
47        }
48        Ok(text)
49    }
50
51    /// Returns, per page in document order, the `/Contents` stream ids and the
52    /// page dictionary.
53    fn page_content_streams(
54        &self,
55        doc: &paperforge_pdf::PdfDocument,
56    ) -> Vec<(Vec<ObjectId>, paperforge_pdf::PdfDictionary)> {
57        let mut out = Vec::new();
58        let Some(catalog) = doc.catalog() else {
59            return out;
60        };
61        let Some(PdfObject::Dictionary(catalog_dict)) = doc.get_object(catalog) else {
62            return out;
63        };
64        let Some(PdfObject::Reference(root)) = catalog_dict.get("Pages") else {
65            return out;
66        };
67        let mut stack = vec![*root];
68        while let Some(id) = stack.pop() {
69            let Some(PdfObject::Dictionary(dict)) = doc.get_object(id) else {
70                continue;
71            };
72            match dict.get_name("Type").map(|n| n.as_str()) {
73                Some("Pages") => {
74                    if let Some(PdfObject::Array(kids)) = dict.get("Kids") {
75                        for kid in kids.0.iter().rev() {
76                            if let PdfObject::Reference(r) = kid {
77                                stack.push(*r);
78                            }
79                        }
80                    }
81                }
82                Some("Page") => {
83                    let ids = self.contents_of(dict);
84                    out.push((ids, dict.clone()));
85                }
86                _ => {}
87            }
88        }
89        out
90    }
91
92    fn contents_of(&self, page: &paperforge_pdf::PdfDictionary) -> Vec<ObjectId> {
93        let Some(contents) = page.get("Contents") else {
94            return Vec::new();
95        };
96        match contents {
97            PdfObject::Reference(id) => vec![*id],
98            PdfObject::Array(arr) => arr.0.iter().filter_map(|o| o.as_reference()).collect(),
99            _ => Vec::new(),
100        }
101    }
102
103    fn decode(&self, stream: &paperforge_pdf::PdfStream) -> ExtractResult<Vec<u8>> {
104        let filter = stream.dictionary.get_name("Filter");
105        match filter {
106            Some(f) if f.as_str() == "FlateDecode" || f.as_str() == "Fl" => StreamDecoder::new()
107                .decode_with_limit(&stream.data, StreamFilter::Flate, 100 * 1024 * 1024)
108                .map_err(|e| ExtractError::Parse(e.to_string())),
109            Some(_) => Err(ExtractError::Unsupported(
110                "unsupported content stream filter".into(),
111            )),
112            None => Ok(stream.data.clone()),
113        }
114    }
115
116    /// Collects the fonts of one page by walking its `/Resources` (including
117    /// the inherited chain up to the Pages tree) and parses each `/ToUnicode`
118    /// CMap. Resources on page level win over inherited ones.
119    fn page_fonts(
120        &self,
121        doc: &paperforge_pdf::PdfDocument,
122        page: &paperforge_pdf::PdfDictionary,
123    ) -> HashMap<String, FontInfo> {
124        let mut fonts = HashMap::new();
125        let mut cur: Option<&paperforge_pdf::PdfDictionary> = Some(page);
126        while let Some(dict) = cur {
127            if let Some(res) = dict.get("Resources") {
128                let res_dict = match res {
129                    PdfObject::Dictionary(d) => Some(d),
130                    PdfObject::Reference(id) => doc.get_object(*id).and_then(|o| o.as_dict()),
131                    _ => None,
132                };
133                if let Some(res_dict) = res_dict {
134                    if let Some(PdfObject::Dictionary(font_dict)) = res_dict.get("Font") {
135                        for (name, obj) in font_dict.iter() {
136                            let font_obj = match obj {
137                                PdfObject::Dictionary(d) => Some(d),
138                                PdfObject::Reference(id) => {
139                                    doc.get_object(*id).and_then(|o| o.as_dict())
140                                }
141                                _ => None,
142                            };
143                            if let Some(font) = font_obj.and_then(|f| self.font_info(doc, f)) {
144                                fonts.insert(name.as_str().to_string(), font);
145                            }
146                        }
147                    }
148                }
149                break; // innermost /Resources wins
150            }
151            cur = dict
152                .get("Parent")
153                .and_then(|p| p.as_reference())
154                .and_then(|id| doc.get_object(id).and_then(|o| o.as_dict()));
155        }
156        fonts
157    }
158
159    fn font_info(
160        &self,
161        doc: &paperforge_pdf::PdfDocument,
162        font: &paperforge_pdf::PdfDictionary,
163    ) -> Option<FontInfo> {
164        let to_unicode_ref = font.get("ToUnicode").and_then(|o| o.as_reference())?;
165        let stream = doc.get_object(to_unicode_ref).and_then(|o| o.as_stream())?;
166        let data = self.decode(stream).ok()?;
167        Some(FontInfo {
168            two_byte: font
169                .get_name("Subtype")
170                .is_some_and(|n| n.as_str() == "Type0"),
171            to_unicode: parse_tounicode_cmap(&data),
172        })
173    }
174
175    /// Renders the /Info dictionary as `Key: value` lines.
176    pub fn extract_metadata(&self, data: &[u8]) -> ExtractResult<Vec<u8>> {
177        let doc = paperforge_pdf::Parser::new()
178            .parse(data)
179            .map_err(|e| ExtractError::Parse(e.to_string()))?;
180
181        let Some(info_id) = doc.info() else {
182            return Ok(Vec::new());
183        };
184        let Some(obj) = doc.get_object(info_id) else {
185            return Ok(Vec::new());
186        };
187        let Some(dict) = obj.as_dict() else {
188            return Ok(Vec::new());
189        };
190        let mut out = String::new();
191        for key in [
192            "Title", "Author", "Subject", "Keywords", "Creator", "Producer",
193        ] {
194            if let Some(bytes) = dict.get_string_bytes(key) {
195                out.push_str(&format!("{key}: {}\n", String::from_utf8_lossy(bytes)));
196            }
197        }
198        Ok(out.into_bytes())
199    }
200}
201
202impl Default for Extractor {
203    fn default() -> Self {
204        Self
205    }
206}
207
208/// Scans a decoded content stream for text-showing operators and returns the
209/// concatenated text. Handles literal strings (with escapes and nesting), hex
210/// strings, `Tj`/`TJ`, and UTF-16BE payloads; bracket depth is tracked so
211/// strings inside `TJ` arrays are space-separated. When a `/Font ... Tf`
212/// pair selects an embedded font, text strings are decoded through that
213/// font's `/ToUnicode` CMap (2-byte codes for `/Type0` fonts).
214fn extract_text_operators(data: &[u8], fonts: &HashMap<String, FontInfo>) -> String {
215    let mut out = String::new();
216    let mut i = 0usize;
217    let mut bracket_depth = 0usize;
218    let mut current_font: Option<&FontInfo> = None;
219    while i < data.len() {
220        match data[i] {
221            b' ' | b'\t' | b'\r' | b'\n' | 0x0c => i += 1,
222            b'%' => {
223                while i < data.len() && data[i] != b'\n' {
224                    i += 1;
225                }
226            }
227            b'[' => {
228                bracket_depth += 1;
229                i += 1;
230            }
231            b']' => {
232                bracket_depth = bracket_depth.saturating_sub(1);
233                i += 1;
234            }
235            b'/' => {
236                // Font resource name, e.g. `/F2 12 Tf`.
237                let start = i;
238                while i < data.len()
239                    && !matches!(data[i], b' ' | b'\t' | b'\r' | b'\n' | 0x0c | b'[' | b'(')
240                {
241                    i += 1;
242                }
243                let name = &data[start + 1..i]; // strip the '/'
244                let mut j = skip_ws(data, i);
245                // Optional numeric parameter (font size), then the operator.
246                if data
247                    .get(j)
248                    .is_some_and(|b| matches!(b, b'-' | b'+' | b'.' | b'0'..=b'9'))
249                {
250                    while data
251                        .get(j)
252                        .is_some_and(|b| matches!(b, b'-' | b'+' | b'.' | b'0'..=b'9'))
253                    {
254                        j += 1;
255                    }
256                    j = skip_ws(data, j);
257                }
258                if data[j..].starts_with(b"Tf") {
259                    current_font = fonts.get(std::str::from_utf8(name).unwrap_or_default());
260                }
261            }
262            b'(' => {
263                if let Some(s) = read_literal_string(data, &mut i) {
264                    out.push_str(&decode_text_bytes(&s, current_font));
265                    if bracket_depth > 0 {
266                        out.push(' ');
267                    } else if next_operator_is_text_show(data, i) {
268                        out.push('\n');
269                    }
270                }
271            }
272            b'<' => {
273                if data.get(i + 1) == Some(&b'<') {
274                    // Inline dictionary: skip to the closing `>>`.
275                    while i + 1 < data.len() && !(data[i] == b'>' && data[i + 1] == b'>') {
276                        i += 1;
277                    }
278                    i = (i + 2).min(data.len());
279                } else if let Some(s) = read_hex_string(data, &mut i) {
280                    out.push_str(&decode_text_bytes(&s, current_font));
281                    if bracket_depth > 0 {
282                        out.push(' ');
283                    } else if next_operator_is_text_show(data, i) {
284                        out.push('\n');
285                    }
286                }
287            }
288            _ => i += 1,
289        }
290    }
291    out
292}
293
294fn skip_ws(data: &[u8], mut i: usize) -> usize {
295    while data
296        .get(i)
297        .is_some_and(|b| matches!(b, b' ' | b'\t' | b'\r' | b'\n' | 0x0c))
298    {
299        i += 1;
300    }
301    i
302}
303
304/// Decodes one text-showing string. If the active font has a `/ToUnicode`
305/// CMap, codes are mapped through it (two bytes per code for `/Type0`);
306/// otherwise the raw bytes fall back to the BOM-aware UTF-16BE/lossy UTF-8
307/// path.
308fn decode_text_bytes(bytes: &[u8], font: Option<&FontInfo>) -> String {
309    if let Some(font) = font {
310        if !font.to_unicode.is_empty() {
311            let mut out = String::new();
312            let step = if font.two_byte { 2 } else { 1 };
313            for code in bytes.chunks(step) {
314                if code.len() < step {
315                    break;
316                }
317                let value = if font.two_byte {
318                    u16::from_be_bytes([code[0], code[1]])
319                } else {
320                    u16::from(code[0])
321                };
322                match font.to_unicode.get(&value) {
323                    Some(s) => out.push_str(s),
324                    None => out.push('\u{FFFD}'),
325                }
326            }
327            return out;
328        }
329    }
330    let mut out = String::new();
331    push_text(bytes, &mut out);
332    out
333}
334
335fn next_operator_is_text_show(data: &[u8], mut i: usize) -> bool {
336    while i < data.len() && matches!(data[i], b' ' | b'\t' | b'\r' | b'\n') {
337        i += 1;
338    }
339    data[i..].starts_with(b"Tj") || data[i..].starts_with(b"TJ")
340}
341
342/// Parses a `/ToUnicode` CMap stream into a code -> unicode string map.
343/// Handles `bfchar` entries and `bfrange` entries where the destination is a
344/// single UTF-16BE string (incremented per code) or an array of strings.
345fn parse_tounicode_cmap(data: &[u8]) -> HashMap<u16, String> {
346    let mut map = HashMap::new();
347    let mut i = 0usize;
348    let mut in_bfchar = false;
349    let mut in_bfrange = false;
350    let mut range_array = false;
351    let mut range_lo: u16 = 0;
352    let mut pending_src: Option<Vec<u8>> = None;
353    let mut pending: Vec<Vec<u8>> = Vec::new();
354
355    while i < data.len() {
356        match data[i] {
357            b'a'..=b'z' => {
358                let start = i;
359                while i < data.len() && data[i].is_ascii_alphanumeric() {
360                    i += 1;
361                }
362                match &data[start..i] {
363                    b"beginbfchar" => {
364                        in_bfchar = true;
365                        pending_src = None;
366                    }
367                    b"endbfchar" => {
368                        in_bfchar = false;
369                        pending_src = None;
370                    }
371                    b"beginbfrange" => {
372                        in_bfrange = true;
373                        pending.clear();
374                    }
375                    b"endbfrange" => {
376                        in_bfrange = false;
377                        pending.clear();
378                    }
379                    _ => {}
380                }
381            }
382            b'[' => {
383                if in_bfrange && pending.len() == 2 {
384                    range_array = true;
385                    range_lo = u16::from_be_bytes([pending[0][0], pending[0][1]]);
386                    pending.clear();
387                }
388                i += 1;
389            }
390            b']' => {
391                range_array = false;
392                pending.clear();
393                i += 1;
394            }
395            b'<' => {
396                if let Some(s) = read_hex_string(data, &mut i) {
397                    if in_bfchar {
398                        // `<src> <dst>` pairs: the first string of each pair
399                        // is held until its destination arrives.
400                        match pending_src.take() {
401                            Some(src) => insert_tounicode(&mut map, &src, &s),
402                            None => pending_src = Some(s),
403                        }
404                    } else if in_bfrange && range_array {
405                        // `<lo> <hi> [ <d0> <d1> ... ]`: each array entry maps
406                        // to one code, starting at `lo`.
407                        let code = range_lo.wrapping_add(pending.len() as u16);
408                        insert_tounicode(&mut map, &code.to_be_bytes(), &s);
409                        pending.push(s);
410                    } else if in_bfrange {
411                        pending.push(s);
412                        if pending.len() == 3 {
413                            apply_bfrange(&mut map, &pending[0], &pending[1], &pending[2]);
414                            pending.clear();
415                        }
416                    }
417                }
418            }
419            _ => i += 1,
420        }
421    }
422    map
423}
424
425fn insert_tounicode(map: &mut HashMap<u16, String>, src: &[u8], dst: &[u8]) {
426    if src.len() == 2 {
427        let code = u16::from_be_bytes([src[0], src[1]]);
428        let s = utf16be_to_string(dst);
429        if !s.is_empty() {
430            map.insert(code, s);
431        }
432    }
433}
434
435/// `bfrange` with a single destination string: `<lo> <hi> <dst>` maps
436/// `lo..=hi` to `dst` with its last UTF-16BE unit incremented per code.
437fn apply_bfrange(map: &mut HashMap<u16, String>, lo: &[u8], hi: &[u8], dst: &[u8]) {
438    if lo.len() != 2 || hi.len() != 2 || dst.len() < 2 {
439        return;
440    }
441    let lo = u16::from_be_bytes([lo[0], lo[1]]);
442    let hi = u16::from_be_bytes([hi[0], hi[1]]);
443    let units: Vec<u16> = dst
444        .chunks_exact(2)
445        .map(|c| u16::from_be_bytes([c[0], c[1]]))
446        .collect();
447    let mut cur = *units.last().unwrap_or(&0);
448    for code in lo..=hi {
449        let mut s = String::new();
450        for (idx, u) in units.iter().enumerate() {
451            let v = if idx == units.len() - 1 { cur } else { *u };
452            s.push(char::from_u32(u32::from(v)).unwrap_or('\u{FFFD}'));
453        }
454        map.insert(code, s);
455        cur = cur.wrapping_add(1);
456    }
457}
458
459fn utf16be_to_string(bytes: &[u8]) -> String {
460    bytes
461        .chunks_exact(2)
462        .map(|c| {
463            let u = u16::from_be_bytes([c[0], c[1]]);
464            char::from_u32(u32::from(u)).unwrap_or('\u{FFFD}')
465        })
466        .collect()
467}
468
469fn push_text(bytes: &[u8], out: &mut String) {
470    // UTF-16BE strings carry a BOM; decode them properly, else lossy UTF-8.
471    if bytes.starts_with(&[0xfe, 0xff]) {
472        let utf16: Vec<u16> = bytes[2..]
473            .chunks_exact(2)
474            .map(|c| u16::from_be_bytes([c[0], c[1]]))
475            .collect();
476        out.push_str(&String::from_utf16_lossy(&utf16));
477    } else {
478        out.push_str(&String::from_utf8_lossy(bytes));
479    }
480}
481
482/// Reads a `( ... )` literal string starting at `data[i] == b'('`, honoring
483/// nested parentheses and backslash escapes. Returns the raw bytes.
484fn read_literal_string(data: &[u8], i: &mut usize) -> Option<Vec<u8>> {
485    debug_assert_eq!(data[*i], b'(');
486    *i += 1;
487    let mut out = Vec::new();
488    let mut depth = 1u32;
489    while *i < data.len() {
490        match data[*i] {
491            b'(' => {
492                depth += 1;
493                out.push(b'(');
494                *i += 1;
495            }
496            b')' => {
497                depth -= 1;
498                *i += 1;
499                if depth == 0 {
500                    return Some(out);
501                }
502                out.push(b')');
503            }
504            b'\\' => {
505                *i += 1;
506                let Some(&esc) = data.get(*i) else {
507                    return Some(out);
508                };
509                *i += 1;
510                match esc {
511                    b'n' => out.push(b'\n'),
512                    b'r' => out.push(b'\r'),
513                    b't' => out.push(b'\t'),
514                    b'b' => out.push(8),
515                    b'f' => out.push(12),
516                    b'(' => out.push(b'('),
517                    b')' => out.push(b')'),
518                    b'\\' => out.push(b'\\'),
519                    b'0'..=b'7' => {
520                        let mut v = esc - b'0';
521                        for _ in 0..2 {
522                            match data.get(*i) {
523                                Some(e @ b'0'..=b'7') => {
524                                    v = v * 8 + (e - b'0');
525                                    *i += 1;
526                                }
527                                _ => break,
528                            }
529                        }
530                        out.push(v);
531                    }
532                    b'\r' => {
533                        if data.get(*i) == Some(&b'\n') {
534                            *i += 1;
535                        }
536                    }
537                    b'\n' => {}
538                    other => out.push(other),
539                }
540            }
541            b'\r' => {
542                *i += 1;
543                if data.get(*i) == Some(&b'\n') {
544                    *i += 1;
545                }
546                out.push(b'\n');
547            }
548            other => {
549                out.push(other);
550                *i += 1;
551            }
552        }
553    }
554    None
555}
556
557/// Reads a `< ... >` hex string starting at `data[i] == b'<'`.
558fn read_hex_string(data: &[u8], i: &mut usize) -> Option<Vec<u8>> {
559    debug_assert_eq!(data[*i], b'<');
560    *i += 1;
561    let mut out = Vec::new();
562    let mut hi: Option<u8> = None;
563    while *i < data.len() {
564        match data[*i] {
565            b'>' => {
566                *i += 1;
567                if let Some(h) = hi {
568                    out.push(h << 4);
569                }
570                return Some(out);
571            }
572            b' ' | b'\t' | b'\r' | b'\n' => *i += 1,
573            b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' => {
574                let v = match data[*i] {
575                    b'0'..=b'9' => data[*i] - b'0',
576                    b'a'..=b'f' => data[*i] - b'a' + 10,
577                    _ => data[*i] - b'A' + 10,
578                };
579                *i += 1;
580                match hi {
581                    None => hi = Some(v),
582                    Some(h) => {
583                        out.push((h << 4) | v);
584                        hi = None;
585                    }
586                }
587            }
588            _ => *i += 1,
589        }
590    }
591    None
592}
593
594#[cfg(test)]
595mod tests {
596    use super::*;
597
598    fn extract(text: &str, compressed: bool) -> String {
599        // Build a minimal one-page PDF with a content stream.
600        let mut doc = paperforge_pdf::PdfDocument::new();
601        doc.set_catalog(ObjectId::new(1, 0));
602        doc.add_object(
603            ObjectId::new(1, 0),
604            PdfObject::Dictionary({
605                let mut d = paperforge_pdf::PdfDictionary::new();
606                d.insert(
607                    "Type",
608                    PdfObject::Name(paperforge_pdf::PdfName::new("Catalog")),
609                );
610                d.insert("Pages", PdfObject::Reference(ObjectId::new(2, 0)));
611                d
612            }),
613        );
614        doc.add_object(
615            ObjectId::new(2, 0),
616            PdfObject::Dictionary({
617                let mut d = paperforge_pdf::PdfDictionary::new();
618                d.insert(
619                    "Type",
620                    PdfObject::Name(paperforge_pdf::PdfName::new("Pages")),
621                );
622                d.insert("Count", PdfObject::Integer(1));
623                let mut kids = paperforge_pdf::PdfArray::new();
624                kids.push(PdfObject::Reference(ObjectId::new(4, 0)));
625                d.insert("Kids", PdfObject::Array(kids));
626                d
627            }),
628        );
629        // Escape the text the way the generator does, so the stream models a
630        // real PaperForge document.
631        let mut escaped = Vec::new();
632        paperforge_pdf::escape_literal_string_into(&mut escaped, text.as_bytes());
633        let content = format!(
634            "BT /F1 12 Tf 50 700 Td ({}) Tj ET",
635            String::from_utf8_lossy(&escaped)
636        );
637        let data = if compressed {
638            paperforge_pdf::StreamEncoder::new()
639                .encode(content.as_bytes(), StreamFilter::Flate)
640                .unwrap()
641        } else {
642            content.as_bytes().to_vec()
643        };
644        let stream =
645            paperforge_pdf::PdfStream::with_dict(paperforge_pdf::PdfDictionary::new(), data);
646        doc.add_object(ObjectId::new(3, 0), PdfObject::Stream(stream));
647        doc.add_object(
648            ObjectId::new(4, 0),
649            PdfObject::Dictionary({
650                let mut d = paperforge_pdf::PdfDictionary::new();
651                d.insert(
652                    "Type",
653                    PdfObject::Name(paperforge_pdf::PdfName::new("Page")),
654                );
655                d.insert("Parent", PdfObject::Reference(ObjectId::new(2, 0)));
656                d.insert("Contents", PdfObject::Reference(ObjectId::new(3, 0)));
657                d
658            }),
659        );
660
661        let mut buf = std::io::Cursor::new(Vec::new());
662        paperforge_pdf::Serializer::new()
663            .serialize(&doc, &mut buf)
664            .unwrap();
665        Extractor::extract_text(&Extractor, buf.get_ref()).unwrap()
666    }
667
668    #[test]
669    fn extracts_simple_text() {
670        assert_eq!(extract("Hello, world!", false), "Hello, world!\n");
671    }
672
673    #[test]
674    fn extracts_escaped_parens_and_backslashes() {
675        // Generator escapes `(`/`)`/`\`; extraction must undo that exactly.
676        assert_eq!(extract(r"a(b)c\d", false), "a(b)c\\d\n");
677    }
678
679    #[test]
680    fn extracts_from_compressed_streams() {
681        assert_eq!(extract("compressed", true), "compressed\n");
682    }
683
684    #[test]
685    fn metadata_does_not_leak_into_text() {
686        // The Producer string lives in /Info; extraction must not include it.
687        let mut doc = paperforge_pdf::PdfDocument::new();
688        doc.set_catalog(ObjectId::new(1, 0));
689        doc.add_object(
690            ObjectId::new(1, 0),
691            PdfObject::Dictionary({
692                let mut d = paperforge_pdf::PdfDictionary::new();
693                d.insert(
694                    "Type",
695                    PdfObject::Name(paperforge_pdf::PdfName::new("Catalog")),
696                );
697                d.insert("Pages", PdfObject::Reference(ObjectId::new(2, 0)));
698                d
699            }),
700        );
701        doc.add_object(
702            ObjectId::new(2, 0),
703            PdfObject::Dictionary({
704                let mut d = paperforge_pdf::PdfDictionary::new();
705                d.insert(
706                    "Type",
707                    PdfObject::Name(paperforge_pdf::PdfName::new("Pages")),
708                );
709                d.insert("Count", PdfObject::Integer(1));
710                let mut kids = paperforge_pdf::PdfArray::new();
711                kids.push(PdfObject::Reference(ObjectId::new(3, 0)));
712                d.insert("Kids", PdfObject::Array(kids));
713                d
714            }),
715        );
716        doc.add_object(
717            ObjectId::new(3, 0),
718            PdfObject::Dictionary({
719                let mut d = paperforge_pdf::PdfDictionary::new();
720                d.insert(
721                    "Type",
722                    PdfObject::Name(paperforge_pdf::PdfName::new("Page")),
723                );
724                d.insert("Parent", PdfObject::Reference(ObjectId::new(2, 0)));
725                d.insert("Contents", PdfObject::Reference(ObjectId::new(4, 0)));
726                d
727            }),
728        );
729        doc.add_object(
730            ObjectId::new(4, 0),
731            PdfObject::Stream(paperforge_pdf::PdfStream::with_dict(
732                paperforge_pdf::PdfDictionary::new(),
733                b"BT (page text) Tj ET".to_vec(),
734            )),
735        );
736        doc.set_info(ObjectId::new(5, 0));
737        doc.add_object(
738            ObjectId::new(5, 0),
739            PdfObject::Dictionary({
740                let mut d = paperforge_pdf::PdfDictionary::new();
741                d.insert(
742                    "Producer",
743                    PdfObject::String(paperforge_pdf::PdfString::from_literal("PaperForge")),
744                );
745                d
746            }),
747        );
748        let mut buf = std::io::Cursor::new(Vec::new());
749        paperforge_pdf::Serializer::new()
750            .serialize(&doc, &mut buf)
751            .unwrap();
752        let text = Extractor::extract_text(&Extractor, buf.get_ref()).unwrap();
753        assert_eq!(text, "page text\n");
754    }
755
756    /// Builds a one-page document whose content stream shows a 2-byte hex
757    /// string with `/F2 12 Tf` (Type0/Identity-H style) and whose font
758    /// `/ToUnicode` CMap uses `bfchar` and both `bfrange` forms.
759    fn embedded_font_fixture() -> Vec<u8> {
760        let mut doc = paperforge_pdf::PdfDocument::new();
761        doc.set_catalog(ObjectId::new(1, 0));
762        doc.add_object(
763            ObjectId::new(1, 0),
764            PdfObject::Dictionary({
765                let mut d = paperforge_pdf::PdfDictionary::new();
766                d.insert(
767                    "Type",
768                    PdfObject::Name(paperforge_pdf::PdfName::new("Catalog")),
769                );
770                d.insert("Pages", PdfObject::Reference(ObjectId::new(2, 0)));
771                d
772            }),
773        );
774        doc.add_object(
775            ObjectId::new(2, 0),
776            PdfObject::Dictionary({
777                let mut d = paperforge_pdf::PdfDictionary::new();
778                d.insert(
779                    "Type",
780                    PdfObject::Name(paperforge_pdf::PdfName::new("Pages")),
781                );
782                d.insert("Count", PdfObject::Integer(1));
783                let mut kids = paperforge_pdf::PdfArray::new();
784                kids.push(PdfObject::Reference(ObjectId::new(4, 0)));
785                d.insert("Kids", PdfObject::Array(kids));
786                d
787            }),
788        );
789
790        // ToUnicode CMap stream: bfchar entries plus a bfrange (single string
791        // and array forms).
792        let cmap = concat!(
793            "/CIDInit /ProcSet findresource begin\n",
794            "12 dict begin\nbegincmap\n",
795            "/CMapType 2 def\n1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n",
796            "2 beginbfchar\n<0048> <0048>\n<0063> <0063>\nendbfchar\n", // H, c
797            "1 beginbfrange\n<0041> <0043> <0041>\nendbfrange\n",       // A B C
798            "1 beginbfrange\n<0061> <0062> [<00E9> <00FC>]\nendbfrange\n", // é, ü
799            "endcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n",
800        );
801        doc.add_object(
802            ObjectId::new(5, 0),
803            PdfObject::Stream(paperforge_pdf::PdfStream::with_dict(
804                paperforge_pdf::PdfDictionary::new(),
805                cmap.as_bytes().to_vec(),
806            )),
807        );
808
809        // Type0 font referencing the CMap.
810        let mut font = paperforge_pdf::PdfDictionary::new();
811        font.insert(
812            "Type",
813            PdfObject::Name(paperforge_pdf::PdfName::new("Font")),
814        );
815        font.insert(
816            "Subtype",
817            PdfObject::Name(paperforge_pdf::PdfName::new("Type0")),
818        );
819        font.insert(
820            "Encoding",
821            PdfObject::Name(paperforge_pdf::PdfName::new("Identity-H")),
822        );
823        font.insert("ToUnicode", PdfObject::Reference(ObjectId::new(5, 0)));
824        doc.add_object(ObjectId::new(6, 0), PdfObject::Dictionary(font));
825
826        doc.add_object(
827            ObjectId::new(3, 0),
828            PdfObject::Stream(paperforge_pdf::PdfStream::with_dict(
829                paperforge_pdf::PdfDictionary::new(),
830                b"BT /F2 12 Tf 50 700 Td <0048006300410042004300610062> Tj ET".to_vec(),
831            )),
832        );
833        doc.add_object(
834            ObjectId::new(4, 0),
835            PdfObject::Dictionary({
836                let mut d = paperforge_pdf::PdfDictionary::new();
837                d.insert(
838                    "Type",
839                    PdfObject::Name(paperforge_pdf::PdfName::new("Page")),
840                );
841                d.insert("Parent", PdfObject::Reference(ObjectId::new(2, 0)));
842                d.insert("Contents", PdfObject::Reference(ObjectId::new(3, 0)));
843                let mut resources = paperforge_pdf::PdfDictionary::new();
844                let mut fonts = paperforge_pdf::PdfDictionary::new();
845                fonts.insert("F2", PdfObject::Reference(ObjectId::new(6, 0)));
846                resources.insert("Font", PdfObject::Dictionary(fonts));
847                d.insert("Resources", PdfObject::Dictionary(resources));
848                d
849            }),
850        );
851
852        let mut buf = std::io::Cursor::new(Vec::new());
853        paperforge_pdf::Serializer::new()
854            .serialize(&doc, &mut buf)
855            .unwrap();
856        buf.into_inner()
857    }
858
859    /// Glyph-id hex strings (Identity-H) must come back as Unicode through
860    /// the `/ToUnicode` CMap: bfchar, incrementing bfrange and array bfrange.
861    #[test]
862    fn extracts_embedded_font_text_via_tounicode() {
863        let data = embedded_font_fixture();
864        let text = Extractor::extract_text(&Extractor, &data).unwrap();
865        assert_eq!(text, "HcABCéü\n");
866    }
867}
paperforge_extract/extractor.rs

paperforge_extract/
extractor.rs
