1mod ir;
5mod markdown;
6mod output;
7mod structure;
8
9use pdfboss_core::{AsyncObjectSource, Document, OcState, Page, Result};
10
11pub use ir::{BBox, Block, Cell, Inline, Line, ListItem, Marker, PageLayout, Role};
12pub use markdown::Markdown;
13pub use output::{Output, Text};
14pub use pdfboss_text::{
15 ExtractReport, FontCache, Ruling, SkipCause, SkippedText, SkippedTextKind, TextSpan,
16};
17pub use structure::{
18 document_layout, document_layout_with_rulings, layout, page_layout, page_layout_with_rulings,
19};
20
21pub fn extract_text(doc: &Document, page: &Page) -> Result<String> {
35 let (text, _) = extract_text_reporting(doc, page)?;
36 Ok(text)
37}
38
39pub async fn extract_text_with<S: AsyncObjectSource>(
50 src: S,
51 page: &Page,
52 oc: Option<&OcState>,
53) -> Result<String> {
54 let (text, _) = extract_text_reporting_with(src, page, oc).await?;
55 Ok(text)
56}
57
58pub fn extract_text_reporting(doc: &Document, page: &Page) -> Result<(String, ExtractReport)> {
64 let (spans, report) = pdfboss_text::extract_spans_reporting(doc, page)?;
65 Ok((Text.render(&[page_layout(&spans)]), report))
66}
67
68pub async fn extract_text_reporting_with<S: AsyncObjectSource>(
71 src: S,
72 page: &Page,
73 oc: Option<&OcState>,
74) -> Result<(String, ExtractReport)> {
75 let (spans, report) = pdfboss_text::extract_spans_reporting_with(src, page, oc).await?;
76 Ok((Text.render(&[page_layout(&spans)]), report))
77}
78
79pub fn extract_text_reporting_cached(
88 doc: &Document,
89 page: &Page,
90 fonts: &FontCache,
91) -> Result<(String, ExtractReport)> {
92 let (spans, report) = pdfboss_text::extract_spans_reporting_cached(doc, page, fonts)?;
93 Ok((Text.render(&[page_layout(&spans)]), report))
94}
95
96pub fn extract_markdown(doc: &Document) -> Result<String> {
107 let (markdown, _) = extract_markdown_reporting(doc)?;
108 Ok(markdown)
109}
110
111pub fn extract_markdown_reporting(doc: &Document) -> Result<(String, Vec<ExtractReport>)> {
116 let fonts = FontCache::default();
117 let per_page = pdfboss_core::map_pages(doc, |doc: &Document, page: &Page| {
118 pdfboss_text::extract_spans_and_rulings_reporting_cached(doc, page, &fonts)
119 });
120 let mut pages = Vec::with_capacity(per_page.len());
121 let mut reports = Vec::with_capacity(per_page.len());
122 for outcome in per_page {
123 let (spans, rulings, report) = outcome?;
124 pages.push((spans, rulings));
125 reports.push(report);
126 }
127 Ok((
128 Markdown.render(&document_layout_with_rulings(&pages)),
129 reports,
130 ))
131}
132
133pub fn extract_page_markdown(doc: &Document, page: &Page) -> Result<String> {
137 let (spans, rulings, _) = pdfboss_text::extract_spans_and_rulings_reporting(doc, page)?;
138 Ok(Markdown.render(&[page_layout_with_rulings(&spans, &rulings)]))
139}
140
141pub async fn extract_page_markdown_with<S: AsyncObjectSource>(
150 src: S,
151 page: &Page,
152 oc: Option<&OcState>,
153) -> Result<String> {
154 let (spans, rulings, _) =
155 pdfboss_text::extract_spans_and_rulings_reporting_with(src, page, oc).await?;
156 Ok(Markdown.render(&[page_layout_with_rulings(&spans, &rulings)]))
157}
158
159#[cfg(test)]
160mod tests {
161 use super::*;
162 use pdfboss_core::{block_on, resolve_with, BoxFuture, ObjRef, Object, Stream};
163 use pdfboss_testkit::{doc_with_graphics, multi_page_doc, simple_doc, PdfBuilder};
164 use std::future::Future;
165
166 fn page_text(doc: &Document, index: usize) -> String {
167 let page = doc.page(index).unwrap();
168 extract_text(doc, &page).unwrap()
169 }
170
171 fn token_counts(text: &str) -> std::collections::BTreeMap<&str, usize> {
174 let mut counts = std::collections::BTreeMap::new();
175 for token in text.split_whitespace() {
176 *counts.entry(token).or_default() += 1;
177 }
178 counts
179 }
180
181 #[test]
191 fn text_adapter_matches_layout_on_fixtures() {
192 let mut headings = 0usize;
193 let mut splits = 0usize;
194 let mut tables = 0usize;
195 let mut ruled_tables = 0usize;
196 for content in structure::tests::fixture_contents() {
197 let doc = Document::load(doc_with_graphics(&content)).unwrap();
198 let page = doc.page(0).unwrap();
199 let (spans, rulings, report) =
200 pdfboss_text::extract_spans_and_rulings_reporting(&doc, &page).unwrap();
201 assert!(report.is_complete(), "unexpected skips: {report:?}");
202 let layout = page_layout_with_rulings(&spans, &rulings);
203 headings += layout
204 .blocks
205 .iter()
206 .filter(|block| matches!(block, Block::Heading { .. }))
207 .count();
208 let paragraphs = layout
209 .blocks
210 .iter()
211 .filter(|block| matches!(block, Block::Paragraph { .. }))
212 .count();
213 splits += usize::from(paragraphs > 1);
214 let fixture_tables = layout
215 .blocks
216 .iter()
217 .filter(|block| matches!(block, Block::Table { .. }))
218 .count();
219 tables += fixture_tables;
220 if !rulings.is_empty() {
221 ruled_tables += fixture_tables;
222 }
223 let via_ir = Text.render(&[layout]);
224 let flat = structure::layout_reference(&spans);
225 if rulings.is_empty() {
226 assert_eq!(via_ir, flat, "content: {content}");
227 } else {
228 assert_eq!(
229 token_counts(&via_ir),
230 token_counts(&flat),
231 "content: {content}\nvia IR: {via_ir}\nflat flow: {flat}"
232 );
233 }
234 }
235 assert!(headings > 0, "no fixture produced a heading block");
238 assert!(splits > 0, "no fixture split into several paragraphs");
239 assert!(tables > 0, "no fixture produced a table block");
240 assert!(ruled_tables > 0, "no fixture produced a ruled table block");
241 }
242
243 fn markdown_of(content: &str) -> String {
246 let doc = Document::load(doc_with_graphics(content)).unwrap();
247 let page = doc.page(0).unwrap();
248 let (spans, report) = pdfboss_text::extract_spans_reporting(&doc, &page).unwrap();
249 assert!(report.is_complete(), "unexpected skips: {report:?}");
250 Markdown.render(&document_layout(&[spans]))
251 }
252
253 fn markdown_of_two_fonts(content: &str) -> String {
256 let mut b = PdfBuilder::new();
257 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
258 b.object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>");
259 b.object(
260 3,
261 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
262 /Resources << /Font << /F1 5 0 R /F2 6 0 R >> >> /Contents 4 0 R >>",
263 );
264 b.stream(4, "", content.as_bytes());
265 b.object(
266 5,
267 "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica \
268 /Encoding /WinAnsiEncoding >>",
269 );
270 b.object(
271 6,
272 "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold \
273 /Encoding /WinAnsiEncoding >>",
274 );
275 let doc = Document::load(b.build(1)).unwrap();
276 let page = doc.page(0).unwrap();
277 let (spans, report) = pdfboss_text::extract_spans_reporting(&doc, &page).unwrap();
278 assert!(report.is_complete(), "unexpected skips: {report:?}");
279 Markdown.render(&document_layout(&[spans]))
280 }
281
282 #[test]
285 fn heading_levels_by_size_rank() {
286 let content = "BT /F1 24 Tf 72 740 Td (Title) Tj \
287 /F1 16 Tf 0 -40 Td (Section) Tj \
288 /F1 12 Tf 0 -30 Td (Body text long enough to look like body.) Tj \
289 0 -14 Td (More body keeps twelve the dominant size.) Tj \
290 0 -14 Td (And a third line for good measure.) Tj ET";
291 let md = markdown_of(content);
292 assert!(md.contains("# Title\n"), "md: {md}");
293 assert!(md.contains("## Section\n"), "md: {md}");
294 assert!(!md.contains("# Body"), "md: {md}");
295 }
296
297 #[test]
301 fn ranks_past_six_clamp_to_level_six() {
302 let heads = [36, 28, 24, 20, 18, 16, 14, 12, 11]
303 .iter()
304 .enumerate()
305 .map(|(index, size)| {
306 let y = 750 - 50 * index;
307 format!("BT /F1 {size} Tf 72 {y} Td (Head {size}) Tj ET ")
308 })
309 .collect::<String>();
310 let body = (0..3)
311 .map(|index| {
312 let y = 260 - 14 * index;
313 format!(
314 "BT /F1 10 Tf 72 {y} Td (Body line {index} is long enough to be body.) Tj ET "
315 )
316 })
317 .collect::<String>();
318 let md = markdown_of(&format!("{heads}{body}"));
319 assert!(md.starts_with("# Head 36"), "md: {md}");
320 assert!(md.contains("###### Head 16"), "md: {md}");
321 assert!(md.contains("###### Head 14"), "md: {md}");
322 assert!(md.contains("###### Head 12"), "md: {md}");
323 assert!(md.contains("###### Head 11"), "md: {md}");
324 }
325
326 #[test]
329 fn blank_heading_line_emits_nothing() {
330 let md = markdown_of(
331 "BT /F1 24 Tf 72 740 Td ( ) Tj \
332 /F1 12 Tf 0 -40 Td (Body line one is long enough to be body.) Tj \
333 0 -14 Td (Body line two keeps twelve the dominant size.) Tj \
334 0 -14 Td (And a third body line seals it.) Tj ET",
335 );
336 assert!(!md.contains('#'), "md: {md:?}");
337 }
338
339 #[test]
342 fn bold_run_renders_as_strong() {
343 let md = markdown_of_two_fonts(
344 "BT /F1 12 Tf 72 720 Td (plain ) Tj /F2 12 Tf (loud) Tj /F1 12 Tf ( tail) Tj \
345 0 -14 Td (body body body body) Tj 0 -14 Td (body body body body) Tj ET",
346 );
347 assert!(md.contains("plain **loud** tail"), "md: {md}");
348 }
349
350 #[test]
351 fn bullet_lines_become_list_items() {
352 let content = "BT /F1 12 Tf 72 720 Td (\\225 first item) Tj \
353 0 -14 Td (\\225 second item) Tj \
354 0 -14 Td (Body sentence after the list ends here.) Tj ET";
355 let md = markdown_of(content);
357 assert!(md.contains("- first item\n- second item"), "md: {md}");
358 assert!(!md.contains('\u{2022}'), "marker replaced, not kept: {md}");
359 }
360
361 #[test]
365 fn a_lone_marker_line_stays_prose() {
366 let content = "BT /F1 12 Tf 72 720 Td (- stray dash line) Tj \
367 0 -14 Td (Body sentence at the same indent.) Tj \
368 0 -14 Td (- alpha) Tj 0 -14 Td (- beta) Tj ET";
369 let md = markdown_of(content);
370 assert!(md.contains("- alpha\n- beta"), "md: {md}");
371 assert!(
372 md.contains("- stray dash line\nBody sentence at the same indent."),
373 "the stray marker line stays in the paragraph: {md}"
374 );
375 }
376
377 #[test]
378 fn numbered_items_keep_their_numbers() {
379 let content = "BT /F1 12 Tf 72 720 Td (1. alpha) Tj 0 -14 Td (2. beta) Tj \
380 0 -14 Td (12) Tj ET";
381 let md = markdown_of(content);
382 assert!(md.contains("1. alpha\n2. beta"), "md: {md}");
383 assert!(
384 md.contains("12"),
385 "a bare number line is not a list item: {md}"
386 );
387 }
388
389 #[test]
390 fn hanging_indent_continues_an_item() {
391 let content = "BT /F1 12 Tf 72 720 Td (\\225 a long item that) Tj \
393 10 -14 Td (wraps to a second line) Tj ET";
394 let md = markdown_of(content);
395 assert!(
396 md.contains("- a long item that\nwraps to a second line")
397 || md.contains("- a long item that wraps to a second line"),
398 "md: {md}"
399 );
400 }
401
402 #[test]
404 fn lane_grid_becomes_pipe_table() {
405 let md = markdown_of(&structure::tests::lane_grid_content());
406 assert!(md.contains("| r0c0 | r0c1 | r0c2 |"), "md: {md}");
407 assert!(
408 md.contains("| --- | --- | --- |"),
409 "separator after header: {md}"
410 );
411 assert!(md.contains("| r3c0 | r3c1 | r3c2 |"), "md: {md}");
412 }
413
414 #[test]
418 fn a_narrow_column_gap_still_opens_a_lane() {
419 let md = markdown_of(&structure::tests::narrow_gap_lane_grid_content());
420 assert!(md.contains("| r0c0 | r0c1 | r0c2 |"), "md: {md}");
421 assert!(md.contains("| r3c0 | r3c1 | r3c2 |"), "md: {md}");
422 }
423
424 #[test]
430 fn page_edge_lines_around_the_grid_stay_prose() {
431 let md = markdown_of(&structure::tests::grid_with_edge_lines_content());
432 let header = structure::tests::RUNNING_HEADER;
433 assert!(
434 !md.contains("<table>"),
435 "an edge line flipped the dialect: {md}"
436 );
437 assert!(
438 md.contains(&format!(
439 "{header}\n\n| r0c0 | r0c1 | r0c2 |\n| --- | --- | --- |"
440 )),
441 "md: {md}"
442 );
443 assert!(
444 md.contains("| r1c0 | r1c1 | r1c2 |\n| wrapped cell | | |\n| r2c0 |"),
445 "wrapped cell is a row: {md}"
446 );
447 assert!(md.contains("| r3c0 | r3c1 | r3c2 |"), "md: {md}");
448 assert!(!md.contains("| 24 |"), "page number is not a row: {md}");
449 assert!(md.ends_with("\n\n24"), "md: {md}");
450 }
451
452 #[test]
457 fn a_margin_page_number_does_not_manufacture_a_column() {
458 let md = markdown_of(&structure::tests::margin_number_grid_content());
459 assert!(!md.contains('|'), "two columns are not a table: {md}");
460 assert!(!md.contains("<table>"), "two columns are not a table: {md}");
461 assert!(md.contains("r0c0 r0c1"), "rows still read as prose: {md}");
462 assert!(md.ends_with("\n\n3"), "the page number survives: {md}");
463 }
464
465 #[test]
467 fn spanning_cell_switches_to_html_table() {
468 let mut content = String::from(
472 "BT /F1 10 Tf 1 0 0 1 72 700 Tm (a merged header cell spanning two lanes xx) Tj \
473 1 0 0 1 430 700 Tm (r0c2) Tj ",
474 );
475 for (row, y) in [(1, 680.0), (2, 660.0), (3, 640.0)] {
476 for (col, x) in [(0, 72.0), (1, 250.0), (2, 430.0)] {
477 content += &format!("1 0 0 1 {x} {y} Tm (r{row}c{col}) Tj ");
478 }
479 }
480 content += "ET";
481 let md = markdown_of(&content);
482 assert!(md.contains("<table>"), "md: {md}");
483 assert!(md.contains("colspan=\"2\""), "md: {md}");
484 assert!(!md.contains("| r1c0 |"), "one table, one dialect: {md}");
485 }
486
487 fn markdown_of_drawn(content: &str) -> String {
490 let doc = Document::load(doc_with_graphics(content)).unwrap();
491 extract_markdown(&doc).unwrap()
492 }
493
494 #[test]
497 fn a_ruled_grid_becomes_a_pipe_table() {
498 let md = markdown_of_drawn(&structure::tests::ruled_grid_content());
499 assert!(
500 md.contains("| a1 | b1 |\n| --- | --- |\n| a2 | b2 |"),
501 "md: {md}"
502 );
503 }
504
505 #[test]
508 fn a_single_column_boxed_list_becomes_a_table() {
509 let md = markdown_of_drawn(&structure::tests::ruled_boxed_list_content());
510 assert!(
511 md.contains(
512 "| first item |\n| --- |\n| second item |\n| third item |\n| fourth item |"
513 ),
514 "md: {md}"
515 );
516 }
517
518 #[test]
522 fn a_wrapped_band_merges_into_one_logical_row() {
523 let md = markdown_of_drawn(&structure::tests::ruled_wrapped_band_content());
524 assert!(
525 md.contains("| h1 | h2 | h3 |\n| --- | --- | --- |\n| m1 | m2 | m3 |"),
526 "md: {md}"
527 );
528 assert!(
529 md.contains("| wrap one wrap two wrap three | solo | tail |"),
530 "the band's lines merge into one row: {md}"
531 );
532 assert!(
533 !md.contains("| wrap two |"),
534 "no fragmentary row survives: {md}"
535 );
536 }
537
538 #[test]
543 fn an_open_edged_grid_becomes_a_full_table() {
544 let md = markdown_of_drawn(&structure::tests::ruled_open_grid_content());
545 assert!(
546 md.contains(
547 "| name | count | note |\n| --- | --- | --- |\n\
548 | alpha | one | xx |\n| beta | two | yy |\n\
549 | gamma | three | zz |\n| delta | four | ww |"
550 ),
551 "md: {md}"
552 );
553 }
554
555 #[test]
559 fn wrapped_records_fold_behind_their_anchors() {
560 let md = markdown_of_drawn(&structure::tests::ruled_wrapped_records_content());
561 assert!(
562 md.contains(
563 "| name | org | count |\n| --- | --- | --- |\n\
564 | one | recordaa wrapa | c1 |\n| two | recordbb wrapb | c2 |"
565 ),
566 "md: {md}"
567 );
568 }
569
570 #[test]
574 fn a_centered_record_band_merges_whole() {
575 let md = markdown_of_drawn(&structure::tests::ruled_centered_record_content());
576 assert!(
577 md.contains(
578 "| name | org | count |\n| --- | --- | --- |\n\
579 | actlinea actlineb actlinec actlined | union | c9 |"
580 ),
581 "md: {md}"
582 );
583 }
584
585 #[test]
589 fn a_ruled_grid_and_a_lane_grid_share_a_segment() {
590 let content = structure::tests::ruled_grid_above_lane_grid_content();
591 let md = markdown_of_drawn(&content);
592 assert!(
593 md.contains("| a1 | b1 |\n| --- | --- |\n| a2 | b2 |"),
594 "the drawn grid stays a table: {md}"
595 );
596 let lane_table = [
597 "| r0c0 | r0c1 | r0c2 |",
598 "| --- | --- | --- |",
599 "| r1c0 | r1c1 | r1c2 |",
600 "| r2c0 | r2c1 | r2c2 |",
601 "| r3c0 | r3c1 | r3c2 |",
602 ]
603 .join("\n");
604 assert!(
605 md.contains(&lane_table),
606 "the laned rows stay a table: {md}"
607 );
608 assert!(
609 md.find("| a1 |").unwrap() < md.find("| r0c0 |").unwrap(),
610 "reading order: {md}"
611 );
612 assert!(
613 markdown_of(&content).contains(&lane_table),
614 "the lane path alone emits the same table"
615 );
616 }
617
618 #[test]
621 fn a_ruling_inside_a_sub_word_gap_rejects_the_grid() {
622 let md = markdown_of_drawn(&structure::tests::ruled_sub_word_gap_content());
623 assert!(!md.contains('|'), "no table: {md}");
624 assert!(!md.contains("<table>"), "no table: {md}");
625 assert!(md.contains("world"), "the word survives whole: {md}");
626 }
627
628 #[test]
631 fn spans_only_layout_ignores_drawn_grids() {
632 let doc = Document::load(doc_with_graphics(
633 &structure::tests::ruled_boxed_list_content(),
634 ))
635 .unwrap();
636 let page = doc.page(0).unwrap();
637 let (spans, report) = pdfboss_text::extract_spans_reporting(&doc, &page).unwrap();
638 assert!(report.is_complete(), "unexpected skips: {report:?}");
639 let layout = page_layout(&spans);
640 assert!(
641 layout
642 .blocks
643 .iter()
644 .all(|block| !matches!(block, Block::Table { .. })),
645 "no rulings, no table: {layout:?}"
646 );
647 }
648
649 #[test]
652 fn document_stats_beat_page_stats() {
653 let mut b = PdfBuilder::new();
654 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
655 b.object(2, "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>");
656 b.object(
657 3,
658 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
659 /Resources << /Font << /F1 7 0 R >> >> /Contents 5 0 R >>",
660 );
661 b.object(
662 4,
663 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
664 /Resources << /Font << /F1 7 0 R >> >> /Contents 6 0 R >>",
665 );
666 b.stream(
667 5,
668 "",
669 b"BT /F1 12 Tf 72 720 Td (Body line one is long enough.) Tj \
670 0 -14 Td (Body line two keeps twelve dominant.) Tj \
671 0 -14 Td (Body line three seals it.) Tj ET",
672 );
673 b.stream(6, "", b"BT /F1 24 Tf 72 720 Td (Chapter Two) Tj ET");
674 b.object(
675 7,
676 "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica \
677 /Encoding /WinAnsiEncoding >>",
678 );
679 let doc = Document::load(b.build(1)).unwrap();
680 let pages: Vec<Vec<TextSpan>> = (0..2)
681 .map(|i| {
682 let page = doc.page(i).unwrap();
683 pdfboss_text::extract_spans_reporting(&doc, &page)
684 .unwrap()
685 .0
686 })
687 .collect();
688 let md = Markdown.render(&document_layout(&pages));
689 assert!(
690 md.contains("# Chapter Two"),
691 "doc stats make it a heading: {md}"
692 );
693 let alone = Markdown.render(&[page_layout(&pages[1])]);
694 assert!(
695 !alone.contains("# "),
696 "page stats alone see 24pt as body: {alone}"
697 );
698 }
699
700 #[test]
703 fn running_headers_and_page_numbers_are_tagged() {
704 let mut b = PdfBuilder::new();
705 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
706 b.object(2, "<< /Type /Pages /Kids [3 0 R 4 0 R 5 0 R] /Count 3 >>");
707 for (page_obj, contents_obj) in [(3u32, 6u32), (4, 7), (5, 8)] {
708 b.object(
709 page_obj,
710 &format!(
711 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
712 /Resources << /Font << /F1 9 0 R >> >> /Contents {contents_obj} 0 R >>"
713 ),
714 );
715 }
716 for (contents_obj, n) in [(6u32, 1u32), (7, 2), (8, 3)] {
717 b.stream(
718 contents_obj,
719 "",
720 format!(
721 "BT /F1 10 Tf 72 770 Td (ACME REPORT) Tj \
722 /F1 12 Tf 0 -50 Td (Page {n} body text differs everywhere.) Tj \
723 0 -14 Td (A second body line pads the page.) Tj \
724 /F1 10 Tf 200 -666 Td ({n}) Tj ET"
725 )
726 .as_bytes(),
727 );
728 }
729 b.object(
730 9,
731 "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica \
732 /Encoding /WinAnsiEncoding >>",
733 );
734 let doc = Document::load(b.build(1)).unwrap();
735 let pages: Vec<Vec<TextSpan>> = (0..3)
736 .map(|i| {
737 let page = doc.page(i).unwrap();
738 pdfboss_text::extract_spans_reporting(&doc, &page)
739 .unwrap()
740 .0
741 })
742 .collect();
743 let layouts = document_layout(&pages);
744 let md = Markdown.render(&layouts);
745 assert!(!md.contains("ACME REPORT"), "md: {md}");
746 assert!(!md.contains("\n1\n"), "page number dropped: {md}");
747 assert!(md.contains("body text differs"), "body survives: {md}");
748 let text = Text.render(&layouts);
749 assert!(
750 text.contains("ACME REPORT"),
751 "text keeps everything: {text}"
752 );
753 }
754
755 #[test]
756 fn simple_doc_exact_text() {
757 let doc = Document::load(simple_doc("Hello, world!")).unwrap();
758 assert_eq!(page_text(&doc, 0), "Hello, world!");
759 }
760
761 #[test]
762 fn multi_page_doc_per_page() {
763 let doc = Document::load(multi_page_doc(&["Page one", "Page two", "Page three"])).unwrap();
764 assert_eq!(doc.page_count(), 3);
765 assert_eq!(page_text(&doc, 0), "Page one");
766 assert_eq!(page_text(&doc, 1), "Page two");
767 assert_eq!(page_text(&doc, 2), "Page three");
768 }
769
770 #[test]
771 fn differences_remap_in_extraction() {
772 let mut b = PdfBuilder::new();
773 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
774 b.object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>");
775 b.object(
776 3,
777 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
778 /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>",
779 );
780 b.stream(4, "", b"BT /F1 12 Tf 72 720 Td (AB) Tj ET");
781 b.object(
782 5,
783 "<< /Type /Font /Subtype /Type1 /BaseFont /Custom \
784 /Encoding << /BaseEncoding /WinAnsiEncoding \
785 /Differences [65 /alpha] >> >>",
786 );
787 let doc = Document::load(b.build(1)).unwrap();
788 assert_eq!(page_text(&doc, 0), "\u{3B1}B");
789 }
790
791 #[test]
792 fn type0_font_with_tounicode_stream() {
793 let mut b = PdfBuilder::new();
794 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
795 b.object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>");
796 b.object(
797 3,
798 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
799 /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>",
800 );
801 b.stream(4, "", b"BT /F1 12 Tf 72 720 Td <00010001> Tj ET");
802 b.object(
803 5,
804 "<< /Type /Font /Subtype /Type0 /BaseFont /X /Encoding /Identity-H \
805 /DescendantFonts [6 0 R] /ToUnicode 7 0 R >>",
806 );
807 b.object(
808 6,
809 "<< /Type /Font /Subtype /CIDFontType2 /BaseFont /X /DW 600 >>",
810 );
811 b.stream(
812 7,
813 "",
814 b"1 begincodespacerange <0000> <FFFF> endcodespacerange\n\
815 1 beginbfchar <0001> <03A9> endbfchar",
816 );
817 let doc = Document::load(b.build(1)).unwrap();
818 assert_eq!(page_text(&doc, 0), "\u{3A9}\u{3A9}");
819 }
820
821 #[test]
822 fn form_xobject_recursion() {
823 let mut b = PdfBuilder::new();
824 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
825 b.object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>");
826 b.object(
827 3,
828 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
829 /Resources << /Font << /F1 5 0 R >> /XObject << /Fx 6 0 R >> >> \
830 /Contents 4 0 R >>",
831 );
832 b.stream(4, "", b"BT /F1 12 Tf 72 720 Td (out) Tj ET /Fx Do");
833 b.object(
834 5,
835 "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica \
836 /Encoding /WinAnsiEncoding >>",
837 );
838 b.stream(
840 6,
841 "/Type /XObject /Subtype /Form /BBox [0 0 612 792] \
842 /Matrix [1 0 0 1 0 -20]",
843 b"BT /F1 12 Tf 72 720 Td (in) Tj ET",
844 );
845 let doc = Document::load(b.build(1)).unwrap();
846 assert_eq!(page_text(&doc, 0), "out\nin");
847 }
848
849 #[test]
860 fn form_with_partial_resources_still_sees_the_page_font() {
861 let mut b = PdfBuilder::new();
862 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
863 b.object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>");
864 b.object(
865 3,
866 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
867 /Resources << /Font << /F1 5 0 R >> /XObject << /Fx 6 0 R >> >> \
868 /Contents 4 0 R >>",
869 );
870 b.stream(4, "", b"/Fx Do");
871 b.object(
872 5,
873 "<< /Type /Font /Subtype /Type1 /BaseFont /Custom \
874 /Encoding << /BaseEncoding /WinAnsiEncoding \
875 /Differences [65 /alpha] >> >>",
876 );
877 b.stream(
879 6,
880 "/Type /XObject /Subtype /Form /BBox [0 0 612 792] \
881 /Resources << /ProcSet [/PDF /Text] >>",
882 b"BT /F1 12 Tf 72 720 Td (A) Tj ET",
883 );
884 let doc = Document::load(b.build(1)).unwrap();
885 assert_eq!(page_text(&doc, 0), "\u{3B1}");
886 }
887
888 fn hex(data: &[u8]) -> Vec<u8> {
892 data.iter()
893 .flat_map(|b| format!("{b:02X}").into_bytes())
894 .chain(*b">")
895 .collect()
896 }
897
898 fn contents_doc(stream_dict: &str, content: &[u8]) -> Document {
901 let mut b = PdfBuilder::new();
902 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
903 b.object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>");
904 b.object(
905 3,
906 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
907 /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>",
908 );
909 b.stream(4, stream_dict, content);
910 b.object(
911 5,
912 "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica \
913 /Encoding /WinAnsiEncoding >>",
914 );
915 Document::load(b.build(1)).unwrap()
916 }
917
918 #[test]
924 fn image_codec_page_contents_yield_no_text_and_one_report_entry() {
925 let doc = contents_doc(
926 "/Filter /JPXDecode",
927 b"BT /F1 12 Tf 72 720 Td (ghost) Tj ET",
928 );
929 let page = doc.page(0).unwrap();
930 let (text, report) = extract_text_reporting(&doc, &page).unwrap();
931 assert_eq!(text, "", "the passthrough bytes must not be parsed");
932 assert_eq!(
933 report.skipped,
934 vec![SkippedText {
935 kind: SkippedTextKind::PageContents,
936 cause: SkipCause::UnsupportedFilter("JPXDecode".to_string()),
937 }],
938 );
939 assert_eq!(extract_text(&doc, &page).unwrap(), "");
941 }
942
943 #[test]
947 fn a_filtered_page_contents_still_extracts() {
948 let doc = contents_doc(
949 "/Filter /ASCIIHexDecode",
950 &hex(b"BT /F1 12 Tf 72 720 Td (plain sight) Tj ET"),
951 );
952 let page = doc.page(0).unwrap();
953 let (text, report) = extract_text_reporting(&doc, &page).unwrap();
954 assert_eq!(text, "plain sight");
955 assert!(report.is_complete(), "nothing was skipped: {report:?}");
956 }
957
958 #[test]
963 fn image_codec_form_content_is_refused_and_reported() {
964 let mut b = PdfBuilder::new();
965 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
966 b.object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>");
967 b.object(
968 3,
969 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
970 /Resources << /Font << /F1 5 0 R >> /XObject << /Fx 6 0 R >> >> \
971 /Contents 4 0 R >>",
972 );
973 b.stream(4, "", b"BT /F1 12 Tf 72 720 Td (kept) Tj ET /Fx Do");
974 b.object(
975 5,
976 "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica \
977 /Encoding /WinAnsiEncoding >>",
978 );
979 b.stream(
980 6,
981 "/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Filter /DCTDecode",
982 b"BT /F1 12 Tf 72 700 Td (ghost) Tj ET",
983 );
984 let doc = Document::load(b.build(1)).unwrap();
985 let page = doc.page(0).unwrap();
986 let (text, report) = extract_text_reporting(&doc, &page).unwrap();
987 assert_eq!(text, "kept", "the page's own text survives the refusal");
988 assert_eq!(
989 report.skipped,
990 vec![SkippedText {
991 kind: SkippedTextKind::Form,
992 cause: SkipCause::UnsupportedFilter("DCTDecode".to_string()),
993 }],
994 );
995 }
996
997 #[test]
1000 fn a_filtered_form_still_extracts() {
1001 let mut b = PdfBuilder::new();
1002 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
1003 b.object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>");
1004 b.object(
1005 3,
1006 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
1007 /Resources << /Font << /F1 5 0 R >> /XObject << /Fx 6 0 R >> >> \
1008 /Contents 4 0 R >>",
1009 );
1010 b.stream(4, "", b"/Fx Do");
1011 b.object(
1012 5,
1013 "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica \
1014 /Encoding /WinAnsiEncoding >>",
1015 );
1016 b.stream(
1017 6,
1018 "/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Filter /ASCIIHexDecode",
1019 &hex(b"BT /F1 12 Tf 72 720 Td (decoded) Tj ET"),
1020 );
1021 let doc = Document::load(b.build(1)).unwrap();
1022 let page = doc.page(0).unwrap();
1023 let (text, report) = extract_text_reporting(&doc, &page).unwrap();
1024 assert_eq!(text, "decoded");
1025 assert!(report.is_complete(), "nothing was skipped: {report:?}");
1026 }
1027
1028 #[test]
1031 fn exhausted_form_depth_is_reported() {
1032 let mut b = PdfBuilder::new();
1033 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
1034 b.object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>");
1035 b.object(
1036 3,
1037 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
1038 /Resources << /Font << /F1 5 0 R >> /XObject << /Fx 6 0 R >> >> \
1039 /Contents 4 0 R >>",
1040 );
1041 b.stream(4, "", b"/Fx Do");
1042 b.object(
1043 5,
1044 "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica \
1045 /Encoding /WinAnsiEncoding >>",
1046 );
1047 b.stream(
1050 6,
1051 "/Type /XObject /Subtype /Form /BBox [0 0 612 792]",
1052 b"BT /F1 12 Tf 72 720 Td (x) Tj ET /Fx Do",
1053 );
1054 let doc = Document::load(b.build(1)).unwrap();
1055 let page = doc.page(0).unwrap();
1056 let (text, report) = extract_text_reporting(&doc, &page).unwrap();
1057 assert!(!text.is_empty(), "the levels above the cap still extract");
1058 assert_eq!(
1059 report.skipped,
1060 vec![SkippedText {
1061 kind: SkippedTextKind::Form,
1062 cause: SkipCause::LimitExceeded,
1063 }],
1064 );
1065 }
1066
1067 #[test]
1070 fn a_missing_xobject_is_reported() {
1071 let doc = contents_doc("", b"BT /F1 12 Tf 72 720 Td (here) Tj ET /Nope Do");
1072 let page = doc.page(0).unwrap();
1073 let (text, report) = extract_text_reporting(&doc, &page).unwrap();
1074 assert_eq!(text, "here");
1075 assert_eq!(
1076 report.skipped,
1077 vec![SkippedText {
1078 kind: SkippedTextKind::XObject,
1079 cause: SkipCause::Missing,
1080 }],
1081 );
1082 }
1083
1084 #[test]
1090 fn a_form_whose_subtype_is_indirect_still_extracts() {
1091 let mut b = PdfBuilder::new();
1092 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
1093 b.object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>");
1094 b.object(
1095 3,
1096 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
1097 /Resources << /Font << /F1 5 0 R >> /XObject << /Fx 6 0 R >> >> \
1098 /Contents 4 0 R >>",
1099 );
1100 b.stream(4, "", b"/Fx Do");
1101 b.object(
1102 5,
1103 "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica \
1104 /Encoding /WinAnsiEncoding >>",
1105 );
1106 b.stream(
1107 6,
1108 "/Type /XObject /Subtype 8 0 R /BBox [0 0 612 792]",
1109 b"BT /F1 12 Tf 72 720 Td (via ref) Tj ET",
1110 );
1111 b.object(8, "/Form");
1112 let doc = Document::load(b.build(1)).unwrap();
1113 assert_eq!(page_text(&doc, 0), "via ref");
1114 }
1115
1116 #[test]
1120 fn form_with_partial_resources_still_sees_the_page_xobject() {
1121 let mut b = PdfBuilder::new();
1122 b.object(1, "<< /Type /Catalog /Pages 2 0 R >>");
1123 b.object(2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>");
1124 b.object(
1125 3,
1126 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \
1127 /Resources << /Font << /F1 5 0 R >> \
1128 /XObject << /Outer 6 0 R /Inner 7 0 R >> >> /Contents 4 0 R >>",
1129 );
1130 b.stream(4, "", b"/Outer Do");
1131 b.object(
1132 5,
1133 "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica \
1134 /Encoding /WinAnsiEncoding >>",
1135 );
1136 b.stream(
1138 6,
1139 "/Type /XObject /Subtype /Form /BBox [0 0 612 792] \
1140 /Resources << /ProcSet [/PDF /Text] >>",
1141 b"/Inner Do",
1142 );
1143 b.stream(
1144 7,
1145 "/Type /XObject /Subtype /Form /BBox [0 0 612 792]",
1146 b"BT /F1 12 Tf 72 720 Td (deep) Tj ET",
1147 );
1148 let doc = Document::load(b.build(1)).unwrap();
1149 assert_eq!(page_text(&doc, 0), "deep");
1150 }
1151
1152 struct NullSource {
1164 payload: Vec<u8>,
1165 }
1166
1167 impl AsyncObjectSource for NullSource {
1168 fn get(&self, _r: ObjRef) -> BoxFuture<'_, Result<Object>> {
1169 Box::pin(std::future::ready(Ok(Object::Null)))
1170 }
1171
1172 fn stream_data<'a>(&'a self, _s: &'a Stream) -> BoxFuture<'a, Result<Vec<u8>>> {
1173 Box::pin(std::future::ready(Ok(self.payload.clone())))
1174 }
1175
1176 fn resolve<'a>(&'a self, o: &'a Object) -> BoxFuture<'a, Result<Object>> {
1177 Box::pin(resolve_with(self, o))
1178 }
1179 }
1180
1181 #[test]
1195 fn the_async_entry_point_yields_a_spawnable_future() {
1196 fn assert_send_static<F: Future + Send + 'static>(_: &F) {}
1197
1198 let doc = Document::load(simple_doc("Hello")).unwrap();
1199 let text_page = doc.page(0).unwrap();
1200 drop(doc);
1201
1202 let text = async move {
1203 extract_text_with(
1204 NullSource {
1205 payload: Vec::new(),
1206 },
1207 &text_page,
1208 None,
1209 )
1210 .await
1211 };
1212 assert_send_static(&text);
1213
1214 assert_eq!(block_on(text).unwrap(), "");
1217 }
1218
1219 #[test]
1220 fn committed_fixture_files() {
1221 let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../tests/fixtures");
1222 let hello = std::fs::read(format!("{dir}/hello.pdf")).unwrap();
1223 let doc = Document::load(hello).unwrap();
1224 assert_eq!(page_text(&doc, 0), "Hello, world!");
1225
1226 let three = std::fs::read(format!("{dir}/three-pages.pdf")).unwrap();
1227 let doc = Document::load(three).unwrap();
1228 assert_eq!(doc.page_count(), 3);
1229 assert_eq!(page_text(&doc, 0), "Page one");
1230 assert_eq!(page_text(&doc, 1), "Page two");
1231 assert_eq!(page_text(&doc, 2), "Page three");
1232
1233 let xs = std::fs::read(format!("{dir}/xref-stream.pdf")).unwrap();
1234 let doc = Document::load(xs).unwrap();
1235 assert_eq!(page_text(&doc, 0), "Hello, world!");
1236 }
1237}