1use std::collections::HashMap;
20
21pub use docling_core::PdfRepair;
22
23pub fn pdf_repairs(bytes: &[u8]) -> Vec<PdfRepair> {
32 let mut repairs = Vec::new();
33 let doc = load(bytes, &mut repairs, None).ok();
34 scan_repairs(bytes, &scan_objects(bytes), doc.as_ref(), repairs)
35}
36
37#[derive(Debug, Clone, PartialEq, Eq)]
39#[non_exhaustive]
40pub enum RepairInspectError {
41 LimitExceeded {
45 limit: usize,
47 },
48 ObjectStreamSkipped {
52 object: (u32, u16),
54 },
55 Unreadable,
57}
58
59impl std::fmt::Display for RepairInspectError {
60 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61 match self {
62 Self::LimitExceeded { limit } => {
63 write!(f, "a stream decodes past the {limit}-byte limit")
64 }
65 Self::ObjectStreamSkipped { object: (num, gen) } => {
66 write!(f, "object stream {num} {gen} was not read within the limit")
67 }
68 Self::Unreadable => f.write_str("the PDF cannot be loaded"),
69 }
70 }
71}
72
73impl std::error::Error for RepairInspectError {}
74
75pub fn try_pdf_repairs(
90 bytes: &[u8],
91 max_decompressed_size: Option<usize>,
92) -> Result<Vec<PdfRepair>, RepairInspectError> {
93 let mut repairs = Vec::new();
94 let doc = load(bytes, &mut repairs, max_decompressed_size)?;
95 let objects = scan_objects(bytes);
96 if max_decompressed_size.is_some() {
97 if let Some(o) = objects
104 .iter()
105 .find(|o| o.objstm && !doc.objects.contains_key(&o.id))
106 {
107 return Err(RepairInspectError::ObjectStreamSkipped { object: o.id });
108 }
109 }
110 Ok(scan_repairs(bytes, &objects, Some(&doc), repairs))
111}
112
113fn load(
116 bytes: &[u8],
117 repairs: &mut Vec<PdfRepair>,
118 limit: Option<usize>,
119) -> Result<lopdf::Document, RepairInspectError> {
120 let options = lopdf::LoadOptions {
121 max_decompressed_size: limit,
122 ..lopdf::LoadOptions::default()
123 };
124 let over = |e: &lopdf::Error| {
125 matches!(
126 e,
127 lopdf::Error::Decompress(lopdf::DecompressError::MemoryLimitExceeded { .. })
128 )
129 };
130 let err = match lopdf::Document::load_mem_with_options(bytes, options.clone()) {
131 Ok(doc) => return Ok(doc),
132 Err(err) => err,
133 };
134 if let (Some(limit), true) = (limit, over(&err)) {
135 return Err(RepairInspectError::LimitExceeded { limit });
136 }
137 let fixed = crate::textparse::pad_short_xref_entries(bytes)
138 .map_err(|_| RepairInspectError::Unreadable)?;
139 match lopdf::Document::load_mem_with_options(&fixed, options) {
140 Ok(doc) => {
141 repairs.push(PdfRepair::XrefEntriesPadded);
142 Ok(doc)
143 }
144 Err(e) => Err(match (limit, over(&e)) {
145 (Some(limit), true) => RepairInspectError::LimitExceeded { limit },
146 _ => RepairInspectError::Unreadable,
147 }),
148 }
149}
150
151fn scan_repairs(
154 bytes: &[u8],
155 objects: &[RawObject],
156 doc: Option<&lopdf::Document>,
157 mut repairs: Vec<PdfRepair>,
158) -> Vec<PdfRepair> {
159 let integers: HashMap<(u32, u16), i64> = objects
160 .iter()
161 .filter_map(|o| Some((o.id, o.integer?)))
162 .collect();
163 let resolve = |id: (u32, u16)| -> Option<i64> {
167 if let Some(&n) = integers.get(&id) {
168 return Some(n);
169 }
170 doc?.get_object(id).ok()?.as_i64().ok()
171 };
172 for o in objects {
173 let Some(stream) = &o.stream else { continue };
174 let declared = match stream.length {
175 Some(LengthValue::Direct(n)) => Some(n),
176 Some(LengthValue::Ref(id)) => resolve(id),
177 None => None,
178 }
179 .filter(|&n| n >= 0);
180 let ends_at_endstream = declared
181 .and_then(|n| usize::try_from(n).ok())
182 .and_then(|n| stream.data.checked_add(n))
183 .is_some_and(|end| ends_at_endstream(bytes, end));
184 if !ends_at_endstream {
185 repairs.push(PdfRepair::StreamLength {
186 object: o.id,
187 offset: o.offset,
188 declared,
189 recovered: recover_length(bytes, stream.data),
190 });
191 }
192 }
193 repairs
194}
195
196fn ends_at_endstream(bytes: &[u8], end: usize) -> bool {
198 let Some(rest) = bytes.get(end..) else {
199 return false;
200 };
201 let rest = rest
202 .strip_prefix(b"\r\n")
203 .or_else(|| rest.strip_prefix(b"\n"))
204 .or_else(|| rest.strip_prefix(b"\r"))
205 .unwrap_or(rest);
206 rest.starts_with(b"endstream")
207}
208
209fn recover_length(bytes: &[u8], data: usize) -> Option<usize> {
214 let window = &bytes[data..];
215 let (first_end, after_first) = boundary(window, 0)?;
216 let bound = next_object_header(window, after_first).unwrap_or(window.len());
217 match boundary(&window[..bound], after_first) {
218 Some(_) => None,
219 None => Some(first_end),
220 }
221}
222
223fn boundary(window: &[u8], from: usize) -> Option<(usize, usize)> {
226 const ENDSTREAM: &[u8] = b"endstream";
227 let mut at = from;
228 while let Some(rel) = find(&window[at..], ENDSTREAM) {
229 let pos = at + rel;
230 at = pos + ENDSTREAM.len();
231 let before = &window[..pos];
232 let data_end = if before.ends_with(b"\r\n") {
233 pos - 2
234 } else if before.ends_with(b"\n") || before.ends_with(b"\r") {
235 pos - 1
236 } else {
237 continue;
238 };
239 let after = &window[at..];
240 let ws = after.iter().take_while(|&&c| is_whitespace(c)).count();
241 if !after[ws..].starts_with(b"endobj")
242 || after.get(ws + 6).is_some_and(|&c| !is_whitespace(c))
243 {
244 continue;
245 }
246 return Some((data_end, at));
247 }
248 None
249}
250
251fn next_object_header(window: &[u8], from: usize) -> Option<usize> {
253 let mut at = from;
254 while let Some(rel) = find(&window[at..], b"obj") {
255 let kw = at + rel;
256 at = kw + 3;
257 if header_before(window, kw).is_some() {
258 return Some(kw);
259 }
260 }
261 None
262}
263
264enum LengthValue {
266 Direct(i64),
267 Ref((u32, u16)),
268}
269
270struct StreamInfo {
271 length: Option<LengthValue>,
273 data: usize,
275}
276
277struct RawObject {
278 id: (u32, u16),
279 offset: usize,
280 integer: Option<i64>,
282 stream: Option<StreamInfo>,
283 objstm: bool,
285}
286
287fn scan_objects(bytes: &[u8]) -> Vec<RawObject> {
291 let mut objects = Vec::new();
292 let mut at = 0;
293 while let Some(rel) = find(&bytes[at..], b"obj") {
294 let kw = at + rel;
295 at = kw + 3;
296 if bytes
298 .get(kw + 3)
299 .is_some_and(|&c| !is_whitespace(c) && !is_delimiter(c))
300 {
301 continue;
302 }
303 let Some((id, offset)) = header_before(bytes, kw) else {
304 continue;
305 };
306 let body = skip_space(bytes, kw + 3);
307 let mut object = RawObject {
308 id,
309 offset,
310 integer: None,
311 stream: None,
312 objstm: false,
313 };
314 if bytes[body..].starts_with(b"<<") {
315 let Some((dict_end, length, objstm)) = dictionary(bytes, body) else {
316 objects.push(object);
317 continue;
318 };
319 let after = skip_space(bytes, dict_end);
320 if bytes[after..].starts_with(b"stream") {
321 let mut data = after + 6;
322 while matches!(bytes.get(data), Some(b' ' | b'\t')) {
323 data += 1;
324 }
325 if bytes[data..].starts_with(b"\r\n") {
326 data += 2;
327 } else if matches!(bytes.get(data), Some(b'\n' | b'\r')) {
328 data += 1;
329 } else {
330 objects.push(object);
332 at = after + 6;
333 continue;
334 }
335 at = match &length {
339 Some(LengthValue::Direct(n))
340 if usize::try_from(*n)
341 .ok()
342 .and_then(|n| data.checked_add(n))
343 .is_some_and(|end| ends_at_endstream(bytes, end)) =>
344 {
345 data + *n as usize
346 }
347 _ => find(&bytes[data..], b"endstream").map_or(bytes.len(), |r| data + r),
348 };
349 object.stream = Some(StreamInfo { length, data });
350 object.objstm = objstm;
351 } else {
352 at = dict_end;
353 }
354 } else {
355 let (value, end) = integer(bytes, body);
356 if let Some(v) = value {
357 if bytes[skip_space(bytes, end)..].starts_with(b"endobj") {
358 object.integer = Some(v);
359 }
360 }
361 }
362 objects.push(object);
363 }
364 objects
365}
366
367fn header_before(bytes: &[u8], kw: usize) -> Option<((u32, u16), usize)> {
370 let mut i = kw;
371 let ws_end = i;
372 while i > 0 && is_whitespace(bytes[i - 1]) {
373 i -= 1;
374 }
375 if i == ws_end {
376 return None;
377 }
378 let gen_end = i;
379 while i > 0 && bytes[i - 1].is_ascii_digit() {
380 i -= 1;
381 }
382 let gen = std::str::from_utf8(&bytes[i..gen_end])
383 .ok()?
384 .parse::<u16>()
385 .ok()?;
386 let sp = i;
387 while i > 0 && is_whitespace(bytes[i - 1]) {
388 i -= 1;
389 }
390 if i == sp {
391 return None;
392 }
393 let num_end = i;
394 while i > 0 && bytes[i - 1].is_ascii_digit() {
395 i -= 1;
396 }
397 let num = std::str::from_utf8(&bytes[i..num_end])
398 .ok()?
399 .parse::<u32>()
400 .ok()?;
401 if i > 0 && !is_whitespace(bytes[i - 1]) && !is_delimiter(bytes[i - 1]) {
402 return None;
403 }
404 Some(((num, gen), i))
405}
406
407fn dictionary(bytes: &[u8], start: usize) -> Option<(usize, Option<LengthValue>, bool)> {
412 let mut depth = 0usize;
413 let mut i = start;
414 let mut length = None;
415 let mut objstm = false;
416 while i < bytes.len() {
417 match bytes[i] {
418 b'<' if bytes.get(i + 1) == Some(&b'<') => {
419 depth += 1;
420 i += 2;
421 }
422 b'>' if bytes.get(i + 1) == Some(&b'>') => {
423 depth -= 1;
424 i += 2;
425 if depth == 0 {
426 return Some((i, length, objstm));
427 }
428 }
429 b'<' => {
430 i = bytes[i..]
431 .iter()
432 .position(|&c| c == b'>')
433 .map(|p| i + p + 1)?
434 }
435 b'(' => i = literal_string_end(bytes, i)?,
436 b'%' => {
437 while i < bytes.len() && !matches!(bytes[i], b'\r' | b'\n') {
438 i += 1;
439 }
440 }
441 b'/' => {
442 let name_end = i
443 + 1
444 + bytes[i + 1..]
445 .iter()
446 .take_while(|&&c| !is_whitespace(c) && !is_delimiter(c))
447 .count();
448 if depth == 1 && &bytes[i + 1..name_end] == b"Length" {
449 length = length_value(bytes, name_end);
450 }
451 if depth == 1 && &bytes[i + 1..name_end] == b"Type" {
452 let at = skip_space(bytes, name_end);
453 objstm = bytes[at..].starts_with(b"/ObjStm");
454 }
455 i = name_end;
456 }
457 _ => i += 1,
458 }
459 }
460 None
461}
462
463fn length_value(bytes: &[u8], at: usize) -> Option<LengthValue> {
465 let (first, end) = integer(bytes, skip_space(bytes, at));
466 let first = first?;
467 let (gen, gen_end) = integer(bytes, skip_space(bytes, end));
468 if let Some(gen) = gen {
469 let r = skip_space(bytes, gen_end);
470 if bytes.get(r) == Some(&b'R')
471 && bytes
472 .get(r + 1)
473 .is_none_or(|&c| is_whitespace(c) || is_delimiter(c))
474 {
475 return Some(LengthValue::Ref((
476 u32::try_from(first).ok()?,
477 u16::try_from(gen).ok()?,
478 )));
479 }
480 }
481 Some(LengthValue::Direct(first))
482}
483
484fn integer(bytes: &[u8], at: usize) -> (Option<i64>, usize) {
487 let mut i = at;
488 if matches!(bytes.get(i), Some(b'+' | b'-')) {
489 i += 1;
490 }
491 let digits = bytes[i.min(bytes.len())..]
492 .iter()
493 .take_while(|c| c.is_ascii_digit())
494 .count();
495 let end = i + digits;
496 if digits == 0 || bytes.get(end) == Some(&b'.') {
497 return (None, at);
498 }
499 let value = std::str::from_utf8(&bytes[at..end])
500 .ok()
501 .and_then(|s| s.parse().ok());
502 (value, end)
503}
504
505fn literal_string_end(bytes: &[u8], start: usize) -> Option<usize> {
508 let mut depth = 0usize;
509 let mut i = start;
510 while i < bytes.len() {
511 match bytes[i] {
512 b'\\' => i += 1,
513 b'(' => depth += 1,
514 b')' => {
515 depth -= 1;
516 if depth == 0 {
517 return Some(i + 1);
518 }
519 }
520 _ => {}
521 }
522 i += 1;
523 }
524 None
525}
526
527fn skip_space(bytes: &[u8], mut i: usize) -> usize {
529 loop {
530 while i < bytes.len() && is_whitespace(bytes[i]) {
531 i += 1;
532 }
533 if bytes.get(i) == Some(&b'%') {
534 while i < bytes.len() && !matches!(bytes[i], b'\r' | b'\n') {
535 i += 1;
536 }
537 } else {
538 return i;
539 }
540 }
541}
542
543fn is_whitespace(c: u8) -> bool {
544 b" \t\n\r\0\x0C".contains(&c)
545}
546
547fn is_delimiter(c: u8) -> bool {
548 b"()<>[]{}/%".contains(&c)
549}
550
551fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
552 haystack.windows(needle.len()).position(|w| w == needle)
553}
554
555#[cfg(test)]
556mod tests {
557 use super::*;
558
559 fn pdf(length: &str, extra: &[&str]) -> Vec<u8> {
563 let mut objects = vec![
564 "<< /Type /Catalog /Pages 2 0 R >>".to_owned(),
565 "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_owned(),
566 "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 100 100] /Contents 4 0 R >>".to_owned(),
567 format!("<< /Length {length} >>\nstream\nq Q\nendstream"),
568 ];
569 objects.extend(extra.iter().map(|s| s.to_string()));
570 let mut bytes = b"%PDF-1.4\n".to_vec();
571 let mut offsets = vec![0];
572 for (i, object) in objects.iter().enumerate() {
573 offsets.push(bytes.len());
574 bytes.extend_from_slice(format!("{} 0 obj\n{object}\nendobj\n", i + 1).as_bytes());
575 }
576 let xref = bytes.len();
577 let n = objects.len() + 1;
578 bytes.extend_from_slice(format!("xref\n0 {n}\n0000000000 65535 f \n").as_bytes());
579 for offset in &offsets[1..] {
580 bytes.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
581 }
582 bytes.extend_from_slice(
583 format!("trailer\n<< /Size {n} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes(),
584 );
585 bytes
586 }
587
588 #[test]
589 fn a_well_formed_file_reports_nothing() {
590 assert_eq!(pdf_repairs(&pdf("4", &[])), []);
593 assert_eq!(pdf_repairs(&pdf("3", &[])), []);
594 }
595
596 #[test]
597 fn an_overstated_length_is_reported_with_both_lengths() {
598 let bytes = pdf("5", &[]);
599 let offset = find(&bytes, b"4 0 obj").unwrap();
600 assert_eq!(
601 pdf_repairs(&bytes),
602 [PdfRepair::StreamLength {
603 object: (4, 0),
604 offset,
605 declared: Some(5),
606 recovered: Some(3),
607 }]
608 );
609 let doc = lopdf::Document::load_mem(&bytes).unwrap();
612 let stream = doc.get_object((4, 0)).unwrap().as_stream().unwrap();
613 assert_eq!(stream.content, b"q Q");
614 assert_eq!(stream.dict.get(b"Length").unwrap().as_i64().unwrap(), 3);
615 }
616
617 #[test]
618 fn understated_missing_and_indirect_lengths() {
619 let repair = |length: &str, extra: &[&str]| match &pdf_repairs(&pdf(length, extra))[..] {
620 [PdfRepair::StreamLength {
621 declared,
622 recovered,
623 ..
624 }] => Some((*declared, *recovered)),
625 [] => None,
626 other => panic!("{other:?}"),
627 };
628 assert_eq!(repair("2", &[]), Some((Some(2), Some(3))));
629 assert_eq!(repair("-1", &[]), Some((None, Some(3))));
630 assert_eq!(repair("/Foo", &[]), Some((None, Some(3))));
631 assert_eq!(repair("5 0 R", &["4"]), None);
633 assert_eq!(repair("5 0 R", &["9"]), Some((Some(9), Some(3))));
634 assert_eq!(repair("7 0 R", &["4"]), Some((None, Some(3))));
636 }
637
638 #[test]
639 fn short_xref_entries_lopdf_reads_are_no_repair() {
640 let bytes = pdf("5", &[]);
644 let text = String::from_utf8(bytes).unwrap();
645 let short = text.replace(" n \n", " n\n").replace(" f \n", " f\n");
646 assert!(lopdf::Document::load_mem(short.as_bytes()).is_ok());
647 assert!(matches!(
648 &pdf_repairs(short.as_bytes())[..],
649 [PdfRepair::StreamLength {
650 object: (4, 0),
651 declared: Some(5),
652 recovered: Some(3),
653 ..
654 }]
655 ));
656 }
657
658 #[test]
659 fn binary_data_and_other_length_keys_are_not_syntax() {
660 let font = "<< /Length1 99 /Length 12 >>\nstream\n1 0 obj >> (\nendstream";
663 assert_eq!(pdf_repairs(&pdf("4", &[font])), []);
664 }
665
666 #[test]
667 fn a_stream_without_an_unambiguous_end_recovers_nothing() {
668 let bytes = pdf("5", &[]);
671 let at = find(&bytes, b"q Q\nendstream").unwrap() + 4;
672 let mut twice = bytes[..at].to_vec();
673 twice.extend_from_slice(b"endstream\nendobj\nQ\n");
674 twice.extend_from_slice(&bytes[at..]);
675 assert!(matches!(
676 &pdf_repairs(&twice)[..],
677 [PdfRepair::StreamLength {
678 recovered: None,
679 ..
680 }]
681 ));
682 }
683
684 fn pdf_with_object_stream(declared: &str) -> Vec<u8> {
688 let mut data = b"10 0 7 ".to_vec();
689 data.resize(4096, b' ');
690 let mut stream = lopdf::Stream::new(lopdf::Dictionary::new(), data);
691 stream.compress().unwrap();
692 let mut objstm = format!(
693 "<< /Type /ObjStm /N 1 /First 5 /Filter /FlateDecode /Length {} >>\nstream\n",
694 stream.content.len()
695 )
696 .into_bytes();
697 objstm.extend_from_slice(&stream.content);
698 objstm.extend_from_slice(b"\nendstream");
699 let bodies: Vec<Vec<u8>> = vec![
700 b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
701 b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(),
702 b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 20 20] /Contents 4 0 R >>".to_vec(),
703 format!("<< /Length {declared} >>\nstream\nq Q\nendstream").into_bytes(),
704 b"4".to_vec(),
705 objstm,
706 ];
707 let mut bytes = b"%PDF-1.5\n".to_vec();
708 let mut offsets = Vec::new();
709 for (i, body) in bodies.iter().enumerate() {
710 offsets.push(bytes.len());
711 bytes.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes());
712 bytes.extend_from_slice(body);
713 bytes.extend_from_slice(b"\nendobj\n");
714 }
715 let xref = bytes.len();
716 let n = bodies.len() + 1;
717 bytes.extend_from_slice(format!("xref\n0 {n}\n0000000000 65535 f \n").as_bytes());
718 for offset in &offsets {
719 bytes.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
720 }
721 bytes.extend_from_slice(
722 format!("trailer\n<< /Root 1 0 R /Size {n} >>\nstartxref\n{xref}\n%%EOF\n").as_bytes(),
723 );
724 bytes
725 }
726
727 #[test]
728 fn a_bounded_inspection_still_reports_the_repair() {
729 let bytes = pdf("5", &[]);
732 assert_eq!(
733 try_pdf_repairs(&bytes, Some(256)).unwrap(),
734 pdf_repairs(&bytes)
735 );
736 assert!(!try_pdf_repairs(&bytes, Some(256)).unwrap().is_empty());
737 assert_eq!(try_pdf_repairs(&pdf("4", &[]), Some(256)).unwrap(), []);
738 }
739
740 #[test]
741 fn an_oversized_object_stream_is_an_error_not_a_silent_gap() {
742 let bytes = pdf_with_object_stream("5");
743 for limit in [None, Some(1 << 20)] {
745 assert!(matches!(
746 &try_pdf_repairs(&bytes, limit).unwrap()[..],
747 [PdfRepair::StreamLength { object: (4, 0), .. }]
748 ));
749 }
750 assert_eq!(
753 try_pdf_repairs(&bytes, Some(256)),
754 Err(RepairInspectError::ObjectStreamSkipped { object: (6, 0) })
755 );
756 assert_eq!(pdf_repairs(&bytes).len(), 1);
758 }
759
760 #[test]
761 fn an_unreadable_file_is_an_error() {
762 assert_eq!(
763 try_pdf_repairs(b"not a pdf", None),
764 Err(RepairInspectError::Unreadable)
765 );
766 assert_eq!(pdf_repairs(b"not a pdf"), []);
767 }
768}