use crate::diagnostics::Diagnostic;
use crate::object_handle::{DocumentResolver, ObjectValue};
use crate::parser::{
parse_qpdf_direct_object_handle_with_diagnostics,
parse_qpdf_file_object_handle_with_diagnostics, HandleResolver, ParserDiagnostic,
};
use crate::reader::file_object::{
finish_file_object_handle, parse_file_object_handle_syntax, parse_file_object_header,
FileObjectDiagnostic, FileObjectDiagnosticKind, HandleFileObjectRead, RecoveryPolicy,
ResolvedStreamLength,
};
use crate::tokenizer::{Token, TokenType, Tokenizer};
use crate::{filters, Diagnostics, Error, ObjectHandle, ObjectRef, Result, XrefEntry};
use std::cell::{OnceCell, RefCell};
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::io::{Read, Seek, SeekFrom};
use std::rc::{Rc, Weak};
const XREF_STACK_RED_ZONE: usize = 128 * 1024;
const XREF_STACK_GROWTH_SIZE: usize = 1024 * 1024;
#[derive(Debug, Clone)]
pub struct LoadedXref {
pub version: String,
pub startxref: u64,
pub entries: BTreeMap<ObjectRef, XrefEntry>,
pub trailer: ObjectHandle,
pub last_xref_form: XrefForm,
pub repair_diagnostics: Diagnostics,
}
#[derive(Debug, Default)]
struct BootstrapHandleState {
handles: BTreeMap<ObjectRef, ObjectHandle>,
resolving: BTreeSet<ObjectRef>,
resolved_object_streams: BTreeSet<u32>,
diagnostics: Diagnostics,
reconstruction_trigger: Option<(u64, String)>,
}
#[derive(Debug, Default)]
pub(crate) struct BootstrapCache {
handle_state: Rc<RefCell<BootstrapHandleState>>,
handle_document: Option<Rc<BootstrapHandleDocument>>,
handle_document_owners: Vec<Rc<BootstrapHandleDocument>>,
}
impl Drop for BootstrapCache {
fn drop(&mut self) {
let mut states: Vec<(Rc<RefCell<BootstrapHandleState>>, usize)> = Vec::new();
let mut add_state = |state: Rc<RefCell<BootstrapHandleState>>| {
if let Some((_, references)) = states
.iter_mut()
.find(|(candidate, _)| Rc::ptr_eq(candidate, &state))
{
*references += 1;
} else {
states.push((state, 1));
}
};
add_state(Rc::clone(&self.handle_state));
if let Some(document) = &self.handle_document {
add_state(Rc::clone(&document.state));
}
for document in &self.handle_document_owners {
add_state(Rc::clone(&document.state));
}
for (state, internal_references) in states {
if Rc::strong_count(&state) != internal_references + 1 {
continue;
}
let handles: Vec<_> = state.borrow().handles.values().cloned().collect();
for handle in handles {
handle.disconnect();
}
}
}
}
type SharedBootstrapCache = Rc<RefCell<BootstrapCache>>;
fn empty_bootstrap_cache() -> SharedBootstrapCache {
Rc::new(RefCell::new(BootstrapCache::default()))
}
#[derive(Debug, Clone)]
pub(crate) struct LoadedXrefState {
pub(crate) loaded: LoadedXref,
pub(crate) first_xref_item_offset: u64,
pub(crate) classic_trailer_offset: Option<usize>,
pub(crate) pending_reconstruction_trigger: Option<(u64, String)>,
pub(crate) trailer_references: BTreeSet<ObjectRef>,
pub(crate) parsed_xref_streams: BTreeMap<ObjectRef, ObjectHandle>,
pub(crate) bootstrap_cache: SharedBootstrapCache,
pub(crate) header_offset: usize,
pub(crate) already_reconstructed: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum XrefForm {
Table,
Stream,
}
#[derive(Debug, Clone, Copy)]
enum ParsedXrefEntry {
Live {
object_ref: ObjectRef,
entry: XrefEntry,
},
Free {
object_ref: ObjectRef,
},
}
#[derive(Debug, Default)]
struct XrefRegistration {
entries: BTreeMap<ObjectRef, XrefEntry>,
deleted_objects: BTreeSet<u32>,
}
impl XrefRegistration {
fn insert_xref_entry(&mut self, object_ref: ObjectRef, entry: XrefEntry) {
if self.deleted_objects.contains(&object_ref.number) {
return;
}
self.entries.entry(object_ref).or_insert(entry);
}
fn insert_free_xref_entry(&mut self, object_ref: ObjectRef) {
if !self.entries.contains_key(&object_ref) {
self.deleted_objects.insert(object_ref.number);
}
}
fn snapshot(&self) -> BTreeMap<ObjectRef, XrefEntry> {
self.entries.clone()
}
}
fn discard_lower_generations(
entries: &mut BTreeMap<ObjectRef, XrefEntry>,
parsed_xref_streams: &mut BTreeMap<ObjectRef, ObjectHandle>,
) {
let mut previous: Option<ObjectRef> = None;
let mut lower_generations = Vec::new();
for &object_ref in entries.keys() {
if let Some(previous_ref) = previous {
if previous_ref.number == object_ref.number {
lower_generations.push(previous_ref);
}
}
previous = Some(object_ref);
}
for object_ref in lower_generations {
entries.remove(&object_ref);
parsed_xref_streams.remove(&object_ref);
}
}
#[derive(Debug, Clone, Copy, Default)]
pub(crate) struct XrefLoadOptions {
pub(crate) allow_repair: bool,
pub(crate) ignore_xref_streams: bool,
}
#[derive(Debug, Clone, Copy)]
enum XrefReadContextSpec<'a> {
ActiveSection,
ActiveSectionWithCache {
bootstrap_cache: &'a SharedBootstrapCache,
},
Reconstruction {
line_scan_entries: &'a BTreeMap<ObjectRef, XrefEntry>,
reference_offsets: &'a Rc<[u64]>,
},
ReconstructionWithCache {
line_scan_entries: &'a BTreeMap<ObjectRef, XrefEntry>,
reference_offsets: &'a Rc<[u64]>,
bootstrap_cache: &'a SharedBootstrapCache,
},
}
#[derive(Debug, Clone, Copy)]
enum XrefObjectDescription {
Ordinary,
XrefStream,
}
impl XrefObjectDescription {
fn warning_prefix(self) -> &'static str {
match self {
Self::Ordinary => "",
Self::XrefStream => "xref stream: ",
}
}
}
#[derive(Debug, Clone, Copy)]
enum XrefEntryLookup<'a> {
Registration(&'a BTreeMap<ObjectRef, XrefEntry>),
Reconstruction {
line_scan_entries: &'a BTreeMap<ObjectRef, XrefEntry>,
registration_entries: &'a BTreeMap<ObjectRef, XrefEntry>,
},
}
impl XrefEntryLookup<'_> {
fn get(&self, object_ref: &ObjectRef) -> Option<XrefEntry> {
let entry = match self {
Self::Registration(entries) => entries.get(object_ref),
Self::Reconstruction {
line_scan_entries,
registration_entries,
} => line_scan_entries
.get(object_ref)
.or_else(|| registration_entries.get(object_ref)),
};
entry
.filter(|entry| !matches!(entry, XrefEntry::Free { .. }))
.copied()
}
fn owned_entries(&self) -> BTreeMap<ObjectRef, XrefEntry> {
let mut entries = BTreeMap::new();
match self {
Self::Registration(source) => {
entries.extend(source.iter().map(|(key, value)| (*key, *value)))
}
Self::Reconstruction {
line_scan_entries,
registration_entries,
} => {
entries.extend(
registration_entries
.iter()
.map(|(key, value)| (*key, *value)),
);
entries.extend(line_scan_entries.iter().map(|(key, value)| (*key, *value)));
}
}
entries.retain(|object_ref, entry| {
!matches!(entry, XrefEntry::Free { .. }) && self.get(object_ref).is_some()
});
entries
}
}
fn reconstructed_reference_offsets(entries: &BTreeMap<ObjectRef, XrefEntry>) -> Rc<[u64]> {
let mut offsets: Vec<u64> = entries
.values()
.filter_map(|entry| match entry {
XrefEntry::Uncompressed { offset } => Some(*offset),
XrefEntry::Compressed { .. } | XrefEntry::Free { .. } => None,
})
.collect();
offsets.sort_unstable();
offsets.dedup();
Rc::from(offsets)
}
struct BootstrapHandleDocument {
bytes: OnceCell<Rc<[u8]>>,
entry_lookup: RefCell<BTreeMap<ObjectRef, XrefEntry>>,
reference_offsets: RefCell<Option<Rc<[u64]>>>,
options: XrefLoadOptions,
state: Rc<RefCell<BootstrapHandleState>>,
resolver: RefCell<Option<Weak<dyn DocumentResolver>>>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ReferenceReadWindow {
end: usize,
fallback_end: usize,
}
struct UncompressedObjectRead {
value: ObjectValue,
parsed_offset: i64,
used_heuristic_recovery: bool,
diagnostics: Vec<Diagnostic>,
}
impl std::fmt::Debug for BootstrapHandleDocument {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BootstrapHandleDocument")
.field("cache_len", &self.state.borrow().handles.len())
.field("resolving", &self.state.borrow().resolving)
.finish()
}
}
impl BootstrapHandleDocument {
fn new_with_state(
bytes: Option<&[u8]>,
entry_lookup: XrefEntryLookup<'_>,
options: XrefLoadOptions,
state: Rc<RefCell<BootstrapHandleState>>,
) -> Rc<Self> {
let source_bytes = OnceCell::new();
if let Some(bytes) = bytes {
let _ = source_bytes.set(Rc::from(bytes));
}
let document = Rc::new(Self {
bytes: source_bytes,
entry_lookup: RefCell::new(entry_lookup.owned_entries()),
reference_offsets: RefCell::new(None),
options,
state,
resolver: RefCell::new(None),
});
let resolver: Rc<dyn DocumentResolver> = document.clone();
*document.resolver.borrow_mut() = Some(Rc::downgrade(&resolver));
document
}
fn ensure_source_bytes(&self, bytes: &[u8]) {
let _ = self.bytes.get_or_init(|| Rc::from(bytes));
}
fn refresh_entry_lookup(&self, entry_lookup: XrefEntryLookup<'_>) {
*self.entry_lookup.borrow_mut() = entry_lookup.owned_entries();
}
fn set_reference_offsets(&self, reference_offsets: Option<Rc<[u64]>>) {
*self.reference_offsets.borrow_mut() = reference_offsets;
}
fn reference_read_window(&self, offset: u64, source_len: usize) -> ReferenceReadWindow {
let Some(offsets) = self.reference_offsets.borrow().as_ref().cloned() else {
return ReferenceReadWindow {
end: source_len,
fallback_end: source_len,
};
};
let next_offset_index = offsets.partition_point(|candidate| *candidate <= offset);
let end = offsets
.get(next_offset_index)
.and_then(|offset| usize::try_from(*offset).ok())
.unwrap_or(source_len)
.min(source_len);
let fallback_index = next_offset_index
.saturating_add(XREF_RECONSTRUCTION_FALLBACK_SPAN)
.min(offsets.len());
let fallback_end = offsets
.get(fallback_index)
.and_then(|offset| usize::try_from(*offset).ok())
.unwrap_or(source_len)
.min(source_len);
ReferenceReadWindow { end, fallback_end }
}
fn resolver_weak(&self) -> Weak<dyn DocumentResolver> {
self.resolver
.borrow()
.clone()
.expect("bootstrap document resolver is installed before parsing")
}
fn handle_for_reference(&self, object_ref: ObjectRef) -> ObjectHandle {
if let Some(handle) = self.state.borrow().handles.get(&object_ref).cloned() {
return handle;
}
let handle = ObjectHandle::new_indirect_with_resolver(object_ref, self.resolver_weak());
self.state
.borrow_mut()
.handles
.insert(object_ref, handle.clone());
handle
}
fn push_warning(&self, message: impl Into<String>, offset: Option<u64>) {
self.state
.borrow_mut()
.diagnostics
.push(Diagnostic::warning(message, offset));
}
fn push_diagnostic(&self, diagnostic: Diagnostic) {
self.state.borrow_mut().diagnostics.push(diagnostic);
}
fn object_policy(&self) -> RecoveryPolicy {
if self.options.allow_repair {
RecoveryPolicy::Bounded
} else {
RecoveryPolicy::RequireEndstream
}
}
fn read_file_object(
&self,
input: &[u8],
absolute_offset: u64,
policy: RecoveryPolicy,
description: XrefObjectDescription,
source_bytes: &[u8],
) -> Result<HandleFileObjectRead> {
let mut parser = BootstrapHandleParser {
document: self,
description,
};
let pending = parse_file_object_handle_syntax(input, &mut parser)?;
let resolved_length = pending.indirect_length_ref().map(|object_ref| {
self.ensure_source_bytes(source_bytes);
stacker::maybe_grow(XREF_STACK_RED_ZONE, XREF_STACK_GROWTH_SIZE, || {
self.resolve_length(object_ref)
})
});
let _ = (absolute_offset, description);
finish_file_object_handle(input, pending, resolved_length, policy)
}
fn resolve_length(&self, object_ref: ObjectRef) -> ResolvedStreamLength {
let handle = self.handle_for_reference(object_ref);
match handle.try_as_integer() {
Ok(Some(value)) => ResolvedStreamLength::Integer(value),
Ok(None) => {
if handle.try_is_null().unwrap_or(true) {
ResolvedStreamLength::Missing
} else {
ResolvedStreamLength::Invalid
}
}
Err(_) => ResolvedStreamLength::Missing,
}
}
fn read_uncompressed_object(
&self,
object_ref: ObjectRef,
offset: u64,
) -> Result<(ObjectValue, i64)> {
let start = usize::try_from(offset)
.ok()
.ok_or_else(|| Error::parse(0, "object offset does not fit usize"))?;
let source_bytes = Rc::clone(self.bytes.get().ok_or_else(|| {
Error::Internal("bootstrap resolver source bytes were not initialized".to_owned())
})?);
let window = self.reference_read_window(offset, source_bytes.len());
let narrow = self.read_uncompressed_object_with_end(
object_ref,
offset,
start,
window.end,
&source_bytes,
);
let should_retry = window.fallback_end > window.end
&& match &narrow {
Err(Error::Parse { .. }) => true,
Ok(read) => read.used_heuristic_recovery,
Err(_) => false, };
let accepted = if should_retry {
self.read_uncompressed_object_with_end(
object_ref,
offset,
start,
window.fallback_end,
&source_bytes,
)
} else {
narrow
}?;
for diagnostic in accepted.diagnostics {
self.push_diagnostic(diagnostic);
}
Ok((accepted.value, accepted.parsed_offset))
}
fn read_uncompressed_object_with_end(
&self,
object_ref: ObjectRef,
offset: u64,
start: usize,
end: usize,
source_bytes: &[u8],
) -> Result<UncompressedObjectRead> {
let input = source_bytes
.get(start..end)
.ok_or_else(|| Error::parse(start, "object is beyond the end of the file"))?;
let policy = self.object_policy();
let actual_object_ref =
parse_file_object_header(input).map_err(|error| error.rebase_offset(start))?;
if actual_object_ref.number == 0 {
return Err(Error::parse(start, "object with ID 0"));
}
if actual_object_ref != object_ref {
let message = format!(
"expected {} {} obj",
object_ref.number, object_ref.generation
);
if self.options.allow_repair || policy == RecoveryPolicy::Bounded {
let mut state = self.state.borrow_mut();
let trigger = &mut state.reconstruction_trigger;
if trigger.is_none() {
*trigger = Some((offset, message.clone()));
}
return Err(Error::parse(start, message));
}
self.push_warning(
format!(
"(object {} {}, offset {}): {}",
object_ref.number, object_ref.generation, offset, message
),
Some(offset),
);
}
let mut completed = self
.read_file_object(
input,
offset,
policy,
XrefObjectDescription::Ordinary,
source_bytes,
)
.map_err(|error| error.rebase_offset(start))?;
let used_heuristic_recovery = completed.diagnostics.iter().any(|diagnostic| {
matches!(
diagnostic.kind,
FileObjectDiagnosticKind::AttemptingStreamLengthRecovery
)
});
let diagnostics = completed
.diagnostics
.iter()
.map(|diagnostic| {
xref_file_object_diagnostic(
XrefObjectDescription::Ordinary,
completed.object_ref,
offset,
diagnostic.clone(),
)
})
.collect();
let parsed_offset = completed.object.get_parsed_offset();
let _ = completed.remove_included_recovery_eol_for_decryption();
let value = completed.object.into_direct_value().ok_or_else(|| {
Error::Internal(format!(
"bootstrap object {} {} did not produce a direct value",
object_ref.number, object_ref.generation
))
})?;
Ok(UncompressedObjectRead {
value: value.0,
parsed_offset,
used_heuristic_recovery,
diagnostics,
})
}
fn resolve_objects_in_stream(&self, stream_number: u32) -> Result<()> {
if !self
.state
.borrow_mut()
.resolved_object_streams
.insert(stream_number)
{
return Ok(());
}
let stream_handle = self.handle_for_reference(ObjectRef::new(stream_number, 0));
stream_handle.try_dereference()?;
let stream_dict = stream_handle.as_stream_dict().ok_or_else(|| {
Error::parse(
0,
format!("supposed object stream {stream_number} is not a stream"),
)
})?;
if stream_dict.try_get_key(b"/Type")?.try_as_name()?.as_deref() != Some(b"ObjStm") {
self.push_warning(
format!("supposed object stream {stream_number} has wrong type"),
None,
);
}
let object_count = self.handle_integer(&stream_dict, b"/N", "object stream /N")?;
let first = self.handle_integer(&stream_dict, b"/First", "object stream /First")?;
let stream_data = stream_handle
.as_stream_data()
.ok_or_else(|| Error::parse(0, "supposed object stream has no data"))?;
let decoded = filters::decode_stream_data_from_handle(
&stream_dict,
&stream_data,
filters::DecodeLimits::default(),
)?;
let mut tokenizer = Tokenizer::new(&decoded);
let mut members = BTreeMap::new();
for _ in 0..object_count {
let object_number = u32::try_from(tokenizer.next_integer()?)
.map_err(|_| Error::parse(0, "object stream object number is invalid"))?;
let object_offset = usize::try_from(tokenizer.next_integer()?)
.map_err(|_| Error::parse(0, "object stream object offset is invalid"))?;
members.insert(object_number, object_offset);
}
for (object_number, object_offset) in members {
let object_ref = ObjectRef::new(object_number, 0);
if !matches!(
self.entry_lookup.borrow().get(&object_ref).copied(),
Some(XrefEntry::Compressed { stream, .. }) if stream == stream_number
) {
continue;
}
let member_start = first
.checked_add(object_offset)
.ok_or_else(|| Error::parse(0, "object stream member offset overflow"))?;
if member_start > decoded.len() {
return Err(Error::parse(
member_start,
"object stream member offset is out of range",
));
}
let mut parser = BootstrapHandleParser {
document: self,
description: XrefObjectDescription::Ordinary,
};
let parsed = match parse_qpdf_file_object_handle_with_diagnostics(
&decoded[member_start..],
i64::try_from(member_start).unwrap_or(i64::MAX),
Some(i64::try_from(member_start).unwrap_or(i64::MAX)),
&mut parser,
) {
Ok(parsed) => parsed,
Err(error) => {
return Err(match error.rebase_offset(member_start) {
Error::Parse { offset, message } => Error::parse(
offset,
format!(
"object stream {stream_number} (object {} 0, offset {offset}): {message}",
object_ref.number
),
),
other => other, });
}
};
for diagnostic in &parsed.diagnostics {
let offset = member_start.saturating_add(diagnostic.relative_offset);
self.push_warning(
format!(
"object stream {stream_number} (object {} 0, offset {offset}): {}",
object_ref.number, diagnostic.message
),
Some(offset as u64),
);
}
if let Some(empty_offset) = parsed.empty_offset {
let offset = member_start.saturating_add(empty_offset);
self.push_warning(
format!(
"object stream {stream_number} (object {} 0, offset {offset}): empty object treated as null",
object_ref.number
),
Some(offset as u64),
);
}
let member_handle = self.handle_for_reference(object_ref);
let parsed_offset = parsed.parsed_offset;
let value = parsed.value.into_direct_value().ok_or_else(|| {
Error::Internal(format!(
"object stream member {} {} did not produce a direct value",
object_ref.number, object_ref.generation
))
})?;
member_handle.set_resolved(value.0);
member_handle.set_parsed_offset_if_unset(parsed_offset);
}
Ok(())
}
fn handle_integer(&self, dictionary: &ObjectHandle, key: &[u8], label: &str) -> Result<usize> {
let value = dictionary.try_get_key(key)?.try_as_integer()?;
let value = value.ok_or_else(|| Error::parse(0, format!("{label} is not an integer")))?;
usize::try_from(value).map_err(|_| Error::parse(0, format!("{label} is invalid")))
}
}
struct BootstrapHandleParser<'document> {
document: &'document BootstrapHandleDocument,
description: XrefObjectDescription,
}
impl HandleResolver for BootstrapHandleParser<'_> {
fn indirect_handle(&mut self, object_ref: ObjectRef) -> ObjectHandle {
self.document.handle_for_reference(object_ref)
}
fn direct_handle(&mut self, value: ObjectValue) -> ObjectHandle {
ObjectHandle::from_parsed_value_with_resolver(value, self.document.resolver_weak())
}
fn description_template(&self) -> Option<Vec<u8>> {
Some(match self.description {
XrefObjectDescription::Ordinary => b"object $OG".to_vec(),
XrefObjectDescription::XrefStream => b"xref stream: object $OG".to_vec(),
})
}
}
impl DocumentResolver for BootstrapHandleDocument {
fn resolve_indirect(&self, object_ref: ObjectRef, handle: &ObjectHandle) -> Result<()> {
stacker::maybe_grow(XREF_STACK_RED_ZONE, XREF_STACK_GROWTH_SIZE, || {
self.resolve_indirect_inner(object_ref, handle)
})
}
}
impl BootstrapHandleDocument {
#[inline(never)]
fn resolve_indirect_inner(&self, object_ref: ObjectRef, handle: &ObjectHandle) -> Result<()> {
if handle.is_resolved() {
return Ok(());
}
if !self.state.borrow_mut().resolving.insert(object_ref) {
self.push_warning(
format!(
"loop detected resolving object {} {}",
object_ref.number, object_ref.generation
),
None,
);
handle.set_resolved(ObjectValue::Null);
return Ok(());
}
let result = match self.entry_lookup.borrow().get(&object_ref).copied() {
Some(XrefEntry::Uncompressed { offset }) if offset != 0 => {
self.read_uncompressed_object(object_ref, offset)
}
Some(XrefEntry::Uncompressed { .. }) => {
self.push_warning("object has offset 0", Some(0));
Ok((ObjectValue::Null, -1))
}
Some(XrefEntry::Compressed { stream, .. }) => {
match self.resolve_objects_in_stream(stream) {
Ok(()) => {
if handle.is_resolved() {
self.state.borrow_mut().resolving.remove(&object_ref);
return Ok(());
}
Ok((ObjectValue::Null, -1))
}
Err(error) => Err(error),
}
}
Some(XrefEntry::Free { .. }) | None => Ok((ObjectValue::Null, -1)),
};
self.state.borrow_mut().resolving.remove(&object_ref);
match result {
Ok((value, parsed_offset)) => {
handle.set_resolved(value);
handle.set_parsed_offset_if_unset(parsed_offset);
}
Err(error) => {
if self.state.borrow().reconstruction_trigger.is_some() {
return Err(error); } let offset = match &error {
Error::Parse { offset, .. } => Some(*offset as u64),
_ => None, };
self.push_warning(error.to_string(), offset);
handle.set_resolved(ObjectValue::Null);
}
}
Ok(())
}
}
#[derive(Debug, Clone)]
struct XrefHandleCache {
shared: SharedBootstrapCache,
}
impl XrefHandleCache {
fn get(&self, object_ref: &ObjectRef) -> Option<ObjectHandle> {
self.shared
.borrow()
.handle_state
.borrow()
.handles
.get(object_ref)
.cloned()
.filter(|handle| handle.is_resolved())
}
fn insert(&self, object_ref: ObjectRef, handle: ObjectHandle) {
self.shared
.borrow_mut()
.handle_state
.borrow_mut()
.handles
.insert(object_ref, handle);
}
fn commit(&mut self) {}
fn shared(&self) -> SharedBootstrapCache {
Rc::clone(&self.shared)
}
}
struct XrefReadContext<'bytes> {
bytes: &'bytes [u8],
document: Rc<BootstrapHandleDocument>,
cache: XrefHandleCache,
diagnostics: Diagnostics,
handle_diagnostics_len: usize,
}
struct XrefDetachedHandles;
impl HandleResolver for XrefDetachedHandles {
fn indirect_handle(&mut self, object_ref: ObjectRef) -> ObjectHandle {
ObjectHandle::new_indirect_unresolved(object_ref, -1)
}
fn direct_handle(&mut self, value: ObjectValue) -> ObjectHandle {
ObjectHandle::from_value(value)
}
}
impl<'bytes> XrefReadContext<'bytes> {
fn new(
bytes: &'bytes [u8],
spec: XrefReadContextSpec<'_>,
registration: &XrefRegistration,
options: XrefLoadOptions,
) -> Self {
let (entry_lookup, shared, reference_offsets) = match spec {
XrefReadContextSpec::ActiveSection => (
XrefEntryLookup::Registration(®istration.entries),
empty_bootstrap_cache(),
None,
),
XrefReadContextSpec::ActiveSectionWithCache { bootstrap_cache } => (
XrefEntryLookup::Registration(®istration.entries),
Rc::clone(bootstrap_cache),
None,
),
XrefReadContextSpec::Reconstruction {
line_scan_entries,
reference_offsets,
} => (
XrefEntryLookup::Reconstruction {
line_scan_entries,
registration_entries: ®istration.entries,
},
empty_bootstrap_cache(),
Some(Rc::clone(reference_offsets)),
),
XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries,
reference_offsets,
bootstrap_cache,
} => (
XrefEntryLookup::Reconstruction {
line_scan_entries,
registration_entries: ®istration.entries,
},
Rc::clone(bootstrap_cache),
Some(Rc::clone(reference_offsets)),
),
};
let document = {
let mut cache = shared.borrow_mut();
if let Some(document) = cache.handle_document.as_ref() {
document.refresh_entry_lookup(entry_lookup);
Rc::clone(document)
} else {
let document = BootstrapHandleDocument::new_with_state(
None,
entry_lookup,
options,
Rc::clone(&cache.handle_state),
);
cache.handle_document = Some(Rc::clone(&document));
document
}
};
document.set_reference_offsets(reference_offsets);
let handle_diagnostics_len = shared
.borrow()
.handle_state
.borrow()
.diagnostics
.entries()
.len();
Self {
bytes,
document,
cache: XrefHandleCache { shared },
diagnostics: Diagnostics::default(),
handle_diagnostics_len,
}
}
fn read_file_object_handle(
&mut self,
input: &[u8],
absolute_offset: u64,
policy: RecoveryPolicy,
description: XrefObjectDescription,
) -> Result<HandleFileObjectRead> {
let result =
self.document
.read_file_object(input, absolute_offset, policy, description, self.bytes);
self.sync_handle_diagnostics();
result
}
fn resolve_dictionary_value(
&mut self,
dictionary: &ObjectHandle,
key: &str,
) -> Option<ObjectHandle> {
let mut name = Vec::with_capacity(key.len() + 1);
name.push(b'/');
name.extend_from_slice(key.as_bytes());
self.ensure_source_for_resolution(dictionary);
let value = dictionary.try_get_key(&name).ok()?;
self.ensure_source_for_resolution(&value);
let _ = value.try_dereference();
self.sync_handle_diagnostics();
Some(value)
}
fn ensure_source_for_resolution(&self, handle: &ObjectHandle) {
if handle.object_ref().is_some() && !handle.is_resolved() {
self.document.ensure_source_bytes(self.bytes);
}
}
fn sync_handle_diagnostics(&mut self) {
let shared = self.cache.shared.borrow();
let state = shared.handle_state.borrow();
for diagnostic in state
.diagnostics
.entries()
.iter()
.skip(self.handle_diagnostics_len)
{
self.diagnostics.push(diagnostic.clone());
}
self.handle_diagnostics_len = state.diagnostics.entries().len();
}
fn append_diagnostics_to(&mut self, diagnostics: &mut Diagnostics) {
self.sync_handle_diagnostics();
for diagnostic in self.diagnostics.entries() {
diagnostics.push(diagnostic.clone());
}
}
fn take_reconstruction_trigger(&self) -> Option<Error> {
let shared = self.cache.shared.borrow();
let trigger = shared
.handle_state
.borrow_mut()
.reconstruction_trigger
.take()
.map(|(offset, message)| Error::parse(offset as usize, message));
trigger
}
}
pub fn load_xref_and_trailer<R: Read + Seek>(reader: &mut R) -> Result<LoadedXref> {
load_xref_and_trailer_with_repair(reader, false)
}
pub fn load_xref_and_trailer_with_repair<R: Read + Seek>(
reader: &mut R,
allow_repair: bool,
) -> Result<LoadedXref> {
load_xref_state_with_options(
reader,
XrefLoadOptions {
allow_repair,
..XrefLoadOptions::default()
},
)
.and_then(|mut state| {
let bootstrap_cache = state.bootstrap_cache;
state.loaded.trailer = detach_bootstrap_handle(&state.loaded.trailer)?;
drop(bootstrap_cache);
Ok(state.loaded)
})
}
fn detach_bootstrap_handle(source: &ObjectHandle) -> Result<ObjectHandle> {
if let Some(object_ref) = source.object_ref() {
return Ok(ObjectHandle::new_indirect_unresolved(
object_ref,
source.get_parsed_offset(),
));
}
source.try_dereference()?;
let value = source
.with_value(|value| value.cloned())
.ok_or_else(|| Error::Internal("bootstrap trailer has no value".to_owned()))?;
let value = match value {
ObjectValue::Array(children) => ObjectValue::Array(
children
.iter()
.map(detach_bootstrap_handle)
.collect::<Result<Vec<_>>>()?,
),
ObjectValue::Dictionary(entries) => ObjectValue::Dictionary(
entries
.into_iter()
.map(|(key, child)| Ok((key, detach_bootstrap_handle(&child)?)))
.collect::<Result<BTreeMap<_, _>>>()?,
),
ObjectValue::Stream {
stream_dict,
stream_data,
stream_provider,
filter_on_write,
stream_length,
} => ObjectValue::Stream {
stream_dict: detach_bootstrap_handle(&stream_dict)?,
stream_data,
stream_provider,
filter_on_write,
stream_length,
},
other => other,
};
Ok(ObjectHandle::from_value(value))
}
pub(crate) fn load_xref_state_with_options<R: Read + Seek>(
reader: &mut R,
options: XrefLoadOptions,
) -> Result<LoadedXrefState> {
let allow_repair = options.allow_repair;
let mut source_bytes = Vec::new();
reader.seek(SeekFrom::Start(0))?;
reader.read_to_end(&mut source_bytes)?;
let mut initial_diagnostics = Diagnostics::default();
let (version, header_offset) = if allow_repair {
match find_qpdf_header(&source_bytes) {
Some((offset, version)) => (version, offset),
None => {
initial_diagnostics.push(Diagnostic::warning("can't find PDF header", None));
("1.2".to_string(), 0)
}
}
} else {
(parse_header(&source_bytes)?, 0)
};
let bytes = &source_bytes[header_offset..];
let mut parse_errors = Vec::new();
let startxref = match parse_startxref(bytes) {
Ok(offset) => offset,
Err(error) if allow_repair => {
parse_errors.push(error);
0
}
Err(error) => return Err(error),
};
let xref_pos = match usize::try_from(startxref) {
Ok(xref_pos) => xref_pos,
Err(_) if allow_repair => {
parse_errors.push(Error::parse(0, "startxref does not fit usize"));
0
}
Err(_) => return Err(Error::parse(0, "startxref does not fit usize")), };
let mut registration = XrefRegistration::default();
let mut initial_parse_diagnostics = Diagnostics::default();
let mut observed_first_xref_item_offset = None;
let initial_bootstrap_cache = empty_bootstrap_cache();
let mut loaded = match parse_xref_from_start(
bytes,
xref_pos,
startxref,
&version,
options,
&mut registration,
Some(&mut initial_parse_diagnostics),
XrefReadContextSpec::ActiveSectionWithCache {
bootstrap_cache: &initial_bootstrap_cache,
},
Some(&mut observed_first_xref_item_offset),
true,
)
{
Ok(loaded) => loaded,
Err(error) if allow_repair => {
let trigger = parse_errors.into_iter().next().unwrap_or_else(|| {
if startxref == 0 {
Error::parse(0, "xref not found")
} else {
error
}
});
if startxref != 0 {
for diagnostic in initial_parse_diagnostics.entries() {
initial_diagnostics.push(diagnostic.clone());
}
}
let preexisting_entries = (startxref != 0).then_some(®istration.entries);
let preexisting_bootstrap_cache = (startxref != 0).then_some(&initial_bootstrap_cache);
let mut recovered = recover_xref_from_linear_scan(
bytes,
version,
startxref,
trigger,
None,
preexisting_entries,
preexisting_bootstrap_cache,
options,
initial_diagnostics,
observed_first_xref_item_offset,
)?;
discard_lower_generations(
&mut recovered.loaded.entries,
&mut recovered.parsed_xref_streams,
);
recovered.header_offset = header_offset;
return Ok(recovered);
}
Err(error) => return Err(error),
};
prepend_repair_diagnostics(&mut loaded.loaded.repair_diagnostics, initial_diagnostics);
if let Some((offset, message)) = loaded.pending_reconstruction_trigger.take() {
let trigger = Error::parse(offset as usize, message);
let diagnostics = std::mem::take(&mut loaded.loaded.repair_diagnostics);
let recovered = recover_xref_from_linear_scan(
bytes,
version.clone(),
startxref,
trigger,
Some(&loaded.loaded.trailer),
Some(®istration.entries),
Some(&loaded.bootstrap_cache),
options,
diagnostics,
Some(loaded.first_xref_item_offset),
)?; let deleted_objects = std::mem::take(&mut registration.deleted_objects);
loaded = merge_recovered_qpdf_state(recovered, loaded, &deleted_objects);
registration.entries = loaded.loaded.entries.clone();
registration.deleted_objects.clear();
}
let mut previous_parse_diagnostics = Diagnostics::default();
if let Err(error) = merge_previous_xref_sections_with_observer(
bytes,
&version,
&mut loaded,
options,
&mut registration,
Some(&mut previous_parse_diagnostics),
XrefReadContextSpec::ActiveSection,
Some(&mut observed_first_xref_item_offset),
) {
if allow_repair {
let deleted_objects = std::mem::take(&mut registration.deleted_objects);
let trigger = parse_errors.into_iter().next().unwrap_or(error);
let recovered = recover_xref_from_linear_scan(
bytes,
version,
startxref,
trigger,
Some(&loaded.loaded.trailer),
Some(®istration.entries),
Some(&loaded.bootstrap_cache),
options,
previous_parse_diagnostics,
observed_first_xref_item_offset,
)?;
let mut recovered = merge_recovered_qpdf_state(recovered, loaded, &deleted_objects);
discard_lower_generations(
&mut recovered.loaded.entries,
&mut recovered.parsed_xref_streams,
);
recovered.header_offset = header_offset;
return Ok(recovered);
}
return Err(error);
}
loaded.loaded.entries = registration.snapshot();
let mut size_context = XrefReadContext::new(
bytes,
XrefReadContextSpec::ActiveSectionWithCache {
bootstrap_cache: &loaded.bootstrap_cache,
},
®istration,
options,
);
let resolved_size = size_context.resolve_dictionary_value(&loaded.loaded.trailer, "Size");
let size_reconstruction_trigger = size_context.take_reconstruction_trigger();
if size_reconstruction_trigger.is_none() {
size_context.cache.commit();
}
size_context.append_diagnostics_to(&mut loaded.loaded.repair_diagnostics);
if let Some(error) = size_reconstruction_trigger {
let diagnostics = std::mem::take(&mut loaded.loaded.repair_diagnostics);
let recovered = recover_xref_from_linear_scan(
bytes,
version.clone(),
startxref,
error,
Some(&loaded.loaded.trailer),
None, Some(&loaded.bootstrap_cache), options,
diagnostics,
None,
)?; let deleted_objects = std::mem::take(&mut registration.deleted_objects);
let mut recovered = merge_recovered_qpdf_state(recovered, loaded, &deleted_objects);
recovered.header_offset = header_offset;
let (recovered_size, recovered_size_diagnostics) = {
let recovered_reference_offsets =
reconstructed_reference_offsets(&recovered.loaded.entries);
let reconstruction_registration = XrefRegistration::default();
let mut recovered_size_context = XrefReadContext::new(
bytes,
XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries: &recovered.loaded.entries,
reference_offsets: &recovered_reference_offsets,
bootstrap_cache: &recovered.bootstrap_cache,
},
&reconstruction_registration,
options,
);
let recovered_size =
recovered_size_context.resolve_dictionary_value(&recovered.loaded.trailer, "Size");
recovered_size_context.cache.commit();
(recovered_size, recovered_size_context.diagnostics.clone())
};
for diagnostic in recovered_size_diagnostics.entries() {
recovered.loaded.repair_diagnostics.push(diagnostic.clone());
}
append_xref_size_warning_for(
recovered_size.as_ref(),
&recovered.loaded.entries,
&BTreeSet::new(),
&mut recovered.loaded.repair_diagnostics,
);
discard_lower_generations(
&mut recovered.loaded.entries,
&mut recovered.parsed_xref_streams,
);
return Ok(recovered);
}
append_xref_size_warning_for(
resolved_size.as_ref(),
&loaded.loaded.entries,
®istration.deleted_objects,
&mut loaded.loaded.repair_diagnostics,
);
registration.deleted_objects.clear();
if let Some(error) = parse_errors.into_iter().next() {
push_repair_diagnostics(&mut loaded.loaded.repair_diagnostics, &error, startxref);
}
discard_lower_generations(&mut loaded.loaded.entries, &mut loaded.parsed_xref_streams);
loaded.header_offset = header_offset;
Ok(loaded)
}
#[allow(clippy::too_many_arguments)]
fn parse_xref_from_start(
bytes: &[u8],
xref_pos: usize,
startxref: u64,
version: &str,
options: XrefLoadOptions,
registration: &mut XrefRegistration,
mut error_diagnostics_sink: Option<&mut Diagnostics>,
context_spec: XrefReadContextSpec<'_>,
first_xref_item_offset_sink: Option<&mut Option<u64>>,
validate_current_classic_trailer: bool,
) -> Result<LoadedXrefState> {
if bytes
.get(xref_pos..)
.is_some_and(|tail| tail.starts_with(b"xref"))
{
let mut cursor = ByteCursor::new(bytes, xref_pos + 4);
let (entries, trailer_start, table_diagnostics, first_xref_item_offset) = parse_xref_table(
&mut cursor,
bytes,
error_diagnostics_sink.as_deref_mut(),
first_xref_item_offset_sink,
)?;
let mut deferred_free = Vec::new();
for entry in entries {
match entry {
ParsedXrefEntry::Live { object_ref, entry } => {
registration.insert_xref_entry(object_ref, entry);
}
ParsedXrefEntry::Free { object_ref } => deferred_free.push(object_ref),
}
}
let mut trailer_context = XrefReadContext::new(bytes, context_spec, registration, options);
let trailer_slice = bytes
.get(trailer_start..)
.ok_or_else(|| Error::parse(trailer_start, "trailer is not a dictionary"))?;
let mut trailer_parser = BootstrapHandleParser {
document: &trailer_context.document,
description: XrefObjectDescription::Ordinary,
};
let (trailer_value, _, trailer_parser_diagnostics) =
parse_qpdf_direct_object_handle_with_diagnostics(
trailer_slice,
i64::try_from(trailer_start).unwrap_or(i64::MAX),
None,
&mut trailer_parser,
)
.map_err(|error| error.rebase_offset(trailer_start))?;
let trailer = ObjectHandle::from_parsed_value_with_resolver(
trailer_value,
trailer_context.document.resolver_weak(),
);
if !trailer.try_is_dictionary()? {
return Err(Error::parse(trailer_start, "trailer is not a dictionary"));
}
let mut trailer_diags = table_diagnostics;
trailer_diags.extend(trailer_diagnostics(
trailer_start,
trailer_parser_diagnostics,
));
let mut bootstrap_diagnostics = Diagnostics::default();
trailer_context.append_diagnostics_to(&mut bootstrap_diagnostics);
if let Some(sink) = error_diagnostics_sink.as_deref_mut() {
for diagnostic in bootstrap_diagnostics.entries() {
sink.push(diagnostic.clone());
}
}
let bootstrap_cache = trailer_context.cache.shared();
let trailer_references = collect_trailer_references(&trailer);
let mut loaded = LoadedXrefState {
loaded: LoadedXref {
version: version.to_string(),
startxref,
entries: registration.snapshot(),
trailer,
last_xref_form: XrefForm::Table,
repair_diagnostics: Diagnostics::default(),
},
first_xref_item_offset,
classic_trailer_offset: Some(trailer_start),
pending_reconstruction_trigger: None,
trailer_references,
parsed_xref_streams: BTreeMap::new(),
bootstrap_cache,
header_offset: 0,
already_reconstructed: false,
};
for diagnostic in trailer_diags {
loaded.loaded.repair_diagnostics.push(diagnostic);
}
for diagnostic in bootstrap_diagnostics.entries() {
loaded.loaded.repair_diagnostics.push(diagnostic.clone());
}
if validate_current_classic_trailer {
let validation = validate_classic_trailer(
&mut trailer_context,
&loaded.loaded.trailer,
trailer_start,
);
match validation {
Ok(ClassicTrailerValidation::Valid) => {}
Ok(ClassicTrailerValidation::NeedsReconstruction(error)) => {
let Error::Parse { offset, message } = error else {
unreachable!("classic trailer reconstruction trigger is a parse error")
};
loaded.pending_reconstruction_trigger = Some((offset as u64, message));
}
Err(error) => {
if let Some(sink) = error_diagnostics_sink.as_deref_mut() {
for diagnostic in loaded.loaded.repair_diagnostics.entries() {
sink.push(diagnostic.clone());
}
}
return Err(error);
}
}
}
merge_xref_stream_from_classic_trailer(
bytes,
xref_pos,
&mut loaded,
options,
registration,
error_diagnostics_sink,
context_spec,
)?;
for object_ref in deferred_free {
registration.insert_free_xref_entry(object_ref);
}
loaded.loaded.entries = registration.snapshot();
return Ok(loaded);
}
parse_xref_stream(
bytes,
xref_pos,
startxref,
version.to_string(),
options,
registration,
error_diagnostics_sink,
context_spec,
)
}
enum ClassicTrailerValidation {
Valid,
NeedsReconstruction(Error),
}
fn validate_classic_trailer(
context: &mut XrefReadContext<'_>,
trailer: &ObjectHandle,
trailer_offset: usize,
) -> Result<ClassicTrailerValidation> {
let size = trailer
.as_dictionary()
.and_then(|entries| entries.get(b"/Size".as_slice()).cloned());
let Some(size) = size else {
return Err(Error::parse(
trailer_offset,
"trailer dictionary lacks /Size key",
));
};
if size.is_null() {
return Err(Error::parse(
trailer_offset,
"trailer dictionary lacks /Size key",
));
}
context.ensure_source_for_resolution(&size);
let is_integer = size.try_is_integer();
context.sync_handle_diagnostics();
if let Some(error) = context.take_reconstruction_trigger() {
return Ok(ClassicTrailerValidation::NeedsReconstruction(error));
}
if !is_integer? {
return Err(Error::parse(
trailer_offset,
"/Size key in trailer dictionary is not an integer",
));
}
Ok(ClassicTrailerValidation::Valid)
}
fn merge_bootstrap_handle_state_prefer_source(
destination: &Rc<RefCell<BootstrapHandleState>>,
source: &Rc<RefCell<BootstrapHandleState>>,
) {
if Rc::ptr_eq(destination, source) {
return;
}
let destination = destination.borrow();
let mut source = source.borrow_mut();
let mut handles = destination.handles.clone();
handles.extend(source.handles.clone());
source.handles = handles;
source
.resolving
.extend(destination.resolving.iter().copied());
source
.resolved_object_streams
.extend(destination.resolved_object_streams.iter().copied());
let mut diagnostics = destination.diagnostics.clone();
for diagnostic in source.diagnostics.entries() {
diagnostics.push(diagnostic.clone());
}
source.diagnostics = diagnostics;
if source.reconstruction_trigger.is_none() {
source.reconstruction_trigger = destination.reconstruction_trigger.clone();
}
}
fn merge_bootstrap_cache_prefer_source(
destination: &SharedBootstrapCache,
source: &SharedBootstrapCache,
) {
if Rc::ptr_eq(destination, source) {
return;
}
let source = source.borrow();
let mut destination = destination.borrow_mut();
destination
.handle_document_owners
.extend(source.handle_document_owners.iter().cloned());
if let Some(source_document) = source.handle_document.as_ref() {
if let Some(destination_document) = destination.handle_document.clone() {
if !Rc::ptr_eq(&destination_document, source_document) {
destination
.handle_document_owners
.push(destination_document);
}
}
merge_bootstrap_handle_state_prefer_source(&destination.handle_state, &source.handle_state);
destination.handle_state = Rc::clone(&source.handle_state);
destination.handle_document = Some(Rc::clone(source_document));
} else {
merge_bootstrap_handle_state_prefer_source(&destination.handle_state, &source.handle_state);
}
}
fn merge_xref_stream_from_classic_trailer(
bytes: &[u8],
classic_xref_pos: usize,
loaded: &mut LoadedXrefState,
options: XrefLoadOptions,
registration: &mut XrefRegistration,
mut error_diagnostics_sink: Option<&mut Diagnostics>,
context_spec: XrefReadContextSpec<'_>,
) -> Result<()> {
let has_xref_stream_key = loaded
.loaded
.trailer
.as_dictionary()
.is_some_and(|entries| entries.contains_key(b"/XRefStm".as_slice()));
if !has_xref_stream_key {
return Ok(());
}
if options.ignore_xref_streams {
return Ok(());
}
let hybrid_bootstrap_cache = match context_spec {
XrefReadContextSpec::ActiveSection | XrefReadContextSpec::Reconstruction { .. } => {
&loaded.bootstrap_cache
}
XrefReadContextSpec::ActiveSectionWithCache { bootstrap_cache }
| XrefReadContextSpec::ReconstructionWithCache {
bootstrap_cache, ..
} => bootstrap_cache,
};
let hybrid_context_spec = match context_spec {
XrefReadContextSpec::ActiveSection | XrefReadContextSpec::ActiveSectionWithCache { .. } => {
XrefReadContextSpec::ActiveSectionWithCache {
bootstrap_cache: hybrid_bootstrap_cache,
}
}
XrefReadContextSpec::Reconstruction {
line_scan_entries,
reference_offsets,
}
| XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries,
reference_offsets,
..
} => XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries,
reference_offsets,
bootstrap_cache: hybrid_bootstrap_cache,
},
};
let mut context = XrefReadContext::new(bytes, hybrid_context_spec, registration, options);
let xref_stream_value = context.resolve_dictionary_value(&loaded.loaded.trailer, "XRefStm");
if let Some(error) = context.take_reconstruction_trigger() {
context.append_diagnostics_to(&mut loaded.loaded.repair_diagnostics);
if let Some(sink) = error_diagnostics_sink.as_mut() {
for diagnostic in loaded.loaded.repair_diagnostics.entries() {
sink.push(diagnostic.clone());
}
}
return Err(error);
}
context.cache.commit();
let Some(xref_stream_offset) = xref_stream_value.and_then(|value| value.try_as_integer().ok().flatten())
else {
context.append_diagnostics_to(&mut loaded.loaded.repair_diagnostics);
if let Some(sink) = error_diagnostics_sink.as_mut() {
for diagnostic in loaded.loaded.repair_diagnostics.entries() {
sink.push(diagnostic.clone());
}
}
return Err(Error::parse(classic_xref_pos, "invalid /XRefStm"));
};
context.append_diagnostics_to(&mut loaded.loaded.repair_diagnostics);
let xref_stream_pos = match usize::try_from(xref_stream_offset) {
Ok(xref_stream_pos) => xref_stream_pos,
Err(_) => {
let error = Error::Io(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
format!("xref stream offset {xref_stream_offset} is before the file start"),
));
if let Some(sink) = error_diagnostics_sink.as_mut() {
for diagnostic in loaded.loaded.repair_diagnostics.entries() {
sink.push(diagnostic.clone());
}
}
return Err(error);
}
};
let mut hybrid_error_diagnostics = Diagnostics::default();
let hybrid = match parse_xref_stream(
bytes,
xref_stream_pos,
xref_stream_pos as u64,
loaded.loaded.version.clone(),
options,
registration,
Some(&mut hybrid_error_diagnostics),
hybrid_context_spec,
) {
Ok(hybrid) => hybrid,
Err(error) => {
if let Some(sink) = error_diagnostics_sink.as_mut() {
for diagnostic in loaded.loaded.repair_diagnostics.entries() {
sink.push(diagnostic.clone());
}
for diagnostic in hybrid_error_diagnostics.entries() {
sink.push(diagnostic.clone());
}
}
return Err(error);
}
};
for diagnostic in hybrid.loaded.repair_diagnostics.entries() {
loaded.loaded.repair_diagnostics.push(diagnostic.clone());
}
if hybrid.first_xref_item_offset != 0 {
loaded.first_xref_item_offset = hybrid.first_xref_item_offset;
}
loaded
.trailer_references
.extend(hybrid.trailer_references.iter().copied());
loaded
.parsed_xref_streams
.extend(hybrid.parsed_xref_streams);
merge_bootstrap_cache_prefer_source(&loaded.bootstrap_cache, &hybrid.bootstrap_cache);
loaded.loaded.entries = registration.snapshot();
Ok(())
}
fn merge_previous_xref_sections(
bytes: &[u8],
version: &str,
loaded: &mut LoadedXrefState,
options: XrefLoadOptions,
registration: &mut XrefRegistration,
error_diagnostics_sink: Option<&mut Diagnostics>,
context_spec: XrefReadContextSpec<'_>,
) -> Result<()> {
merge_previous_xref_sections_with_observer(
bytes,
version,
loaded,
options,
registration,
error_diagnostics_sink,
context_spec,
None,
)
}
#[allow(clippy::too_many_arguments)]
fn merge_previous_xref_sections_with_observer(
bytes: &[u8],
version: &str,
loaded: &mut LoadedXrefState,
options: XrefLoadOptions,
registration: &mut XrefRegistration,
mut error_diagnostics_sink: Option<&mut Diagnostics>,
context_spec: XrefReadContextSpec<'_>,
mut first_xref_item_offset_sink: Option<&mut Option<u64>>,
) -> Result<()> {
let mut visited = HashSet::new();
let chain_bootstrap_cache = match context_spec {
XrefReadContextSpec::ActiveSection | XrefReadContextSpec::Reconstruction { .. } => {
Rc::clone(&loaded.bootstrap_cache)
}
XrefReadContextSpec::ActiveSectionWithCache { bootstrap_cache }
| XrefReadContextSpec::ReconstructionWithCache {
bootstrap_cache, ..
} => Rc::clone(bootstrap_cache),
};
let section_context_spec = match context_spec {
XrefReadContextSpec::ActiveSection | XrefReadContextSpec::ActiveSectionWithCache { .. } => {
XrefReadContextSpec::ActiveSectionWithCache {
bootstrap_cache: &chain_bootstrap_cache,
}
}
XrefReadContextSpec::Reconstruction {
line_scan_entries,
reference_offsets,
}
| XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries,
reference_offsets,
..
} => XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries,
reference_offsets,
bootstrap_cache: &chain_bootstrap_cache,
},
};
let (mut previous_offset, previous_diagnostics, reconstruction_trigger) =
resolve_previous_xref_offset(
bytes,
options,
registration,
section_context_spec,
&loaded.loaded.trailer,
loaded.classic_trailer_offset,
)?;
for diagnostic in previous_diagnostics.entries() {
loaded.loaded.repair_diagnostics.push(diagnostic.clone());
}
if let Some(error) = reconstruction_trigger {
return Err(error);
}
while let Some(offset) = previous_offset {
let previous_pos = usize::try_from(offset)
.map_err(|_| Error::parse(0, "xref /Prev does not fit usize"))?;
if !visited.insert(offset) {
return Err(Error::parse(0, "loop detected following xref tables"));
}
let previous = parse_xref_from_start(
bytes,
previous_pos,
offset,
version,
options,
registration,
error_diagnostics_sink.as_deref_mut(),
section_context_spec,
first_xref_item_offset_sink.as_deref_mut(),
false,
)?;
for diagnostic in previous.loaded.repair_diagnostics.entries() {
loaded.loaded.repair_diagnostics.push(diagnostic.clone());
}
if previous.first_xref_item_offset != 0 {
loaded.first_xref_item_offset = previous.first_xref_item_offset;
}
loaded
.trailer_references
.extend(previous.trailer_references.iter().copied());
for (object_ref, object) in previous.parsed_xref_streams {
let newer_live = matches!(
loaded.loaded.entries.get(&object_ref),
Some(XrefEntry::Uncompressed { .. } | XrefEntry::Compressed { .. })
);
if !newer_live {
loaded
.parsed_xref_streams
.entry(object_ref)
.or_insert(object);
}
}
let (next_previous_offset, previous_diagnostics, reconstruction_trigger) =
resolve_previous_xref_offset(
bytes,
options,
registration,
section_context_spec,
&previous.loaded.trailer,
previous.classic_trailer_offset,
)?;
for diagnostic in previous_diagnostics.entries() {
loaded.loaded.repair_diagnostics.push(diagnostic.clone());
}
if let Some(error) = reconstruction_trigger {
return Err(error);
}
previous_offset = next_previous_offset;
}
loaded.loaded.entries = registration.snapshot();
Ok(())
}
fn resolve_previous_xref_offset(
bytes: &[u8],
options: XrefLoadOptions,
registration: &XrefRegistration,
context_spec: XrefReadContextSpec<'_>,
trailer: &ObjectHandle,
trailer_offset: Option<usize>,
) -> Result<(Option<u64>, Diagnostics, Option<Error>)> {
let mut context = XrefReadContext::new(bytes, context_spec, registration, options);
context.ensure_source_for_resolution(trailer);
if let Some(previous) = trailer
.as_dictionary()
.and_then(|entries| entries.get(b"/Prev".as_slice()).cloned())
{
context.ensure_source_for_resolution(&previous);
}
let has_previous = trailer.try_has_key(b"/Prev")?;
context.sync_handle_diagnostics();
let previous = has_previous
.then(|| context.resolve_dictionary_value(trailer, "Prev"))
.flatten();
let (offset, validation_error) = match previous {
Some(offset) => match parse_non_negative_u64_handle(&offset, "/Prev") {
Ok(offset) => (Some(offset).filter(|&offset| offset != 0), None),
Err(_) if trailer_offset.is_some() => (
None,
Some(Error::parse(
trailer_offset.expect("classic trailer offset is present"),
"/Prev key in trailer dictionary is not an integer",
)),
),
Err(_) => (None, None),
},
None => (None, None),
};
let reconstruction_trigger = context.take_reconstruction_trigger();
context.cache.commit();
let diagnostics = context.diagnostics.clone();
Ok((
offset,
diagnostics,
reconstruction_trigger.or(validation_error),
))
}
fn collect_trailer_references(trailer: &ObjectHandle) -> BTreeSet<ObjectRef> {
let mut references = BTreeSet::new();
let mut stack = vec![trailer.clone()];
while let Some(handle) = stack.pop() {
if let Some(object_ref) = handle.object_ref() {
references.insert(object_ref);
continue;
}
if let Some(children) = handle.as_array() {
stack.extend(children);
} else if let Some(entries) = handle.as_dictionary() {
stack.extend(entries.into_values());
} else if let Some(stream_dict) = handle.as_stream_dict() {
stack.push(stream_dict);
}
}
references
}
fn append_xref_size_warning_for(
size: Option<&ObjectHandle>,
entries: &BTreeMap<ObjectRef, XrefEntry>,
deleted_objects: &BTreeSet<u32>,
repair_diagnostics: &mut Diagnostics,
) {
let Some(size) = size.and_then(|value| value.try_as_integer().ok().flatten()) else {
return;
};
let max_live = entries
.keys()
.map(|object_ref| object_ref.number)
.max()
.unwrap_or(0);
let max_deleted = deleted_objects.iter().copied().max().unwrap_or(0);
let max_object = max_live.max(max_deleted);
if size < 1 || size - 1 != i64::from(max_object) {
repair_diagnostics.push(Diagnostic::warning(
format!(
"reported number of objects ({size}) is not one plus the highest object number ({max_object})"
),
None,
));
}
}
#[allow(clippy::too_many_arguments)]
fn recover_xref_from_linear_scan(
bytes: &[u8],
version: String,
startxref: u64,
trigger_error: Error,
fallback_trailer: Option<&ObjectHandle>,
preexisting_entries: Option<&BTreeMap<ObjectRef, XrefEntry>>,
preexisting_bootstrap_cache: Option<&SharedBootstrapCache>,
options: XrefLoadOptions,
mut repair_diagnostics: Diagnostics,
observed_first_xref_item_offset: Option<u64>,
) -> Result<LoadedXrefState> {
push_repair_diagnostics(&mut repair_diagnostics, &trigger_error, startxref);
let recovered = recover_xref_entries(bytes, fallback_trailer.is_none())
.map_err(|error| Error::with_open_diagnostics(error, repair_diagnostics.clone()))?;
let mut entries = recovered.entries;
if let Some(preexisting_entries) = preexisting_entries {
for (&object_ref, &entry) in preexisting_entries {
if !matches!(entry, XrefEntry::Uncompressed { .. }) {
entries.entry(object_ref).or_insert(entry);
}
}
}
for diagnostic in recovered.trailer_diagnostics {
repair_diagnostics.push(diagnostic);
}
let mut parsed_xref_streams = BTreeMap::new();
let mut extra_trailer_references = BTreeSet::new();
let mut bootstrap_cache = preexisting_bootstrap_cache
.map(Rc::clone)
.unwrap_or_else(empty_bootstrap_cache);
let (trailer, recovered_startxref, recovered_form, recovered_first_xref_item_offset) =
if let Some(trailer) = fallback_trailer {
(trailer.clone(), startxref, XrefForm::Table, 0)
} else {
match recovered.trailer {
Some(trailer) => (trailer, startxref, XrefForm::Table, 0),
None => match recover_trailer_from_xref_stream_candidate(
bytes,
&version,
options,
&mut entries,
&mut parsed_xref_streams,
&mut repair_diagnostics,
&mut extra_trailer_references,
preexisting_bootstrap_cache,
) {
Ok((
trailer,
max_offset,
form,
_deleted_objects,
first_xref_item_offset,
candidate_bootstrap_cache,
)) => {
bootstrap_cache = candidate_bootstrap_cache;
(trailer, max_offset, form, first_xref_item_offset)
}
Err(candidate_error) => {
return Err(Error::with_open_diagnostics(
candidate_error,
repair_diagnostics,
));
}
},
}
};
let recovered_first_xref_item_offset =
observed_first_xref_item_offset.unwrap_or(recovered_first_xref_item_offset);
let mut trailer_references = collect_trailer_references(&trailer);
trailer_references.extend(extra_trailer_references);
entries.retain(|_, entry| !matches!(entry, XrefEntry::Free { .. }));
Ok(LoadedXrefState {
loaded: LoadedXref {
version,
startxref: recovered_startxref,
entries,
trailer,
last_xref_form: recovered_form,
repair_diagnostics,
},
first_xref_item_offset: recovered_first_xref_item_offset,
classic_trailer_offset: None,
pending_reconstruction_trigger: None,
trailer_references,
parsed_xref_streams,
bootstrap_cache,
header_offset: 0,
already_reconstructed: true,
})
}
fn prepend_repair_diagnostics(target: &mut Diagnostics, initial: Diagnostics) {
if initial.entries().is_empty() {
return;
}
let existing = std::mem::take(target);
*target = initial;
for diagnostic in existing.entries() {
target.push(diagnostic.clone());
}
}
fn merge_recovered_qpdf_state(
mut recovered: LoadedXrefState,
mut accumulated: LoadedXrefState,
accumulated_deleted_objects: &BTreeSet<u32>,
) -> LoadedXrefState {
let mut repair_diagnostics = std::mem::take(&mut accumulated.loaded.repair_diagnostics);
for diagnostic in recovered.loaded.repair_diagnostics.entries() {
repair_diagnostics.push(diagnostic.clone());
}
recovered.loaded.repair_diagnostics = repair_diagnostics;
recovered.loaded.last_xref_form = accumulated.loaded.last_xref_form;
if recovered.first_xref_item_offset == 0 {
recovered.first_xref_item_offset = accumulated.first_xref_item_offset;
}
recovered.classic_trailer_offset = accumulated
.classic_trailer_offset
.or(recovered.classic_trailer_offset);
recovered
.loaded
.entries
.retain(|object_ref, _| !accumulated_deleted_objects.contains(&object_ref.number));
recovered
.trailer_references
.extend(accumulated.trailer_references);
recovered
.parsed_xref_streams
.append(&mut accumulated.parsed_xref_streams);
merge_bootstrap_cache_prefer_source(&recovered.bootstrap_cache, &accumulated.bootstrap_cache);
recovered
}
pub fn load_xref_and_trailer_best_effort<R: Read + Seek>(reader: &mut R) -> Result<LoadedXref> {
load_xref_and_trailer_with_repair(reader, true)
}
pub(crate) struct RecoveredXref {
pub(crate) entries: BTreeMap<ObjectRef, XrefEntry>,
pub(crate) trailer: Option<ObjectHandle>,
pub(crate) trailer_diagnostics: Vec<Diagnostic>,
}
pub(crate) fn recover_xref_entries(bytes: &[u8], capture_trailer: bool) -> Result<RecoveredXref> {
let mut entries = BTreeMap::new();
let mut trailer = None;
let mut trailer_diagnostics = Vec::new();
let mut line_start = 0usize;
while line_start < bytes.len() {
let next_line_start = next_line_start(bytes, line_start);
if let Some(first_token) = read_scan_token(bytes, line_start, next_line_start) {
if capture_trailer && trailer.is_none() && first_token.is_word_value(b"trailer") {
let (candidate, diagnostics) = parse_trailer_candidate(bytes, first_token.end);
trailer = candidate;
trailer_diagnostics.extend(diagnostics);
} else if let Some((object_ref, offset)) =
scan_object_header_after_first_token(bytes, &first_token)
{
entries.insert(object_ref, XrefEntry::Uncompressed { offset });
}
}
line_start = next_line_start;
}
Ok(RecoveredXref {
entries,
trailer,
trailer_diagnostics,
})
}
const XREF_RECONSTRUCTION_FALLBACK_SPAN: usize = 64;
#[allow(clippy::too_many_arguments)]
fn recover_trailer_from_xref_stream_candidate(
bytes: &[u8],
version: &str,
options: XrefLoadOptions,
entries: &mut BTreeMap<ObjectRef, XrefEntry>,
parsed_xref_streams: &mut BTreeMap<ObjectRef, ObjectHandle>,
repair_diagnostics: &mut Diagnostics,
trailer_references: &mut BTreeSet<ObjectRef>,
preexisting_bootstrap_cache: Option<&SharedBootstrapCache>,
) -> Result<(
ObjectHandle,
u64,
XrefForm,
BTreeSet<u32>,
u64,
SharedBootstrapCache,
)> {
let reference_offsets = reconstructed_reference_offsets(entries);
let (candidate, discovery_diagnostics) = find_xref_stream_trailer_candidate(
bytes,
entries,
options,
&reference_offsets,
preexisting_bootstrap_cache,
);
for diagnostic in discovery_diagnostics.entries() {
repair_diagnostics.push(diagnostic.clone());
}
let Some(candidate) = candidate else {
return Err(Error::parse(
0,
"unable to find trailer dictionary while recovering damaged file",
));
};
let max_offset = candidate.max_offset;
let mut reentry_registration = XrefRegistration {
entries: entries.clone(),
..XrefRegistration::default()
};
let mut reentry = match parse_xref_from_start(
bytes,
max_offset as usize,
max_offset,
version,
options,
&mut reentry_registration,
Some(&mut *repair_diagnostics),
XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries: entries,
reference_offsets: &reference_offsets,
bootstrap_cache: &candidate.bootstrap_cache,
},
None,
false,
) {
Ok(reentry) => reentry,
Err(_) => {
return Err(Error::parse(
0,
"error decoding candidate xref stream while recovering damaged file",
));
}
};
let candidate_diagnostic_count = reentry.loaded.repair_diagnostics.entries().len();
for diagnostic in reentry.loaded.repair_diagnostics.entries() {
repair_diagnostics.push(diagnostic.clone());
}
let mut previous_failure_diagnostics = Diagnostics::default();
if merge_previous_xref_sections(
bytes,
version,
&mut reentry,
options,
&mut reentry_registration,
Some(&mut previous_failure_diagnostics),
XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries: entries,
reference_offsets: &reference_offsets,
bootstrap_cache: &candidate.bootstrap_cache,
},
)
.is_err()
{
for diagnostic in reentry
.loaded
.repair_diagnostics
.entries()
.iter()
.skip(candidate_diagnostic_count)
{
repair_diagnostics.push(diagnostic.clone());
}
for diagnostic in previous_failure_diagnostics.entries() {
repair_diagnostics.push(diagnostic.clone());
}
return Err(Error::parse(
0,
"error decoding candidate xref stream while recovering damaged file",
));
}
let first_xref_item_offset = reentry.first_xref_item_offset;
let deleted_objects = reentry_registration.deleted_objects.clone();
for (object_ref, xref_entry) in reentry.loaded.entries {
entries.entry(object_ref).or_insert(xref_entry);
}
entries.retain(|object_ref, _| !deleted_objects.contains(&object_ref.number));
parsed_xref_streams.extend(reentry.parsed_xref_streams);
trailer_references.extend(reentry.trailer_references);
for diagnostic in reentry
.loaded
.repair_diagnostics
.entries()
.iter()
.skip(candidate_diagnostic_count)
{
repair_diagnostics.push(diagnostic.clone());
}
let merged_reference_offsets = reconstructed_reference_offsets(entries);
let mut size_context = XrefReadContext::new(
bytes,
XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries: entries,
reference_offsets: &merged_reference_offsets,
bootstrap_cache: &candidate.bootstrap_cache,
},
&reentry_registration,
options,
);
let resolved_size = size_context.resolve_dictionary_value(&candidate.trailer, "Size");
size_context.cache.commit();
size_context.append_diagnostics_to(repair_diagnostics);
append_xref_size_warning_for(
resolved_size.as_ref(),
entries,
&deleted_objects,
repair_diagnostics,
);
Ok((
candidate.trailer,
max_offset,
reentry.loaded.last_xref_form,
deleted_objects,
first_xref_item_offset,
candidate.bootstrap_cache,
))
}
struct XrefStreamCandidate {
trailer: ObjectHandle,
max_offset: u64,
bootstrap_cache: SharedBootstrapCache,
}
fn find_xref_stream_trailer_candidate(
bytes: &[u8],
entries: &BTreeMap<ObjectRef, XrefEntry>,
options: XrefLoadOptions,
reference_offsets: &Rc<[u64]>,
preexisting_bootstrap_cache: Option<&SharedBootstrapCache>,
) -> (Option<XrefStreamCandidate>, Diagnostics) {
let mut max_offset = 0u64;
let mut trailer: Option<ObjectHandle> = None;
let mut discovery_diagnostics = Diagnostics::default();
let empty_registration = XrefRegistration::default();
let context_spec = match preexisting_bootstrap_cache {
Some(bootstrap_cache) => XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries: entries,
reference_offsets,
bootstrap_cache,
},
None => XrefReadContextSpec::Reconstruction {
line_scan_entries: entries,
reference_offsets,
},
};
let mut context = XrefReadContext::new(bytes, context_spec, &empty_registration, options);
let mut emitted_diagnostics = 0usize;
for (&object_ref, entry) in entries {
let XrefEntry::Uncompressed { offset } = *entry else {
continue;
};
let start = offset as usize;
let next_offset_index = reference_offsets.partition_point(|&candidate| candidate <= offset);
let window_end = reference_offsets
.get(next_offset_index)
.map_or(bytes.len(), |&next| next as usize);
let parsed = if let Some(cached) = context.cache.get(&object_ref) {
Some(cached)
} else {
let narrow =
read_xref_candidate(&mut context, bytes, start, window_end, offset, object_ref);
let retry =
window_end < bytes.len() && narrow.as_ref().is_none_or(is_recovered_null_candidate);
let accepted = if retry {
let wide_index = next_offset_index
.saturating_add(XREF_RECONSTRUCTION_FALLBACK_SPAN)
.min(reference_offsets.len());
let wide_end = reference_offsets
.get(wide_index)
.map_or(bytes.len(), |&next| next as usize);
read_xref_candidate(&mut context, bytes, start, wide_end, offset, object_ref)
} else {
narrow
};
accepted.and_then(|completed| {
commit_xref_candidate(&mut context, completed, offset, object_ref)
})
};
append_new_context_diagnostics(
&context,
&mut discovery_diagnostics,
&mut emitted_diagnostics,
);
let Some(object) = parsed else {
continue;
};
let Some(stream_dict) = object.as_stream_dict() else {
continue;
};
if !is_xref_stream_dict(&mut context, &stream_dict) {
append_new_context_diagnostics(
&context,
&mut discovery_diagnostics,
&mut emitted_diagnostics,
);
continue;
}
append_new_context_diagnostics(
&context,
&mut discovery_diagnostics,
&mut emitted_diagnostics,
);
if offset > max_offset {
max_offset = offset;
if trailer.is_none() {
trailer = Some(stream_dict);
}
}
}
context.cache.commit();
let candidate = trailer.map(|dict| XrefStreamCandidate {
trailer: dict,
max_offset,
bootstrap_cache: context.cache.shared(),
});
append_new_context_diagnostics(
&context,
&mut discovery_diagnostics,
&mut emitted_diagnostics,
);
(candidate, discovery_diagnostics)
}
fn read_xref_candidate(
context: &mut XrefReadContext,
bytes: &[u8],
start: usize,
end: usize,
offset: u64,
object_ref: ObjectRef,
) -> Option<HandleFileObjectRead> {
let input = bytes.get(start..end)?;
let mut completed = context
.read_file_object_handle(
input,
offset,
RecoveryPolicy::Bounded,
XrefObjectDescription::Ordinary,
)
.ok()?;
if completed.object_ref != object_ref {
return None;
}
let _ = completed.remove_included_recovery_eol_for_decryption();
Some(completed)
}
fn is_recovered_null_candidate(completed: &HandleFileObjectRead) -> bool {
completed.object.is_null() && !completed.diagnostics.is_empty()
}
fn commit_xref_candidate(
context: &mut XrefReadContext,
completed: HandleFileObjectRead,
offset: u64,
object_ref: ObjectRef,
) -> Option<ObjectHandle> {
for diagnostic in completed.diagnostics {
context.diagnostics.push(xref_file_object_diagnostic(
XrefObjectDescription::Ordinary,
object_ref,
offset,
diagnostic,
));
}
let canonical = context.document.handle_for_reference(object_ref);
let value = completed.object.into_direct_value()?.0;
canonical.set_resolved(value);
context.cache.insert(object_ref, canonical.clone());
Some(canonical)
}
fn append_new_context_diagnostics(
context: &XrefReadContext,
diagnostics: &mut Diagnostics,
emitted: &mut usize,
) {
for diagnostic in context.diagnostics.entries().iter().skip(*emitted) {
diagnostics.push(diagnostic.clone());
}
*emitted = context.diagnostics.entries().len();
}
fn is_xref_stream_dict(context: &mut XrefReadContext, dict: &ObjectHandle) -> bool {
context
.resolve_dictionary_value(dict, "Type")
.and_then(|value| value.try_as_name().ok())
.flatten()
.is_some_and(|name| name.as_slice() == b"XRef")
}
fn push_repair_diagnostics(diagnostics: &mut Diagnostics, trigger_error: &Error, startxref: u64) {
diagnostics.push(Diagnostic::warning("file is damaged", None));
let (message, offset) = match trigger_error {
Error::Parse { offset: 0, message } if message == "xref not found" => {
("can't find startxref".to_string(), None)
}
Error::Parse { message, .. } if message == "can't find startxref" => {
(message.clone(), None)
}
Error::Parse { message, .. } if message == "loop detected following xref tables" => {
(message.clone(), None)
}
Error::Parse { offset, message } if is_classic_trailer_validation_message(message) => {
(format!("(trailer, offset {offset}): {message}"), None)
}
Error::Parse { offset, message }
if message.starts_with("unknown xref stream entry type ") =>
{
(format!("(xref stream, offset {offset}): {message}"), None)
}
Error::Parse { offset, message } => (message.clone(), Some(*offset as u64)),
Error::Io(error) => (format!("error reading xref: {error}"), None),
other => (other.to_string(), Some(startxref)),
};
diagnostics.push(Diagnostic::warning(message, offset));
diagnostics.push(Diagnostic::warning(
"Attempting to reconstruct cross-reference table",
None,
));
}
fn is_classic_trailer_validation_message(message: &str) -> bool {
matches!(
message,
"trailer dictionary lacks /Size key"
| "/Size key in trailer dictionary is not an integer"
| "/Prev key in trailer dictionary is not an integer"
)
}
fn parse_trailer_candidate(bytes: &[u8], start: usize) -> (Option<ObjectHandle>, Vec<Diagnostic>) {
let Some(slice) = bytes.get(start..) else {
return (None, Vec::new());
};
let mut resolver = XrefDetachedHandles;
let result = parse_qpdf_direct_object_handle_with_diagnostics(
slice,
i64::try_from(start).unwrap_or(i64::MAX),
None,
&mut resolver,
);
let (trailer, parser_diagnostics) = match result {
Ok((value, _, diagnostics)) => {
let handle = ObjectHandle::from_value(value);
(
handle
.try_is_dictionary()
.ok()
.filter(|is_dict| *is_dict)
.map(|_| handle),
diagnostics,
)
}
Err(_) => (None, Vec::new()), };
let diagnostics = trailer_diagnostics(start, parser_diagnostics);
(trailer, diagnostics)
}
fn trailer_diagnostics(start: usize, diagnostics: Vec<ParserDiagnostic>) -> Vec<Diagnostic> {
diagnostics
.into_iter()
.map(|diagnostic| {
let offset = (start as u64).saturating_add(diagnostic.relative_offset as u64);
Diagnostic::warning(
format!("(trailer, offset {offset}): {}", diagnostic.message),
Some(offset),
)
})
.collect()
}
fn next_line_start(bytes: &[u8], from: usize) -> usize {
let mut pos = from;
while pos < bytes.len() && !matches!(bytes[pos], b'\n' | b'\r') {
pos += 1;
}
while pos < bytes.len() && matches!(bytes[pos], b'\n' | b'\r') {
pos += 1;
}
pos
}
const XREF_RECONSTRUCTION_MAX_TOKEN_LEN: usize = 100;
fn read_scan_token(bytes: &[u8], from: usize, limit: usize) -> Option<Token> {
let bounded = bytes.get(..limit)?;
let mut tokenizer = Tokenizer::new(bounded);
tokenizer.allow_eof();
tokenizer.set_position(from).ok()?;
let token = tokenizer
.read_token(true, XREF_RECONSTRUCTION_MAX_TOKEN_LEN)
.ok()?;
(token.token_type != TokenType::Eof && token.start < limit).then_some(token)
}
fn parse_scan_integer(token: &Token) -> Option<i64> {
if !token.is_integer() {
return None;
}
std::str::from_utf8(&token.value).ok()?.parse().ok()
}
fn scan_object_header_after_first_token(
bytes: &[u8],
number_token: &Token,
) -> Option<(ObjectRef, u64)> {
let obj = parse_scan_integer(number_token)?;
let gen_token = read_scan_token(bytes, number_token.end, bytes.len())?;
let gen = parse_scan_integer(&gen_token)?;
let obj_token = read_scan_token(bytes, gen_token.end, bytes.len())?;
if !obj_token.is_word_value(b"obj") {
return None;
}
if obj <= 0 || !(0..65535).contains(&gen) {
return None;
}
let number = u32::try_from(obj).ok()?;
let generation = u16::try_from(gen).ok()?;
Some((
ObjectRef::new(number, generation),
number_token.start as u64,
))
}
fn parse_xref_table(
cursor: &mut ByteCursor<'_>,
bytes: &[u8],
_error_diagnostics_sink: Option<&mut Diagnostics>,
mut first_xref_item_offset_sink: Option<&mut Option<u64>>,
) -> Result<(Vec<ParsedXrefEntry>, usize, Vec<Diagnostic>, u64)> {
let mut entries = Vec::new();
let mut first_xref_item_offset = 0;
loop {
let first_token = cursor.read_token()?;
if first_token.is_word_value(b"trailer") {
break;
}
let first = parse_xref_subsection_u32(&first_token)?;
let count = cursor.read_u32()?;
if first == 0 && count > 0 {
cursor.skip_ws();
first_xref_item_offset = cursor.pos as u64;
if let Some(sink) = first_xref_item_offset_sink.as_deref_mut() {
*sink = Some(first_xref_item_offset);
}
}
for index in 0..count {
cursor.skip_ws();
let offset = cursor.read_fixed_u64(10)?;
cursor.skip_ws();
let generation = cursor.read_fixed_u16(5)?;
cursor.skip_ws();
let in_use = cursor.read_byte()?;
cursor.skip_line();
match in_use {
b'f' => {
let _next = offset;
entries.push(ParsedXrefEntry::Free {
object_ref: ObjectRef::new(first + index, generation),
});
}
b'n' => {
entries.push(ParsedXrefEntry::Live {
object_ref: ObjectRef::new(first + index, generation),
entry: XrefEntry::Uncompressed { offset },
});
}
_ => return Err(Error::parse(0, "xref table entry status is not f or n")),
}
}
}
let trailer_start = cursor.pos;
let _ = bytes;
Ok((entries, trailer_start, Vec::new(), first_xref_item_offset))
}
#[allow(clippy::too_many_arguments)]
fn parse_xref_stream(
bytes: &[u8],
xref_pos: usize,
startxref: u64,
version: String,
options: XrefLoadOptions,
registration: &mut XrefRegistration,
error_diagnostics_sink: Option<&mut Diagnostics>,
context_spec: XrefReadContextSpec<'_>,
) -> Result<LoadedXrefState> {
if options.ignore_xref_streams {
return Err(Error::parse(xref_pos, "xref not found"));
}
let allow_repair = options.allow_repair;
let tail = bytes
.get(xref_pos..)
.filter(|slice| !slice.is_empty())
.ok_or_else(|| Error::parse(xref_pos, "xref stream offset is beyond end of file"))?;
let policy = if allow_repair {
RecoveryPolicy::Bounded
} else {
RecoveryPolicy::RequireEndstream
};
let mut repair_diagnostics = Diagnostics::default();
let (build_result, reconstruction_trigger, bootstrap_cache) = {
let mut context = XrefReadContext::new(bytes, context_spec, registration, options);
let mut handle_completed = match context.read_file_object_handle(
tail,
xref_pos as u64,
policy,
XrefObjectDescription::XrefStream,
) {
Ok(completed) => completed,
Err(error) => {
context.append_diagnostics_to(&mut repair_diagnostics);
if let Some(sink) = error_diagnostics_sink {
for diagnostic in repair_diagnostics.entries() {
sink.push(diagnostic.clone());
}
}
return Err(error.rebase_offset(xref_pos));
}
};
let _recovered_handle_eol = handle_completed.remove_included_recovery_eol_for_decryption();
let stream_data_offset = handle_completed
.stream_data_offset
.map(|offset| xref_pos.saturating_add(offset));
let handle_object = handle_completed.object;
let object_ref = handle_completed.object_ref;
for diagnostic in &handle_completed.diagnostics {
context.diagnostics.push(xref_file_object_diagnostic(
XrefObjectDescription::XrefStream,
object_ref,
xref_pos as u64,
diagnostic.clone(),
));
}
let mut build = || -> Result<XrefStreamBuild> {
let handle_stream_dict = handle_object
.as_stream_dict()
.ok_or_else(|| Error::parse(xref_pos, "xref not found"))?;
if !is_xref_stream_handle(&mut context, &handle_object)? {
return Err(Error::parse(xref_pos, "xref not found"));
}
let trailer = handle_stream_dict.clone();
let size_value = handle_stream_dict
.as_dictionary()
.and_then(|entries| entries.get(b"/Size".as_slice()).cloned())
.ok_or(Error::Missing("XRef stream /Size"))?;
context.ensure_source_for_resolution(&size_value);
let size = parse_non_negative_u64_handle(&size_value, "/Size")?;
let size =
u32::try_from(size).map_err(|_| Error::parse(0, "/Size does not fit u32"))?;
let widths = parse_xref_widths_handle(&mut context, &handle_stream_dict)?;
let index = parse_xref_index_handle(&mut context, &handle_stream_dict, size)?;
let ranges = build_xref_ranges(index)?;
let has_first_xref_item = ranges.iter().any(|&(start, count)| start == 0 && count > 0);
let handle_stream_data = handle_object
.as_stream_data()
.ok_or_else(|| Error::parse(xref_pos, "xref stream has no data"))?;
let stream_dictionary = handle_stream_dict.as_dictionary();
if let Some(filter) = stream_dictionary
.as_ref()
.and_then(|dictionary| dictionary.get(b"/Filter".as_slice()))
{
context.ensure_source_for_resolution(filter);
}
if let Some(decode_parms) = stream_dictionary
.as_ref()
.and_then(|dictionary| dictionary.get(b"/DecodeParms".as_slice()))
{
context.ensure_source_for_resolution(decode_parms);
}
let stream_data = filters::decode_stream_data_from_handle(
&handle_stream_dict,
&handle_stream_data,
filters::DecodeLimits::default(),
)?;
let entry_size = widths
.0
.checked_add(widths.1)
.and_then(|size| size.checked_add(widths.2))
.ok_or_else(|| Error::parse(xref_pos, "xref stream entry size overflow"))?;
let expected_size = ranges.iter().try_fold(0usize, |total, &(_, count)| {
let count = usize::try_from(count)
.map_err(|_| Error::parse(xref_pos, "xref stream entry count overflow"))?;
let range_size = entry_size.checked_mul(count).ok_or_else(|| {
Error::parse(xref_pos, "xref stream data size calculation overflow")
})?; total.checked_add(range_size).ok_or_else(|| {
Error::parse(xref_pos, "xref stream data size calculation overflow")
}) })?;
if stream_data.len() < expected_size {
return Err(Error::parse(
xref_pos,
format!(
"Cross-reference stream data has the wrong size; expected = {expected_size}; actual = {}",
stream_data.len()
),
));
}
if stream_data.len() > expected_size {
context.diagnostics.push(Diagnostic::warning(
format!(
"(xref stream, offset {xref_pos}): Cross-reference stream data has the wrong size; expected = {expected_size}; actual = {}",
stream_data.len()
),
None,
));
}
let mut cursor = ByteCursor::new(&stream_data, 0);
let entries = parse_xref_entries(
&mut cursor,
size,
&ranges,
widths,
stream_data_offset,
registration,
)?;
let trailer_references = collect_trailer_references(&trailer);
Ok((trailer, entries, trailer_references, has_first_xref_item))
};
let build_result = build().map(
|(trailer, entries, trailer_references, has_first_xref_item)| {
(
object_ref,
handle_object.clone(),
trailer,
entries,
trailer_references,
has_first_xref_item,
)
},
);
let reconstruction_trigger = context.take_reconstruction_trigger();
context.append_diagnostics_to(&mut repair_diagnostics);
context.cache.commit();
let bootstrap_cache = context.cache.shared();
(build_result, reconstruction_trigger, bootstrap_cache)
};
let (object_ref, handle_object, trailer, entries, trailer_references, has_first_xref_item) =
match build_result {
Ok(built) => built,
Err(error) => {
let error = reconstruction_trigger.unwrap_or(error);
if let Some(sink) = error_diagnostics_sink {
for diagnostic in repair_diagnostics.entries() {
sink.push(diagnostic.clone());
}
} return Err(error);
}
};
for entry in entries {
match entry {
ParsedXrefEntry::Live { object_ref, entry } => {
registration.insert_xref_entry(object_ref, entry);
}
ParsedXrefEntry::Free { object_ref } => {
registration.insert_free_xref_entry(object_ref);
}
}
}
let parsed_xref_streams = BTreeMap::from([(object_ref, handle_object)]);
let state = LoadedXrefState {
loaded: LoadedXref {
version,
startxref,
entries: registration.snapshot(),
trailer,
last_xref_form: XrefForm::Stream,
repair_diagnostics,
},
first_xref_item_offset: if has_first_xref_item {
xref_pos as u64
} else {
0
},
classic_trailer_offset: None,
pending_reconstruction_trigger: None,
trailer_references,
parsed_xref_streams,
bootstrap_cache,
header_offset: 0,
already_reconstructed: false,
};
if let Some(error) = reconstruction_trigger {
if let Some(sink) = error_diagnostics_sink {
for diagnostic in state.loaded.repair_diagnostics.entries() {
sink.push(diagnostic.clone());
}
}
return Err(error);
}
Ok(state)
}
fn xref_file_object_diagnostic(
description: XrefObjectDescription,
object_ref: ObjectRef,
offset: u64,
diagnostic: FileObjectDiagnostic,
) -> Diagnostic {
Diagnostic::warning(
format!(
"({}object {} {}, offset {}): {}",
description.warning_prefix(),
object_ref.number,
object_ref.generation,
offset.saturating_add(diagnostic.relative_offset as u64),
diagnostic.kind.message()
),
Some(offset.saturating_add(diagnostic.relative_offset as u64)),
)
}
type XrefWidths = (usize, usize, usize);
type XrefStreamBuild = (
ObjectHandle,
Vec<ParsedXrefEntry>,
BTreeSet<ObjectRef>,
bool,
);
const MAX_XREF_FIELD_WIDTH: usize = std::mem::size_of::<i64>();
fn is_xref_stream_handle(context: &mut XrefReadContext<'_>, stream: &ObjectHandle) -> Result<bool> {
let Some(stream_dict) = stream.as_stream_dict() else {
return Ok(false);
};
context.ensure_source_for_resolution(&stream_dict);
let type_value = stream_dict.try_get_key(b"/Type")?;
context.ensure_source_for_resolution(&type_value);
Ok(type_value.try_as_name()?.as_deref() == Some(b"XRef"))
}
fn parse_xref_widths_handle(
context: &mut XrefReadContext<'_>,
stream_dict: &ObjectHandle,
) -> Result<XrefWidths> {
context.ensure_source_for_resolution(stream_dict);
let value = stream_dict
.as_dictionary()
.and_then(|entries| entries.get(b"/W".as_slice()).cloned())
.ok_or(Error::Missing("XRef stream /W"))?;
context.ensure_source_for_resolution(&value);
let values = value
.try_as_array()?
.ok_or_else(|| Error::parse(0, "/W must be array"))?;
if values.len() != 3 {
return Err(Error::parse(0, "/W must contain three integers"));
}
for value in &values {
context.ensure_source_for_resolution(value);
}
let w0 = parse_usize(parse_non_negative_u64_handle(&values[0], "/W[0]")?, "/W[0]")?;
let w1 = parse_usize(parse_non_negative_u64_handle(&values[1], "/W[1]")?, "/W[1]")?;
let w2 = parse_usize(parse_non_negative_u64_handle(&values[2], "/W[2]")?, "/W[2]")?;
if w0 > MAX_XREF_FIELD_WIDTH || w1 > MAX_XREF_FIELD_WIDTH || w2 > MAX_XREF_FIELD_WIDTH {
return Err(Error::parse(
0,
"Cross-reference stream's /W contains impossibly large values",
));
}
Ok((w0, w1, w2))
}
fn parse_xref_index_handle(
context: &mut XrefReadContext<'_>,
stream_dict: &ObjectHandle,
size: u32,
) -> Result<Vec<u32>> {
context.ensure_source_for_resolution(stream_dict);
let value = stream_dict.try_get_key(b"/Index")?;
context.ensure_source_for_resolution(&value);
if value.try_is_null()? {
return Ok(vec![0, size]);
}
let values = value
.try_as_array()?
.ok_or_else(|| Error::parse(0, "/Index must be array"))?;
if values.len() % 2 != 0 {
return Err(Error::parse(
0,
"/Index must contain an even number of integers",
));
}
values
.iter()
.map(|value| {
context.ensure_source_for_resolution(value);
parse_non_negative_u64_handle(value, "/Index").and_then(|integer| {
integer
.try_into()
.map_err(|_| Error::parse(0, "xref /Index value must fit u32"))
})
})
.collect()
}
fn parse_non_negative_u64_handle(value: &ObjectHandle, name: &str) -> Result<u64> {
let integer = value
.try_as_integer()?
.ok_or_else(|| Error::parse(0, format!("{name} is not integer")))?;
if integer < 0 {
return Err(Error::parse(0, format!("{name} is negative")));
}
Ok(integer as u64)
}
fn build_xref_ranges(index: Vec<u32>) -> Result<Vec<(u32, u32)>> {
let mut ranges = Vec::with_capacity(index.len() / 2);
for chunk in index.chunks_exact(2) {
if chunk[1] == 0 {
continue;
}
ranges.push((chunk[0], chunk[1]));
}
Ok(ranges)
}
fn parse_xref_entries(
cursor: &mut ByteCursor<'_>,
size: u32,
ranges: &[(u32, u32)],
widths: XrefWidths,
stream_data_offset: Option<usize>,
registration: &mut XrefRegistration,
) -> Result<Vec<ParsedXrefEntry>> {
let (w0, w1, w2) = widths;
let entry_width = w0 + w1 + w2;
if entry_width == 0 {
return Err(Error::parse(0, "invalid cross-reference stream widths"));
}
let mut entries = Vec::new();
for &(start, count) in ranges {
let start =
usize::try_from(start).map_err(|_| Error::parse(0, "object number too large"))?;
let count = usize::try_from(count).map_err(|_| Error::parse(0, "range count too large"))?;
for index in 0..count {
if start + index >= usize::try_from(size).unwrap_or(usize::MAX) {
return Err(Error::parse(0, "xref range exceeds /Size"));
}
if cursor.pos + entry_width > cursor.bytes.len() {
return Err(Error::parse(cursor.pos, "xref stream data truncated"));
}
let object_type = if w0 == 0 {
1
} else {
let value = cursor.read_be_u64(w0)?;
u8::try_from(value).map_err(|_| {
Error::parse(cursor.pos, "xref stream object type does not fit u8")
})?
};
let field1 = if w1 == 0 { 0 } else { cursor.read_be_u64(w1)? };
let field2 = if w2 == 0 { 0 } else { cursor.read_be_u64(w2)? };
let object_number = (start + index) as u32;
let object_ref = match object_type {
0 | 2 => ObjectRef::new(object_number, 0),
_ => ObjectRef::new(
object_number,
u16::try_from(field2)
.map_err(|_| Error::parse(0, "generation does not fit u16"))?,
),
};
if registration.deleted_objects.contains(&object_number)
|| registration.entries.contains_key(&object_ref)
{
continue;
}
match object_type {
0 => {
let _next = field1;
let _generation = field2;
registration.insert_free_xref_entry(object_ref);
entries.push(ParsedXrefEntry::Free { object_ref });
}
1 => {
let entry = XrefEntry::Uncompressed { offset: field1 };
registration.insert_xref_entry(object_ref, entry);
entries.push(ParsedXrefEntry::Live { object_ref, entry });
}
2 => {
let stream = u32::try_from(field1).map_err(|_| {
Error::parse(0, "xref stream object number does not fit u32")
})?;
let index = u32::try_from(field2)
.map_err(|_| Error::parse(0, "xref stream index does not fit u32"))?;
let entry = XrefEntry::Compressed { stream, index };
registration.insert_xref_entry(object_ref, entry);
entries.push(ParsedXrefEntry::Live { object_ref, entry });
}
_ => {
registration
.entries
.insert(object_ref, XrefEntry::Free { next: 0 });
let offset = stream_data_offset.unwrap_or_default();
return Err(Error::parse(
offset,
format!("unknown xref stream entry type {object_type}"),
));
}
}
}
}
Ok(entries)
}
fn parse_usize(value: u64, name: &str) -> Result<usize> {
usize::try_from(value).map_err(|_| Error::parse(0, format!("{name} does not fit usize")))
}
fn parse_header(bytes: &[u8]) -> Result<String> {
if !bytes.starts_with(b"%PDF-") {
return Err(Error::parse(0, "missing PDF header"));
}
let end = bytes
.iter()
.position(|byte| *byte == b'\n' || *byte == b'\r')
.unwrap_or(bytes.len());
let header = std::str::from_utf8(&bytes[5..end])
.map_err(|_| Error::parse(5, "PDF version is not utf-8"))?;
Ok(header.to_string())
}
fn find_qpdf_header(bytes: &[u8]) -> Option<(usize, String)> {
let search_end = bytes.len().min(1024);
(0..search_end).find_map(|offset| {
if !bytes[offset..].starts_with(b"%PDF-") {
return None;
}
parse_qpdf_header_version(&bytes[offset..]).map(|version| (offset, version))
})
}
fn parse_qpdf_header_version(bytes: &[u8]) -> Option<String> {
let line = &bytes[..bytes.len().min(1024)];
let line_end = line
.iter()
.position(|byte| *byte == b'\n' || *byte == b'\r')
.unwrap_or(line.len());
let version = line.get(5..line_end)?;
let major_end = version
.iter()
.position(|byte| !byte.is_ascii_digit())
.unwrap_or(version.len());
let minor_start = major_end.checked_add(1)?;
if major_end == 0
|| version.get(major_end) != Some(&b'.')
|| !version.get(minor_start).is_some_and(u8::is_ascii_digit)
{
return None;
}
let minor_len = version[minor_start..]
.iter()
.take_while(|byte| byte.is_ascii_digit())
.count();
let version_end = minor_start + minor_len;
Some(
std::str::from_utf8(&version[..version_end])
.expect("validated PDF version contains only ASCII digits and a dot")
.to_string(),
)
}
fn parse_startxref(bytes: &[u8]) -> Result<u64> {
let marker = b"startxref";
let search_start = bytes.len().saturating_sub(1054);
let Some(relative_pos) = bytes[search_start..]
.windows(marker.len())
.rposition(|window| window == marker)
else {
return Err(Error::parse(0, "can't find startxref"));
};
let pos = search_start + relative_pos;
let mut cursor = ByteCursor::new(bytes, pos + marker.len());
let token = cursor.read_token()?;
if !token.is_integer() {
return Err(Error::parse(0, "can't find startxref"));
}
let text = std::str::from_utf8(&token.value)
.map_err(|_| Error::parse(token.start, "number is not utf-8"))?;
text.parse::<u64>()
.map_err(|_| Error::parse(token.start, "invalid unsigned integer"))
}
struct ByteCursor<'a> {
bytes: &'a [u8],
pos: usize,
}
impl<'a> ByteCursor<'a> {
fn new(bytes: &'a [u8], pos: usize) -> Self {
Self { bytes, pos }
}
fn skip_ws(&mut self) {
while matches!(
self.bytes.get(self.pos),
Some(b'\0' | b'\t' | b'\n' | b'\x0c' | b'\r' | b' ')
) {
self.pos += 1;
}
}
fn skip_line(&mut self) {
while !matches!(self.bytes.get(self.pos), None | Some(b'\n' | b'\r')) {
self.pos += 1;
}
while matches!(self.bytes.get(self.pos), Some(b'\n' | b'\r')) {
self.pos += 1;
}
}
fn read_byte(&mut self) -> Result<u8> {
let Some(byte) = self.bytes.get(self.pos).copied() else {
return Err(Error::parse(self.pos, "unexpected end of input"));
};
self.pos += 1;
Ok(byte)
}
fn read_u32(&mut self) -> Result<u32> {
let token = self.read_token()?;
parse_xref_subsection_u32(&token)
}
fn read_token(&mut self) -> Result<Token> {
let mut tokenizer = Tokenizer::new(self.bytes);
tokenizer.allow_eof();
tokenizer.set_position(self.pos)?;
let token = tokenizer.read_token(false, 0)?;
self.pos = tokenizer.position();
Ok(token)
}
fn read_fixed_u64(&mut self, width: usize) -> Result<u64> {
self.read_fixed(width)?
.parse::<u64>()
.map_err(|_| Error::parse(self.pos, "invalid fixed-width u64"))
}
fn read_be_u64(&mut self, width: usize) -> Result<u64> {
if self.pos + width > self.bytes.len() {
return Err(Error::parse(self.pos, "unexpected end of stream field"));
}
let mut value = 0u64;
for _ in 0..width {
value = (value << 8) | u64::from(self.bytes[self.pos]);
self.pos += 1;
}
Ok(value)
}
fn read_fixed_u16(&mut self, width: usize) -> Result<u16> {
self.read_fixed(width)?
.parse::<u16>()
.map_err(|_| Error::parse(self.pos, "invalid fixed-width u16"))
}
fn read_fixed(&mut self, width: usize) -> Result<&str> {
if self.pos + width > self.bytes.len() {
return Err(Error::parse(
self.pos,
"unexpected end of fixed-width field",
));
}
let text = std::str::from_utf8(&self.bytes[self.pos..self.pos + width])
.map_err(|_| Error::parse(self.pos, "field is not utf-8"))?;
self.pos += width;
Ok(text)
}
}
fn parse_xref_subsection_u32(token: &Token) -> Result<u32> {
if !token.is_integer() || !token.value.iter().all(u8::is_ascii_digit) {
return Err(Error::parse(token.start, "expected unsigned integer"));
}
let value = std::str::from_utf8(&token.value)
.map_err(|_| Error::parse(token.start, "number is not utf-8"))?
.parse::<u64>()
.map_err(|_| Error::parse(token.start, "invalid unsigned integer"))?;
u32::try_from(value).map_err(|_| Error::parse(token.start, "number does not fit u32"))
}
#[cfg(test)]
mod final_handle_tests {
use super::*;
use std::time::{Duration, Instant};
fn malformed_candidate_fixture(count: usize) -> Vec<u8> {
let mut bytes = b"%PDF-1.5\n".to_vec();
for index in 1..=count {
let target = count + index;
bytes.extend_from_slice(
format!(
"{index} 0 obj\n<< /Type {target} 0 R /Length 0 /W [1 1 1] /Size 0 >>\nstream\nendstream\nendobj\n"
)
.as_bytes(),
);
}
for index in 1..=count {
let target = count + index;
bytes.extend_from_slice(
format!("{target} 0 obj\n(unterminated target {target}\n").as_bytes(),
);
}
bytes.extend_from_slice(b"\nstartxref\n999999\n%%EOF\n");
bytes
}
fn chained_indirect_length_bootstrap_fixture(
links: u32,
) -> (Vec<u8>, BTreeMap<ObjectRef, XrefEntry>) {
let mut bytes = Vec::from(*b"%PDF-1.4\n");
let mut entries = BTreeMap::new();
for number in 1..=links {
entries.insert(
ObjectRef::new(number, 0),
XrefEntry::Uncompressed {
offset: bytes.len() as u64,
},
);
bytes.extend_from_slice(
format!(
"{number} 0 obj\n<< /Length {} 0 R >>\nstream\n\nendstream\nendobj\n",
number + 1
)
.as_bytes(),
);
}
entries.insert(
ObjectRef::new(links + 1, 0),
XrefEntry::Uncompressed {
offset: bytes.len() as u64,
},
);
bytes.extend_from_slice(format!("{} 0 obj\n0\nendobj\n", links + 1).as_bytes());
(bytes, entries)
}
#[test]
fn bootstrap_long_indirect_length_chain_grows_the_stack_instead_of_aborting() {
#[cfg(windows)]
let stack_size = 32 * 1024 * 1024;
#[cfg(not(windows))]
let stack_size = 256 * 1024;
std::thread::Builder::new()
.stack_size(stack_size)
.spawn(|| {
let (bytes, entries) = chained_indirect_length_bootstrap_fixture(4_000);
let mut context = XrefReadContext::new(
&bytes,
XrefReadContextSpec::ActiveSection,
&XrefRegistration {
entries,
..XrefRegistration::default()
},
XrefLoadOptions::default(),
);
let indirect = context.document.handle_for_reference(ObjectRef::new(1, 0));
let dictionary = ObjectHandle::dictionary(vec![(b"/Size".to_vec(), indirect)]);
let value = context
.resolve_dictionary_value(&dictionary, "Size")
.expect("the chained reference must be resolved");
assert!(value.is_resolved());
assert!(value.as_stream_dict().is_some());
})
.expect("spawn")
.join()
.expect("a 4,000-link bootstrap chain must not overflow a small stack");
}
#[test]
fn bootstrap_document_construction_defers_the_source_snapshot() {
let entries = BTreeMap::new();
let document = BootstrapHandleDocument::new_with_state(
None,
XrefEntryLookup::Registration(&entries),
XrefLoadOptions::default(),
Rc::new(RefCell::new(BootstrapHandleState::default())),
);
assert!(
document.bytes.get().is_none(),
"constructing the bootstrap owner must not copy the complete input"
);
}
fn timed_malformed_candidate_recovery(count: usize) -> Duration {
let bytes = malformed_candidate_fixture(count);
let started = Instant::now();
let error = load_xref_and_trailer_with_repair(&mut std::io::Cursor::new(&bytes), true)
.expect_err("the fixture has no trailer dictionary");
assert!(error
.to_string()
.contains("unable to find trailer dictionary while recovering damaged file"));
started.elapsed()
}
#[test]
fn reconstruction_bounds_referenced_reads_for_malformed_candidates() {
let small = timed_malformed_candidate_recovery(500);
let large = timed_malformed_candidate_recovery(5_000);
assert!(
large < small * 20 + Duration::from_millis(200),
"elapsed time scaled worse than the bounded-window guard allows: \
small (500 candidates) = {small:?}, large (5,000 candidates) = {large:?}"
);
}
#[test]
fn reconstruction_reference_read_window_uses_adjacent_offset_and_fallback_positions() {
let mut entries = BTreeMap::new();
for index in 0..=70 {
let object_ref = ObjectRef::new(index + 1, 0);
entries.insert(
object_ref,
XrefEntry::Uncompressed {
offset: u64::from(index) * 10,
},
);
}
entries.insert(ObjectRef::new(100, 0), XrefEntry::Free { next: 0 });
entries.insert(
ObjectRef::new(101, 0),
XrefEntry::Compressed {
stream: 1,
index: 0,
},
);
let reference_offsets = reconstructed_reference_offsets(&entries);
let context = XrefReadContext::new(
&[0; 800],
XrefReadContextSpec::Reconstruction {
line_scan_entries: &entries,
reference_offsets: &reference_offsets,
},
&XrefRegistration::default(),
XrefLoadOptions::default(),
);
assert_eq!(
context.document.reference_read_window(10, 800),
ReferenceReadWindow {
end: 20,
fallback_end: 660,
}
);
let shared = empty_bootstrap_cache();
let cached_context = XrefReadContext::new(
&[0; 800],
XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries: &entries,
reference_offsets: &reference_offsets,
bootstrap_cache: &shared,
},
&XrefRegistration::default(),
XrefLoadOptions::default(),
);
assert_eq!(
cached_context.document.reference_read_window(10, 800),
ReferenceReadWindow {
end: 20,
fallback_end: 660,
}
);
let context_offsets = context
.document
.reference_offsets
.borrow()
.as_ref()
.expect("reconstruction context must retain its offset index")
.clone();
let cached_offsets = cached_context
.document
.reference_offsets
.borrow()
.as_ref()
.expect("cached reconstruction context must retain its offset index")
.clone();
assert!(Rc::ptr_eq(&context_offsets, &reference_offsets));
assert!(Rc::ptr_eq(&cached_offsets, &reference_offsets));
}
#[test]
fn active_reference_read_window_keeps_qpdf_unbounded_view() {
let context = XrefReadContext::new(
&[0; 800],
XrefReadContextSpec::ActiveSection,
&XrefRegistration::default(),
XrefLoadOptions::default(),
);
assert_eq!(
context.document.reference_read_window(10, 800),
ReferenceReadWindow {
end: 800,
fallback_end: 800,
}
);
}
#[test]
fn reconstruction_reference_read_retries_with_the_controlled_fallback_window() {
let mut bytes = b"1 0 obj\n<< /Length 27 >>\nstream\n".to_vec();
bytes.extend_from_slice(b"aaaa\n2 0 obj\nbbbbbbbbbbbbb\n");
let stream_length = b"aaaa\n2 0 obj\nbbbbbbbbbbbbb\n".len();
bytes.extend_from_slice(b"endstream\nendobj\n");
assert_eq!(stream_length, 27);
let first_ref = ObjectRef::new(1, 0);
let second_ref = ObjectRef::new(2, 0);
let second_offset = b"1 0 obj\n<< /Length 27 >>\nstream\n".len() + b"aaaa\n".len();
let mut entries = BTreeMap::new();
entries.insert(first_ref, XrefEntry::Uncompressed { offset: 0 });
entries.insert(
second_ref,
XrefEntry::Uncompressed {
offset: second_offset as u64,
},
);
let reference_offsets = reconstructed_reference_offsets(&entries);
let context = XrefReadContext::new(
&bytes,
XrefReadContextSpec::Reconstruction {
line_scan_entries: &entries,
reference_offsets: &reference_offsets,
},
&XrefRegistration::default(),
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
);
context.document.ensure_source_bytes(&bytes);
let (value, _) = context
.document
.read_uncompressed_object(first_ref, 0)
.expect("the fallback window must recover a valid stream");
let ObjectValue::Stream { stream_data, .. } = value else {
panic!("expected a recovered stream, got {value:?}"); };
assert_eq!(
stream_data.as_deref().map(Vec::len),
Some(stream_length),
"the fallback window must recover the stream's real content, not an empty placeholder"
);
}
#[test]
fn xref_candidate_retry_widens_a_literal_truncated_by_a_false_header() {
let mut bytes = b"%PDF-1.4\n".to_vec();
let first_offset = bytes.len();
bytes.extend_from_slice(
b"1 0 obj\n<< /Type /XRef /W [1 0 1] /Size 1 /Length 4 /Info (prefix\n",
);
let false_header_offset = bytes.len();
bytes.extend_from_slice(b"2 0 obj\nsuffix) >>\nstream\nabcd\nendstream\nendobj\n%%EOF\n");
let first_ref = ObjectRef::new(1, 0);
let false_ref = ObjectRef::new(2, 0);
let mut entries = BTreeMap::new();
entries.insert(
first_ref,
XrefEntry::Uncompressed {
offset: first_offset as u64,
},
);
entries.insert(
false_ref,
XrefEntry::Uncompressed {
offset: false_header_offset as u64,
},
);
let reference_offsets = reconstructed_reference_offsets(&entries);
let (candidate, diagnostics) = find_xref_stream_trailer_candidate(
&bytes,
&entries,
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
&reference_offsets,
None,
);
let candidate = candidate.expect("wide retry must recover the xref stream candidate");
assert_eq!(candidate.max_offset, first_offset as u64);
assert_eq!(
candidate
.trailer
.try_get_key(b"/Type")
.expect("candidate dictionary has a /Type key")
.as_name(),
Some(b"XRef".to_vec())
);
assert!(
diagnostics
.entries()
.iter()
.all(|diagnostic| !diagnostic.message.starts_with("(object 1 0,")),
"the discarded narrow attempt's diagnostics must not leak once the \
wider retry recovers the candidate: {diagnostics:?}"
);
}
#[test]
fn xref_candidate_scan_keeps_the_diagnostics_of_an_unretryable_recovered_null() {
let mut bytes = b"%PDF-1.4\n".to_vec();
let first_offset = bytes.len();
bytes.extend_from_slice(b"1 0 obj\n<< /Type /XRef /W [1 0 1] /Size 1 /Info (unterminated");
let first_ref = ObjectRef::new(1, 0);
let mut entries = BTreeMap::new();
entries.insert(
first_ref,
XrefEntry::Uncompressed {
offset: first_offset as u64,
},
);
let reference_offsets = reconstructed_reference_offsets(&entries);
let (candidate, diagnostics) = find_xref_stream_trailer_candidate(
&bytes,
&entries,
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
&reference_offsets,
None,
);
assert!(
candidate.is_none(),
"a null recovered from a truncated object is not an xref-stream candidate"
);
assert!(
diagnostics
.entries()
.iter()
.any(|diagnostic| diagnostic.message.starts_with("(object 1 0,")),
"the accepted recovered-null read must keep its diagnostics: {diagnostics:?}"
);
}
#[test]
fn xref_candidate_scan_commits_the_wide_retry_s_recovered_null_with_its_diagnostics() {
let mut bytes = b"%PDF-1.4\n".to_vec();
let first_offset = bytes.len();
bytes.extend_from_slice(b"1 0 obj\n<< /Type /XRef /Info (prefix\n");
let false_header_offset = bytes.len();
bytes.extend_from_slice(b"2 0 obj\nsuffix) /Deep ");
bytes.extend(std::iter::repeat_n(
b'[',
crate::parser::MAX_PARSE_DEPTH + 1,
));
bytes.extend_from_slice(b"\n%%EOF\n");
let mut entries = BTreeMap::new();
entries.insert(
ObjectRef::new(1, 0),
XrefEntry::Uncompressed {
offset: first_offset as u64,
},
);
entries.insert(
ObjectRef::new(2, 0),
XrefEntry::Uncompressed {
offset: false_header_offset as u64,
},
);
let reference_offsets = reconstructed_reference_offsets(&entries);
let (candidate, diagnostics) = find_xref_stream_trailer_candidate(
&bytes,
&entries,
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
&reference_offsets,
None,
);
assert!(candidate.is_none());
assert!(
diagnostics
.entries()
.iter()
.any(|diagnostic| diagnostic.message.starts_with("(object 1 0,")),
"a wide retry that recovers a null must keep its diagnostics: {diagnostics:?}"
);
}
#[test]
fn reconstruction_reference_read_retries_a_narrow_window_that_recovers_nonempty_but_truncated_data(
) {
let mut bytes = b"1 0 obj\n<< /Length 40 >>\nstream\n".to_vec();
let header_len = bytes.len();
let embedded_false_terminator = b"AAAAAAAAAendobj ";
let mut payload = embedded_false_terminator.to_vec();
payload.resize(40, b'B');
assert_eq!(payload.len(), 40);
bytes.extend_from_slice(&payload);
bytes.extend_from_slice(b"\nendstream\nendobj\n");
let first_ref = ObjectRef::new(1, 0);
let second_ref = ObjectRef::new(2, 0);
let second_offset = header_len + embedded_false_terminator.len();
let mut entries = BTreeMap::new();
entries.insert(first_ref, XrefEntry::Uncompressed { offset: 0 });
entries.insert(
second_ref,
XrefEntry::Uncompressed {
offset: second_offset as u64,
},
);
let reference_offsets = reconstructed_reference_offsets(&entries);
let mut context = XrefReadContext::new(
&bytes,
XrefReadContextSpec::Reconstruction {
line_scan_entries: &entries,
reference_offsets: &reference_offsets,
},
&XrefRegistration::default(),
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
);
context.document.ensure_source_bytes(&bytes);
let (value, _) = context
.document
.read_uncompressed_object(first_ref, 0)
.expect("the fallback window must recover a valid stream");
let ObjectValue::Stream { stream_data, .. } = value else {
panic!("expected a recovered stream, got {value:?}"); };
assert_eq!(
stream_data.as_deref().map(Vec::len),
Some(payload.len()),
"a narrow window's false, non-empty terminator match must not be \
accepted over the wider window's real declared-length boundary"
);
let mut collected = Diagnostics::default();
context.append_diagnostics_to(&mut collected);
assert!(
collected.entries().is_empty(),
"the discarded narrow attempt's recovery diagnostics must not leak \
once the wider retry recovers the stream cleanly: {collected:?}"
);
}
#[test]
fn read_uncompressed_object_pushes_the_accepted_read_s_own_diagnostics() {
let bytes = b"1 0 obj\n<< >>\nstream\nhello\nendstream\nendobj\n".to_vec();
let object_ref = ObjectRef::new(1, 0);
let mut entries = BTreeMap::new();
entries.insert(object_ref, XrefEntry::Uncompressed { offset: 0 });
let mut context = XrefReadContext::new(
&bytes,
XrefReadContextSpec::ActiveSection,
&XrefRegistration::default(),
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
);
context.document.ensure_source_bytes(&bytes);
let (value, _) = context
.document
.read_uncompressed_object(object_ref, 0)
.expect("a missing /Length recovers through the endstream scan");
assert!(matches!(value, ObjectValue::Stream { .. }));
let mut collected = Diagnostics::default();
context.append_diagnostics_to(&mut collected);
assert!(
collected.entries().iter().any(|diagnostic| diagnostic
.message
.contains("stream dictionary lacks /Length key")),
"the accepted read's own diagnostics must reach the shared list: {collected:?}"
);
}
#[test]
fn direct_only_bootstrap_access_does_not_initialize_the_source_snapshot() {
let registration = XrefRegistration::default();
let mut context = XrefReadContext::new(
b"source bytes are not needed",
XrefReadContextSpec::ActiveSection,
®istration,
XrefLoadOptions::default(),
);
let dictionary =
ObjectHandle::dictionary(vec![(b"/Size".to_vec(), ObjectHandle::integer(7))]);
let value = context
.resolve_dictionary_value(&dictionary, "Size")
.expect("direct dictionary value");
assert_eq!(value.try_as_integer().unwrap(), Some(7));
assert!(context.document.bytes.get().is_none());
context.sync_handle_diagnostics();
assert!(context.diagnostics.entries().is_empty());
let input = b"1 0 obj\n<< /Root 2 0 R >>\nendobj\n";
let mut context = XrefReadContext::new(
input,
XrefReadContextSpec::ActiveSection,
®istration,
XrefLoadOptions::default(),
);
let parsed = context
.read_file_object_handle(
input,
0,
RecoveryPolicy::RequireEndstream,
XrefObjectDescription::Ordinary,
)
.expect("a direct dictionary with an unused reference");
assert!(parsed.object.as_dictionary().is_some());
assert!(context.document.bytes.get().is_none());
}
#[test]
fn indirect_bootstrap_resolution_initializes_the_source_snapshot_once() {
let bytes = b"\n1 0 obj\n7\nendobj\n";
let object_ref = ObjectRef::new(1, 0);
let mut registration = XrefRegistration::default();
registration.insert_xref_entry(object_ref, XrefEntry::Uncompressed { offset: 1 });
let mut context = XrefReadContext::new(
bytes,
XrefReadContextSpec::ActiveSection,
®istration,
XrefLoadOptions::default(),
);
let indirect = context.document.handle_for_reference(object_ref);
let dictionary = ObjectHandle::dictionary(vec![(b"/Size".to_vec(), indirect)]);
assert!(context.document.bytes.get().is_none());
let value = context
.resolve_dictionary_value(&dictionary, "Size")
.expect("indirect dictionary value");
assert_eq!(value.try_as_integer().unwrap(), Some(7));
let snapshot = context.document.bytes.get().expect("source snapshot");
let snapshot_ptr = Rc::as_ptr(snapshot);
context.document.ensure_source_bytes(b"different bytes");
assert_eq!(
Rc::as_ptr(context.document.bytes.get().unwrap()),
snapshot_ptr
);
}
#[test]
fn bootstrap_object_read_reports_an_uninitialized_source_snapshot() {
let entries = BTreeMap::new();
let document = BootstrapHandleDocument::new_with_state(
None,
XrefEntryLookup::Registration(&entries),
XrefLoadOptions::default(),
Rc::new(RefCell::new(BootstrapHandleState::default())),
);
let error = document
.read_uncompressed_object(ObjectRef::new(1, 0), 1)
.expect_err("a direct resolver call without a source must fail internally");
assert!(matches!(
error,
Error::Internal(message)
if message == "bootstrap resolver source bytes were not initialized"
));
}
#[test]
fn indirect_stream_length_initializes_the_source_snapshot_before_resolution() {
let mut bytes = b"1 0 obj\n<< /Length 2 0 R >>\nstream\nabc\nendstream\nendobj\n".to_vec();
let object_two_offset = bytes.len();
bytes.extend_from_slice(b"2 0 obj\n3\nendobj\n");
let object_one = ObjectRef::new(1, 0);
let object_two = ObjectRef::new(2, 0);
let mut entries = BTreeMap::new();
entries.insert(
object_two,
XrefEntry::Uncompressed {
offset: object_two_offset as u64,
},
);
let document = BootstrapHandleDocument::new_with_state(
None,
XrefEntryLookup::Registration(&entries),
XrefLoadOptions::default(),
Rc::new(RefCell::new(BootstrapHandleState::default())),
);
assert!(document.bytes.get().is_none());
let result = document
.read_file_object(
&bytes,
0,
RecoveryPolicy::RequireEndstream,
XrefObjectDescription::Ordinary,
&bytes,
)
.expect("indirect stream length should resolve");
assert_eq!(result.object_ref, object_one);
assert_eq!(result.object.as_stream_data().unwrap().as_slice(), b"abc");
assert!(document.bytes.get().is_some());
}
fn classic_xref_with_trailer(trailer: &str) -> (Vec<u8>, usize) {
let mut bytes = b"%PDF-1.4\n".to_vec();
let xref = bytes.len();
bytes.extend_from_slice(b"xref\n0 1\n0000000000 65535 f \ntrailer\n");
bytes.extend_from_slice(trailer.as_bytes());
bytes.extend_from_slice(b"\nstartxref\n");
bytes.extend_from_slice(xref.to_string().as_bytes());
bytes.extend_from_slice(b"\n%%EOF\n");
(bytes, xref)
}
fn trailer_value_offset(bytes: &[u8]) -> usize {
bytes
.windows(b"trailer".len())
.position(|window| window == b"trailer")
.expect("classic fixture has a trailer keyword")
+ b"trailer".len()
}
fn loaded_state_with_trailer(trailer: ObjectHandle) -> LoadedXrefState {
LoadedXrefState {
loaded: LoadedXref {
version: "1.4".to_owned(),
startxref: 0,
entries: BTreeMap::new(),
trailer,
last_xref_form: XrefForm::Table,
repair_diagnostics: Diagnostics::default(),
},
first_xref_item_offset: 0,
classic_trailer_offset: None,
pending_reconstruction_trigger: None,
trailer_references: BTreeSet::new(),
parsed_xref_streams: BTreeMap::new(),
bootstrap_cache: empty_bootstrap_cache(),
header_offset: 0,
already_reconstructed: false,
}
}
fn classic_xref_with_malformed_previous() -> (Vec<u8>, usize) {
let mut bytes = b"%PDF-1.4\n".to_vec();
let object_offset = bytes.len();
bytes.extend_from_slice(b"1 0 obj\n<< /Type /Catalog >>\nendobj\n");
let previous_xref = bytes.len();
bytes.extend_from_slice(b"xref\n0 2\n0000000000 65535 f \n");
bytes.extend_from_slice(format!("{object_offset:010} 00000 n \n").as_bytes());
bytes.extend_from_slice(b"trailer\n<< /Size 2 /Root 1 0 R /Prev (bad) >>\n");
let previous_trailer = trailer_value_offset(&bytes);
let current_xref = bytes.len();
bytes.extend_from_slice(b"xref\n0 2\n0000000000 65535 f \n");
bytes.extend_from_slice(format!("{object_offset:010} 00000 n \n").as_bytes());
bytes.extend_from_slice(
format!("trailer\n<< /Size 2 /Root 1 0 R /Prev {previous_xref} >>\n").as_bytes(),
);
bytes.extend_from_slice(format!("startxref\n{current_xref}\n%%EOF\n").as_bytes());
(bytes, previous_trailer)
}
fn classic_xref_with_indirect_previous_in_older_section() -> Vec<u8> {
let mut bytes = b"%PDF-1.4\n".to_vec();
let object_offset = bytes.len();
bytes.extend_from_slice(b"1 0 obj\n<< /Type /Catalog >>\nendobj\n");
let previous_xref = bytes.len();
bytes.extend_from_slice(b"xref\n0 3\n0000000000 65535 f \n");
bytes.extend_from_slice(format!("{object_offset:010} 00000 n \n").as_bytes());
bytes.extend_from_slice(format!("{object_offset:010} 00000 n \n").as_bytes());
bytes.extend_from_slice(b"trailer\n<< /Size 3 /Root 1 0 R /Prev 2 0 R >>\n");
let current_xref = bytes.len();
bytes.extend_from_slice(b"xref\n0 2\n0000000000 65535 f \n");
bytes.extend_from_slice(format!("{object_offset:010} 00000 n \n").as_bytes());
bytes.extend_from_slice(
format!("trailer\n<< /Size 3 /Root 1 0 R /Prev {previous_xref} >>\n").as_bytes(),
);
bytes.extend_from_slice(format!("startxref\n{current_xref}\n%%EOF\n").as_bytes());
bytes
}
fn assert_strict_classic_trailer_error(trailer: &str, message: &str) {
let (bytes, _) = classic_xref_with_trailer(trailer);
let offset = trailer_value_offset(&bytes);
let mut reader = std::io::Cursor::new(bytes);
let error = load_xref_and_trailer(&mut reader)
.expect_err("strict classic trailer validation must reject the fixture");
assert!(matches!(
error,
Error::Parse {
offset: actual_offset,
message: actual_message,
} if actual_offset == offset && actual_message == message
));
}
#[test]
fn strict_classic_xref_rejects_missing_trailer_size() {
assert_strict_classic_trailer_error(
"<< /Root 1 0 R >>",
"trailer dictionary lacks /Size key",
);
}
#[test]
fn strict_classic_xref_rejects_non_integer_trailer_size() {
assert_strict_classic_trailer_error(
"<< /Size (bad) /Root 1 0 R >>",
"/Size key in trailer dictionary is not an integer",
);
}
#[test]
fn strict_classic_xref_rejects_null_trailer_size() {
assert_strict_classic_trailer_error(
"<< /Size null /Root 1 0 R >>",
"trailer dictionary lacks /Size key",
);
}
#[test]
fn strict_classic_validation_forwards_trailer_parser_diagnostics() {
let (bytes, xref) = classic_xref_with_trailer("<< /Root 1 0 R /Broken >>");
let mut registration = XrefRegistration::default();
let mut diagnostics = Diagnostics::default();
let error = parse_xref_from_start(
&bytes,
xref,
xref as u64,
"1.4",
XrefLoadOptions::default(),
&mut registration,
Some(&mut diagnostics),
XrefReadContextSpec::ActiveSection,
None,
true,
)
.expect_err("missing /Size must remain a strict validation error");
assert!(error
.to_string()
.contains("trailer dictionary lacks /Size key"));
assert!(
diagnostics
.entries()
.iter()
.any(|diagnostic| diagnostic.message.contains("dictionary ended prematurely")),
"strict validation must forward parser diagnostics collected before the error"
);
let mut registration = XrefRegistration::default();
let error = parse_xref_from_start(
&bytes,
xref,
xref as u64,
"1.4",
XrefLoadOptions::default(),
&mut registration,
None,
XrefReadContextSpec::ActiveSection,
None,
true,
)
.expect_err("the same validation error must not require a diagnostic sink");
assert!(error
.to_string()
.contains("trailer dictionary lacks /Size key"));
}
#[test]
fn malformed_xref_stream_framing_error_is_forwarded() {
let mut registration = XrefRegistration::default();
let mut diagnostics = Diagnostics::default();
let error = parse_xref_stream(
b"not an indirect object",
0,
0,
"1.5".to_owned(),
XrefLoadOptions::default(),
&mut registration,
Some(&mut diagnostics),
XrefReadContextSpec::ActiveSection,
)
.expect_err("a malformed xref-stream object header must fail framing");
assert!(matches!(error, Error::Parse { .. }));
assert!(diagnostics.entries().is_empty());
}
#[test]
fn candidate_xref_stream_wrong_size_warning_survives_later_decode_failure() {
let bytes = b"%PDF-1.4\n1 0 obj\n<< /Type /XRef /W [1 0 1] /Size 1 /Length 4 >>\nstream\nabcd\nendstream\nendobj\n%%EOF\n";
let error = load_xref_and_trailer_with_repair(&mut std::io::Cursor::new(bytes), true)
.expect_err("the malformed candidate must fail after warning");
let (_, diagnostics) = error
.open_failure()
.expect("permissive candidate failure carries repair diagnostics");
let messages: Vec<_> = diagnostics
.entries()
.iter()
.map(|diagnostic| diagnostic.message.as_str())
.collect();
assert_eq!(
messages,
vec![
"file is damaged",
"can't find startxref",
"Attempting to reconstruct cross-reference table",
"(xref stream, offset 9): Cross-reference stream data has the wrong size; expected = 2; actual = 4",
]
);
}
#[test]
fn nonzero_xref_stream_decode_warning_is_kept_before_recovery() {
let bytes = b"%PDF-1.4\n1 0 obj\n<< /Type /XRef /W [1 0 1] /Size 1 /Length 4 >>\nstream\nabcd\nendstream\nendobj\nstartxref\n9\n%%EOF\n";
let loaded = load_xref_and_trailer_with_repair(&mut std::io::Cursor::new(bytes), true)
.expect("the reconstruction re-entry must skip the already-registered entry");
let messages: Vec<_> = loaded
.repair_diagnostics
.entries()
.iter()
.map(|diagnostic| diagnostic.message.as_str())
.collect();
assert_eq!(
messages,
vec![
"(xref stream, offset 9): Cross-reference stream data has the wrong size; expected = 2; actual = 4",
"file is damaged",
"(xref stream, offset 71): unknown xref stream entry type 97",
"Attempting to reconstruct cross-reference table",
"(xref stream, offset 9): Cross-reference stream data has the wrong size; expected = 2; actual = 4",
"reported number of objects (1) is not one plus the highest object number (1)",
]
);
}
#[test]
fn xref_stream_previous_offset_validation_ignores_non_integer_prev() {
let trailer = ObjectHandle::dictionary(vec![(
b"/Prev".to_vec(),
ObjectHandle::string(b"bad".to_vec()),
)]);
let registration = XrefRegistration::default();
let (offset, diagnostics, error) = resolve_previous_xref_offset(
b"",
XrefLoadOptions::default(),
®istration,
XrefReadContextSpec::ActiveSection,
&trailer,
None,
)
.expect("xref-stream /Prev validation should be non-fatal");
assert_eq!(offset, None);
assert!(diagnostics.entries().is_empty());
assert!(
error.is_none(),
"xref-stream trailers have no classic trailer error context"
);
}
#[test]
fn previous_xref_merge_propagates_an_uninitialized_trailer_error() {
let mut loaded = LoadedXrefState {
loaded: LoadedXref {
version: "1.4".to_owned(),
startxref: 0,
entries: BTreeMap::new(),
trailer: ObjectHandle::uninitialized(),
last_xref_form: XrefForm::Table,
repair_diagnostics: Diagnostics::default(),
},
first_xref_item_offset: 0,
classic_trailer_offset: None,
pending_reconstruction_trigger: None,
trailer_references: BTreeSet::new(),
parsed_xref_streams: BTreeMap::new(),
bootstrap_cache: empty_bootstrap_cache(),
header_offset: 0,
already_reconstructed: false,
};
let mut registration = XrefRegistration::default();
let error = merge_previous_xref_sections(
b"",
"1.4",
&mut loaded,
XrefLoadOptions::default(),
&mut registration,
None,
XrefReadContextSpec::ActiveSection,
)
.expect_err("an uninitialized trailer must fail at the dictionary boundary");
assert!(matches!(error, Error::Internal(message) if message.contains("uninitialized")));
}
#[test]
fn previous_xref_merge_propagates_an_indirect_prev_resolution_error() {
let bytes = classic_xref_with_indirect_previous_in_older_section();
let current_xref = bytes
.windows(b"startxref\n".len())
.position(|window| window == b"startxref\n")
.expect("current xref has a startxref marker");
let startxref = bytes[current_xref + b"startxref\n".len()..]
.split(|byte| *byte == b'\n')
.next()
.and_then(|value| std::str::from_utf8(value).ok())
.and_then(|value| value.parse::<usize>().ok())
.expect("startxref contains the current xref offset");
let mut registration = XrefRegistration::default();
let mut loaded = parse_xref_from_start(
&bytes,
startxref,
startxref as u64,
"1.4",
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
&mut registration,
None,
XrefReadContextSpec::ActiveSection,
None,
true,
)
.expect("current xref section should parse before walking /Prev");
let error = merge_previous_xref_sections(
&bytes,
"1.4",
&mut loaded,
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
&mut registration,
None,
XrefReadContextSpec::ActiveSection,
)
.expect_err("an indirect /Prev with a stale xref row must propagate its trigger");
assert!(matches!(error, Error::Parse { message, .. } if message == "expected 2 0 obj"));
}
#[test]
fn reconstruction_contexts_retain_the_precomputed_index_across_xref_routes() {
let mut entries = BTreeMap::new();
entries.insert(ObjectRef::new(1, 0), XrefEntry::Uncompressed { offset: 0 });
let reference_offsets = reconstructed_reference_offsets(&entries);
let mut previous_loaded = loaded_state_with_trailer(ObjectHandle::dictionary(vec![(
b"/Size".to_vec(),
ObjectHandle::integer(1),
)]));
let mut registration = XrefRegistration::default();
merge_previous_xref_sections(
b"",
"1.4",
&mut previous_loaded,
XrefLoadOptions::default(),
&mut registration,
None,
XrefReadContextSpec::Reconstruction {
line_scan_entries: &entries,
reference_offsets: &reference_offsets,
},
)
.expect("a reconstruction context without /Prev should be a no-op");
let mut hybrid_loaded = loaded_state_with_trailer(ObjectHandle::dictionary(vec![(
b"/XRefStm".to_vec(),
ObjectHandle::integer(1),
)]));
let result = merge_xref_stream_from_classic_trailer(
b"",
0,
&mut hybrid_loaded,
XrefLoadOptions::default(),
&mut registration,
None,
XrefReadContextSpec::Reconstruction {
line_scan_entries: &entries,
reference_offsets: &reference_offsets,
},
);
assert!(matches!(result, Err(Error::Parse { .. })));
let mut cached_hybrid_loaded = loaded_state_with_trailer(ObjectHandle::dictionary(vec![(
b"/XRefStm".to_vec(),
ObjectHandle::integer(1),
)]));
let shared = empty_bootstrap_cache();
let result = merge_xref_stream_from_classic_trailer(
b"",
0,
&mut cached_hybrid_loaded,
XrefLoadOptions::default(),
&mut registration,
None,
XrefReadContextSpec::ReconstructionWithCache {
line_scan_entries: &entries,
reference_offsets: &reference_offsets,
bootstrap_cache: &shared,
},
);
assert!(matches!(result, Err(Error::Parse { .. })));
}
#[test]
fn strict_classic_xref_rejects_non_integer_previous_offset() {
let (bytes, offset) = classic_xref_with_malformed_previous();
let mut reader = std::io::Cursor::new(bytes);
let error = load_xref_and_trailer(&mut reader)
.expect_err("strict xref chain validation must reject malformed /Prev");
assert!(matches!(
error,
Error::Parse {
offset: actual_offset,
message,
} if actual_offset == offset
&& message == "/Prev key in trailer dictionary is not an integer"
));
}
#[test]
fn repair_mode_reports_classic_trailer_validation_before_recovery() {
let (bytes, _) = classic_xref_with_trailer("<< /Root 1 0 R >>");
let offset = trailer_value_offset(&bytes);
let mut reader = std::io::Cursor::new(bytes);
let loaded = load_xref_and_trailer_with_repair(&mut reader, true)
.expect("repair mode must recover a trailer missing /Size");
let messages: Vec<_> = loaded
.repair_diagnostics
.entries()
.iter()
.map(|diagnostic| diagnostic.message.as_str())
.collect();
assert_eq!(
messages,
vec![
"file is damaged",
&format!("(trailer, offset {offset}): trailer dictionary lacks /Size key"),
"Attempting to reconstruct cross-reference table",
]
);
}
#[test]
fn ordinary_classic_xref_loading_keeps_the_bootstrap_source_lazy() {
let (bytes, _) = classic_xref_with_trailer("<< /Size 1 /Root 1 0 R >>");
let mut reader = std::io::Cursor::new(bytes);
let state = load_xref_state_with_options(&mut reader, XrefLoadOptions::default())
.expect("ordinary classic xref should load");
let cache = state.bootstrap_cache.borrow();
let document = cache
.handle_document
.as_ref()
.expect("handle parser creates the shared bootstrap owner");
assert!(
document.bytes.get().is_none(),
"unused trailer references must not force the source snapshot"
);
}
fn hybrid_xref_with_indirect_filter() -> Vec<u8> {
fn entry(kind: u8, offset: u16, gen: u8) -> [u8; 4] {
let [hi, lo] = offset.to_be_bytes();
[kind, hi, lo, gen]
}
let mut bytes = b"%PDF-1.5\n%\xe2\xe3\xcf\xd3\n".to_vec();
let mut offsets = [0usize; 6];
offsets[1] = bytes.len();
bytes.extend_from_slice(b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n");
offsets[2] = bytes.len();
bytes.extend_from_slice(b"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n");
offsets[3] = bytes.len();
bytes.extend_from_slice(
b"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>\nendobj\n",
);
offsets[4] = bytes.len();
bytes.extend_from_slice(b"4 0 obj\n/ASCIIHexDecode\nendobj\n");
offsets[5] = bytes.len();
let mut raw_entries = Vec::new();
raw_entries.extend_from_slice(&entry(0, 0, 0));
for &offset in &offsets[1..=5] {
raw_entries.extend_from_slice(&entry(1, offset as u16, 0));
}
let mut stream_data = raw_entries
.iter()
.map(|byte| format!("{byte:02X}"))
.collect::<String>()
.into_bytes();
stream_data.push(b'>');
bytes.extend_from_slice(b"5 0 obj\n");
bytes.extend_from_slice(
format!(
"<< /Type /XRef /Filter 4 0 R /W [1 2 1] /Size 6 /Root 1 0 R /Length {} >>",
stream_data.len()
)
.as_bytes(),
);
bytes.extend_from_slice(b"\nstream\n");
bytes.extend_from_slice(&stream_data);
bytes.extend_from_slice(b"\nendstream\nendobj\n");
let xref_stream_offset = offsets[5];
let classic_xref_offset = bytes.len();
bytes.extend_from_slice(b"xref\n0 6\n0000000000 65535 f \n");
for &offset in &offsets[1..=5] {
bytes.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
}
bytes.extend_from_slice(
format!("trailer\n<< /Size 6 /Root 1 0 R /XRefStm {xref_stream_offset} >>\n")
.as_bytes(),
);
bytes.extend_from_slice(format!("startxref\n{classic_xref_offset}\n%%EOF\n").as_bytes());
bytes
}
#[test]
fn hybrid_xref_stream_with_indirect_filter_loads_without_reconstruction() {
let bytes = hybrid_xref_with_indirect_filter();
let mut reader = std::io::Cursor::new(bytes);
let state = load_xref_state_with_options(&mut reader, XrefLoadOptions::default())
.expect("hybrid xref with an indirect /Filter must load");
assert!(
!state.already_reconstructed,
"a valid indirect /Filter on the /XRefStm stream must not force \
cross-reference reconstruction"
);
}
#[test]
fn candidate_recovery_passes_one_offset_index_through_reentry_and_size() {
let bytes = hybrid_xref_with_indirect_filter();
let recovered = recover_xref_entries(&bytes, false).expect("scan candidate entries");
let mut entries = recovered.entries;
let mut parsed_xref_streams = BTreeMap::new();
let mut repair_diagnostics = Diagnostics::default();
let mut trailer_references = BTreeSet::new();
let recovered = recover_trailer_from_xref_stream_candidate(
&bytes,
"1.5",
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
&mut entries,
&mut parsed_xref_streams,
&mut repair_diagnostics,
&mut trailer_references,
None,
)
.expect("the scanned xref-stream candidate must be recoverable");
assert!(recovered.1 > 0);
assert!(!entries.is_empty());
}
#[test]
fn reconstructed_size_revalidation_uses_the_recovery_offset_index() {
let mut bytes = b"%PDF-1.4\n".to_vec();
let object_offset = bytes.len();
bytes.extend_from_slice(b"1 0 obj\n<< /Type /Catalog >>\nendobj\n");
let xref = bytes.len();
bytes.extend_from_slice(b"xref\n0 3\n0000000000 65535 f \n");
bytes.extend_from_slice(format!("{object_offset:010} 00000 n \n").as_bytes());
bytes.extend_from_slice(format!("{object_offset:010} 00000 n \n").as_bytes());
bytes.extend_from_slice(
format!("trailer\n<< /Size 2 0 R /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n")
.as_bytes(),
);
let mut reader = std::io::Cursor::new(bytes);
let mut initial_registration = XrefRegistration::default();
let mut initial_diagnostics = Diagnostics::default();
let mut initial_first_xref_item_offset = None;
let initial = parse_xref_from_start(
reader.get_ref(),
xref,
xref as u64,
"1.4",
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
&mut initial_registration,
Some(&mut initial_diagnostics),
XrefReadContextSpec::ActiveSection,
Some(&mut initial_first_xref_item_offset),
true,
)
.expect("the initial classic section must defer the stale /Size mismatch");
assert!(initial.pending_reconstruction_trigger.is_some());
let recovered = load_xref_state_with_options(
&mut reader,
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
)
.expect("repair mode must complete the recovered size revalidation");
assert!(recovered.already_reconstructed);
assert!(recovered
.loaded
.repair_diagnostics
.entries()
.iter()
.any(|diagnostic| diagnostic.message.contains("expected 2 0 obj")));
}
#[test]
fn bootstrap_trigger_and_diagnostic_append_stay_on_handle_state() {
let object_ref = ObjectRef::new(1, 0);
let mut entries = BTreeMap::new();
entries.insert(object_ref, XrefEntry::Uncompressed { offset: 1 });
let state = Rc::new(RefCell::new(BootstrapHandleState {
reconstruction_trigger: Some((1, "header mismatch".to_owned())),
..BootstrapHandleState::default()
}));
let document = BootstrapHandleDocument::new_with_state(
Some(b"x"),
XrefEntryLookup::Registration(&entries),
XrefLoadOptions::default(),
state,
);
let handle = document.handle_for_reference(object_ref);
let error = <BootstrapHandleDocument as DocumentResolver>::resolve_indirect(
&document, object_ref, &handle,
)
.expect_err("a reconstruction trigger propagates the parse error");
assert!(matches!(error, Error::Parse { .. }));
let registration = XrefRegistration::default();
let mut context = XrefReadContext::new(
b"",
XrefReadContextSpec::ActiveSection,
®istration,
XrefLoadOptions::default(),
);
context
.document
.push_diagnostic(Diagnostic::warning("late diagnostic", None));
let mut diagnostics = Diagnostics::default();
context.append_diagnostics_to(&mut diagnostics);
assert_eq!(diagnostics.entries().len(), 1);
}
#[test]
fn resolve_length_maps_a_reconstruction_trigger_error_to_missing() {
let object_ref = ObjectRef::new(1, 0);
let mut entries = BTreeMap::new();
entries.insert(object_ref, XrefEntry::Uncompressed { offset: 1 });
let state = Rc::new(RefCell::new(BootstrapHandleState {
reconstruction_trigger: Some((1, "header mismatch".to_owned())),
..BootstrapHandleState::default()
}));
let document = BootstrapHandleDocument::new_with_state(
Some(b"x"),
XrefEntryLookup::Registration(&entries),
XrefLoadOptions::default(),
state,
);
assert_eq!(
document.resolve_length(object_ref),
ResolvedStreamLength::Missing
);
}
#[test]
fn classic_xref_validation_covers_non_dictionary_and_invalid_hybrid_trailers() {
let (bytes, xref) = classic_xref_with_trailer("42");
let mut registration = XrefRegistration::default();
let error = parse_xref_from_start(
&bytes,
xref,
xref as u64,
"1.4",
XrefLoadOptions::default(),
&mut registration,
None,
XrefReadContextSpec::ActiveSection,
None,
true,
)
.expect_err("classic trailer must be a dictionary");
assert!(error.to_string().contains("trailer is not a dictionary"));
let (bytes, xref) = classic_xref_with_trailer("<< /Size 1 /XRefStm (bad) >>");
let mut registration = XrefRegistration::default();
let error = parse_xref_from_start(
&bytes,
xref,
xref as u64,
"1.4",
XrefLoadOptions::default(),
&mut registration,
None,
XrefReadContextSpec::ActiveSection,
None,
true,
)
.expect_err("a non-integer hybrid offset is invalid");
assert!(error.to_string().contains("invalid /XRefStm"));
}
#[test]
fn trailer_reference_collection_and_candidate_reader_keep_canonical_handles() {
let stream = ObjectHandle::stream(
ObjectHandle::dictionary(vec![(
b"/Child".to_vec(),
ObjectHandle::new_indirect_unresolved(ObjectRef::new(7, 0), -1),
)]),
Rc::new(b"data".to_vec()),
);
assert!(collect_trailer_references(&stream).contains(&ObjectRef::new(7, 0)));
let registration = XrefRegistration::default();
let bytes = b"1 0 obj\n42\nendobj\n";
let mut context = XrefReadContext::new(
bytes,
XrefReadContextSpec::ActiveSection,
®istration,
XrefLoadOptions::default(),
);
assert!(
read_xref_candidate(&mut context, bytes, 0, bytes.len(), 0, ObjectRef::new(2, 0),)
.is_none()
);
let bytes = b"1 0 obj\n42 extra\nendobj\n";
let mut context = XrefReadContext::new(
bytes,
XrefReadContextSpec::ActiveSection,
®istration,
XrefLoadOptions::default(),
);
let completed =
read_xref_candidate(&mut context, bytes, 0, bytes.len(), 0, ObjectRef::new(1, 0))
.expect("a well-framed object is read before it is committed");
assert!(context.diagnostics.entries().is_empty());
assert!(commit_xref_candidate(&mut context, completed, 0, ObjectRef::new(1, 0)).is_some());
assert!(!context.diagnostics.entries().is_empty());
let bytes = b"1 0 obj\n<< /Info (unterminated";
let mut context = XrefReadContext::new(
bytes,
XrefReadContextSpec::ActiveSection,
®istration,
XrefLoadOptions {
allow_repair: true,
..XrefLoadOptions::default()
},
);
let completed =
read_xref_candidate(&mut context, bytes, 0, bytes.len(), 0, ObjectRef::new(1, 0))
.expect("a truncated object is recovered as a null read");
assert!(is_recovered_null_candidate(&completed));
let cached = commit_xref_candidate(&mut context, completed, 0, ObjectRef::new(1, 0))
.expect("a recovered null is still committed to the cache");
assert!(cached.is_null());
assert!(context.cache.get(&ObjectRef::new(1, 0)).is_some());
assert!(!context.diagnostics.entries().is_empty());
let malformed = vec![b'['; crate::parser::MAX_PARSE_DEPTH + 1];
let (trailer, diagnostics) = parse_trailer_candidate(&malformed, 0);
assert!(trailer.is_none());
assert!(!diagnostics.is_empty());
}
#[test]
fn bootstrap_cache_disconnects_reference_cycles_before_drop() {
let first_ref = ObjectRef::new(1, 0);
let second_ref = ObjectRef::new(2, 0);
let first = ObjectHandle::new_indirect_unresolved(first_ref, -1);
let second = ObjectHandle::new_indirect_unresolved(second_ref, -1);
first.set_resolved(ObjectValue::Dictionary(BTreeMap::from([(
b"/next".to_vec(),
second.clone(),
)])));
second.set_resolved(ObjectValue::Dictionary(BTreeMap::from([(
b"/next".to_vec(),
first.clone(),
)])));
let cache = BootstrapCache {
handle_state: Rc::new(RefCell::new(BootstrapHandleState {
handles: BTreeMap::from([(first_ref, first.clone()), (second_ref, second.clone())]),
..BootstrapHandleState::default()
})),
handle_document: None,
handle_document_owners: Vec::new(),
};
drop(cache);
assert!(!first.is_indirect());
assert!(!second.is_indirect());
}
#[test]
fn bootstrap_cache_cleans_owned_states_and_detaches_stream_values() {
let state = Rc::new(RefCell::new(BootstrapHandleState::default()));
let entries = BTreeMap::new();
let document = BootstrapHandleDocument::new_with_state(
Some(b""),
XrefEntryLookup::Registration(&entries),
XrefLoadOptions::default(),
Rc::clone(&state),
);
let cache = BootstrapCache {
handle_state: state.clone(),
handle_document: None,
handle_document_owners: vec![document],
};
drop(state);
drop(cache);
let stream = ObjectHandle::from_value(ObjectValue::Stream {
stream_dict: ObjectHandle::dictionary(vec![(
b"/Length".to_vec(),
ObjectHandle::integer(0),
)]),
stream_data: Some(Rc::new(Vec::new())),
stream_provider: None,
filter_on_write: true,
stream_length: 0,
});
let detached = detach_bootstrap_handle(&stream).expect("stream detaches");
assert!(detached.get_filter_on_write().expect("stream flag"));
assert_eq!(
detached
.as_stream_dict()
.expect("detached stream dictionary")
.get_key(b"/Length")
.as_integer(),
Some(0)
);
}
}