use crate::page_object_helper::PageObjectHelper;
use crate::qpdf_obj_gen::QpdfObjGen;
use crate::resource_finder::{ResourceFinder, ResourceNamesByType};
use crate::{Error, ObjectHandle, ObjectRef, Pdf, Result};
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::io::{Read, Seek};
type UsedNames = BTreeMap<Vec<u8>, BTreeSet<Vec<u8>>>;
fn diagnostic_count<R: Read + Seek>(pdf: &Pdf<R>) -> usize {
pdf.num_warnings()
}
const BAD_TOKEN_WARNING: &str =
"Bad token found while scanning content stream; not attempting to remove unreferenced objects from this object";
fn warn_resource_parse_failure(handle: &ObjectHandle, parse_error: Option<&str>) -> Result<()> {
if let Some(parse_error) = parse_error {
let warning = format!(
"Unable to parse content stream: {parse_error}; not attempting to remove unreferenced objects from this object"
);
handle.warn_if_possible(&warning)
} else {
handle.warn_if_possible(BAD_TOKEN_WARNING)
}
}
pub(crate) fn remove_unreferenced_resources_on_page<R: Read + Seek>(
pdf: &mut Pdf<R>,
page_ref: ObjectRef,
) -> Result<()> {
let (unresolved, any_failures) = remove_unreferenced_resources_in_form_xobjects(pdf, page_ref)?;
if any_failures {
return Ok(());
}
let diagnostics_before = diagnostic_count(pdf);
let (finder, parse_error, pending_operands) = {
let mut helper = PageObjectHelper::new(page_ref, pdf);
let mut finder = ResourceFinder::default();
let parse_error = helper
.parse_contents(&mut finder)
.err()
.map(|error| error.to_string());
let pending_operands = finder.has_pending_operands();
(finder, parse_error, pending_operands)
};
let page_handle = pdf.get_object_handle(page_ref);
if let Some(parse_error) = parse_error {
warn_resource_parse_failure(&page_handle, Some(&parse_error))?;
return Ok(());
}
if pending_operands || diagnostic_count(pdf) > diagnostics_before {
warn_resource_parse_failure(&page_handle, None)?;
return Ok(());
}
let resources = PageObjectHelper::new(page_ref, pdf).get_resources(true)?;
if resources.try_is_null()? {
return Ok(());
}
for category in [b"/Font".as_slice(), b"/XObject".as_slice()] {
let value = resources.try_get_key(category)?;
if value.try_is_null()? {
continue;
}
value.try_dereference()?;
if !value.try_is_dictionary()? {
continue;
}
let dictionary = if value.is_indirect() {
let copy = value.shallow_copy()?;
resources.replace_key(category, copy.clone())?;
copy
} else {
value
};
let names = finder.names();
let remove = dictionary
.try_get_keys()?
.into_iter()
.filter(|name| {
let resource_name = name.strip_prefix(b"/").unwrap_or(name.as_slice());
!names.contains(resource_name)
&& !unresolved
.iter()
.any(|unresolved_name| unresolved_name.as_slice() == resource_name)
})
.collect::<Vec<_>>();
for name in remove {
dictionary.remove_key(&name);
}
}
Ok(())
}
pub(crate) fn remove_unreferenced_resources_on_form<R: Read + Seek>(
pdf: &mut Pdf<R>,
form: ObjectHandle,
) -> Result<()> {
let mut nested_forms = Vec::new();
{
let mut helper = PageObjectHelper::from_object_handle(form.clone(), pdf);
helper.for_each_form_xobject(true, |nested, _, _| {
nested_forms.push(nested);
Ok(())
})?;
}
for nested in nested_forms {
prune_canonical_resource_target(pdf, nested)?;
}
prune_canonical_resource_target(pdf, form)
}
fn prune_canonical_resource_target<R: Read + Seek>(
pdf: &mut Pdf<R>,
target: ObjectHandle,
) -> Result<()> {
let diagnostics_before = diagnostic_count(pdf);
let (finder, parse_error, pending_operands) = {
let mut helper = PageObjectHelper::from_object_handle(target.clone(), pdf);
let mut finder = ResourceFinder::default();
let parse_error = helper
.parse_contents(&mut finder)
.err()
.map(|error| error.to_string());
let pending_operands = finder.has_pending_operands();
(finder, parse_error, pending_operands)
};
if let Some(parse_error) = parse_error {
warn_resource_parse_failure(&target, Some(&parse_error))?;
return Ok(());
}
if pending_operands || diagnostic_count(pdf) > diagnostics_before {
warn_resource_parse_failure(&target, None)?;
return Ok(());
}
let resources = PageObjectHelper::from_object_handle(target, pdf).get_resources(true)?;
if resources.try_is_null()? {
return Ok(());
}
let categories = [b"/Font".as_slice(), b"/XObject".as_slice()];
let mut dictionaries = Vec::new();
let mut known_names = BTreeSet::new();
for category in categories {
let value = resources.try_get_key(category)?;
if value.try_is_null()? {
continue;
}
value.try_dereference()?;
if !value.try_is_dictionary()? {
continue;
}
let dictionary = if value.is_indirect() {
let copy = value.shallow_copy()?;
resources.replace_key(category, copy.clone())?;
copy
} else {
value
};
let live_keys = dictionary.try_get_keys()?;
known_names.extend(
live_keys
.iter()
.map(|key| key.strip_prefix(b"/").unwrap_or(key.as_slice()).to_vec()),
);
dictionaries.push((category, dictionary, live_keys));
}
let local_unresolved = categories
.iter()
.filter_map(|category| finder.names_by_resource_type().get(&category[1..]))
.flat_map(|entries| entries.keys())
.filter(|name| !known_names.contains(*name))
.cloned()
.collect::<BTreeSet<_>>();
if !local_unresolved.is_empty() && resources.try_is_dictionary()? {
return Ok(());
}
for (_category, dictionary, live_keys) in dictionaries {
let names = finder.names();
let remove = dictionary
.try_get_keys()?
.into_iter()
.filter(|key| live_keys.contains(key))
.filter(|key| {
let name = key.strip_prefix(b"/").unwrap_or(key.as_slice());
!names.contains(name)
})
.collect::<Vec<_>>();
for key in remove {
dictionary.remove_key(&key);
}
}
Ok(())
}
fn remove_unreferenced_resources_in_form_xobjects<R: Read + Seek>(
pdf: &mut Pdf<R>,
page_ref: ObjectRef,
) -> Result<(BTreeSet<Vec<u8>>, bool)> {
let page_resources = {
let mut helper = PageObjectHelper::new(page_ref, pdf);
helper.get_resources(false)?
};
if page_resources.try_is_null()? {
return Ok((BTreeSet::new(), false));
}
let mut pending = VecDeque::from(form_xobjects_in_resources(&page_resources)?);
let mut visited: BTreeSet<QpdfObjGen> = BTreeSet::new();
let mut unresolved = BTreeSet::new();
let mut any_failures = false;
while let Some(holder_handle) = pending.pop_front() {
let Some(object_gen) = holder_handle
.qpdf_obj_gen()
.filter(|object_gen| object_gen.is_indirect())
else {
continue;
};
if !visited.insert(object_gen) {
continue;
}
holder_handle.try_dereference()?;
let form_handle = holder_handle;
if !form_handle.is_form_xobject()? {
continue; }
let stream_dict = form_stream_dict(&form_handle)?;
let resources = stream_dict.try_get_key(b"/Resources")?;
let resources = resources.try_is_dictionary()?.then_some(resources);
let diagnostics_before = diagnostic_count(pdf);
let (finder, parse_error, pending_operands) = {
let mut helper = PageObjectHelper::from_object_handle(form_handle.clone(), pdf);
let mut finder = ResourceFinder::default();
let parse_error = helper
.parse_contents(&mut finder)
.err()
.map(|error| error.to_string());
let pending_operands = finder.has_pending_operands();
(finder, parse_error, pending_operands)
};
if let Some(parse_error) = parse_error {
warn_resource_parse_failure(&form_handle, Some(&parse_error))?;
any_failures = true;
if let Some(resources) = resources.as_ref() {
pending.extend(form_xobjects_in_resources(resources)?);
} continue;
}
if pending_operands || diagnostic_count(pdf) > diagnostics_before {
warn_resource_parse_failure(&form_handle, None)?;
any_failures = true;
if let Some(resources) = resources.as_ref() {
pending.extend(form_xobjects_in_resources(resources)?);
} continue;
}
let mut used = BTreeMap::new();
record_direct_names(&mut used, finder.names_by_resource_type(), true);
let local_unresolved = unresolved_resource_names(resources.as_ref(), &used)?;
unresolved.extend(local_unresolved.iter().cloned());
let child_forms = match resources.as_ref() {
Some(resources) => form_xobjects_in_resources(resources)?,
None => Vec::new(),
};
if !local_unresolved.is_empty() && resources.is_some() {
any_failures = true;
} else if let Some(resources) = &resources {
let resources = if resources.is_indirect() {
let copy = resources.shallow_copy()?;
stream_dict.replace_key(b"/Resources", copy.clone())?;
copy
} else {
resources.clone()
};
prune_font_and_xobject_dictionaries(&resources, &used)?;
}
pending.extend(child_forms);
}
Ok((unresolved, any_failures))
}
fn unresolved_resource_names(
resources: Option<&ObjectHandle>,
used: &UsedNames,
) -> Result<BTreeSet<Vec<u8>>> {
let mut known_names: BTreeSet<Vec<u8>> = BTreeSet::new();
if let Some(resources) = resources {
for category in [b"Font".as_slice(), b"XObject".as_slice()] {
let mut key = Vec::with_capacity(category.len() + 1);
key.push(b'/');
key.extend_from_slice(category);
let value = resources.try_get_key(&key)?;
value.try_dereference()?;
let Some(dictionary) = value.try_as_dictionary()? else {
continue;
};
known_names.extend(
dictionary
.keys()
.map(|name| name.strip_prefix(b"/").unwrap_or(name.as_slice()).to_vec()),
);
}
}
let mut unresolved = BTreeSet::new();
for category in [b"Font".as_slice(), b"XObject".as_slice()] {
for name in used
.get(category)
.into_iter()
.flat_map(|names| names.iter())
{
if !known_names.contains(&name.to_vec()) {
unresolved.insert(name.clone());
}
}
}
Ok(unresolved)
}
fn form_xobjects_in_resources(resources: &ObjectHandle) -> Result<Vec<ObjectHandle>> {
let xobjects = resources.try_get_key(b"/XObject")?;
xobjects.try_dereference()?;
let Some(xobjects) = xobjects.try_as_dictionary()? else {
return Ok(Vec::new());
};
let mut forms = Vec::new();
for value in xobjects.values() {
if value
.qpdf_obj_gen()
.filter(|object_gen| object_gen.is_indirect())
.is_none()
{
continue;
}
value.try_dereference()?;
if !value.is_form_xobject()? {
continue;
}
forms.push(value.clone());
}
Ok(forms)
}
fn prune_font_and_xobject_dictionaries(resources: &ObjectHandle, used: &UsedNames) -> Result<()> {
for category in [b"Font".as_slice(), b"XObject".as_slice()] {
let mut key = Vec::with_capacity(category.len() + 1);
key.push(b'/');
key.extend_from_slice(category);
let value = resources.try_get_key(&key)?;
value.try_dereference()?;
if !value.try_is_dictionary()? {
continue;
}
let dictionary = if value.is_indirect() {
let copy = value.shallow_copy()?;
resources.replace_key(&key, copy.clone())?;
copy
} else {
value
};
let names = used.get(category).cloned().unwrap_or_default();
let remove = dictionary
.try_get_keys()?
.into_iter()
.filter(|name| !names.contains(name.strip_prefix(b"/").unwrap_or(name.as_slice())))
.collect::<Vec<_>>();
for name in remove {
dictionary.remove_key(&name);
}
}
Ok(())
}
const RESOURCE_CATEGORIES: &[&str] = &[
"Font",
"XObject",
"ColorSpace",
"Pattern",
"Shading",
"ExtGState",
"Properties",
];
fn is_builtin_color_space_cs_op(name: &[u8]) -> bool {
matches!(
name,
b"DeviceGray" | b"DeviceRGB" | b"DeviceCMYK" | b"Pattern"
)
}
fn record_direct_names(used: &mut UsedNames, names: &ResourceNamesByType, record_direct: bool) {
if !record_direct {
return;
}
for &category in RESOURCE_CATEGORIES {
let category = category.as_bytes();
for name in names
.get(category)
.into_iter()
.flat_map(|by_name| by_name.keys())
{
if category == b"ColorSpace" && is_builtin_color_space_cs_op(name) {
continue;
}
used.entry(category.to_vec())
.or_default()
.insert(name.clone());
}
}
}
fn form_stream_dict(handle: &ObjectHandle) -> Result<ObjectHandle> {
handle.as_stream_dict().ok_or_else(|| {
Error::Internal("Form XObject handle did not resolve to a stream".to_owned())
})
}
#[cfg(test)]
mod final_handle_tests {
use super::{
form_xobjects_in_resources, remove_unreferenced_resources_in_form_xobjects,
remove_unreferenced_resources_on_form,
};
use crate::{ObjectHandle, ObjectRef, Pdf};
use std::io::Cursor;
use std::rc::Rc;
fn fixture() -> Pdf<Cursor<Vec<u8>>> {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../tests/fixtures/compat/direct-root-one-page.pdf");
Pdf::open_mem_owned(std::fs::read(path).expect("fixture exists")).expect("fixture opens")
}
fn form_with_resources(pdf: &Pdf<Cursor<Vec<u8>>>, contents: &[u8]) -> ObjectHandle {
let form = pdf
.new_stream_with_data(Rc::new(contents.to_vec()))
.expect("form stream");
let form_dict = form.as_stream_dict().expect("form dictionary");
form_dict
.replace_key(b"/Type", ObjectHandle::name(b"XObject".to_vec()))
.expect("form type");
form_dict
.replace_key(b"/Subtype", ObjectHandle::name(b"Form".to_vec()))
.expect("form subtype");
form_dict
.replace_key(
b"/Resources",
ObjectHandle::dictionary(vec![(
b"/Font".to_vec(),
ObjectHandle::dictionary(vec![
(b"/F1".to_vec(), ObjectHandle::dictionary(Vec::new())),
(b"/Unused".to_vec(), ObjectHandle::dictionary(Vec::new())),
]),
)]),
)
.expect("form resources");
form
}
#[test]
fn form_resource_prepass_resolves_form_and_resource_handles() {
let mut pdf = fixture();
let page_ref = crate::pages::page_refs(&mut pdf).expect("page refs")[0];
let page_handle = pdf.get_object_handle(page_ref);
page_handle.try_is_scalar().expect("page resolves");
let replacement = page_handle.shallow_copy().expect("page is copyable");
let form = pdf
.new_stream_with_data(Rc::new(b"q Q".to_vec()))
.expect("form stream");
let form_dict = form.as_stream_dict().expect("form dictionary");
form_dict
.replace_key(b"/Type", ObjectHandle::name(b"XObject".to_vec()))
.expect("form type");
form_dict
.replace_key(b"/Subtype", ObjectHandle::name(b"Form".to_vec()))
.expect("form subtype");
form_dict
.replace_key(
b"/Resources",
ObjectHandle::dictionary(vec![(
b"/Font".to_vec(),
ObjectHandle::dictionary(vec![(b"/Unused".to_vec(), ObjectHandle::integer(1))]),
)]),
)
.expect("form resources");
let resources = ObjectHandle::dictionary(vec![(
b"/XObject".to_vec(),
ObjectHandle::dictionary(vec![(b"/Fm0".to_vec(), form)]),
)]);
replacement
.replace_key(b"/Resources", resources)
.expect("page resources");
pdf.replace_object(page_ref, replacement)
.expect("replace page");
let (unresolved, failures) =
remove_unreferenced_resources_in_form_xobjects(&mut pdf, page_ref)
.expect("form resource prepass");
assert!(unresolved.is_empty());
assert!(!failures);
}
#[test]
fn form_resource_prepass_does_not_drop_a_raw_generation_form() {
let mut pdf = fixture();
let raw_ref = ObjectRef::new(50, 65_535);
let form = ObjectHandle::stream(
ObjectHandle::dictionary(vec![
(b"/Type".to_vec(), ObjectHandle::name(b"XObject".to_vec())),
(b"/Subtype".to_vec(), ObjectHandle::name(b"Form".to_vec())),
]),
Rc::new(Vec::new()),
);
pdf.replace_object(raw_ref, form).expect("install raw form");
let resources = ObjectHandle::dictionary(vec![(
b"/XObject".to_vec(),
ObjectHandle::dictionary(vec![(b"/Fm0".to_vec(), pdf.get_object_handle(raw_ref))]),
)]);
let forms = form_xobjects_in_resources(&resources).expect("find raw form");
assert_eq!(forms.len(), 1);
}
#[test]
fn canonical_form_pruning_uses_the_clean_parse_boundary() {
let mut pdf = fixture();
let form = form_with_resources(&pdf, b"/F1 12 Tf");
remove_unreferenced_resources_on_form(&mut pdf, form.clone())
.expect("clean Form content should be pruned");
let resources = form
.as_stream_dict()
.expect("form dictionary")
.try_get_key(b"/Resources")
.expect("resources");
let fonts = resources.try_get_key(b"/Font").expect("fonts");
assert!(!fonts.try_get_key(b"/F1").expect("used font").is_null());
assert!(fonts
.try_get_key(b"/Unused")
.expect("unused font")
.is_null());
}
#[test]
fn canonical_form_pruning_skips_a_scope_with_parser_warnings() {
let mut pdf = fixture();
let form = form_with_resources(&pdf, b"<0g>");
remove_unreferenced_resources_on_form(&mut pdf, form.clone())
.expect("recoverable Form warnings should skip pruning");
assert!(pdf
.repair_diagnostics()
.entries()
.iter()
.any(|diagnostic| diagnostic
.message_string()
.contains("invalid character (g) in hexstring")));
let resources = form
.as_stream_dict()
.expect("form dictionary")
.try_get_key(b"/Resources")
.expect("resources");
let fonts = resources.try_get_key(b"/Font").expect("fonts");
assert!(!fonts
.try_get_key(b"/Unused")
.expect("unused font")
.is_null());
}
#[test]
fn canonical_form_pruning_reports_a_parse_exception() {
let mut pdf = fixture();
let form = form_with_resources(&pdf, b"not-flate");
form.as_stream_dict()
.expect("form dictionary")
.replace_key(b"/Filter", ObjectHandle::name(b"FlateDecode".to_vec()))
.expect("filter");
remove_unreferenced_resources_on_form(&mut pdf, form)
.expect("recoverable Form parse failures should be warnings");
assert!(pdf
.repair_diagnostics()
.entries()
.iter()
.any(|diagnostic| diagnostic
.message_string()
.contains("Unable to parse content stream: stream inflate")));
}
#[test]
fn page_resource_pruning_reports_a_parse_exception() {
let mut pdf = fixture();
let page_ref = crate::pages::page_refs(&mut pdf).expect("page refs")[0];
let page_handle = pdf.get_object_handle(page_ref);
page_handle.try_is_scalar().expect("page resolves");
let replacement = page_handle.shallow_copy().expect("page is copyable");
let contents = pdf
.new_stream_with_data(Rc::new(b"not-flate".to_vec()))
.expect("content stream");
contents
.as_stream_dict()
.expect("content stream dictionary")
.replace_key(b"/Filter", ObjectHandle::name(b"FlateDecode".to_vec()))
.expect("filter");
replacement
.replace_key(b"/Contents", contents)
.expect("page contents");
pdf.replace_object(page_ref, replacement)
.expect("replace page");
super::remove_unreferenced_resources_on_page(&mut pdf, page_ref)
.expect("recoverable page parse failures should be warnings");
assert!(pdf
.repair_diagnostics()
.entries()
.iter()
.any(|diagnostic| diagnostic
.message_string()
.contains("Unable to parse content stream: stream inflate")));
}
fn page_with_malformed_form_and_child(
outer_stream: &[u8],
outer_filter: Option<&[u8]>,
) -> (Pdf<std::io::Cursor<Vec<u8>>>, crate::ObjectRef) {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../tests/fixtures/compat/direct-root-one-page.pdf");
let mut pdf = Pdf::open_mem_owned(std::fs::read(path).expect("fixture exists"))
.expect("fixture opens");
let page_ref = crate::pages::page_refs(&mut pdf).expect("page refs")[0];
let page_handle = pdf.get_object_handle(page_ref);
page_handle.try_is_scalar().expect("page resolves");
let replacement = page_handle.shallow_copy().expect("page is copyable");
let child_form = pdf
.new_stream_with_data(Rc::new(b"/Ghost Do".to_vec()))
.expect("child form stream");
let child_dict = child_form.as_stream_dict().expect("child form dictionary");
child_dict
.replace_key(b"/Type", ObjectHandle::name(b"XObject".to_vec()))
.expect("child form type");
child_dict
.replace_key(b"/Subtype", ObjectHandle::name(b"Form".to_vec()))
.expect("child form subtype");
child_dict
.replace_key(b"/Resources", ObjectHandle::dictionary(vec![]))
.expect("child form resources");
let outer_form = pdf
.new_stream_with_data(Rc::new(outer_stream.to_vec()))
.expect("outer form stream");
let outer_dict = outer_form.as_stream_dict().expect("outer form dictionary");
outer_dict
.replace_key(b"/Type", ObjectHandle::name(b"XObject".to_vec()))
.expect("outer form type");
outer_dict
.replace_key(b"/Subtype", ObjectHandle::name(b"Form".to_vec()))
.expect("outer form subtype");
if let Some(filter) = outer_filter {
outer_dict
.replace_key(b"/Filter", ObjectHandle::name(filter.to_vec()))
.expect("outer form filter");
}
outer_dict
.replace_key(
b"/Resources",
ObjectHandle::dictionary(vec![(
b"/XObject".to_vec(),
ObjectHandle::dictionary(vec![(b"/FmChild".to_vec(), child_form)]),
)]),
)
.expect("outer form resources");
let resources = ObjectHandle::dictionary(vec![(
b"/XObject".to_vec(),
ObjectHandle::dictionary(vec![(b"/Fm0".to_vec(), outer_form)]),
)]);
replacement
.replace_key(b"/Resources", resources)
.expect("page resources");
pdf.replace_object(page_ref, replacement)
.expect("replace page");
(pdf, page_ref)
}
#[test]
fn form_walk_continues_into_declared_children_after_a_decode_failure() {
let (mut pdf, page_ref) =
page_with_malformed_form_and_child(b"not-flate", Some(b"FlateDecode"));
let (unresolved, failures) =
remove_unreferenced_resources_in_form_xobjects(&mut pdf, page_ref)
.expect("form resource prepass tolerates an undecodable Form");
assert!(failures, "the undecodable outer Form must set any_failures");
assert!(
unresolved.contains(b"Ghost".as_slice()),
"the child Form's own unresolved /Ghost use must surface, proving \
the walk descended past the undecodable parent: {unresolved:?}"
);
}
#[test]
fn form_walk_continues_into_declared_children_after_a_parse_failure() {
let (mut pdf, page_ref) = page_with_malformed_form_and_child(b"<0g>", None);
let (unresolved, failures) =
remove_unreferenced_resources_in_form_xobjects(&mut pdf, page_ref)
.expect("form resource prepass tolerates an unparseable Form");
assert!(failures, "the unparseable outer Form must set any_failures");
assert!(
unresolved.contains(b"Ghost".as_slice()),
"the child Form's own unresolved /Ghost use must surface, proving \
the walk descended past the unparseable parent: {unresolved:?}"
);
}
}