use crate::linearization::renumber::RenumberMap;
use crate::object_handle::ObjectHandle;
use crate::parser::MAX_PARSE_DEPTH;
use crate::writer::object_streams::{
compressible_objgens_qpdf_plan, eligibility_context, is_eligible_for_objstm_handle,
ObjectStreamMode, PlannerConfig,
};
use crate::{ObjectRef, Pdf, Result};
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::io::{Read, Seek};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PageHintEntry {
pub page_ref: ObjectRef,
pub first_object_index: u32,
pub object_count: u32,
pub byte_length: u64,
}
impl PageHintEntry {
pub fn placeholder(page_ref: ObjectRef) -> Self {
Self {
page_ref,
first_object_index: 0,
object_count: 0,
byte_length: 0,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SharedObjectHintEntry {
pub object_ref: ObjectRef,
pub referencing_pages: Vec<u32>,
}
impl SharedObjectHintEntry {
pub fn new(object_ref: ObjectRef) -> Self {
Self {
object_ref,
referencing_pages: Vec::new(),
}
}
}
fn linearization_content_normalize_refs<R: Read + Seek>(
pdf: &mut Pdf<R>,
options: &crate::writer::WriterOptions,
) -> Result<BTreeSet<ObjectRef>> {
if !options.content_normalization {
return Ok(BTreeSet::new());
}
let mut refs = BTreeSet::new();
for page_ref in crate::pages::page_refs(pdf)? {
for content_ref in crate::writer::collect_content_stream_refs(pdf, page_ref)? {
refs.insert(content_ref);
}
}
Ok(refs)
}
fn stream_has_indirect_parameter_edge(handle: &ObjectHandle) -> Result<bool> {
let Some(stream_dict) = handle.as_stream_dict() else {
return Ok(false); };
for key in [b"/Filter".as_slice(), b"/DecodeParms".as_slice()] {
let value = stream_dict.try_get_key(key)?;
let mut refs = Vec::new();
collect_direct_handle_refs(&value, 0, &mut refs)?;
if !refs.is_empty() {
return Ok(true);
}
}
Ok(false)
}
fn stream_parameters_removed_for_linearization(
handle: &ObjectHandle,
stream_ref: Option<ObjectRef>,
options: &crate::writer::WriterOptions,
content_normalize_refs: &BTreeSet<ObjectRef>,
) -> Result<bool> {
let normalize_content =
stream_ref.is_some_and(|object_ref| content_normalize_refs.contains(&object_ref));
if handle.is_data_modified() {
crate::writer::plain::body::canonical_stream_filter_probe_for_linearization(
handle,
options,
normalize_content,
) } else {
crate::writer::plain::body::canonical_stream_will_be_refiltered_with_policy(
handle,
options,
true,
normalize_content,
) }
}
fn collect_direct_handle_refs(
handle: &ObjectHandle,
depth: usize,
out: &mut Vec<ObjectRef>,
) -> Result<()> {
if depth > MAX_PARSE_DEPTH {
return Err(crate::Error::Unsupported(format!(
"linearization plan: inline object nesting exceeds maximum of {MAX_PARSE_DEPTH}"
)));
}
let mut contextual = Vec::new();
collect_direct_handle_refs_with_context(handle, depth, false, &mut contextual)?;
out.extend(contextual.into_iter().map(|(object_ref, _)| object_ref));
Ok(())
}
fn collect_direct_handle_refs_with_stream_parameters(
handle: &ObjectHandle,
depth: usize,
out: &mut Vec<ObjectRef>,
skipped_stream_parameter_streams: &BTreeSet<ObjectRef>,
) -> Result<()> {
let mut contextual = Vec::new();
collect_direct_handle_refs_with_stream_parameters_context(
handle,
depth,
false,
&mut contextual,
skipped_stream_parameter_streams,
)?; out.extend(contextual.into_iter().map(|(object_ref, _)| object_ref));
Ok(())
}
fn collect_direct_handle_refs_with_context(
handle: &ObjectHandle,
depth: usize,
in_array: bool,
out: &mut Vec<(ObjectRef, bool)>,
) -> Result<()> {
if depth > MAX_PARSE_DEPTH {
return Err(crate::Error::Unsupported(format!(
"linearization plan: inline object nesting exceeds maximum of {MAX_PARSE_DEPTH}"
)));
}
if let Some(object_ref) = handle.object_ref() {
out.push((object_ref, in_array));
return Ok(());
}
collect_direct_handle_children(
handle,
depth,
in_array,
&mut |child, child_depth, child_in_array| {
collect_direct_handle_refs_with_context(child, child_depth, child_in_array, out)
},
)
}
fn collect_direct_handle_refs_with_stream_parameters_context(
handle: &ObjectHandle,
depth: usize,
in_array: bool,
out: &mut Vec<(ObjectRef, bool)>,
skipped_stream_parameter_streams: &BTreeSet<ObjectRef>,
) -> Result<()> {
if depth > MAX_PARSE_DEPTH {
return Err(crate::Error::Unsupported(format!(
"linearization plan: inline object nesting exceeds maximum of {MAX_PARSE_DEPTH}"
)));
}
if let Some(object_ref) = handle.object_ref() {
out.push((object_ref, in_array));
return Ok(());
}
collect_direct_handle_children_with_stream_parameters(
handle,
depth,
in_array,
skipped_stream_parameter_streams,
&mut |child, child_depth, child_in_array| {
collect_direct_handle_refs_with_stream_parameters_context(
child,
child_depth,
child_in_array,
out,
skipped_stream_parameter_streams,
)
},
)
}
fn collect_direct_handle_children<F>(
handle: &ObjectHandle,
depth: usize,
_parent_in_array: bool,
visit: &mut F,
) -> Result<()>
where
F: FnMut(&ObjectHandle, usize, bool) -> Result<()>,
{
collect_direct_handle_children_with_stream_parameters(
handle,
depth,
_parent_in_array,
&BTreeSet::new(),
visit,
)
}
fn collect_direct_handle_children_with_stream_parameters<F>(
handle: &ObjectHandle,
depth: usize,
_parent_in_array: bool,
skipped_stream_parameter_streams: &BTreeSet<ObjectRef>,
visit: &mut F,
) -> Result<()>
where
F: FnMut(&ObjectHandle, usize, bool) -> Result<()>,
{
handle.try_dereference()?;
if let Some(stream_dict) = handle.as_stream_dict() {
let Some(entries) = stream_dict.try_as_dictionary()? else {
return Ok(());
};
let skip_stream_parameters =
handle_has_stream_parameter_skip(handle, skipped_stream_parameter_streams)?;
for (key, child) in entries {
if key == b"/Length"
|| (skip_stream_parameters
&& matches!(key.as_slice(), b"/Filter" | b"/DecodeParms"))
{
continue;
}
visit(&child, depth + 1, false)?;
}
return Ok(());
}
if let Some(children) = handle.try_as_array()? {
for child in children {
visit(&child, depth + 1, true)?;
}
return Ok(());
}
if let Some(entries) = handle.try_as_dictionary()? {
for (_key, child) in entries {
visit(&child, depth + 1, false)?;
}
}
Ok(())
}
fn handle_has_stream_parameter_skip(
handle: &ObjectHandle,
skipped_stream_parameter_streams: &BTreeSet<ObjectRef>,
) -> Result<bool> {
Ok(handle
.object_ref()
.is_some_and(|object_ref| skipped_stream_parameter_streams.contains(&object_ref)))
}
fn collect_handle_children_with_stream_parameters(
handle: &ObjectHandle,
depth: usize,
out: &mut Vec<(ObjectRef, bool)>,
skipped_stream_parameter_streams: &BTreeSet<ObjectRef>,
) -> Result<()> {
collect_direct_handle_children_with_stream_parameters(
handle,
depth,
false,
skipped_stream_parameter_streams,
&mut |child, child_depth, in_array| {
collect_direct_handle_refs_with_stream_parameters_context(
child,
child_depth,
in_array,
out,
skipped_stream_parameter_streams,
)
},
)
}
fn is_page_tree_handle(handle: &ObjectHandle) -> Result<bool> {
Ok(handle.try_is_dictionary_of_type(b"Pages", b"")?
|| handle.try_is_dictionary_of_type(b"Page", b"")?)
}
fn compute_closure_with_stream_parameters<R: Read + Seek>(
pdf: &mut Pdf<R>,
root: ObjectRef,
live: &BTreeSet<ObjectRef>,
resurrectable: &BTreeSet<ObjectRef>,
skipped_stream_parameter_streams: &BTreeSet<ObjectRef>,
) -> crate::Result<Vec<ObjectRef>> {
let mut visited: BTreeSet<ObjectRef> = BTreeSet::new();
let mut order: Vec<ObjectRef> = Vec::new();
let mut queue: VecDeque<(ObjectRef, bool)> = VecDeque::from([(root, false)]);
let mut deferred_resurrect: BTreeSet<ObjectRef> = BTreeSet::new();
let mut seen_as_array: BTreeSet<ObjectRef> = BTreeSet::new();
let admits_body_object = |r: ObjectRef| -> bool { r.number != 0 && live.contains(&r) };
while let Some((current, via_array)) = queue.pop_front() {
if !visited.insert(current) {
continue;
}
if resurrectable.contains(¤t) {
if via_array || seen_as_array.contains(¤t) {
order.push(current);
} else {
deferred_resurrect.insert(current);
}
continue;
}
if !admits_body_object(current) {
continue;
}
order.push(current);
let current_handle = pdf.get_object_handle(current);
pdf.resolve(¤t_handle)?;
let is_pages_node = current_handle.try_is_dictionary_of_type(b"Pages", b"")?;
let is_page_leaf = current_handle.try_is_dictionary_of_type(b"Page", b"")?;
if is_pages_node || is_page_leaf {
if let Some(dict) = current_handle.try_as_dictionary()? {
if is_page_leaf {
{
let resources = current_handle.try_get_key(b"/Resources")?;
let mut seeds: Vec<(ObjectRef, bool)> = Vec::new();
collect_direct_handle_refs_with_context(&resources, 0, false, &mut seeds)?;
for &(r, va) in &seeds {
if va {
seen_as_array.insert(r);
}
}
let mut stack: Vec<(ObjectRef, bool)> = seeds.into_iter().rev().collect();
while let Some((r, via_array)) = stack.pop() {
if !visited.insert(r) {
continue;
}
if resurrectable.contains(&r) {
if via_array || seen_as_array.contains(&r) {
order.push(r);
} else {
deferred_resurrect.insert(r);
} continue;
}
if !admits_body_object(r) {
continue;
}
let child_handle = pdf.get_object_handle(r);
pdf.resolve(&child_handle)?;
if is_page_tree_handle(&child_handle)? {
continue;
}
order.push(r);
let mut child_refs: Vec<(ObjectRef, bool)> = Vec::new();
collect_handle_children_with_stream_parameters(
&child_handle,
0,
&mut child_refs,
skipped_stream_parameter_streams,
)?;
for cr in child_refs.into_iter().rev() {
if cr.1 {
seen_as_array.insert(cr.0);
}
if !visited.contains(&cr.0) {
stack.push(cr);
}
}
}
}
} let mut refs_raw: Vec<(ObjectRef, bool)> = Vec::new();
for (k, v) in dict.iter() {
if k == b"/Kids" {
continue;
}
if k == b"/Thumb" {
continue;
}
if k == b"/Parent" {
let mut to_visit: Vec<ObjectRef> = Vec::new();
let mut seen_parents: BTreeSet<ObjectRef> = BTreeSet::new();
collect_direct_handle_refs_with_stream_parameters(
v,
0,
&mut to_visit,
skipped_stream_parameter_streams,
)?;
while let Some(parent_ref) = to_visit.pop() {
if !seen_parents.insert(parent_ref) {
continue;
}
let parent_handle = pdf.get_object_handle(parent_ref);
pdf.resolve(&parent_handle)?;
let Some(parent_dict) = parent_handle.try_as_dictionary()? else {
continue;
};
for (pk, pv) in parent_dict.iter() {
if pk == b"/Kids" {
continue;
}
if pk == b"/Parent" {
collect_direct_handle_refs(pv, 0, &mut to_visit)?;
continue;
}
let mut refs: Vec<(ObjectRef, bool)> = Vec::new();
collect_direct_handle_refs_with_stream_parameters_context(
pv,
0,
false,
&mut refs,
skipped_stream_parameter_streams,
)?; for (r, va) in refs {
if va {
seen_as_array.insert(r);
}
if !visited.contains(&r) {
queue.push_back((r, va));
}
}
}
}
continue;
}
collect_direct_handle_refs_with_stream_parameters_context(
v,
0,
false,
&mut refs_raw,
skipped_stream_parameter_streams,
)?; }
for &(r, va) in &refs_raw {
if va {
seen_as_array.insert(r);
}
}
refs_raw.sort_by_key(|(r, _)| pdf.linearization_object_order_key(*r));
for (r, va) in refs_raw {
if !visited.contains(&r) {
queue.push_back((r, va));
}
}
}
} else {
let mut refs: Vec<(ObjectRef, bool)> = Vec::new();
collect_handle_children_with_stream_parameters(
¤t_handle,
0,
&mut refs,
skipped_stream_parameter_streams,
)?; for &(r, va) in &refs {
if va {
seen_as_array.insert(r);
}
}
refs.sort_by_key(|(r, _)| pdf.linearization_object_order_key(*r));
for (r, va) in refs {
if !visited.contains(&r) {
queue.push_back((r, va));
}
}
}
}
if order.len() > 1 {
order[1..].sort_by_key(|r| pdf.linearization_object_order_key(*r));
}
for r in deferred_resurrect {
if seen_as_array.contains(&r) {
let tail_pos = order[1..].partition_point(|existing| existing.number < r.number);
order.insert(1 + tail_pos, r);
} }
Ok(order)
}
#[derive(Debug, Clone)]
pub struct LinearizationPlan {
pub part1_objects: Vec<ObjectRef>,
pub part2_objects: Vec<ObjectRef>,
pub part3_objects: Vec<ObjectRef>,
pub part4_other_pages_private: Vec<ObjectRef>,
pub part4_other_pages_shared: Vec<ObjectRef>,
pub part4_rest: Vec<ObjectRef>,
pub total_object_count: u32,
pub root_ref: Option<ObjectRef>,
pub pages_tree_ref: Option<ObjectRef>,
pub info_ref: Option<ObjectRef>,
pub page_hints: Vec<PageHintEntry>,
pub shared_hints: Vec<SharedObjectHintEntry>,
pub per_page_private_objects: Vec<Vec<ObjectRef>>,
pub all_referenced_pages: BTreeMap<ObjectRef, BTreeSet<u32>>,
pub(crate) outline_first_page_members: Vec<ObjectRef>,
pub(crate) part9_outline_objects: Vec<ObjectRef>,
pub(crate) part6_outline_objects: Vec<ObjectRef>,
pub part4_open_document_plain: Vec<ObjectRef>,
pub(crate) optimization: Option<crate::optimization::Optimization>,
pub(crate) content_normalize_refs: BTreeSet<ObjectRef>,
pub(crate) removed_refs: BTreeSet<ObjectRef>,
pub(crate) object_stream_mode: crate::writer::ObjectStreamMode,
}
impl LinearizationPlan {
pub fn from_pdf<R: Read + Seek>(
pdf: &mut Pdf<R>,
use_generate_objstm: bool,
) -> crate::Result<Self> {
let mode = if use_generate_objstm {
crate::writer::ObjectStreamMode::Generate
} else {
crate::writer::ObjectStreamMode::Disable
};
Self::from_pdf_with_object_stream_mode(pdf, mode)
}
pub fn from_pdf_with_object_stream_mode<R: Read + Seek>(
pdf: &mut Pdf<R>,
object_stream_mode: crate::writer::ObjectStreamMode,
) -> crate::Result<Self> {
let options = crate::writer::WriterOptions {
object_streams: object_stream_mode,
..crate::writer::WriterOptions::default()
};
Self::from_pdf_with_writer_options(pdf, &options)
}
pub(crate) fn from_pdf_with_writer_options<R: Read + Seek>(
pdf: &mut Pdf<R>,
options: &crate::writer::WriterOptions,
) -> crate::Result<Self> {
let object_stream_mode = options.object_streams;
let use_generate_objstm = matches!(
object_stream_mode,
crate::writer::ObjectStreamMode::Generate
);
let capture_pre_optimization_objects =
!matches!(object_stream_mode, crate::writer::ObjectStreamMode::Disable);
let pre_optimization_object_refs =
capture_pre_optimization_objects.then(|| pdf.live_object_refs().into_iter().collect());
let generate_objstm_eligible = use_generate_objstm
.then(|| crate::writer::object_streams::get_compressible_objgens(pdf))
.transpose()?;
if pdf.root_ref().is_some() {
crate::writer::rewrite_renumber::CanonicalCatalogFirstRenumber::build_qpdf_with_stream_policy(
pdf,
true,
false,
&BTreeSet::new(),
None)?;
}
let content_normalize_refs = linearization_content_normalize_refs(pdf, options)?;
let mut skipped_stream_parameter_streams = BTreeSet::new();
let mut optimization = crate::optimization::Optimization::optimize(
pdf,
&BTreeMap::new(),
true,
|stream_ref, stream| {
if stream_ref.is_some_and(|object_ref| {
skipped_stream_parameter_streams.contains(&object_ref)
}) {
return Ok(2);
}
let has_indirect_parameter = stream_has_indirect_parameter_edge(stream)?;
let refiltered = if has_indirect_parameter {
stream_parameters_removed_for_linearization(
stream,
stream_ref,
options,
&content_normalize_refs,
)? } else {
crate::writer::plain::body::canonical_stream_filter_probe_for_linearization(
stream,
options,
stream_ref
.is_some_and(|object_ref| content_normalize_refs.contains(&object_ref)),
)? };
if refiltered {
if has_indirect_parameter {
if let Some(object_ref) = stream_ref {
skipped_stream_parameter_streams.insert(object_ref);
}
}
Ok(2)
} else {
Ok(1)
}
},
)?;
if let Some(eligible) = generate_objstm_eligible {
optimization.set_generate_objstm_eligible(eligible);
}
if let Some(refs) = pre_optimization_object_refs {
optimization.set_pre_optimization_object_refs(refs);
}
if pdf.root_ref().is_some()
&& optimization
.objects_for(&crate::optimization::ObjectUser::Page(0))
.is_empty()
{
return Err(crate::Error::Unsupported(
"no pages found while calculating linearization data".to_string(),
));
}
let reachable =
crate::writer::rewrite_renumber::reachable_object_set_with_stream_parameters(
pdf,
true,
&skipped_stream_parameter_streams,
)?;
let object_refs = pdf.object_refs();
let mut all_refs: Vec<ObjectRef> = Vec::with_capacity(object_refs.len());
for r in object_refs {
if r.number == 0 {
continue;
}
if !reachable.contains(&r) {
continue;
}
let object_handle = pdf.get_object_handle(r);
pdf.resolve(&object_handle)?;
if object_handle.try_is_stream_of_type(b"XRef", b"")?
|| object_handle.try_is_stream_of_type(b"ObjStm", b"")?
{
continue;
}
all_refs.push(r);
}
let source_had_compressed_objects = pdf
.source_xref_entries()
.iter()
.any(|(_reference, offset)| matches!(offset, crate::XrefEntry::Compressed { .. }));
let operation_removes_stale_generations = use_generate_objstm
|| (matches!(
object_stream_mode,
crate::writer::ObjectStreamMode::Preserve
) && source_had_compressed_objects);
let removed_refs = if operation_removes_stale_generations {
crate::writer::object_streams::compressible_objgens_qpdf_plan(pdf)?.removed_refs
} else {
BTreeSet::new()
};
all_refs.retain(|reference| !removed_refs.contains(reference));
let resurrectable =
crate::writer::rewrite_renumber::resurrectable_null_refs_excluding(pdf, &removed_refs)?;
let mut resurrected: Vec<ObjectRef> = resurrectable
.iter()
.filter(|r| all_refs.binary_search(r).is_err())
.copied()
.collect();
if !resurrected.is_empty() {
all_refs.append(&mut resurrected);
all_refs.sort();
}
let total_object_count = all_refs.len() as u32;
let root_ref = pdf.root_ref();
let info_handle = pdf.trailer().try_get_key(b"/Info")?;
let info_ref = info_handle
.object_ref()
.or_else(|| info_handle.object_ref());
let pages_tree_ref = if let Some(root_ref) = root_ref {
let root_handle = pdf.get_object_handle(root_ref);
pdf.resolve(&root_handle)?;
let pages_handle = root_handle.try_get_key(b"/Pages")?;
pages_handle
.object_ref()
.or_else(|| pages_handle.object_ref())
} else {
None };
let page_refs: Vec<ObjectRef> = crate::pages::page_refs(pdf)?;
let live: BTreeSet<ObjectRef> = pdf.live_object_refs().into_iter().collect();
let mut all_referenced_pages: BTreeMap<ObjectRef, BTreeSet<u32>> = BTreeMap::new();
for (object_ref, _) in optimization.object_users() {
let pages = optimization.referenced_pages(object_ref);
if !pages.is_empty() {
all_referenced_pages.insert(object_ref, pages);
}
}
let first_page_users = optimization
.objects_for(&crate::optimization::ObjectUser::Page(0))
.clone();
let root_objects = optimization
.objects_for(&crate::optimization::ObjectUser::Root)
.clone();
let mut open_document_objects = optimization.objects_for_trailer_key(b"Encrypt");
for key in OPEN_DOCUMENT_CATALOG_KEYS {
open_document_objects.extend(optimization.objects_for_root_key(key));
}
let all_outline_refs = optimization.objects_for_root_key(b"Outlines");
let document_other_objects: BTreeSet<ObjectRef> = optimization
.object_users()
.filter_map(|(object_ref, users)| {
users
.iter()
.any(is_document_other_user)
.then_some(object_ref)
})
.collect();
let elig_ctx = if use_generate_objstm {
Some(eligibility_context(pdf)?)
} else {
None
};
let mut first_page_closure: Vec<ObjectRef> = if let Some(&first_page) = page_refs.first() {
compute_closure_with_stream_parameters(
pdf,
first_page,
&live,
&resurrectable,
&skipped_stream_parameter_streams,
)? } else {
Vec::new()
};
first_page_closure.retain(|object_ref| first_page_users.contains(object_ref));
let first_page_set: BTreeSet<ObjectRef> = first_page_closure.iter().copied().collect();
let mut shared_page_indices: BTreeMap<ObjectRef, BTreeSet<u32>> = BTreeMap::new();
let mut other_page_closures: Vec<Vec<ObjectRef>> =
Vec::with_capacity(page_refs.len().saturating_sub(1));
for (page_idx, &page_ref) in page_refs.iter().enumerate().skip(1) {
let mut closure = compute_closure_with_stream_parameters(
pdf,
page_ref,
&live,
&resurrectable,
&skipped_stream_parameter_streams,
)?; let page_users =
optimization.objects_for(&crate::optimization::ObjectUser::Page(page_idx as u32));
closure.retain(|object_ref| page_users.contains(object_ref));
for obj_ref in &closure {
if first_page_set.contains(obj_ref) {
shared_page_indices
.entry(*obj_ref)
.or_default()
.insert(page_idx as u32);
}
}
other_page_closures.push(closure);
}
let thumbnail_user_set = optimization.thumbnail_objects();
let first_page_ref = page_refs.first().copied();
let mut part2_objects: Vec<ObjectRef> = Vec::new();
let mut part3_objects: Vec<ObjectRef> = Vec::new();
for obj_ref in &first_page_closure {
if root_objects.contains(obj_ref)
|| all_outline_refs.contains(obj_ref)
|| open_document_objects.contains(obj_ref)
{
continue;
}
if Some(*obj_ref) == first_page_ref {
part2_objects.push(*obj_ref);
} else if shared_page_indices.contains_key(obj_ref)
|| document_other_objects.contains(obj_ref)
|| thumbnail_user_set.contains(obj_ref)
{
part3_objects.push(*obj_ref);
} else {
part2_objects.push(*obj_ref);
}
}
part3_objects.sort_unstable_by_key(|r| pdf.linearization_object_order_key(*r));
part2_objects.sort_unstable_by_key(|r| {
(
Some(*r) != first_page_ref,
pdf.linearization_object_order_key(*r),
)
});
let moved: BTreeSet<ObjectRef> = part2_objects
.iter()
.chain(&part3_objects)
.copied()
.collect();
let part4_provisional: Vec<ObjectRef> = all_refs
.into_iter()
.filter(|r| !moved.contains(r))
.collect();
let mut page_hints: Vec<PageHintEntry> = page_refs
.iter()
.map(|&r| PageHintEntry::placeholder(r))
.collect();
let part2_set: BTreeSet<ObjectRef> = part2_objects.iter().copied().collect();
let part3_set: BTreeSet<ObjectRef> = part3_objects.iter().copied().collect();
let page0_private = part2_objects.clone();
if !page_hints.is_empty() {
page_hints[0].first_object_index = 0;
page_hints[0].object_count = (page0_private.len() + part3_objects.len()) as u32;
}
let mut per_page_private_objects: Vec<Vec<ObjectRef>> = Vec::with_capacity(page_refs.len());
per_page_private_objects.push(page0_private);
let mut all_closures: Vec<Vec<ObjectRef>> = Vec::with_capacity(page_refs.len());
all_closures.push(first_page_closure.clone());
all_closures.extend(other_page_closures.iter().cloned());
let mut page_reach: BTreeMap<ObjectRef, u32> = BTreeMap::new();
for closure in &all_closures {
let unique: BTreeSet<ObjectRef> = closure.iter().copied().collect();
for r in unique {
*page_reach.entry(r).or_insert(0) += 1;
}
}
for (i, closure) in other_page_closures.into_iter().enumerate() {
let page_idx = i + 1; let private: Vec<ObjectRef> = closure
.into_iter()
.filter(|r| {
if part2_set.contains(r) || part3_set.contains(r) {
return false;
}
if open_document_objects.contains(r) {
return false;
}
if all_outline_refs.contains(r) {
return false;
}
if document_other_objects.contains(r) {
return false;
}
if thumbnail_user_set.contains(r) {
return false;
}
page_reach.get(r).copied() == Some(1)
})
.collect();
if page_idx < page_hints.len() {
let count = private.len().max(1) as u32;
page_hints[page_idx].object_count = count;
}
per_page_private_objects.push(private);
}
let provisional_set: BTreeSet<ObjectRef> = part4_provisional.iter().copied().collect();
let mut part4_other_pages_private: Vec<ObjectRef> = Vec::new();
let mut part4_other_pages_shared: Vec<ObjectRef> = Vec::new();
let mut part4_rest: Vec<ObjectRef> = Vec::new();
let mut part4_open_document_plain: Vec<ObjectRef> = Vec::new();
let mut placed_private: BTreeSet<ObjectRef> = BTreeSet::new();
for privates in per_page_private_objects.iter().skip(1) {
for &r in privates {
if provisional_set.contains(&r) && placed_private.insert(r) {
part4_other_pages_private.push(r);
}
}
}
for &r in &part4_provisional {
if placed_private.contains(&r) {
continue;
}
if all_outline_refs.contains(&r) {
part4_rest.push(r);
continue;
}
let reach = page_reach.get(&r).copied().unwrap_or(0);
let in_first_page = first_page_set.contains(&r)
&& !root_objects.contains(&r)
&& !open_document_objects.contains(&r);
if in_first_page {
continue;
}
if root_objects.contains(&r) || open_document_objects.contains(&r) {
if let Some(ctx) = elig_ctx.as_ref() {
let obj = pdf.get_object_handle(r);
if is_eligible_for_objstm_handle(r, &obj, ctx)? {
part4_rest.push(r);
} else {
part4_open_document_plain.push(r);
}
} else {
part4_open_document_plain.push(r);
}
continue;
}
if reach >= 2 {
part4_other_pages_shared.push(r);
} else {
part4_rest.push(r);
}
}
debug_assert_eq!(
part4_other_pages_private.len()
+ part4_other_pages_shared.len()
+ part4_rest.len()
+ part4_open_document_plain.len(),
part4_provisional.len(),
"Part-4 sub-partition must preserve membership"
);
part4_open_document_plain.sort_unstable();
let outlines_in_first_page = outlines_in_first_page_predicate(pdf)?;
let outline_root_ref: Option<ObjectRef> = if let Some(root_ref) = pdf.root_ref() {
let root_handle = pdf.get_object_handle(root_ref);
pdf.resolve(&root_handle)?;
let outlines = root_handle.try_get_key(b"/Outlines")?;
outlines.object_ref()
} else {
None };
let extract_outlines = |src: &[ObjectRef]| -> Vec<ObjectRef> {
let mut v: Vec<ObjectRef> = src
.iter()
.filter(|r| all_outline_refs.contains(r))
.copied()
.collect();
if let Some(root) = outline_root_ref {
if let Some(pos) = v.iter().position(|&r| r == root) {
v[..=pos].rotate_right(1);
}
}
v
};
let (part6_outline_objects, part9_outline_objects): (Vec<ObjectRef>, Vec<ObjectRef>) =
if outlines_in_first_page {
(extract_outlines(&part4_rest), vec![])
} else {
(vec![], extract_outlines(&part4_rest))
};
let outline_extract_set: BTreeSet<ObjectRef> = part6_outline_objects
.iter()
.chain(&part9_outline_objects)
.copied()
.collect();
part4_rest.retain(|r| !outline_extract_set.contains(r));
if outlines_in_first_page && !page_hints.is_empty() {
page_hints[0].object_count += part6_outline_objects.len() as u32;
}
let outline_first_page_members: Vec<ObjectRef> = if outlines_in_first_page {
part6_outline_objects.clone()
} else {
vec![]
};
let part2_entries = part2_objects.iter().map(|&obj_ref| SharedObjectHintEntry {
object_ref: obj_ref,
referencing_pages: vec![],
});
let part3_entries = part3_objects.iter().map(|&obj_ref| {
let pages: Vec<u32> = shared_page_indices
.get(&obj_ref)
.map(|s| s.iter().copied().collect())
.unwrap_or_default();
SharedObjectHintEntry {
object_ref: obj_ref,
referencing_pages: pages,
}
});
let outline_entries =
outline_first_page_members
.iter()
.map(|&obj_ref| SharedObjectHintEntry {
object_ref: obj_ref,
referencing_pages: vec![],
});
let part4_shared_entries = part4_other_pages_shared.iter().map(|&obj_ref| {
let pages: Vec<u32> = all_referenced_pages
.get(&obj_ref)
.map(|s| s.iter().copied().collect())
.unwrap_or_default();
SharedObjectHintEntry {
object_ref: obj_ref,
referencing_pages: pages,
}
});
let shared_hints: Vec<SharedObjectHintEntry> = part2_entries
.chain(part3_entries)
.chain(outline_entries)
.chain(part4_shared_entries)
.collect();
Ok(Self {
part1_objects: Vec::new(),
part2_objects,
part3_objects,
part4_other_pages_private,
part4_other_pages_shared,
part4_rest,
part4_open_document_plain,
total_object_count,
root_ref,
pages_tree_ref,
info_ref,
page_hints,
shared_hints,
per_page_private_objects,
all_referenced_pages,
outline_first_page_members,
part9_outline_objects,
part6_outline_objects,
object_stream_mode,
removed_refs,
optimization: Some(optimization),
content_normalize_refs,
})
}
pub fn part4_objects(&self) -> Vec<ObjectRef> {
let mut objects: Vec<ObjectRef> = self
.part4_other_pages_private
.iter()
.chain(&self.part4_other_pages_shared)
.copied()
.collect();
let part9_pages: BTreeSet<ObjectRef> = self
.optimization
.as_ref()
.map(|optimization| optimization.objects_for_root_key(b"Pages"))
.filter(|pages| !pages.is_empty())
.unwrap_or_else(|| self.pages_tree_ref.into_iter().collect());
for pages_tree in part9_pages.iter().copied() {
if self.part4_rest.contains(&pages_tree) {
objects.push(pages_tree);
}
}
objects.extend(self.part9_outline_objects.iter().copied());
objects.extend(
self.part4_rest
.iter()
.copied()
.filter(|object| !part9_pages.contains(object)),
);
objects
}
pub(crate) fn part8_container_nums(
&self,
member_to_container: &BTreeMap<ObjectRef, (u32, u32)>,
) -> BTreeSet<u32> {
let first_page: BTreeSet<ObjectRef> = self
.part2_objects
.iter()
.chain(&self.part3_objects)
.copied()
.collect();
let part4_shared: BTreeSet<ObjectRef> =
self.part4_other_pages_shared.iter().copied().collect();
let mut all_containers: BTreeSet<u32> = BTreeSet::new();
let mut container_pages: BTreeMap<u32, BTreeSet<u32>> = BTreeMap::new();
let mut has_first_page_member: BTreeSet<u32> = BTreeSet::new();
let mut has_shared_member: BTreeSet<u32> = BTreeSet::new();
for (member, &(cnum, _)) in member_to_container {
all_containers.insert(cnum);
if first_page.contains(member) {
has_first_page_member.insert(cnum);
}
if part4_shared.contains(member) {
has_shared_member.insert(cnum);
}
if let Some(pages) = self.all_referenced_pages.get(member) {
container_pages
.entry(cnum)
.or_default()
.extend(pages.iter().copied().filter(|&p| p != 0));
}
}
all_containers
.into_iter()
.filter(|cnum| {
!has_first_page_member.contains(cnum)
&& (has_shared_member.contains(cnum)
|| container_pages.get(cnum).is_some_and(|p| p.len() >= 2))
})
.collect()
}
pub(crate) fn rest_container_nums(
&self,
member_to_container: &BTreeMap<ObjectRef, (u32, u32)>,
) -> BTreeSet<u32> {
self.part9_outline_objects
.iter()
.filter_map(|member| member_to_container.get(member).map(|&(cnum, _)| cnum))
.collect()
}
pub(crate) fn canonical_shared_hints(
&self,
member_to_container: &BTreeMap<ObjectRef, (u32, u32)>,
renumber: &RenumberMap,
second_half_container_nums: &BTreeSet<u32>,
open_document_container_nums: &BTreeSet<u32>,
) -> Vec<SharedObjectHintEntry> {
if member_to_container.is_empty() {
return self.shared_hints.clone();
}
let first_page_input = self.part2_objects.len()
+ self.part3_objects.len()
+ self.outline_first_page_members.len();
let rest_container_nums = self.rest_container_nums(member_to_container);
let mut container_pos: BTreeMap<u32, usize> = BTreeMap::new();
let mut out: Vec<SharedObjectHintEntry> = Vec::with_capacity(self.shared_hints.len());
let mut first_page_out_end: Option<usize> = None;
for (input_idx, entry) in self.shared_hints.iter().enumerate() {
if input_idx == first_page_input {
first_page_out_end = Some(out.len());
}
match member_to_container.get(&entry.object_ref) {
Some(&(container_num, _idx)) => {
if open_document_container_nums.contains(&container_num) {
continue;
}
if input_idx < first_page_input
&& second_half_container_nums.contains(&container_num)
{
continue;
}
if input_idx >= first_page_input && rest_container_nums.contains(&container_num)
{
continue;
}
if let Some(&pos) = container_pos.get(&container_num) {
let merged: &mut Vec<u32> = &mut out[pos].referencing_pages;
for &p in &entry.referencing_pages {
if let Err(insert_at) = merged.binary_search(&p) {
merged.insert(insert_at, p);
}
}
} else {
let mut pages = entry.referencing_pages.clone();
pages.sort_unstable();
pages.dedup();
container_pos.insert(container_num, out.len());
out.push(SharedObjectHintEntry {
object_ref: ObjectRef::new(container_num, u16::MAX),
referencing_pages: pages,
});
}
}
None => out.push(entry.clone()),
}
}
let boundary = first_page_out_end.unwrap_or(out.len());
let new_number = |e: &SharedObjectHintEntry| -> u32 {
if e.object_ref.generation == u16::MAX {
e.object_ref.number
} else {
renumber
.new_for_original(e.object_ref)
.map_or(u32::MAX, |r| r.number)
}
};
out[..boundary].sort_unstable_by_key(&new_number);
for cnum in self.part8_container_nums(member_to_container) {
if !container_pos.contains_key(&cnum)
&& !open_document_container_nums.contains(&cnum)
&& !rest_container_nums.contains(&cnum)
{
out.push(SharedObjectHintEntry {
object_ref: ObjectRef::new(cnum, u16::MAX),
referencing_pages: Vec::new(), });
}
}
out[boundary..].sort_unstable_by_key(&new_number);
if !self.all_referenced_pages.is_empty() {
let mut container_members: BTreeMap<u32, Vec<ObjectRef>> = BTreeMap::new();
for (&member, &(cnum, _)) in member_to_container {
container_members.entry(cnum).or_default().push(member);
}
let pages_excluding_first = |refs: &mut dyn Iterator<Item = ObjectRef>| -> Vec<u32> {
let mut pages: BTreeSet<u32> = BTreeSet::new();
for r in refs {
if let Some(ps) = self.all_referenced_pages.get(&r) {
pages.extend(ps.iter().copied().filter(|&p| p != 0));
}
}
pages.into_iter().collect()
};
for entry in &mut out {
entry.referencing_pages = if entry.object_ref.generation == u16::MAX {
let members = container_members
.get(&entry.object_ref.number)
.cloned()
.unwrap_or_default();
pages_excluding_first(&mut members.into_iter())
} else {
pages_excluding_first(&mut std::iter::once(entry.object_ref))
};
}
}
out
}
pub fn all_assigned_refs(&self) -> BTreeSet<ObjectRef> {
self.part1_objects
.iter()
.chain(&self.part2_objects)
.chain(&self.part3_objects)
.chain(&self.part4_other_pages_private)
.chain(&self.part4_other_pages_shared)
.chain(&self.part4_rest)
.copied()
.collect()
}
pub(crate) fn renumber_assigned_refs(&self) -> BTreeSet<ObjectRef> {
self.part2_objects
.iter()
.chain(&self.part3_objects)
.chain(&self.part4_other_pages_private)
.chain(&self.part4_other_pages_shared)
.chain(&self.part4_rest)
.chain(&self.part6_outline_objects)
.chain(&self.part9_outline_objects)
.chain(&self.part4_open_document_plain)
.copied()
.collect()
}
pub fn parts_are_disjoint(&self) -> bool {
let mut seen = BTreeSet::new();
for r in self
.part1_objects
.iter()
.chain(&self.part2_objects)
.chain(&self.part3_objects)
.chain(&self.part4_other_pages_private)
.chain(&self.part4_other_pages_shared)
.chain(&self.part6_outline_objects)
.chain(&self.part9_outline_objects)
.chain(&self.part4_rest)
.chain(&self.part4_open_document_plain)
{
if !seen.insert(*r) {
return false;
}
}
true
}
}
impl Default for LinearizationPlan {
fn default() -> Self {
Self {
part1_objects: Vec::new(),
part2_objects: Vec::new(),
part3_objects: Vec::new(),
part4_other_pages_private: Vec::new(),
part4_other_pages_shared: Vec::new(),
part4_rest: Vec::new(),
part4_open_document_plain: Vec::new(),
total_object_count: 0,
root_ref: None,
pages_tree_ref: None,
info_ref: None,
page_hints: Vec::new(),
shared_hints: Vec::new(),
per_page_private_objects: Vec::new(),
all_referenced_pages: BTreeMap::new(),
outline_first_page_members: Vec::new(),
part9_outline_objects: Vec::new(),
part6_outline_objects: Vec::new(),
optimization: None,
removed_refs: BTreeSet::new(),
object_stream_mode: crate::writer::ObjectStreamMode::Disable,
content_normalize_refs: BTreeSet::new(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub(crate) struct ObjStmBatchPlan {
pub(crate) open_document_batches: Vec<Vec<ObjectRef>>,
pub(crate) part3_batches: Vec<Vec<ObjectRef>>,
pub(crate) part4_batches: Vec<RoutedObjStmBatch>,
}
impl LinearizationPlan {
pub(crate) fn objstm_batches<R: Read + Seek>(
&self,
pdf: &mut Pdf<R>,
config: &PlannerConfig,
) -> crate::Result<ObjStmBatchPlan> {
if config.mode == ObjectStreamMode::Disable {
return Ok(ObjStmBatchPlan::default());
}
let optimization = self.optimization.as_ref().ok_or_else(|| {
crate::Error::Unsupported(
"linearization plan: missing optimization snapshot".to_string(),
)
})?;
let ctx = eligibility_context(pdf)?;
let length_exclusions =
if config.mode == ObjectStreamMode::Preserve && config.preserve_unreferenced_objects {
BTreeSet::new()
} else {
compressible_objgens_qpdf_plan(pdf)?.indirect_objstm_length_refs
};
let plan = match config.mode {
ObjectStreamMode::Disable => unreachable!(),
ObjectStreamMode::Generate => {
self.objstm_batches_generate(pdf, config, &ctx, &length_exclusions, optimization)?
}
ObjectStreamMode::Preserve => {
self.objstm_batches_preserve(pdf, config, &ctx, &length_exclusions, optimization)?
}
};
Ok(plan)
}
fn objstm_batches_generate<R: Read + Seek>(
&self,
pdf: &mut Pdf<R>,
_config: &PlannerConfig,
_ctx: &crate::writer::object_streams::EligibilityContext,
_length_exclusions: &BTreeSet<ObjectRef>,
optimization: &crate::optimization::Optimization,
) -> crate::Result<ObjStmBatchPlan> {
let assigned = self.renumber_assigned_refs();
let containers = objstm_membership_linearized_with_eligibility(
pdf,
&assigned,
optimization.generate_objstm_eligible(),
)?; let routes = route_objstm_containers(
optimization,
!self.outline_first_page_members.is_empty(),
&containers,
);
let mut open_document_batches: Vec<Vec<ObjectRef>> = Vec::new();
let mut part3_private: Vec<Vec<ObjectRef>> = Vec::new();
let mut part3_shared: Vec<Vec<ObjectRef>> = Vec::new();
let mut part3_outlines: Vec<Vec<ObjectRef>> = Vec::new();
let mut part4_private: Vec<RoutedObjStmBatch> = Vec::new();
let mut part4_shared: Vec<RoutedObjStmBatch> = Vec::new();
let mut part4_rest: Vec<RoutedObjStmBatch> = Vec::new();
for (mut members, route) in containers.into_iter().zip(routes) {
members.sort_unstable_by_key(|r| r.number);
push_routed_objstm_batch(
members,
route,
None,
&mut open_document_batches,
&mut part3_private,
&mut part3_shared,
&mut part3_outlines,
&mut part4_private,
&mut part4_shared,
&mut part4_rest,
);
}
let mut part4_batches = part4_private;
part4_batches.extend(part4_shared);
part4_batches.extend(part4_rest);
let mut part3_batches = part3_private;
part3_batches.extend(part3_shared);
part3_batches.extend(part3_outlines);
Ok(ObjStmBatchPlan {
open_document_batches,
part3_batches,
part4_batches,
})
}
fn objstm_batches_preserve<R: Read + Seek>(
&self,
pdf: &mut Pdf<R>,
_config: &PlannerConfig,
ctx: &crate::writer::object_streams::EligibilityContext,
length_exclusions: &BTreeSet<ObjectRef>,
optimization: &crate::optimization::Optimization,
) -> crate::Result<ObjStmBatchPlan> {
use crate::XrefEntry;
let entries = pdf.source_xref_entries();
let mut groups: BTreeMap<u32, Vec<(u32, ObjectRef)>> = BTreeMap::new();
for (obj_ref, offset) in &entries {
if let XrefEntry::Compressed { stream, index } = offset {
groups.entry(*stream).or_default().push((*index, *obj_ref));
}
}
let assigned = self.renumber_assigned_refs();
let page_dicts: BTreeSet<ObjectRef> = crate::pages::page_refs(pdf)?.into_iter().collect();
let catalog = self.root_ref;
let mut containers: Vec<Vec<ObjectRef>> = Vec::new();
let mut source_container_numbers: Vec<u32> = Vec::new();
for (container_num, mut members) in groups {
members.sort_by_key(|(idx, _)| *idx);
let mut surviving = Vec::new();
for (_idx, obj_ref) in members {
if page_dicts.contains(&obj_ref) || Some(obj_ref) == catalog {
continue;
}
if length_exclusions.contains(&obj_ref) || !assigned.contains(&obj_ref) {
continue;
}
let eligible = {
let obj = pdf.get_object_handle(obj_ref);
is_eligible_for_objstm_handle(obj_ref, &obj, ctx)?
};
let obj = pdf.get_object_handle(obj_ref);
if eligible && !crate::writer::object_streams::is_qpdf_signature_dict(pdf, &obj)? {
surviving.push(obj_ref);
}
}
if !surviving.is_empty() {
source_container_numbers.push(container_num);
containers.push(surviving);
}
}
let routes = route_objstm_containers(
optimization,
!self.outline_first_page_members.is_empty(),
&containers,
);
let mut open_document_batches = Vec::new();
let mut part3_private = Vec::new();
let mut part3_shared = Vec::new();
let mut part3_outlines = Vec::new();
let mut part4_private = Vec::new();
let mut part4_shared = Vec::new();
let mut part4_rest = Vec::new();
for ((members, route), source_container_number) in containers
.into_iter()
.zip(routes)
.zip(source_container_numbers)
{
push_routed_objstm_batch(
members,
route,
Some(source_container_number),
&mut open_document_batches,
&mut part3_private,
&mut part3_shared,
&mut part3_outlines,
&mut part4_private,
&mut part4_shared,
&mut part4_rest,
);
}
let mut part3_batches = part3_private;
part3_batches.extend(part3_shared);
part3_batches.extend(part3_outlines);
let mut part4_batches = part4_private;
part4_batches.extend(part4_shared);
part4_batches.extend(part4_rest);
Ok(ObjStmBatchPlan {
open_document_batches,
part3_batches,
part4_batches,
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ContainerPart {
OpenDocument,
FirstPagePrivate,
FirstPageShared,
FirstPageOutlines,
OtherPagePrivate,
OtherPageShared,
Rest,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct RoutedObjStmBatch {
pub(crate) members: Vec<ObjectRef>,
pub(crate) route: ContainerPart,
pub(crate) source_container_number: Option<u32>,
}
#[allow(clippy::too_many_arguments)]
fn push_routed_objstm_batch(
members: Vec<ObjectRef>,
route: ContainerPart,
source_container_number: Option<u32>,
open_document_batches: &mut Vec<Vec<ObjectRef>>,
part3_private: &mut Vec<Vec<ObjectRef>>,
part3_shared: &mut Vec<Vec<ObjectRef>>,
part3_outlines: &mut Vec<Vec<ObjectRef>>,
part4_private: &mut Vec<RoutedObjStmBatch>,
part4_shared: &mut Vec<RoutedObjStmBatch>,
part4_rest: &mut Vec<RoutedObjStmBatch>,
) {
match route {
ContainerPart::OpenDocument => open_document_batches.push(members),
ContainerPart::FirstPagePrivate => part3_private.push(members),
ContainerPart::FirstPageShared => part3_shared.push(members),
ContainerPart::FirstPageOutlines => part3_outlines.push(members),
ContainerPart::OtherPagePrivate => part4_private.push(RoutedObjStmBatch {
members,
route,
source_container_number,
}),
ContainerPart::OtherPageShared => part4_shared.push(RoutedObjStmBatch {
members,
route,
source_container_number,
}),
ContainerPart::Rest => part4_rest.push(RoutedObjStmBatch {
members,
route,
source_container_number,
}),
}
}
impl std::ops::Deref for RoutedObjStmBatch {
type Target = [ObjectRef];
fn deref(&self) -> &Self::Target {
&self.members
}
}
pub(crate) fn objstm_membership_linearized_with_eligibility<R: Read + Seek>(
pdf: &mut Pdf<R>,
assigned: &BTreeSet<ObjectRef>,
eligibility_override: Option<&[ObjectRef]>,
) -> crate::Result<Vec<Vec<ObjectRef>>> {
let mut eligible = match eligibility_override {
Some(eligible) => eligible.to_vec(),
None => crate::writer::object_streams::get_compressible_objgens(pdf)?,
};
eligible.retain(|r| assigned.contains(r));
let streams = crate::writer::object_streams::even_split_into_streams(&eligible);
let mut erase: BTreeSet<ObjectRef> = crate::pages::page_refs(pdf)?.into_iter().collect();
if let Some(root) = pdf.root_ref() {
erase.insert(root);
}
Ok(streams
.into_iter()
.map(|stream| {
stream
.into_iter()
.filter(|r| !erase.contains(r))
.collect::<Vec<ObjectRef>>()
})
.filter(|container| !container.is_empty())
.collect())
}
const OPEN_DOCUMENT_CATALOG_KEYS: [&[u8]; 5] = [
b"ViewerPreferences",
b"PageMode",
b"Threads",
b"OpenAction",
b"AcroForm",
];
fn is_open_document_user(user: &crate::optimization::ObjectUser) -> bool {
match user {
crate::optimization::ObjectUser::TrailerKey(key) => key == b"Encrypt",
crate::optimization::ObjectUser::RootKey(key) => {
OPEN_DOCUMENT_CATALOG_KEYS.contains(&key.as_slice())
}
_ => false,
}
}
fn is_document_other_user(user: &crate::optimization::ObjectUser) -> bool {
match user {
crate::optimization::ObjectUser::TrailerKey(key) => key != b"Encrypt",
crate::optimization::ObjectUser::RootKey(key) => {
key != b"Outlines" && !OPEN_DOCUMENT_CATALOG_KEYS.contains(&key.as_slice())
}
_ => false,
}
}
fn outlines_in_first_page_predicate<R: Read + Seek>(pdf: &mut Pdf<R>) -> crate::Result<bool> {
let Some(root) = pdf.root_ref() else {
return Ok(false); };
let root_handle = pdf.get_object_handle(root);
pdf.resolve(&root_handle)?;
if !root_handle.try_has_key(b"/Outlines")? {
return Ok(false);
}
Ok(root_handle
.try_get_key(b"/PageMode")?
.try_as_name()?
.is_some_and(|name| name == b"UseOutlines"))
}
pub(crate) fn route_objstm_containers(
optimization: &crate::optimization::Optimization,
outlines_in_first_page: bool,
containers: &[Vec<ObjectRef>],
) -> Vec<ContainerPart> {
containers
.iter()
.map(|members| {
let users = optimization.users_for_members(members.iter());
classify_container_users(&users, outlines_in_first_page)
})
.collect()
}
fn classify_container_users(
users: &BTreeSet<crate::optimization::ObjectUser>,
outlines_in_first_page: bool,
) -> ContainerPart {
if users.iter().any(is_outline_user) {
return if outlines_in_first_page {
ContainerPart::FirstPageOutlines
} else {
ContainerPart::Rest
};
}
if users.iter().any(is_open_document_user) {
return ContainerPart::OpenDocument;
}
let page_numbers: BTreeSet<u32> = users
.iter()
.filter_map(|user| match user {
crate::optimization::ObjectUser::Page(page_number) => Some(*page_number),
_ => None,
})
.collect();
let has_document_other = users.iter().any(is_document_other_user);
let has_thumbnail = users.iter().any(is_thumbnail_user);
if page_numbers.contains(&0) {
return if page_numbers.iter().any(|&page| page != 0) || has_document_other || has_thumbnail
{
ContainerPart::FirstPageShared
} else {
ContainerPart::FirstPagePrivate
};
}
match page_numbers.len() {
0 => ContainerPart::Rest,
1 if has_document_other || has_thumbnail => ContainerPart::Rest,
1 => ContainerPart::OtherPagePrivate,
_ => ContainerPart::OtherPageShared,
}
}
fn is_outline_user(user: &crate::optimization::ObjectUser) -> bool {
matches!(
user,
crate::optimization::ObjectUser::RootKey(key) if key == b"Outlines"
)
}
fn is_thumbnail_user(user: &crate::optimization::ObjectUser) -> bool {
matches!(user, crate::optimization::ObjectUser::Thumbnail(_))
}
#[cfg(test)]
mod tests {
use super::{
collect_direct_handle_refs, collect_direct_handle_refs_with_context,
collect_direct_handle_refs_with_stream_parameters_context, LinearizationPlan,
};
use crate::acroform_document_helper::AcroFormDocumentHelper;
use crate::object_handle::ObjectHandle;
use crate::parser::MAX_PARSE_DEPTH;
use crate::writer::{ObjectStreamMode, WriterOptions};
use crate::Pdf;
use flate2::write::ZlibEncoder;
use flate2::Compression;
use std::cell::Cell;
use std::collections::BTreeSet;
use std::io::{Cursor, Write};
use std::rc::Rc;
fn flate(data: &[u8]) -> Vec<u8> {
let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
encoder.write_all(data).expect("compress fixture stream");
encoder.finish().expect("finish fixture stream")
}
fn stream_object(number: u32, dictionary: &str, data: &[u8]) -> Vec<u8> {
let mut object = format!(
"{number} 0 obj\n<< {dictionary} /Length {} >>\nstream\n",
data.len()
)
.into_bytes();
object.extend_from_slice(data);
object.extend_from_slice(b"\nendstream\nendobj\n");
object
}
fn parameter_probe_fixture() -> Vec<u8> {
let page_content_indirect_filter = flate(b"q Q\n");
let page_content_direct_filter = flate(b"q Q\n");
let existing_appearance = flate(b"/Tx BMC\nEMC\n");
let objects = vec![
b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R /AcroForm 6 0 R >>\nendobj\n"
.to_vec(),
b"2 0 obj\n<< /Type /Pages /Count 1 /Kids [3 0 R] /ColorSpace [10 0 R] >>\nendobj\n".to_vec(),
b"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /ProcSet [10 0 R] >> /Contents [4 0 R 12 0 R] /Annots [5 0 R] >>\nendobj\n".to_vec(),
stream_object(4, "/Filter 9 0 R", &page_content_indirect_filter),
b"5 0 obj\n<< /Type /Annot /Subtype /Widget /FT /Tx /T (name) /V (Hello) /DA (/Helv 12 Tf 0 g) /Rect [10 10 200 30] /P 3 0 R /AP << /N 8 0 R >> >>\nendobj\n".to_vec(),
b"6 0 obj\n<< /Fields [5 0 R] /NeedAppearances true /DR << >> /DA (/Helv 12 Tf 0 g) >>\nendobj\n".to_vec(),
b"7 0 obj\nnull\nendobj\n".to_vec(),
stream_object(
8,
"/Type /XObject /Subtype /Form /BBox [0 0 190 20] /Resources << >> /Filter 9 0 R",
&existing_appearance,
),
b"9 0 obj\n/FlateDecode\nendobj\n".to_vec(),
b"10 0 obj\n<< /Type /ExtGState >>\nendobj\n".to_vec(),
b"11 0 obj\nnull\nendobj\n".to_vec(),
stream_object(12, "/Filter /FlateDecode", &page_content_direct_filter),
];
let mut pdf = b"%PDF-1.7\n".to_vec();
let object_count = objects.len();
let mut offsets = Vec::with_capacity(object_count);
for object in objects {
offsets.push(pdf.len());
pdf.extend_from_slice(&object);
}
let xref = pdf.len();
let _ = writeln!(&mut pdf, "xref\n0 {}", object_count + 1);
pdf.extend_from_slice(b"0000000000 65535 f \n");
for offset in offsets {
let _ = writeln!(&mut pdf, "{offset:010} 00000 n ");
}
let _ = writeln!(
&mut pdf,
"trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF",
object_count + 1
);
pdf
}
struct PassThroughTokenFilter;
impl crate::token_filter::TokenFilter for PassThroughTokenFilter {
fn handle_token(
&mut self,
token: &crate::tokenizer::Token,
output: &mut crate::token_filter::TokenFilterOutput<'_>,
) -> crate::pipeline::PipelineResult<()> {
output.write_token(token)
}
}
#[test]
fn qpdf_linearization_probe_consumes_modified_and_parameterized_streams() {
let mut pdf = Pdf::open(Cursor::new(parameter_probe_fixture())).expect("parse fixture");
AcroFormDocumentHelper::new(&mut pdf)
.expect("AcroForm")
.generate_appearances_if_needed()
.expect("generate appearance");
let options = WriterOptions {
object_streams: ObjectStreamMode::Disable,
..WriterOptions::default()
};
let plan = LinearizationPlan::from_pdf_with_writer_options(&mut pdf, &options)
.expect("build linearization plan");
assert_eq!(plan.page_hints.len(), 1);
assert!(!plan.part2_objects.is_empty());
assert!(
plan.all_assigned_refs()
.contains(&crate::ObjectRef::new(10, 0)),
"inline resource and ancestor array references must reach the plan"
);
}
#[test]
fn qpdf_linearization_parameter_probe_runs_once_before_raw_retry() {
let mut pdf = Pdf::open(Cursor::new(parameter_probe_fixture())).expect("parse fixture");
let stream = pdf.get_object_handle(crate::ObjectRef::new(4, 0));
pdf.resolve(&stream).expect("resolve page content");
let token = crate::tokenizer::Token::new(crate::tokenizer::TokenType::Word, b"q".to_vec());
let mut filter = PassThroughTokenFilter;
let mut output = crate::token_filter::TokenFilterOutput::new(None);
crate::token_filter::TokenFilter::handle_token(&mut filter, &token, &mut output)
.expect("pass-through filter token");
let provider_calls = Rc::new(Cell::new(0));
let provider_calls_for_callback = Rc::clone(&provider_calls);
stream
.replace_stream_data_with_retry_callback(
move |_pipeline, _suppress_warnings, _will_retry| {
provider_calls_for_callback.set(provider_calls_for_callback.get() + 1);
Ok(false)
},
None,
None,
)
.expect("install stateful stream provider");
stream
.add_token_filter(Rc::new(std::cell::RefCell::new(PassThroughTokenFilter)))
.expect("register token filter");
pdf.mark_object_handle_dirty(&stream)
.expect("mark modified content");
let options = WriterOptions {
object_streams: ObjectStreamMode::Disable,
..WriterOptions::default()
};
LinearizationPlan::from_pdf_with_writer_options(&mut pdf, &options)
.expect("build linearization plan");
assert_eq!(
provider_calls.get(),
2,
"one qpdf parameter probe may retry once with raw data, but the optimization callback must not probe the same stream again"
);
}
#[test]
fn direct_reference_walkers_reject_depth_beyond_parser_limit() {
let handle = ObjectHandle::null();
let mut refs = Vec::new();
let error = collect_direct_handle_refs(&handle, MAX_PARSE_DEPTH + 1, &mut refs)
.expect_err("the direct reference walk has a parser-depth guard");
assert!(error.to_string().contains("maximum of 500"));
let mut contextual = Vec::new();
let error = collect_direct_handle_refs_with_context(
&handle,
MAX_PARSE_DEPTH + 1,
false,
&mut contextual,
)
.expect_err("the contextual reference walk has a parser-depth guard");
assert!(error.to_string().contains("maximum of 500"));
let mut stream_contextual = Vec::new();
let error = collect_direct_handle_refs_with_stream_parameters_context(
&handle,
MAX_PARSE_DEPTH + 1,
false,
&mut stream_contextual,
&BTreeSet::new(),
)
.expect_err("the stream-policy reference walk has a parser-depth guard");
assert!(error.to_string().contains("maximum of 500"));
}
}