use crate::linearization::renumber::RenumberMap;
use crate::object_handle::ObjectHandle;
use crate::parser::MAX_PARSE_DEPTH;
use crate::pdf::WriterObjectOrderKey;
use crate::qpdf_obj_gen::QpdfObjGen;
use crate::writer::object_streams::{
compressible_objgens_qpdf_plan, eligibility_context, is_eligible_for_objstm_handle,
ObjectStreamMode, PlannerConfig,
};
use crate::{ObjectRef, Pdf, Result};
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::io::{Read, Seek};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PageHintEntry {
pub page_ref: ObjectRef,
pub first_object_index: u32,
pub object_count: u32,
pub byte_length: u64,
}
impl PageHintEntry {
pub fn placeholder(page_ref: ObjectRef) -> Self {
Self {
page_ref,
first_object_index: 0,
object_count: 0,
byte_length: 0,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SharedObjectHintEntry {
pub object_ref: ObjectRef,
pub referencing_pages: Vec<u32>,
}
impl SharedObjectHintEntry {
pub fn new(object_ref: ObjectRef) -> Self {
Self {
object_ref,
referencing_pages: Vec::new(),
}
}
}
fn linearization_content_normalize_refs<R: Read + Seek>(
pdf: &mut Pdf<R>,
options: &crate::writer::WriterOptions,
page_refs: &[ObjectRef],
) -> Result<BTreeSet<QpdfObjGen>> {
if !options.content_normalization {
return Ok(BTreeSet::new());
}
let mut refs = BTreeSet::new();
for page_ref in page_refs {
let page = pdf.get_object_handle(*page_ref);
let contents = page.try_get_key(b"/Contents")?;
let mut record_stream = |stream: &ObjectHandle| -> Result<()> {
if stream.type_code()? == 10 {
if let Some(object_gen) = stream
.qpdf_obj_gen()
.filter(|object_gen| object_gen.is_indirect())
{
refs.insert(object_gen);
}
}
Ok(())
};
if contents.type_code()? == 10 {
record_stream(&contents)?;
} else if let Some(items) = contents.try_as_array()? {
for item in items {
record_stream(&item)?;
}
}
}
Ok(refs)
}
fn stream_has_indirect_parameter_edge(handle: &ObjectHandle) -> Result<bool> {
let Some(stream_dict) = handle.as_stream_dict() else {
return Ok(false); };
for key in [b"/Filter".as_slice(), b"/DecodeParms".as_slice()] {
let value = stream_dict.try_get_key(key)?;
let mut refs = Vec::new();
collect_direct_handle_refs(&value, 0, &mut refs)?;
if !refs.is_empty() {
return Ok(true);
}
}
Ok(false)
}
fn stream_parameters_removed_for_linearization(
handle: &ObjectHandle,
stream_ref: Option<QpdfObjGen>,
options: &crate::writer::WriterOptions,
content_normalize_refs: &BTreeSet<QpdfObjGen>,
) -> Result<bool> {
let normalize_content =
stream_ref.is_some_and(|object_gen| content_normalize_refs.contains(&object_gen));
if handle.is_data_modified() {
crate::writer::plain::body::canonical_stream_filter_probe_for_linearization(
handle,
options,
normalize_content,
) } else {
crate::writer::plain::body::canonical_stream_will_be_refiltered_with_policy(
handle,
options,
true,
normalize_content,
) }
}
fn collect_direct_handle_refs(
handle: &ObjectHandle,
depth: usize,
out: &mut Vec<ObjectRef>,
) -> Result<()> {
if depth > MAX_PARSE_DEPTH {
return Err(crate::Error::Unsupported(format!(
"linearization plan: inline object nesting exceeds maximum of {MAX_PARSE_DEPTH}"
)));
}
let mut contextual = Vec::new();
collect_direct_handle_refs_with_context(handle, depth, false, &mut contextual)?;
out.extend(contextual.into_iter().map(|(object_ref, _)| object_ref));
Ok(())
}
fn collect_direct_handle_refs_with_stream_parameters(
handle: &ObjectHandle,
depth: usize,
out: &mut Vec<ObjectRef>,
skipped_stream_parameter_streams: &BTreeSet<QpdfObjGen>,
) -> Result<()> {
let mut contextual = Vec::new();
collect_direct_handle_refs_with_stream_parameters_context(
handle,
depth,
false,
&mut contextual,
skipped_stream_parameter_streams,
)?; out.extend(contextual.into_iter().map(|(object_ref, _)| object_ref));
Ok(())
}
fn collect_direct_handle_refs_with_context(
handle: &ObjectHandle,
depth: usize,
in_array: bool,
out: &mut Vec<(ObjectRef, bool)>,
) -> Result<()> {
if depth > MAX_PARSE_DEPTH {
return Err(crate::Error::Unsupported(format!(
"linearization plan: inline object nesting exceeds maximum of {MAX_PARSE_DEPTH}"
)));
}
if let Some(object_ref) = handle.object_ref() {
out.push((object_ref, in_array));
return Ok(());
}
collect_direct_handle_children(
handle,
depth,
in_array,
&mut |child, child_depth, child_in_array| {
collect_direct_handle_refs_with_context(child, child_depth, child_in_array, out)
},
)
}
fn collect_direct_handle_refs_with_stream_parameters_context(
handle: &ObjectHandle,
depth: usize,
in_array: bool,
out: &mut Vec<(ObjectRef, bool)>,
skipped_stream_parameter_streams: &BTreeSet<QpdfObjGen>,
) -> Result<()> {
if depth > MAX_PARSE_DEPTH {
return Err(crate::Error::Unsupported(format!(
"linearization plan: inline object nesting exceeds maximum of {MAX_PARSE_DEPTH}"
)));
}
if let Some(object_ref) = handle.object_ref() {
out.push((object_ref, in_array));
return Ok(());
}
collect_direct_handle_children_with_stream_parameters(
handle,
depth,
in_array,
skipped_stream_parameter_streams,
&mut |child, child_depth, child_in_array| {
collect_direct_handle_refs_with_stream_parameters_context(
child,
child_depth,
child_in_array,
out,
skipped_stream_parameter_streams,
)
},
)
}
fn collect_direct_handle_children<F>(
handle: &ObjectHandle,
depth: usize,
_parent_in_array: bool,
visit: &mut F,
) -> Result<()>
where
F: FnMut(&ObjectHandle, usize, bool) -> Result<()>,
{
collect_direct_handle_children_with_stream_parameters(
handle,
depth,
_parent_in_array,
&BTreeSet::new(),
visit,
)
}
fn collect_direct_handle_children_with_stream_parameters<F>(
handle: &ObjectHandle,
depth: usize,
_parent_in_array: bool,
skipped_stream_parameter_streams: &BTreeSet<QpdfObjGen>,
visit: &mut F,
) -> Result<()>
where
F: FnMut(&ObjectHandle, usize, bool) -> Result<()>,
{
handle.try_dereference()?;
if let Some(stream_dict) = handle.as_stream_dict() {
let Some(entries) = stream_dict.try_as_dictionary()? else {
return Ok(());
};
let skip_stream_parameters =
handle_has_stream_parameter_skip(handle, skipped_stream_parameter_streams)?;
for (key, child) in entries {
if key == b"/Length"
|| (skip_stream_parameters
&& matches!(key.as_slice(), b"/Filter" | b"/DecodeParms"))
{
continue;
}
visit(&child, depth + 1, false)?;
}
return Ok(());
}
if let Some(children) = handle.try_as_array()? {
for child in children {
visit(&child, depth + 1, true)?;
}
return Ok(());
}
if let Some(entries) = handle.try_as_dictionary()? {
for (_key, child) in entries {
visit(&child, depth + 1, false)?;
}
}
Ok(())
}
fn handle_has_stream_parameter_skip(
handle: &ObjectHandle,
skipped_stream_parameter_streams: &BTreeSet<QpdfObjGen>,
) -> Result<bool> {
Ok(handle
.qpdf_obj_gen()
.is_some_and(|object_gen| skipped_stream_parameter_streams.contains(&object_gen)))
}
fn collect_handle_children_with_stream_parameters(
handle: &ObjectHandle,
depth: usize,
out: &mut Vec<(ObjectRef, bool)>,
skipped_stream_parameter_streams: &BTreeSet<QpdfObjGen>,
) -> Result<()> {
collect_direct_handle_children_with_stream_parameters(
handle,
depth,
false,
skipped_stream_parameter_streams,
&mut |child, child_depth, in_array| {
collect_direct_handle_refs_with_stream_parameters_context(
child,
child_depth,
in_array,
out,
skipped_stream_parameter_streams,
)
},
)
}
fn is_page_tree_handle(handle: &ObjectHandle) -> Result<bool> {
Ok(handle.try_is_dictionary_of_type(b"Pages", b"")?
|| handle.try_is_dictionary_of_type(b"Page", b"")?)
}
fn compute_closure_with_stream_parameters<R: Read + Seek>(
pdf: &mut Pdf<R>,
root: ObjectRef,
live: &BTreeSet<ObjectRef>,
resurrectable: &BTreeSet<ObjectRef>,
skipped_stream_parameter_streams: &BTreeSet<QpdfObjGen>,
) -> crate::Result<Vec<ObjectRef>> {
let mut visited: BTreeSet<ObjectRef> = BTreeSet::new();
let mut order: Vec<ObjectRef> = Vec::new();
let mut queue: VecDeque<(ObjectRef, bool)> = VecDeque::from([(root, false)]);
let mut deferred_resurrect: BTreeSet<ObjectRef> = BTreeSet::new();
let mut seen_as_array: BTreeSet<ObjectRef> = BTreeSet::new();
let admits_body_object = |r: ObjectRef| -> bool { r.number != 0 && live.contains(&r) };
while let Some((current, via_array)) = queue.pop_front() {
if !visited.insert(current) {
continue;
}
if resurrectable.contains(¤t) {
if via_array || seen_as_array.contains(¤t) {
order.push(current);
} else {
deferred_resurrect.insert(current);
}
continue;
}
if !admits_body_object(current) {
continue;
}
order.push(current);
let current_handle = pdf.get_object_handle(current);
let is_pages_node = current_handle.try_is_dictionary_of_type(b"Pages", b"")?;
let is_page_leaf = current_handle.try_is_dictionary_of_type(b"Page", b"")?;
if is_pages_node || is_page_leaf {
if let Some(dict) = current_handle.try_as_dictionary()? {
if is_page_leaf {
{
let resources = current_handle.try_get_key(b"/Resources")?;
let mut seeds: Vec<(ObjectRef, bool)> = Vec::new();
collect_direct_handle_refs_with_context(&resources, 0, false, &mut seeds)?;
for &(r, va) in &seeds {
if va {
seen_as_array.insert(r);
}
}
let mut stack: Vec<(ObjectRef, bool)> = seeds.into_iter().rev().collect();
while let Some((r, via_array)) = stack.pop() {
if !visited.insert(r) {
continue;
}
if resurrectable.contains(&r) {
if via_array || seen_as_array.contains(&r) {
order.push(r);
} else {
deferred_resurrect.insert(r);
} continue;
}
if !admits_body_object(r) {
continue;
}
let child_handle = pdf.get_object_handle(r);
if is_page_tree_handle(&child_handle)? {
continue;
}
order.push(r);
let mut child_refs: Vec<(ObjectRef, bool)> = Vec::new();
collect_handle_children_with_stream_parameters(
&child_handle,
0,
&mut child_refs,
skipped_stream_parameter_streams,
)?;
for cr in child_refs.into_iter().rev() {
if cr.1 {
seen_as_array.insert(cr.0);
}
if !visited.contains(&cr.0) {
stack.push(cr);
}
}
}
}
} let mut refs_raw: Vec<(ObjectRef, bool)> = Vec::new();
for (k, v) in dict.iter() {
if k == b"/Kids" {
continue;
}
if k == b"/Thumb" {
continue;
}
if k == b"/Parent" {
let mut to_visit: Vec<ObjectRef> = Vec::new();
let mut seen_parents: BTreeSet<ObjectRef> = BTreeSet::new();
collect_direct_handle_refs_with_stream_parameters(
v,
0,
&mut to_visit,
skipped_stream_parameter_streams,
)?;
while let Some(parent_ref) = to_visit.pop() {
if !seen_parents.insert(parent_ref) {
continue;
}
let parent_handle = pdf.get_object_handle(parent_ref);
let Some(parent_dict) = parent_handle.try_as_dictionary()? else {
continue;
};
for (pk, pv) in parent_dict.iter() {
if pk == b"/Kids" {
continue;
}
if pk == b"/Parent" {
collect_direct_handle_refs(pv, 0, &mut to_visit)?;
continue;
}
let mut refs: Vec<(ObjectRef, bool)> = Vec::new();
collect_direct_handle_refs_with_stream_parameters_context(
pv,
0,
false,
&mut refs,
skipped_stream_parameter_streams,
)?; for (r, va) in refs {
if va {
seen_as_array.insert(r);
}
if !visited.contains(&r) {
queue.push_back((r, va));
}
}
}
}
continue;
}
collect_direct_handle_refs_with_stream_parameters_context(
v,
0,
false,
&mut refs_raw,
skipped_stream_parameter_streams,
)?; }
for &(r, va) in &refs_raw {
if va {
seen_as_array.insert(r);
}
}
refs_raw.sort_by_key(|(r, _)| pdf.writer_object_order_key(*r));
for (r, va) in refs_raw {
if !visited.contains(&r) {
queue.push_back((r, va));
}
}
}
} else {
let mut refs: Vec<(ObjectRef, bool)> = Vec::new();
collect_handle_children_with_stream_parameters(
¤t_handle,
0,
&mut refs,
skipped_stream_parameter_streams,
)?; for &(r, va) in &refs {
if va {
seen_as_array.insert(r);
}
}
refs.sort_by_key(|(r, _)| pdf.writer_object_order_key(*r));
for (r, va) in refs {
if !visited.contains(&r) {
queue.push_back((r, va));
}
}
}
}
if order.len() > 1 {
order[1..].sort_by_key(|r| pdf.writer_object_order_key(*r));
}
for r in deferred_resurrect {
if seen_as_array.contains(&r) {
let tail_pos = order[1..].partition_point(|existing| existing.number < r.number);
order.insert(1 + tail_pos, r);
} }
Ok(order)
}
#[allow(dead_code)]
#[derive(Debug, Clone, Default)]
pub(crate) struct RawLinearizationPlan {
pub(crate) part2_objects: Vec<QpdfObjGen>,
pub(crate) part3_objects: Vec<QpdfObjGen>,
pub(crate) part4_other_pages_private: Vec<QpdfObjGen>,
pub(crate) part4_other_pages_shared: Vec<QpdfObjGen>,
pub(crate) part4_rest: Vec<QpdfObjGen>,
pub(crate) part4_open_document_plain: Vec<QpdfObjGen>,
pub(crate) root: Option<QpdfObjGen>,
pub(crate) pages_tree: Option<QpdfObjGen>,
pub(crate) info: Option<QpdfObjGen>,
pub(crate) per_page_private_objects: Vec<Vec<QpdfObjGen>>,
pub(crate) part9_thumbnail_objects: Vec<QpdfObjGen>,
pub(crate) outline_first_page_members: Vec<QpdfObjGen>,
pub(crate) part9_outline_objects: Vec<QpdfObjGen>,
pub(crate) part6_outline_objects: Vec<QpdfObjGen>,
pub(crate) content_normalize_refs: BTreeSet<QpdfObjGen>,
pub(crate) removed_refs: BTreeSet<QpdfObjGen>,
pub(crate) shared_hints: Vec<RawSharedObjectHintEntry>,
pub(crate) projection_gap: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct RawSharedObjectHintEntry {
pub(crate) object: Option<QpdfObjGen>,
pub(crate) container: Option<u32>,
pub(crate) referencing_pages: Vec<u32>,
}
#[derive(Debug, Clone)]
pub struct LinearizationPlan {
pub part1_objects: Vec<ObjectRef>,
pub part2_objects: Vec<ObjectRef>,
pub part3_objects: Vec<ObjectRef>,
pub part4_other_pages_private: Vec<ObjectRef>,
pub part4_other_pages_shared: Vec<ObjectRef>,
pub part4_rest: Vec<ObjectRef>,
pub total_object_count: u32,
pub root_ref: Option<ObjectRef>,
pub pages_tree_ref: Option<ObjectRef>,
pub info_ref: Option<ObjectRef>,
pub page_hints: Vec<PageHintEntry>,
pub shared_hints: Vec<SharedObjectHintEntry>,
pub per_page_private_objects: Vec<Vec<ObjectRef>>,
pub(crate) part9_thumbnail_objects: Vec<ObjectRef>,
pub(crate) outline_first_page_members: Vec<ObjectRef>,
pub(crate) part9_outline_objects: Vec<ObjectRef>,
pub(crate) part6_outline_objects: Vec<ObjectRef>,
pub part4_open_document_plain: Vec<ObjectRef>,
pub(crate) writer_object_order: Option<BTreeMap<ObjectRef, WriterObjectOrderKey>>,
pub(crate) optimization: Option<crate::optimization::Optimization>,
pub(crate) preserve_objstm_plan: Option<crate::writer::object_streams::ObjectStreamPlan>,
#[allow(dead_code)]
pub(crate) content_normalize_refs: BTreeSet<ObjectRef>,
#[allow(dead_code)]
pub(crate) removed_refs: BTreeSet<ObjectRef>,
pub(crate) object_stream_mode: crate::writer::ObjectStreamMode,
pub(crate) raw: RawLinearizationPlan,
}
fn raw_refs_from_object_refs(refs: impl IntoIterator<Item = ObjectRef>) -> Vec<QpdfObjGen> {
refs.into_iter()
.filter_map(|object_ref| QpdfObjGen::try_from_object_ref(object_ref).ok())
.collect()
}
fn raw_refs_with_extras(
refs: impl IntoIterator<Item = ObjectRef>,
extras: impl IntoIterator<Item = QpdfObjGen>,
sort: bool,
) -> Vec<QpdfObjGen> {
let extras: Vec<_> = extras.into_iter().collect();
let mut result = raw_refs_from_object_refs(refs);
result.extend(extras.iter().copied());
if sort && !extras.is_empty() {
result.sort_unstable();
}
result
}
fn canonical_preserve_ref(
source_container_by_member: &BTreeMap<ObjectRef, ObjectRef>,
object_ref: ObjectRef,
) -> ObjectRef {
source_container_by_member
.get(&object_ref)
.copied()
.unwrap_or(object_ref)
}
fn canonicalize_preserve_refs(
source_container_by_member: &BTreeMap<ObjectRef, ObjectRef>,
refs: Vec<ObjectRef>,
) -> Vec<ObjectRef> {
if source_container_by_member.is_empty() {
return refs;
}
let mut seen = BTreeSet::new();
refs.into_iter()
.map(|object_ref| canonical_preserve_ref(source_container_by_member, object_ref))
.filter(|object_ref| seen.insert(*object_ref))
.collect()
}
fn raw_refs_with_extras_preserving_first(
refs: impl IntoIterator<Item = ObjectRef>,
extras: impl IntoIterator<Item = QpdfObjGen>,
) -> Vec<QpdfObjGen> {
let mut result = raw_refs_from_object_refs(refs);
let first = result.first().copied();
let extras: Vec<_> = extras.into_iter().collect();
result.extend(extras.iter().copied());
if !extras.is_empty() && result.len() > 1 {
result[1..].sort_unstable();
}
if let Some(first) = first {
if let Some(position) = result.iter().position(|object| *object == first) {
result.remove(position);
result.insert(0, first);
}
} result
}
fn raw_refs_with_extras_root_first(
refs: impl IntoIterator<Item = ObjectRef>,
extras: impl IntoIterator<Item = QpdfObjGen>,
root: Option<QpdfObjGen>,
) -> Vec<QpdfObjGen> {
let mut result = raw_refs_from_object_refs(refs);
result.extend(extras);
result.sort_unstable();
if let Some(root) = root {
if let Some(position) = result.iter().position(|object| *object == root) {
result.remove(position);
result.insert(0, root);
}
}
result
}
#[allow(clippy::too_many_arguments)]
#[inline(never)]
fn build_raw_linearization_plan<R: Read + Seek>(
pdf: &mut Pdf<R>,
optimization: &crate::optimization::Optimization,
part2_objects: &[ObjectRef],
part3_objects: &[ObjectRef],
part4_other_pages_shared: &[ObjectRef],
part4_rest: &[ObjectRef],
part4_open_document_plain: &[ObjectRef],
per_page_private_objects: &[Vec<ObjectRef>],
page_refs: &[ObjectRef],
part9_outline_objects: &[ObjectRef],
part6_outline_objects: &[ObjectRef],
root_ref: Option<ObjectRef>,
pages_tree_ref: Option<ObjectRef>,
info_ref: Option<ObjectRef>,
content_normalize_refs: &BTreeSet<QpdfObjGen>,
removed_refs: &BTreeSet<ObjectRef>,
outlines_in_first_page: bool,
) -> Result<RawLinearizationPlan> {
let raw_removed_refs: BTreeSet<QpdfObjGen> = removed_refs
.iter()
.filter_map(|object_ref| QpdfObjGen::try_from_object_ref(*object_ref).ok())
.collect();
let outline_refs = optimization.raw_objects_for_root_key(b"Outlines");
let open_document_refs: BTreeSet<QpdfObjGen> = optimization
.raw_objects_for_trailer_key(b"Encrypt")
.into_iter()
.chain(
OPEN_DOCUMENT_CATALOG_KEYS
.into_iter()
.flat_map(|key| optimization.raw_objects_for_root_key(key)),
)
.collect();
let thumbnail_refs = optimization.raw_thumbnail_objects();
let mut raw_part2_extra = Vec::new();
let mut raw_part3_extra = Vec::new();
let mut raw_part4_shared_extra = Vec::new();
let mut raw_part4_rest_extra = Vec::new();
let mut raw_open_document_plain_extra = Vec::new();
let mut raw_private_by_page: BTreeMap<u32, Vec<QpdfObjGen>> = BTreeMap::new();
for (object_gen, users) in optimization.raw_object_users() {
if object_gen.to_object_ref().is_some()
|| object_gen.get_obj() <= 0
|| raw_removed_refs.contains(&object_gen)
{
continue;
}
let object =
pdf.get_object_handle_by_raw_identity(object_gen.get_obj(), object_gen.get_gen());
if object.try_is_stream_of_type(b"XRef", b"")?
|| object.try_is_stream_of_type(b"ObjStm", b"")?
{
continue;
}
let page_users: Vec<u32> = optimization.raw_page_users(object_gen).collect();
let in_first_page = page_users.contains(&0);
let in_other_page = page_users.iter().any(|page| *page != 0);
let in_outline = outline_refs.contains(&object_gen);
let in_open_document = open_document_refs.contains(&object_gen);
let in_document_other = users.iter().any(is_document_other_user);
let in_thumbnail = thumbnail_refs.contains(&object_gen);
if in_outline {
raw_part4_rest_extra.push(object_gen);
} else if in_open_document {
raw_open_document_plain_extra.push(object_gen);
} else if in_first_page {
if in_other_page || in_document_other || in_thumbnail {
raw_part3_extra.push(object_gen);
} else {
raw_part2_extra.push(object_gen);
}
} else if let Some(page) = optimization.raw_other_page_private_owner(object_gen) {
raw_private_by_page
.entry(page)
.or_default()
.push(object_gen);
} else if page_users.len() >= 2 {
raw_part4_shared_extra.push(object_gen);
} else {
raw_part4_rest_extra.push(object_gen);
}
}
raw_part4_shared_extra.sort_unstable();
raw_part4_rest_extra.sort_unstable();
raw_open_document_plain_extra.sort_unstable();
let mut raw_per_page_private_objects = per_page_private_objects
.iter()
.map(|objects| raw_refs_from_object_refs(objects.iter().copied()))
.collect::<Vec<_>>();
for (page, objects) in raw_private_by_page {
if let Some(destination) = raw_per_page_private_objects.get_mut(page as usize) {
destination.extend(objects);
}
}
for objects in raw_per_page_private_objects.iter_mut().skip(1) {
if objects.len() > 1 {
objects[1..].sort_unstable();
}
}
let raw_part4_private: Vec<QpdfObjGen> = raw_per_page_private_objects
.iter()
.skip(1)
.flatten()
.copied()
.collect();
let raw_part6_outline_extra: Vec<_> = if outlines_in_first_page {
raw_part4_rest_extra
.iter()
.copied()
.filter(|object_gen| outline_refs.contains(object_gen))
.collect()
} else {
Vec::new()
};
let raw_part9_outline_extra: Vec<_> = if outlines_in_first_page {
Vec::new()
} else {
raw_part4_rest_extra
.iter()
.copied()
.filter(|object_gen| outline_refs.contains(object_gen))
.collect()
};
raw_part4_rest_extra.retain(|object_gen| !outline_refs.contains(object_gen));
let outline_root = if let Some(root_ref) = pdf.root_ref() {
pdf.get_object_handle(root_ref)
.try_get_key(b"/Outlines")?
.qpdf_obj_gen()
.filter(|object_gen| object_gen.is_indirect())
} else {
None };
let raw_part6_outline_objects = raw_refs_with_extras_root_first(
part6_outline_objects.iter().copied(),
raw_part6_outline_extra.clone(),
outline_root,
);
let raw_part9_outline_objects = raw_refs_with_extras_root_first(
part9_outline_objects.iter().copied(),
raw_part9_outline_extra.clone(),
outline_root,
);
let raw_part4_rest =
raw_refs_with_extras(part4_rest.iter().copied(), raw_part4_rest_extra, true);
let raw_part4_rest_set: BTreeSet<QpdfObjGen> = raw_part4_rest.iter().copied().collect();
let raw_part9_thumbnail_objects =
part9_thumbnail_objects(pdf, optimization, page_refs, &raw_part4_rest_set)?;
let raw_shared_hints = raw_part2_objects_for_hints(
&raw_refs_with_extras_preserving_first(
part2_objects.iter().copied(),
raw_part2_extra.clone(),
),
&raw_refs_with_extras(part3_objects.iter().copied(), raw_part3_extra.clone(), true),
&raw_part6_outline_objects,
&raw_refs_with_extras(
part4_other_pages_shared.iter().copied(),
raw_part4_shared_extra.clone(),
true,
),
optimization,
);
let mut raw = RawLinearizationPlan {
part2_objects: raw_refs_with_extras_preserving_first(
part2_objects.iter().copied(),
raw_part2_extra,
),
part3_objects: raw_refs_with_extras(part3_objects.iter().copied(), raw_part3_extra, true),
part4_other_pages_private: raw_part4_private,
part4_other_pages_shared: raw_refs_with_extras(
part4_other_pages_shared.iter().copied(),
raw_part4_shared_extra,
true,
),
part4_rest: raw_part4_rest,
part4_open_document_plain: raw_refs_with_extras(
part4_open_document_plain.iter().copied(),
raw_open_document_plain_extra,
true,
),
root: root_ref.and_then(|object_ref| QpdfObjGen::try_from_object_ref(object_ref).ok()),
pages_tree: pages_tree_ref
.and_then(|object_ref| QpdfObjGen::try_from_object_ref(object_ref).ok()),
info: info_ref.and_then(|object_ref| QpdfObjGen::try_from_object_ref(object_ref).ok()),
per_page_private_objects: raw_per_page_private_objects,
part9_thumbnail_objects: raw_part9_thumbnail_objects,
outline_first_page_members: raw_part6_outline_objects.clone(),
part9_outline_objects: raw_part9_outline_objects,
part6_outline_objects: raw_part6_outline_objects,
content_normalize_refs: content_normalize_refs.clone(),
removed_refs: raw_removed_refs,
shared_hints: raw_shared_hints,
projection_gap: false,
};
raw.projection_gap = raw.compute_projection_gap();
Ok(raw)
}
impl RawLinearizationPlan {
fn compute_projection_gap(&self) -> bool {
self.part2_objects
.iter()
.chain(&self.part3_objects)
.chain(&self.part4_other_pages_private)
.chain(&self.part4_other_pages_shared)
.chain(&self.part4_rest)
.chain(&self.part4_open_document_plain)
.chain(&self.part6_outline_objects)
.chain(&self.part9_outline_objects)
.any(|object_gen| object_gen.to_object_ref().is_none())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Part9ThumbnailCategory {
Private(u32),
Shared,
}
fn part9_thumbnail_category(
users: &crate::optimization::CompactObjectUserSet,
) -> Option<Part9ThumbnailCategory> {
let mut other_pages = BTreeSet::new();
let mut thumbnail_pages = BTreeSet::new();
let mut has_page_zero = false;
let mut has_root = false;
let mut has_outline = false;
let mut has_open_document = false;
let mut has_document_other = false;
for user in users.iter() {
match user {
crate::optimization::ObjectUser::Page(page) => {
if *page == 0 {
has_page_zero = true;
} else {
other_pages.insert(*page);
}
}
crate::optimization::ObjectUser::Thumbnail(page) => {
thumbnail_pages.insert(*page);
}
crate::optimization::ObjectUser::Root => has_root = true,
user if is_outline_user(user) => has_outline = true,
user if is_open_document_user(user) => has_open_document = true,
user if is_document_other_user(user) => has_document_other = true,
_ => {} }
}
if has_page_zero || has_root || has_outline || has_open_document {
return None;
}
if other_pages.len() > 1 {
return None; }
if other_pages.len() == 1 && thumbnail_pages.is_empty() && !has_document_other {
return None; }
if thumbnail_pages.len() == 1 && !has_document_other {
return Some(Part9ThumbnailCategory::Private(
*thumbnail_pages.first().expect("thumbnail page exists"),
));
}
if thumbnail_pages.len() > 1 {
return Some(Part9ThumbnailCategory::Shared);
}
None
}
fn part9_thumbnail_objects<R: Read + Seek>(
pdf: &mut Pdf<R>,
optimization: &crate::optimization::Optimization,
page_refs: &[ObjectRef],
part4_rest: &BTreeSet<QpdfObjGen>,
) -> Result<Vec<QpdfObjGen>> {
let mut ordered = Vec::new();
let mut placed = BTreeSet::new();
for (page_index, page_ref) in page_refs.iter().enumerate() {
let page_number = page_index as u32;
let page = pdf.get_object_handle(*page_ref);
let thumb_root = page.try_get_key(b"/Thumb")?.qpdf_obj_gen();
let is_private = |object: QpdfObjGen| {
part4_rest.contains(&object)
&& matches!(
part9_thumbnail_category(optimization.raw_users_for(object)),
Some(Part9ThumbnailCategory::Private(page)) if page == page_number
)
};
if let Some(object) = thumb_root.filter(|object| is_private(*object)) {
if placed.insert(object) {
ordered.push(object);
}
}
for &object in
optimization.raw_objects_for(&crate::optimization::ObjectUser::Thumbnail(page_number))
{
if is_private(object) && placed.insert(object) {
ordered.push(object);
}
}
}
let mut shared: Vec<QpdfObjGen> = part4_rest
.iter()
.copied()
.filter(|object| {
matches!(
part9_thumbnail_category(optimization.raw_users_for(*object)),
Some(Part9ThumbnailCategory::Shared)
)
})
.collect();
shared.sort_unstable();
ordered.extend(shared);
Ok(ordered)
}
fn raw_shared_hints_for_objects(
objects: &[QpdfObjGen],
optimization: &crate::optimization::Optimization,
include_first_page: bool,
) -> Vec<RawSharedObjectHintEntry> {
objects
.iter()
.copied()
.map(|object| RawSharedObjectHintEntry {
object: Some(object),
container: None,
referencing_pages: optimization
.raw_page_users(object)
.filter(|page| include_first_page || *page != 0)
.collect(),
})
.collect()
}
fn raw_part2_objects_for_hints(
part2: &[QpdfObjGen],
part3: &[QpdfObjGen],
outline: &[QpdfObjGen],
part4_shared: &[QpdfObjGen],
optimization: &crate::optimization::Optimization,
) -> Vec<RawSharedObjectHintEntry> {
let mut entries = raw_shared_hints_for_objects(part2, optimization, false);
entries.extend(raw_shared_hints_for_objects(part3, optimization, false));
entries.extend(raw_shared_hints_for_objects(outline, optimization, false));
entries.extend(raw_shared_hints_for_objects(
part4_shared,
optimization,
true,
));
entries
}
impl LinearizationPlan {
#[allow(dead_code)]
pub(crate) fn preserve_source_membership(&self) -> Option<BTreeMap<u32, u32>> {
let plan = self.preserve_objstm_plan.as_ref()?;
let mut membership = BTreeMap::new();
for group in &plan.groups {
let crate::writer::object_streams::ObjectStreamGroup::SourceBacked { source, members } =
group
else {
continue;
};
for member in members {
membership.insert(member.number, source.number);
}
}
Some(membership)
}
pub fn from_pdf<R: Read + Seek>(
pdf: &mut Pdf<R>,
use_generate_objstm: bool,
) -> crate::Result<Self> {
let mode = if use_generate_objstm {
crate::writer::ObjectStreamMode::Generate
} else {
crate::writer::ObjectStreamMode::Disable
};
Self::from_pdf_with_object_stream_mode(pdf, mode)
}
pub fn from_pdf_with_object_stream_mode<R: Read + Seek>(
pdf: &mut Pdf<R>,
object_stream_mode: crate::writer::ObjectStreamMode,
) -> crate::Result<Self> {
let options = crate::writer::WriterOptions {
object_streams: object_stream_mode,
..crate::writer::WriterOptions::default()
};
Self::from_pdf_with_writer_options(pdf, &options)
}
pub(crate) fn from_pdf_with_writer_options<R: Read + Seek>(
pdf: &mut Pdf<R>,
options: &crate::writer::WriterOptions,
) -> crate::Result<Self> {
Self::from_pdf_with_writer_options_and_source_membership(pdf, options, None, None)
}
#[inline(never)]
pub(crate) fn from_pdf_with_writer_options_and_source_membership<R: Read + Seek>(
pdf: &mut Pdf<R>,
options: &crate::writer::WriterOptions,
source_membership_snapshot: Option<&BTreeMap<u32, u32>>,
generated_compressible_snapshot: Option<&crate::writer::object_streams::CompressiblePlan>,
) -> crate::Result<Self> {
let object_stream_mode = options.object_streams;
let use_generate_objstm = matches!(
object_stream_mode,
crate::writer::ObjectStreamMode::Generate
);
let capture_pre_optimization_objects =
!matches!(object_stream_mode, crate::writer::ObjectStreamMode::Disable);
let pre_optimization_object_refs = capture_pre_optimization_objects
.then(|| pdf.canonical_live_object_refs().into_iter().collect());
let generate_compressible_plan = if use_generate_objstm {
Some(match generated_compressible_snapshot {
Some(snapshot) => snapshot.clone(),
None => compressible_objgens_qpdf_plan(pdf)?,
})
} else {
None
};
let mut preserve_objstm_plan = if matches!(
object_stream_mode,
crate::writer::ObjectStreamMode::Preserve
) {
Some(
crate::writer::object_streams::plan_qpdf_preserve_object_streams_with_source_membership(
pdf,
options.preserve_unreferenced_objects,
source_membership_snapshot,
)?, )
} else {
None
};
let source_had_compressed_objects = preserve_objstm_plan
.as_ref()
.is_some_and(|plan| plan.source_membership_present);
if let Some(plan) = preserve_objstm_plan.as_mut() {
crate::writer::object_streams::filter_preserve_object_stream_plan_for_output(
pdf, plan, true, false,
)?; }
let preserve_object_stream_data: BTreeMap<u32, u32> = preserve_objstm_plan
.as_ref()
.map(|plan| {
plan.groups
.iter()
.filter_map(|group| match group {
crate::writer::object_streams::ObjectStreamGroup::SourceBacked {
source,
members,
} => Some((source, members)),
crate::writer::object_streams::ObjectStreamGroup::Generated { .. } => None,
})
.flat_map(|(source, members)| {
members
.iter()
.map(move |member| (member.number, source.number))
})
.collect()
})
.unwrap_or_default();
let preserve_source_container_by_member: BTreeMap<ObjectRef, ObjectRef> =
preserve_object_stream_data
.iter()
.map(|(&member, &source)| (ObjectRef::new(member, 0), ObjectRef::new(source, 0)))
.collect();
let generate_objstm_eligible = generate_compressible_plan
.as_ref()
.map(|plan| plan.eligible.clone());
let prepared_page_sequence = if options.content_normalization {
crate::pages::repair::prepare_for_optimization(pdf)?
} else {
None
};
let prepared_page_refs = prepared_page_sequence
.as_ref()
.map_or(&[][..], |prepared| prepared.pages.as_slice());
let content_normalize_refs =
linearization_content_normalize_refs(pdf, options, prepared_page_refs)?;
let mut skipped_raw_stream_parameter_streams: BTreeSet<QpdfObjGen> = BTreeSet::new();
let mut optimization = crate::optimization::Optimization::optimize(
pdf,
&preserve_object_stream_data,
true,
|stream_ref, stream| {
if stream_ref.is_some_and(|object_gen| {
skipped_raw_stream_parameter_streams.contains(&object_gen)
}) {
return Ok(2);
}
let has_indirect_parameter = stream_has_indirect_parameter_edge(stream)?;
let refiltered = if has_indirect_parameter {
stream_parameters_removed_for_linearization(
stream,
stream_ref,
options,
&content_normalize_refs,
)? } else {
crate::writer::plain::body::canonical_stream_filter_probe_for_linearization(
stream,
options,
stream_ref
.is_some_and(|object_gen| content_normalize_refs.contains(&object_gen)),
)? };
if refiltered {
if has_indirect_parameter {
if let Some(object_gen) = stream_ref {
skipped_raw_stream_parameter_streams.insert(object_gen);
}
}
Ok(2)
} else {
Ok(1)
}
},
)?;
if let Some(eligible) = generate_objstm_eligible {
optimization.set_generate_objstm_eligible(eligible);
}
if let Some(refs) = pre_optimization_object_refs {
optimization.set_pre_optimization_object_refs(refs);
}
if pdf.root_ref().is_some()
&& optimization
.objects_for(&crate::optimization::ObjectUser::Page(0))
.next()
.is_none()
{
return Err(crate::Error::Unsupported(
"no pages found while calculating linearization data".to_string(),
));
}
let page_refs: Vec<ObjectRef> = crate::pages::page_refs(pdf)?;
let page_parent_users_are_complete =
page_refs
.iter()
.try_fold(true, |complete, page_ref| -> Result<bool> {
if !complete {
return Ok(false);
}
let page = pdf.get_object_handle(*page_ref);
let parent = page.try_get_key(b"/Parent")?;
if parent.try_is_null()? {
return Ok(true);
}
let Some(parent_gen) = parent.qpdf_obj_gen() else {
return Ok(false);
};
Ok(optimization
.raw_users_for(parent_gen)
.iter()
.next()
.is_some())
})?;
let use_optimization_page_user_map =
preserve_object_stream_data.is_empty() && page_parent_users_are_complete;
let reachable: BTreeSet<ObjectRef> = if use_optimization_page_user_map {
optimization.linearization_reachable_object_refs().collect()
} else {
crate::writer::rewrite_renumber::reachable_object_set_with_stream_parameters(
pdf,
true,
&skipped_raw_stream_parameter_streams,
)? };
let object_refs = pdf.canonical_object_refs();
let mut all_refs: Vec<ObjectRef> = Vec::with_capacity(object_refs.len());
for r in object_refs {
if r.number == 0 {
continue;
}
if !reachable.contains(&r) {
continue;
}
let object_handle = pdf.get_object_handle(r);
if object_handle.try_is_stream_of_type(b"XRef", b"")?
|| object_handle.try_is_stream_of_type(b"ObjStm", b"")?
{
continue;
}
all_refs.push(canonical_preserve_ref(
&preserve_source_container_by_member,
r,
));
}
if object_stream_mode == crate::writer::ObjectStreamMode::Preserve {
all_refs.sort_unstable();
all_refs.dedup();
}
let operation_removes_stale_generations = use_generate_objstm
|| (matches!(
object_stream_mode,
crate::writer::ObjectStreamMode::Preserve
) && source_had_compressed_objects);
let removed_refs = if operation_removes_stale_generations {
if let Some(plan) = generate_compressible_plan.as_ref() {
plan.removed_refs.clone()
} else if let Some(plan) = preserve_objstm_plan.as_ref() {
plan.removed_refs.clone()
} else {
crate::writer::object_streams::compressible_objgens_qpdf_plan(pdf)?.removed_refs
}
} else {
BTreeSet::new()
};
if let Some(plan) = generate_compressible_plan.as_ref() {
all_refs.extend(plan.eligible.iter().copied());
all_refs.sort_unstable();
all_refs.dedup();
}
all_refs.retain(|reference| !removed_refs.contains(reference));
let resurrectable =
crate::writer::rewrite_renumber::resurrectable_null_refs_excluding(pdf, &removed_refs)?;
let mut resurrected: Vec<ObjectRef> = resurrectable
.iter()
.filter(|r| all_refs.binary_search(r).is_err())
.copied()
.collect();
if !resurrected.is_empty() {
all_refs.append(&mut resurrected);
all_refs.sort();
}
let total_object_count = all_refs.len() as u32;
let root_ref = pdf.root_ref();
let info_handle = pdf.trailer().try_get_key(b"/Info")?;
let info_ref = info_handle
.object_ref()
.or_else(|| info_handle.object_ref());
let pages_tree_ref = if let Some(root_ref) = root_ref {
let root_handle = pdf.get_object_handle(root_ref);
let pages_handle = root_handle.try_get_key(b"/Pages")?;
pages_handle
.object_ref()
.or_else(|| pages_handle.object_ref())
.map(|object_ref| {
canonical_preserve_ref(&preserve_source_container_by_member, object_ref)
})
} else {
None };
let live: BTreeSet<ObjectRef> = pdf.canonical_live_object_refs().into_iter().collect();
let first_page_users = optimization
.objects_for(&crate::optimization::ObjectUser::Page(0))
.collect::<BTreeSet<_>>();
let root_objects = optimization
.objects_for(&crate::optimization::ObjectUser::Root)
.collect::<BTreeSet<_>>();
let mut open_document_objects: BTreeSet<ObjectRef> =
optimization.objects_for_trailer_key(b"Encrypt").collect();
for key in OPEN_DOCUMENT_CATALOG_KEYS {
open_document_objects.extend(optimization.objects_for_root_key(key));
}
let all_outline_refs: BTreeSet<ObjectRef> =
optimization.objects_for_root_key(b"Outlines").collect();
let document_other_objects: BTreeSet<ObjectRef> = optimization
.object_users()
.filter_map(|(object_ref, users)| {
users
.iter()
.any(is_document_other_user)
.then_some(object_ref)
})
.collect();
let elig_ctx = if use_generate_objstm {
Some(eligibility_context(pdf)?)
} else {
None
};
let mut first_page_closure: Vec<ObjectRef> = if let Some(&first_page) = page_refs.first() {
compute_closure_with_stream_parameters(
pdf,
first_page,
&live,
&resurrectable,
&skipped_raw_stream_parameter_streams,
)? } else {
Vec::new()
};
first_page_closure =
canonicalize_preserve_refs(&preserve_source_container_by_member, first_page_closure);
first_page_closure.retain(|object_ref| first_page_users.contains(object_ref));
let first_page_set: BTreeSet<ObjectRef> = first_page_closure.iter().copied().collect();
let mut shared_page_indices: BTreeMap<ObjectRef, BTreeSet<u32>> = BTreeMap::new();
let mut other_page_closures: Vec<Vec<ObjectRef>> =
Vec::with_capacity(page_refs.len().saturating_sub(1));
for (page_idx, &page_ref) in page_refs.iter().enumerate().skip(1) {
let closure: Vec<ObjectRef> = if use_optimization_page_user_map {
let mut closure = Vec::new();
closure.push(page_ref);
closure.extend(
optimization
.objects_for(&crate::optimization::ObjectUser::Page(page_idx as u32))
.filter(|object_ref| *object_ref != page_ref),
);
closure
} else {
let mut closure = compute_closure_with_stream_parameters(
pdf,
page_ref,
&live,
&resurrectable,
&skipped_raw_stream_parameter_streams,
)?;
closure = canonicalize_preserve_refs(&preserve_source_container_by_member, closure);
let page_user = crate::optimization::ObjectUser::Page(page_idx as u32);
closure.retain(|object_ref| {
optimization
.users_for(*object_ref)
.iter()
.any(|user| *user == page_user)
});
closure
};
for obj_ref in &closure {
if first_page_set.contains(obj_ref) {
shared_page_indices
.entry(*obj_ref)
.or_default()
.insert(page_idx as u32);
}
}
other_page_closures.push(closure);
}
let thumbnail_user_set: BTreeSet<ObjectRef> = optimization.thumbnail_objects().collect();
let first_page_ref = page_refs.first().copied();
let mut part2_objects: Vec<ObjectRef> = Vec::new();
let mut part3_objects: Vec<ObjectRef> = Vec::new();
for obj_ref in &first_page_closure {
if root_objects.contains(obj_ref)
|| all_outline_refs.contains(obj_ref)
|| open_document_objects.contains(obj_ref)
{
continue;
}
if Some(*obj_ref) == first_page_ref {
part2_objects.push(*obj_ref);
} else if shared_page_indices.contains_key(obj_ref)
|| document_other_objects.contains(obj_ref)
|| thumbnail_user_set.contains(obj_ref)
{
part3_objects.push(*obj_ref);
} else {
part2_objects.push(*obj_ref);
}
}
part3_objects.sort_unstable_by_key(|r| pdf.writer_object_order_key(*r));
part2_objects.sort_unstable_by_key(|r| {
(Some(*r) != first_page_ref, pdf.writer_object_order_key(*r))
});
let moved: BTreeSet<ObjectRef> = part2_objects
.iter()
.chain(&part3_objects)
.copied()
.collect();
let part4_provisional: Vec<ObjectRef> = all_refs
.into_iter()
.filter(|r| !moved.contains(r))
.collect();
let mut page_hints: Vec<PageHintEntry> = page_refs
.iter()
.map(|&r| PageHintEntry::placeholder(r))
.collect();
let part2_set: BTreeSet<ObjectRef> = part2_objects.iter().copied().collect();
let part3_set: BTreeSet<ObjectRef> = part3_objects.iter().copied().collect();
let page0_private = part2_objects.clone();
if !page_hints.is_empty() {
page_hints[0].first_object_index = 0;
page_hints[0].object_count = (page0_private.len() + part3_objects.len()) as u32;
}
let mut per_page_private_objects: Vec<Vec<ObjectRef>> = Vec::with_capacity(page_refs.len());
per_page_private_objects.push(page0_private);
for (i, closure) in other_page_closures.into_iter().enumerate() {
let page_idx = i + 1; let private: Vec<ObjectRef> = closure
.into_iter()
.filter(|r| {
if part2_set.contains(r) || part3_set.contains(r) {
return false;
}
if open_document_objects.contains(r) {
return false;
}
if all_outline_refs.contains(r) {
return false;
}
if document_other_objects.contains(r) {
return false;
}
if thumbnail_user_set.contains(r) {
return false;
}
optimization.page_users(*r).count() == 1
})
.collect();
if page_idx < page_hints.len() {
let count = private.len().max(1) as u32;
page_hints[page_idx].object_count = count;
}
per_page_private_objects.push(private);
}
let provisional_set: BTreeSet<ObjectRef> = part4_provisional.iter().copied().collect();
let mut part4_other_pages_private: Vec<ObjectRef> = Vec::new();
let mut part4_other_pages_shared: Vec<ObjectRef> = Vec::new();
let mut part4_rest: Vec<ObjectRef> = Vec::new();
let mut part4_open_document_plain: Vec<ObjectRef> = Vec::new();
let mut placed_private: BTreeSet<ObjectRef> = BTreeSet::new();
for privates in per_page_private_objects.iter().skip(1) {
for &r in privates {
if provisional_set.contains(&r) && placed_private.insert(r) {
part4_other_pages_private.push(r);
}
}
}
for &r in &part4_provisional {
if placed_private.contains(&r) {
continue;
}
if all_outline_refs.contains(&r) {
part4_rest.push(r);
continue;
}
let reach = optimization.page_users(r).count() as u32;
let in_first_page = first_page_set.contains(&r)
&& !root_objects.contains(&r)
&& !open_document_objects.contains(&r);
if in_first_page {
continue;
}
if root_objects.contains(&r) || open_document_objects.contains(&r) {
if let Some(ctx) = elig_ctx.as_ref() {
let obj = pdf.get_object_handle(r);
if is_eligible_for_objstm_handle(r, &obj, ctx)? {
part4_rest.push(r);
} else {
part4_open_document_plain.push(r);
}
} else {
part4_open_document_plain.push(r);
}
continue;
}
if reach >= 2 {
part4_other_pages_shared.push(r);
} else {
part4_rest.push(r);
}
}
debug_assert_eq!(
part4_other_pages_private.len()
+ part4_other_pages_shared.len()
+ part4_rest.len()
+ part4_open_document_plain.len(),
part4_provisional.len(),
"Part-4 sub-partition must preserve membership"
);
part4_open_document_plain.sort_unstable_by_key(|r| pdf.writer_object_order_key(*r));
let outlines_in_first_page = outlines_in_first_page_predicate(pdf)?;
let outline_root_ref: Option<ObjectRef> = if let Some(root_ref) = pdf.root_ref() {
let root_handle = pdf.get_object_handle(root_ref);
let outlines = root_handle.try_get_key(b"/Outlines")?;
outlines.object_ref().map(|object_ref| {
canonical_preserve_ref(&preserve_source_container_by_member, object_ref)
})
} else {
None };
let extract_outlines = |src: &[ObjectRef]| -> Vec<ObjectRef> {
let mut v: Vec<ObjectRef> = src
.iter()
.filter(|r| all_outline_refs.contains(r))
.copied()
.collect();
if let Some(root) = outline_root_ref {
if let Some(pos) = v.iter().position(|&r| r == root) {
v[..=pos].rotate_right(1);
}
}
v
};
let (part6_outline_objects, part9_outline_objects): (Vec<ObjectRef>, Vec<ObjectRef>) =
if outlines_in_first_page {
(extract_outlines(&part4_rest), vec![])
} else {
(vec![], extract_outlines(&part4_rest))
};
let outline_extract_set: BTreeSet<ObjectRef> = part6_outline_objects
.iter()
.chain(&part9_outline_objects)
.copied()
.collect();
part4_rest.retain(|r| !outline_extract_set.contains(r));
if outlines_in_first_page && !page_hints.is_empty() {
page_hints[0].object_count += part6_outline_objects.len() as u32;
}
let outline_first_page_members: Vec<ObjectRef> = if outlines_in_first_page {
part6_outline_objects.clone()
} else {
vec![]
};
let raw = build_raw_linearization_plan(
pdf,
&optimization,
&part2_objects,
&part3_objects,
&part4_other_pages_shared,
&part4_rest,
&part4_open_document_plain,
&per_page_private_objects,
&page_refs,
&part9_outline_objects,
&part6_outline_objects,
root_ref,
pages_tree_ref,
info_ref,
&content_normalize_refs,
&removed_refs,
outlines_in_first_page,
)?;
if let Some(first_hint) = page_hints.first_mut() {
first_hint.object_count = (raw.part2_objects.len()
+ raw.part3_objects.len()
+ raw.part6_outline_objects.len()) as u32;
}
for (page_index, hint) in page_hints.iter_mut().enumerate().skip(1) {
if let Some(private) = raw.per_page_private_objects.get(page_index) {
hint.object_count = private.len().max(1) as u32;
}
}
let part9_thumbnail_objects: Vec<ObjectRef> = raw
.part9_thumbnail_objects
.iter()
.filter_map(|object| object.to_object_ref())
.collect();
let part2_entries = part2_objects.iter().map(|&obj_ref| SharedObjectHintEntry {
object_ref: obj_ref,
referencing_pages: vec![],
});
let part3_entries = part3_objects.iter().map(|&obj_ref| {
let pages: Vec<u32> = shared_page_indices
.get(&obj_ref)
.map(|s| s.iter().copied().collect())
.unwrap_or_default();
SharedObjectHintEntry {
object_ref: obj_ref,
referencing_pages: pages,
}
});
let outline_entries = outline_first_page_members.iter().map(|&obj_ref| {
let pages: Vec<u32> = optimization
.page_users(obj_ref)
.filter(|&page| page != 0)
.collect();
SharedObjectHintEntry {
object_ref: obj_ref,
referencing_pages: pages,
}
});
let part4_shared_entries = part4_other_pages_shared.iter().map(|&obj_ref| {
let pages: Vec<u32> = optimization.page_users(obj_ref).collect();
SharedObjectHintEntry {
object_ref: obj_ref,
referencing_pages: pages,
}
});
let shared_hints: Vec<SharedObjectHintEntry> = part2_entries
.chain(part3_entries)
.chain(outline_entries)
.chain(part4_shared_entries)
.collect();
Ok(Self {
part1_objects: Vec::new(),
part2_objects,
part3_objects,
part4_other_pages_private,
part4_other_pages_shared,
part4_rest,
part4_open_document_plain,
writer_object_order: pdf.writer_object_order.clone(),
total_object_count,
root_ref,
pages_tree_ref,
info_ref,
page_hints,
shared_hints,
per_page_private_objects,
part9_thumbnail_objects,
outline_first_page_members,
part9_outline_objects,
part6_outline_objects,
object_stream_mode,
removed_refs,
optimization: Some(optimization),
preserve_objstm_plan,
content_normalize_refs: content_normalize_refs
.iter()
.filter_map(|object_gen| object_gen.to_object_ref())
.collect(),
raw,
})
}
pub fn part4_objects(&self) -> Vec<ObjectRef> {
let mut objects: Vec<ObjectRef> = self
.part4_other_pages_private
.iter()
.chain(&self.part4_other_pages_shared)
.copied()
.collect();
let part9_pages: BTreeSet<ObjectRef> = self
.optimization
.as_ref()
.map(|optimization| {
optimization
.objects_for_root_key(b"Pages")
.collect::<BTreeSet<ObjectRef>>()
})
.filter(|pages| !pages.is_empty())
.unwrap_or_else(|| self.pages_tree_ref.into_iter().collect());
for pages_tree in part9_pages.iter().copied() {
if self.part4_rest.contains(&pages_tree) {
objects.push(pages_tree);
}
}
objects.extend(self.part9_outline_objects.iter().copied());
objects.extend(
self.part4_rest
.iter()
.copied()
.filter(|object| !part9_pages.contains(object)),
);
objects
}
pub(crate) fn part8_container_nums(
&self,
member_to_container: &BTreeMap<ObjectRef, (u32, u32)>,
) -> BTreeSet<u32> {
let first_page: BTreeSet<ObjectRef> = self
.part2_objects
.iter()
.chain(&self.part3_objects)
.copied()
.collect();
let part4_shared: BTreeSet<ObjectRef> =
self.part4_other_pages_shared.iter().copied().collect();
let mut all_containers: BTreeSet<u32> = BTreeSet::new();
let mut container_pages: BTreeMap<u32, BTreeSet<u32>> = BTreeMap::new();
let mut has_first_page_member: BTreeSet<u32> = BTreeSet::new();
let mut has_shared_member: BTreeSet<u32> = BTreeSet::new();
for (member, &(cnum, _)) in member_to_container {
all_containers.insert(cnum);
if first_page.contains(member) {
has_first_page_member.insert(cnum);
}
if part4_shared.contains(member) {
has_shared_member.insert(cnum);
}
if let Some(optimization) = self.optimization.as_ref() {
container_pages
.entry(cnum)
.or_default()
.extend(optimization.page_users(*member).filter(|&p| p != 0));
}
}
all_containers
.into_iter()
.filter(|cnum| {
!has_first_page_member.contains(cnum)
&& (has_shared_member.contains(cnum)
|| container_pages.get(cnum).is_some_and(|p| p.len() >= 2))
})
.collect()
}
pub(crate) fn rest_container_nums(
&self,
member_to_container: &BTreeMap<ObjectRef, (u32, u32)>,
) -> BTreeSet<u32> {
self.part9_outline_objects
.iter()
.filter_map(|member| member_to_container.get(member).map(|&(cnum, _)| cnum))
.collect()
}
pub(crate) fn canonical_shared_hints(
&self,
member_to_container: &BTreeMap<ObjectRef, (u32, u32)>,
renumber: &RenumberMap,
second_half_container_nums: &BTreeSet<u32>,
open_document_container_nums: &BTreeSet<u32>,
) -> Vec<SharedObjectHintEntry> {
if member_to_container.is_empty() {
return self.shared_hints.clone();
}
let first_page_input = self.part2_objects.len()
+ self.part3_objects.len()
+ self.outline_first_page_members.len();
let rest_container_nums = self.rest_container_nums(member_to_container);
let mut container_pos: BTreeMap<u32, usize> = BTreeMap::new();
let mut out: Vec<SharedObjectHintEntry> = Vec::with_capacity(self.shared_hints.len());
let mut first_page_out_end: Option<usize> = None;
for (input_idx, entry) in self.shared_hints.iter().enumerate() {
if input_idx == first_page_input {
first_page_out_end = Some(out.len());
}
match member_to_container.get(&entry.object_ref) {
Some(&(container_num, _idx)) => {
if open_document_container_nums.contains(&container_num) {
continue;
}
if input_idx < first_page_input
&& second_half_container_nums.contains(&container_num)
{
continue;
}
if input_idx >= first_page_input && rest_container_nums.contains(&container_num)
{
continue;
}
if let Some(&pos) = container_pos.get(&container_num) {
let merged: &mut Vec<u32> = &mut out[pos].referencing_pages;
for &p in &entry.referencing_pages {
if let Err(insert_at) = merged.binary_search(&p) {
merged.insert(insert_at, p);
}
}
} else {
let mut pages = entry.referencing_pages.clone();
pages.sort_unstable();
pages.dedup();
container_pos.insert(container_num, out.len());
out.push(SharedObjectHintEntry {
object_ref: ObjectRef::new(container_num, u16::MAX),
referencing_pages: pages,
});
}
}
None => out.push(entry.clone()),
}
}
let boundary = first_page_out_end.unwrap_or(out.len());
let new_number = |e: &SharedObjectHintEntry| -> u32 {
if e.object_ref.generation == u16::MAX {
e.object_ref.number
} else {
renumber
.new_for_original(e.object_ref)
.map_or(u32::MAX, |r| r.number)
}
};
out[..boundary].sort_unstable_by_key(&new_number);
for cnum in self.part8_container_nums(member_to_container) {
if !container_pos.contains_key(&cnum)
&& !open_document_container_nums.contains(&cnum)
&& !rest_container_nums.contains(&cnum)
{
out.push(SharedObjectHintEntry {
object_ref: ObjectRef::new(cnum, u16::MAX),
referencing_pages: Vec::new(), });
}
}
out[boundary..].sort_unstable_by_key(&new_number);
if let Some(optimization) = self.optimization.as_ref().filter(|optimization| {
optimization.object_users().any(|(_, users)| {
users
.iter()
.any(|user| matches!(user, crate::optimization::ObjectUser::Page(_)))
})
}) {
let mut container_members: BTreeMap<u32, Vec<ObjectRef>> = BTreeMap::new();
for (&member, &(cnum, _)) in member_to_container {
container_members.entry(cnum).or_default().push(member);
}
let pages_excluding_first = |refs: &mut dyn Iterator<Item = ObjectRef>| -> Vec<u32> {
let mut pages: BTreeSet<u32> = BTreeSet::new();
for r in refs {
pages.extend(optimization.page_users(r).filter(|&p| p != 0));
}
pages.into_iter().collect()
};
for entry in &mut out {
entry.referencing_pages = if entry.object_ref.generation == u16::MAX {
let members = container_members
.get(&entry.object_ref.number)
.cloned()
.unwrap_or_default();
pages_excluding_first(&mut members.into_iter())
} else {
pages_excluding_first(&mut std::iter::once(entry.object_ref))
};
}
}
out
}
pub(crate) fn canonical_raw_shared_hints(
&self,
member_to_container: &BTreeMap<ObjectRef, (u32, u32)>,
renumber: &RenumberMap,
second_half_container_nums: &BTreeSet<u32>,
open_document_container_nums: &BTreeSet<u32>,
) -> Vec<RawSharedObjectHintEntry> {
let has_raw_plan = self.has_raw_projection_gap();
if member_to_container.is_empty() {
if has_raw_plan {
return self.raw.shared_hints.clone();
}
return self
.shared_hints
.iter()
.filter_map(|entry| {
QpdfObjGen::try_from_object_ref(entry.object_ref)
.ok()
.map(|object| RawSharedObjectHintEntry {
object: Some(object),
container: None,
referencing_pages: entry.referencing_pages.clone(),
})
})
.collect();
}
let public = self.canonical_shared_hints(
member_to_container,
renumber,
second_half_container_nums,
open_document_container_nums,
);
let mut result: Vec<RawSharedObjectHintEntry> = public
.into_iter()
.filter_map(|entry| {
if entry.object_ref.generation == u16::MAX {
Some(RawSharedObjectHintEntry {
object: None,
container: Some(entry.object_ref.number),
referencing_pages: entry.referencing_pages,
})
} else {
QpdfObjGen::try_from_object_ref(entry.object_ref)
.ok()
.map(|object| RawSharedObjectHintEntry {
object: Some(object),
container: None,
referencing_pages: entry.referencing_pages,
})
}
})
.collect();
if !has_raw_plan {
return result;
}
let first_page_input = self.part2_objects.len()
+ self.part3_objects.len()
+ self.outline_first_page_members.len();
let mut first_page_count = 0usize;
let mut folded_containers = BTreeSet::new();
for entry in self.shared_hints.iter().take(first_page_input) {
if let Some(&(container_num, _)) = member_to_container.get(&entry.object_ref) {
if open_document_container_nums.contains(&container_num)
|| second_half_container_nums.contains(&container_num)
{
continue; }
if folded_containers.insert(container_num) {
first_page_count += 1;
}
} else {
first_page_count += 1;
}
}
let first_page_raw: BTreeSet<QpdfObjGen> = self
.raw
.part2_objects
.iter()
.chain(&self.raw.part3_objects)
.chain(&self.raw.part6_outline_objects)
.filter(|object| object.to_object_ref().is_none())
.copied()
.collect();
let part8_raw: BTreeSet<QpdfObjGen> = self
.raw
.part4_other_pages_shared
.iter()
.filter(|object| object.to_object_ref().is_none())
.copied()
.collect();
let mut first_extra = Vec::new();
let mut part8_extra = Vec::new();
for entry in &self.raw.shared_hints {
let Some(object) = entry.object else {
continue; };
if first_page_raw.contains(&object) {
first_extra.push(entry.clone());
} else if part8_raw.contains(&object) {
part8_extra.push(entry.clone());
}
}
let raw_number = |entry: &RawSharedObjectHintEntry| {
entry.container.unwrap_or_else(|| {
entry
.object
.and_then(|object| renumber.new_for_raw(object))
.map_or(u32::MAX, |object_ref| object_ref.number)
})
};
first_extra.sort_unstable_by_key(&raw_number);
let first_extra_count = first_extra.len();
result.splice(first_page_count..first_page_count, first_extra);
let first_page_end = first_page_count + first_extra_count;
result[..first_page_end].sort_unstable_by_key(&raw_number);
result.extend(part8_extra);
result[first_page_end..].sort_unstable_by_key(&raw_number);
result
}
pub(crate) fn has_raw_projection_gap(&self) -> bool {
self.raw.projection_gap
}
pub fn all_assigned_refs(&self) -> BTreeSet<ObjectRef> {
self.part1_objects
.iter()
.chain(&self.part2_objects)
.chain(&self.part3_objects)
.chain(&self.part4_other_pages_private)
.chain(&self.part4_other_pages_shared)
.chain(&self.part4_rest)
.copied()
.collect()
}
pub(crate) fn writer_object_order_key(&self, object_ref: ObjectRef) -> WriterObjectOrderKey {
match &self.writer_object_order {
Some(order) => order
.get(&object_ref)
.copied()
.unwrap_or_else(|| WriterObjectOrderKey::fresh(object_ref)),
None => WriterObjectOrderKey::primary(object_ref),
}
}
pub(crate) fn renumber_assigned_refs(&self) -> BTreeSet<ObjectRef> {
self.part2_objects
.iter()
.chain(&self.part3_objects)
.chain(&self.part4_other_pages_private)
.chain(&self.part4_other_pages_shared)
.chain(&self.part4_rest)
.chain(&self.part6_outline_objects)
.chain(&self.part9_outline_objects)
.chain(&self.part4_open_document_plain)
.copied()
.collect()
}
pub(crate) fn renumber_assigned_raw(&self) -> BTreeSet<QpdfObjGen> {
let has_raw_plan = self.raw.root.is_some()
|| !self.raw.part2_objects.is_empty()
|| !self.raw.part3_objects.is_empty()
|| !self.raw.part4_other_pages_private.is_empty()
|| !self.raw.part4_other_pages_shared.is_empty()
|| !self.raw.part4_rest.is_empty()
|| !self.raw.part4_open_document_plain.is_empty();
if has_raw_plan {
self.raw
.part2_objects
.iter()
.chain(&self.raw.part3_objects)
.chain(&self.raw.part4_other_pages_private)
.chain(&self.raw.part4_other_pages_shared)
.chain(&self.raw.part4_rest)
.chain(&self.raw.part6_outline_objects)
.chain(&self.raw.part9_outline_objects)
.chain(&self.raw.part4_open_document_plain)
.copied()
.collect()
} else {
self.renumber_assigned_refs()
.into_iter()
.filter_map(|object_ref| QpdfObjGen::try_from_object_ref(object_ref).ok())
.collect()
}
}
pub fn parts_are_disjoint(&self) -> bool {
let mut seen = BTreeSet::new();
for r in self
.part1_objects
.iter()
.chain(&self.part2_objects)
.chain(&self.part3_objects)
.chain(&self.part4_other_pages_private)
.chain(&self.part4_other_pages_shared)
.chain(&self.part6_outline_objects)
.chain(&self.part9_outline_objects)
.chain(&self.part4_rest)
.chain(&self.part4_open_document_plain)
{
if !seen.insert(*r) {
return false;
}
}
true
}
}
impl Default for LinearizationPlan {
fn default() -> Self {
Self {
part1_objects: Vec::new(),
part2_objects: Vec::new(),
part3_objects: Vec::new(),
part4_other_pages_private: Vec::new(),
part4_other_pages_shared: Vec::new(),
part4_rest: Vec::new(),
part4_open_document_plain: Vec::new(),
writer_object_order: None,
total_object_count: 0,
root_ref: None,
pages_tree_ref: None,
info_ref: None,
page_hints: Vec::new(),
shared_hints: Vec::new(),
per_page_private_objects: Vec::new(),
part9_thumbnail_objects: Vec::new(),
outline_first_page_members: Vec::new(),
part9_outline_objects: Vec::new(),
part6_outline_objects: Vec::new(),
optimization: None,
preserve_objstm_plan: None,
removed_refs: BTreeSet::new(),
object_stream_mode: crate::writer::ObjectStreamMode::Disable,
content_normalize_refs: BTreeSet::new(),
raw: RawLinearizationPlan::default(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub(crate) struct ObjStmBatchPlan {
pub(crate) open_document_batches: Vec<RoutedObjStmBatch>,
pub(crate) part3_batches: Vec<RoutedObjStmBatch>,
pub(crate) part4_batches: Vec<RoutedObjStmBatch>,
}
impl LinearizationPlan {
pub(crate) fn objstm_batches<R: Read + Seek>(
&self,
pdf: &mut Pdf<R>,
config: &PlannerConfig,
) -> crate::Result<ObjStmBatchPlan> {
if config.mode == ObjectStreamMode::Disable {
return Ok(ObjStmBatchPlan::default());
}
let optimization = self.optimization.as_ref().ok_or_else(|| {
crate::Error::Unsupported(
"linearization plan: missing optimization snapshot".to_string(),
)
})?;
let ctx = eligibility_context(pdf)?;
let length_exclusions = if config.mode == ObjectStreamMode::Generate {
compressible_objgens_qpdf_plan(pdf)?.indirect_objstm_length_refs
} else {
BTreeSet::new()
};
let plan = match config.mode {
ObjectStreamMode::Disable => unreachable!(),
ObjectStreamMode::Generate => {
self.objstm_batches_generate(pdf, config, &ctx, &length_exclusions, optimization)?
}
ObjectStreamMode::Preserve => {
self.objstm_batches_preserve(pdf, config, &ctx, &length_exclusions, optimization)?
}
};
Ok(plan)
}
fn objstm_batches_generate<R: Read + Seek>(
&self,
pdf: &mut Pdf<R>,
_config: &PlannerConfig,
_ctx: &crate::writer::object_streams::EligibilityContext,
_length_exclusions: &BTreeSet<ObjectRef>,
optimization: &crate::optimization::Optimization,
) -> crate::Result<ObjStmBatchPlan> {
let assigned = self.renumber_assigned_refs();
let containers = objstm_membership_linearized_with_eligibility(
pdf,
&assigned,
optimization.generate_objstm_eligible(),
)?; let routes = route_objstm_containers(
optimization,
!self.outline_first_page_members.is_empty(),
&containers,
);
let mut open_document_batches: Vec<RoutedObjStmBatch> = Vec::new();
let mut part3_private: Vec<RoutedObjStmBatch> = Vec::new();
let mut part3_shared: Vec<RoutedObjStmBatch> = Vec::new();
let mut part3_outlines: Vec<RoutedObjStmBatch> = Vec::new();
let mut part4_private: Vec<RoutedObjStmBatch> = Vec::new();
let mut part4_shared: Vec<RoutedObjStmBatch> = Vec::new();
let mut part4_rest: Vec<RoutedObjStmBatch> = Vec::new();
for (mut members, route) in containers.into_iter().zip(routes) {
members.sort_unstable_by_key(|r| pdf.writer_object_order_key(*r));
push_routed_objstm_batch(
members,
route,
None,
&mut open_document_batches,
&mut part3_private,
&mut part3_shared,
&mut part3_outlines,
&mut part4_private,
&mut part4_shared,
&mut part4_rest,
);
}
let part9_pages: BTreeSet<ObjectRef> =
optimization.objects_for_root_key(b"Pages").collect();
part4_rest.sort_by_key(|batch| {
part9_category_order_key(optimization, &part9_pages, batch.members.iter())
});
let mut part4_batches = part4_private;
part4_batches.extend(part4_shared);
part4_batches.extend(part4_rest);
let mut part3_batches = part3_private;
part3_batches.extend(part3_shared);
part3_batches.extend(part3_outlines);
Ok(ObjStmBatchPlan {
open_document_batches,
part3_batches,
part4_batches,
})
}
fn objstm_batches_preserve<R: Read + Seek>(
&self,
pdf: &mut Pdf<R>,
_config: &PlannerConfig,
_ctx: &crate::writer::object_streams::EligibilityContext,
_length_exclusions: &BTreeSet<ObjectRef>,
optimization: &crate::optimization::Optimization,
) -> crate::Result<ObjStmBatchPlan> {
let source_plan = match self.preserve_objstm_plan.clone() {
Some(plan) => plan,
None => {
crate::writer::object_streams::plan_qpdf_preserve_object_streams_with_unreferenced(
pdf,
_config.preserve_unreferenced_objects,
)?
} }; let mut assigned = self.renumber_assigned_refs();
for group in &source_plan.groups {
if let crate::writer::object_streams::ObjectStreamGroup::SourceBacked {
members, ..
} = group
{
assigned.extend(members.iter().copied());
}
}
let mut containers: Vec<Vec<ObjectRef>> = Vec::new();
let mut source_container_refs: Vec<Option<ObjectRef>> = Vec::new();
for group in source_plan.groups {
let crate::writer::object_streams::ObjectStreamGroup::SourceBacked { source, members } =
group
else {
return Err(crate::Error::Internal(
"linearized Preserve planner returned a generated ObjStm group".to_string(),
)); };
let mut surviving = Vec::new();
for obj_ref in members {
if !assigned.contains(&obj_ref) {
continue;
}
let obj = pdf.get_object_handle(obj_ref);
if !crate::writer::object_streams::is_qpdf_signature_dict(&obj)? {
surviving.push(obj_ref);
}
}
if !surviving.is_empty() {
source_container_refs.push(Some(source));
containers.push(surviving);
}
}
crate::writer::object_streams::filter_objstm_batches_for_output(
pdf,
&mut containers,
&mut source_container_refs,
true,
false,
)?;
let routes = route_preserve_objstm_containers(
optimization,
!self.outline_first_page_members.is_empty(),
&containers,
&source_container_refs,
);
let mut open_document_batches = Vec::new();
let mut part3_private = Vec::new();
let mut part3_shared = Vec::new();
let mut part3_outlines = Vec::new();
let mut part4_private = Vec::new();
let mut part4_shared = Vec::new();
let mut part4_rest = Vec::new();
for ((members, route), source_container_ref) in containers
.into_iter()
.zip(routes)
.zip(source_container_refs)
{
let source_container_number = source_container_ref
.expect("Preserve ObjStm batch retains its source container")
.number;
push_routed_objstm_batch(
members,
route,
Some(source_container_number),
&mut open_document_batches,
&mut part3_private,
&mut part3_shared,
&mut part3_outlines,
&mut part4_private,
&mut part4_shared,
&mut part4_rest,
);
}
let mut part3_batches = part3_private;
part3_batches.extend(part3_shared);
part3_batches.extend(part3_outlines);
let mut part4_batches = part4_private;
part4_batches.extend(part4_shared);
part4_batches.extend(part4_rest);
Ok(ObjStmBatchPlan {
open_document_batches,
part3_batches,
part4_batches,
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ContainerPart {
OpenDocument,
FirstPagePrivate,
FirstPageShared,
FirstPageOutlines,
OtherPagePrivate,
OtherPageShared,
Rest,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct RoutedObjStmBatch {
pub(crate) members: Vec<ObjectRef>,
pub(crate) route: ContainerPart,
pub(crate) source_container_number: Option<u32>,
}
#[allow(clippy::too_many_arguments)]
fn push_routed_objstm_batch(
members: Vec<ObjectRef>,
route: ContainerPart,
source_container_number: Option<u32>,
open_document_batches: &mut Vec<RoutedObjStmBatch>,
part3_private: &mut Vec<RoutedObjStmBatch>,
part3_shared: &mut Vec<RoutedObjStmBatch>,
part3_outlines: &mut Vec<RoutedObjStmBatch>,
part4_private: &mut Vec<RoutedObjStmBatch>,
part4_shared: &mut Vec<RoutedObjStmBatch>,
part4_rest: &mut Vec<RoutedObjStmBatch>,
) {
match route {
ContainerPart::OpenDocument => open_document_batches.push(RoutedObjStmBatch {
members,
route,
source_container_number,
}),
ContainerPart::FirstPagePrivate => part3_private.push(RoutedObjStmBatch {
members,
route,
source_container_number,
}),
ContainerPart::FirstPageShared => part3_shared.push(RoutedObjStmBatch {
members,
route,
source_container_number,
}),
ContainerPart::FirstPageOutlines => part3_outlines.push(RoutedObjStmBatch {
members,
route,
source_container_number,
}),
ContainerPart::OtherPagePrivate => part4_private.push(RoutedObjStmBatch {
members,
route,
source_container_number,
}),
ContainerPart::OtherPageShared => part4_shared.push(RoutedObjStmBatch {
members,
route,
source_container_number,
}),
ContainerPart::Rest => part4_rest.push(RoutedObjStmBatch {
members,
route,
source_container_number,
}),
}
}
impl std::ops::Deref for RoutedObjStmBatch {
type Target = [ObjectRef];
fn deref(&self) -> &Self::Target {
&self.members
}
}
pub(crate) fn objstm_membership_linearized_with_eligibility<R: Read + Seek>(
pdf: &mut Pdf<R>,
assigned: &BTreeSet<ObjectRef>,
eligibility_override: Option<&[ObjectRef]>,
) -> crate::Result<Vec<Vec<ObjectRef>>> {
let mut eligible = match eligibility_override {
Some(eligible) => eligible.to_vec(),
None => crate::writer::object_streams::get_compressible_objgens(pdf)?,
};
eligible.retain(|r| assigned.contains(r));
if pdf.writer_object_order.is_some() {
eligible.sort_unstable_by_key(|object_ref| pdf.writer_object_order_key(*object_ref));
}
let mut streams = crate::writer::object_streams::even_split_into_streams(&eligible);
let mut source_containers = vec![None; streams.len()];
crate::writer::object_streams::filter_objstm_batches_for_output(
pdf,
&mut streams,
&mut source_containers,
true,
false,
)?; Ok(streams)
}
const OPEN_DOCUMENT_CATALOG_KEYS: [&[u8]; 5] = [
b"ViewerPreferences",
b"PageMode",
b"Threads",
b"OpenAction",
b"AcroForm",
];
fn is_open_document_user(user: &crate::optimization::ObjectUser) -> bool {
match user {
crate::optimization::ObjectUser::TrailerKey(key) => key == b"Encrypt",
crate::optimization::ObjectUser::RootKey(key) => {
OPEN_DOCUMENT_CATALOG_KEYS.contains(&key.as_slice())
}
_ => false,
}
}
fn is_document_other_user(user: &crate::optimization::ObjectUser) -> bool {
match user {
crate::optimization::ObjectUser::TrailerKey(key) => key != b"Encrypt",
crate::optimization::ObjectUser::RootKey(key) => {
key != b"Outlines" && !OPEN_DOCUMENT_CATALOG_KEYS.contains(&key.as_slice())
}
_ => false,
}
}
pub(crate) fn part9_category_order_key<'a>(
optimization: &crate::optimization::Optimization,
part9_pages: &BTreeSet<ObjectRef>,
members: impl IntoIterator<Item = &'a ObjectRef>,
) -> (u8, u32) {
let member_refs: Vec<ObjectRef> = members.into_iter().copied().collect();
let users = optimization.users_for_members(member_refs.iter());
let has_outline = users.iter().any(is_outline_user);
let has_document_other = users.iter().any(is_document_other_user);
let thumbnail_pages: BTreeSet<u32> = users
.iter()
.filter_map(|user| match user {
crate::optimization::ObjectUser::Thumbnail(page) => Some(*page),
_ => None,
})
.collect();
if has_outline {
(3, 0)
} else if thumbnail_pages.len() == 1 && !has_document_other {
(1, *thumbnail_pages.first().expect("thumbnail page exists"))
} else if thumbnail_pages.len() > 1 {
(2, 0)
} else if member_refs
.iter()
.any(|member| part9_pages.contains(member))
{
(0, 0)
} else {
(4, 0)
}
}
fn outlines_in_first_page_predicate<R: Read + Seek>(pdf: &mut Pdf<R>) -> crate::Result<bool> {
let Some(root) = pdf.root_ref() else {
return Ok(false); };
let root_handle = pdf.get_object_handle(root);
if !root_handle.try_has_key(b"/Outlines")? {
return Ok(false);
}
Ok(root_handle
.try_get_key(b"/PageMode")?
.try_as_name()?
.is_some_and(|name| name == b"UseOutlines"))
}
pub(crate) fn route_objstm_containers(
optimization: &crate::optimization::Optimization,
outlines_in_first_page: bool,
containers: &[Vec<ObjectRef>],
) -> Vec<ContainerPart> {
containers
.iter()
.map(|members| {
let users = optimization.users_for_members(members.iter());
classify_container_users(&users, outlines_in_first_page)
})
.collect()
}
fn route_preserve_objstm_containers(
optimization: &crate::optimization::Optimization,
outlines_in_first_page: bool,
containers: &[Vec<ObjectRef>],
source_containers: &[Option<ObjectRef>],
) -> Vec<ContainerPart> {
containers
.iter()
.zip(source_containers)
.map(|(_members, source)| {
let source = source.expect("Preserve ObjStm batch must have a source container");
let users: BTreeSet<crate::optimization::ObjectUser> =
optimization.users_for(source).iter().cloned().collect();
classify_container_users(&users, outlines_in_first_page)
})
.collect()
}
fn classify_container_users(
users: &BTreeSet<crate::optimization::ObjectUser>,
outlines_in_first_page: bool,
) -> ContainerPart {
if users.iter().any(is_outline_user) {
return if outlines_in_first_page {
ContainerPart::FirstPageOutlines
} else {
ContainerPart::Rest
};
}
if users.iter().any(is_open_document_user) {
return ContainerPart::OpenDocument;
}
let page_numbers: BTreeSet<u32> = users
.iter()
.filter_map(|user| match user {
crate::optimization::ObjectUser::Page(page_number) => Some(*page_number),
_ => None,
})
.collect();
let has_document_other = users.iter().any(is_document_other_user);
let has_thumbnail = users.iter().any(is_thumbnail_user);
if page_numbers.contains(&0) {
return if page_numbers.iter().any(|&page| page != 0) || has_document_other || has_thumbnail
{
ContainerPart::FirstPageShared
} else {
ContainerPart::FirstPagePrivate
};
}
match page_numbers.len() {
0 => ContainerPart::Rest,
1 if has_document_other || has_thumbnail => ContainerPart::Rest,
1 => ContainerPart::OtherPagePrivate,
_ => ContainerPart::OtherPageShared,
}
}
fn is_outline_user(user: &crate::optimization::ObjectUser) -> bool {
matches!(
user,
crate::optimization::ObjectUser::RootKey(key) if key == b"Outlines"
)
}
fn is_thumbnail_user(user: &crate::optimization::ObjectUser) -> bool {
matches!(user, crate::optimization::ObjectUser::Thumbnail(_))
}
#[cfg(test)]
mod tests {
use super::{
collect_direct_handle_refs, collect_direct_handle_refs_with_context,
collect_direct_handle_refs_with_stream_parameters_context, LinearizationPlan,
RawLinearizationPlan,
};
use crate::acroform_document_helper::AcroFormDocumentHelper;
use crate::object_handle::ObjectHandle;
use crate::parser::MAX_PARSE_DEPTH;
use crate::writer::{ObjectStreamMode, WriterOptions};
use crate::{Error, ObjectRef, Pdf};
use flate2::write::ZlibEncoder;
use flate2::Compression;
use std::cell::Cell;
use std::collections::{BTreeMap, BTreeSet};
use std::io::{Cursor, Write};
use std::rc::Rc;
#[test]
fn raw_projection_route_is_materialized_at_plan_construction() {
let plan = LinearizationPlan {
raw: RawLinearizationPlan {
projection_gap: true,
..Default::default()
},
..Default::default()
};
assert!(plan.has_raw_projection_gap());
}
#[test]
fn preserve_source_membership_keeps_one_canonical_container_per_group() {
let plan = LinearizationPlan {
preserve_objstm_plan: Some(crate::writer::object_streams::ObjectStreamPlan {
groups: vec![
crate::writer::object_streams::ObjectStreamGroup::SourceBacked {
source: ObjectRef::new(1, 0),
members: vec![ObjectRef::new(2, 0), ObjectRef::new(3, 0)],
},
crate::writer::object_streams::ObjectStreamGroup::SourceBacked {
source: ObjectRef::new(7, 0),
members: vec![ObjectRef::new(8, 0)],
},
crate::writer::object_streams::ObjectStreamGroup::Generated {
source: ObjectRef::new(9, 0),
members: vec![ObjectRef::new(10, 0)],
},
],
removed_refs: BTreeSet::new(),
source_membership_present: true,
}),
..Default::default()
};
assert_eq!(
plan.preserve_source_membership(),
Some(BTreeMap::from([(2, 1), (3, 1), (8, 7)]))
);
}
#[test]
fn preserve_objstm_batches_do_not_rewalk_compressible_objects() {
crate::writer::object_streams::reset_compressible_plan_call_count();
let mut pdf = Pdf::open(Cursor::new(parameter_probe_fixture())).expect("parse fixture");
let options = WriterOptions {
object_streams: ObjectStreamMode::Preserve,
..WriterOptions::default()
};
let plan = LinearizationPlan::from_pdf_with_writer_options(&mut pdf, &options)
.expect("build Preserve linearization plan");
let setup_calls = crate::writer::object_streams::compressible_plan_call_count();
let _ = plan
.objstm_batches(&mut pdf, &super::PlannerConfig::default())
.expect("build Preserve ObjStm batches");
assert_eq!(
crate::writer::object_streams::compressible_plan_call_count(),
setup_calls,
"Preserve batching must reuse setup ownership instead of walking compressible objects"
);
}
fn flate(data: &[u8]) -> Vec<u8> {
let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
encoder.write_all(data).expect("compress fixture stream");
encoder.finish().expect("finish fixture stream")
}
fn stream_object(number: u32, dictionary: &str, data: &[u8]) -> Vec<u8> {
let mut object = format!(
"{number} 0 obj\n<< {dictionary} /Length {} >>\nstream\n",
data.len()
)
.into_bytes();
object.extend_from_slice(data);
object.extend_from_slice(b"\nendstream\nendobj\n");
object
}
fn parameter_probe_fixture() -> Vec<u8> {
let page_content_indirect_filter = flate(b"q Q\n");
let page_content_direct_filter = flate(b"q Q\n");
let existing_appearance = flate(b"/Tx BMC\nEMC\n");
let objects = vec![
b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R /AcroForm 6 0 R >>\nendobj\n"
.to_vec(),
b"2 0 obj\n<< /Type /Pages /Count 1 /Kids [3 0 R] /ColorSpace [10 0 R] >>\nendobj\n".to_vec(),
b"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /ProcSet [10 0 R] >> /Contents [4 0 R 12 0 R] /Annots [5 0 R] >>\nendobj\n".to_vec(),
stream_object(4, "/Filter 9 0 R", &page_content_indirect_filter),
b"5 0 obj\n<< /Type /Annot /Subtype /Widget /FT /Tx /T (name) /V (Hello) /DA (/Helv 12 Tf 0 g) /Rect [10 10 200 30] /P 3 0 R /AP << /N 8 0 R >> >>\nendobj\n".to_vec(),
b"6 0 obj\n<< /Fields [5 0 R] /NeedAppearances true /DR << >> /DA (/Helv 12 Tf 0 g) >>\nendobj\n".to_vec(),
b"7 0 obj\nnull\nendobj\n".to_vec(),
stream_object(
8,
"/Type /XObject /Subtype /Form /BBox [0 0 190 20] /Resources << >> /Filter 9 0 R",
&existing_appearance,
),
b"9 0 obj\n/FlateDecode\nendobj\n".to_vec(),
b"10 0 obj\n<< /Type /ExtGState >>\nendobj\n".to_vec(),
b"11 0 obj\nnull\nendobj\n".to_vec(),
stream_object(12, "/Filter /FlateDecode", &page_content_direct_filter),
];
let mut pdf = b"%PDF-1.7\n".to_vec();
let object_count = objects.len();
let mut offsets = Vec::with_capacity(object_count);
for object in objects {
offsets.push(pdf.len());
pdf.extend_from_slice(&object);
}
let xref = pdf.len();
let _ = writeln!(&mut pdf, "xref\n0 {}", object_count + 1);
pdf.extend_from_slice(b"0000000000 65535 f \n");
for offset in offsets {
let _ = writeln!(&mut pdf, "{offset:010} 00000 n ");
}
let _ = writeln!(
&mut pdf,
"trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF",
object_count + 1
);
pdf
}
struct PassThroughTokenFilter;
impl crate::token_filter::TokenFilter for PassThroughTokenFilter {
fn handle_token(
&mut self,
token: &crate::tokenizer::Token,
output: &mut crate::token_filter::TokenFilterOutput<'_>,
) -> crate::pipeline::PipelineResult<()> {
output.write_token(token)
}
}
#[test]
fn qpdf_linearization_probe_consumes_modified_and_parameterized_streams() {
let mut pdf = Pdf::open(Cursor::new(parameter_probe_fixture())).expect("parse fixture");
AcroFormDocumentHelper::new(&mut pdf)
.expect("AcroForm")
.generate_appearances_if_needed()
.expect("generate appearance");
let options = WriterOptions {
object_streams: ObjectStreamMode::Disable,
..WriterOptions::default()
};
let plan = LinearizationPlan::from_pdf_with_writer_options(&mut pdf, &options)
.expect("build linearization plan");
assert_eq!(plan.page_hints.len(), 1);
assert!(!plan.part2_objects.is_empty());
assert!(
plan.all_assigned_refs()
.contains(&crate::ObjectRef::new(10, 0)),
"inline resource and ancestor array references must reach the plan"
);
}
#[test]
fn qpdf_linearization_parameter_probe_runs_once_before_raw_retry() {
let mut pdf = Pdf::open(Cursor::new(parameter_probe_fixture())).expect("parse fixture");
let stream = pdf.get_object_handle(crate::ObjectRef::new(4, 0));
stream.try_is_scalar().expect("resolve page content");
let token = crate::tokenizer::Token::new(crate::tokenizer::TokenType::Word, b"q".to_vec());
let mut filter = PassThroughTokenFilter;
let mut output = crate::token_filter::TokenFilterOutput::new(None);
crate::token_filter::TokenFilter::handle_token(&mut filter, &token, &mut output)
.expect("pass-through filter token");
let provider_calls = Rc::new(Cell::new(0));
let provider_calls_for_callback = Rc::clone(&provider_calls);
stream
.replace_stream_data_with_retry_callback(
move |_pipeline, _suppress_warnings, _will_retry| {
provider_calls_for_callback.set(provider_calls_for_callback.get() + 1);
Ok(false)
},
None,
None,
)
.expect("install stateful stream provider");
stream
.add_token_filter(Rc::new(std::cell::RefCell::new(PassThroughTokenFilter)))
.expect("register token filter");
let options = WriterOptions {
object_streams: ObjectStreamMode::Disable,
..WriterOptions::default()
};
LinearizationPlan::from_pdf_with_writer_options(&mut pdf, &options)
.expect("build linearization plan");
assert_eq!(
provider_calls.get(),
2,
"one qpdf parameter probe may retry once with raw data, but the optimization callback must not probe the same stream again"
);
}
#[test]
fn direct_reference_walkers_reject_depth_beyond_parser_limit() {
let handle = ObjectHandle::null();
let mut refs = Vec::new();
let error = collect_direct_handle_refs(&handle, MAX_PARSE_DEPTH + 1, &mut refs)
.expect_err("the direct reference walk has a parser-depth guard");
assert!(error.to_string().contains("maximum of 500"));
let mut contextual = Vec::new();
let error = collect_direct_handle_refs_with_context(
&handle,
MAX_PARSE_DEPTH + 1,
false,
&mut contextual,
)
.expect_err("the contextual reference walk has a parser-depth guard");
assert!(error.to_string().contains("maximum of 500"));
let mut stream_contextual = Vec::new();
let error = collect_direct_handle_refs_with_stream_parameters_context(
&handle,
MAX_PARSE_DEPTH + 1,
false,
&mut stream_contextual,
&BTreeSet::new(),
)
.expect_err("the stream-policy reference walk has a parser-depth guard");
assert!(error.to_string().contains("maximum of 500"));
}
#[test]
fn page_tree_classification_propagates_resolution_errors() {
let unresolved = ObjectHandle::new_indirect_unresolved(ObjectRef::new(91, 0), -1);
let error = super::is_page_tree_handle(&unresolved)
.expect_err("page-tree classification must propagate resolution errors");
assert!(matches!(
error,
Error::Internal(message) if message == "object 91 0 belongs to a dropped PDF"
));
}
#[test]
fn merge_writer_order_missing_ref_uses_fresh_fallback() {
let plan = LinearizationPlan {
writer_object_order: Some(BTreeMap::new()),
..Default::default()
};
let object_ref = ObjectRef::new(91, 0);
assert_eq!(
plan.writer_object_order_key(object_ref),
crate::pdf::WriterObjectOrderKey::fresh(object_ref)
);
}
}