use std::collections::{BTreeMap, BTreeSet};
use std::io::{Read, Seek, Write};
use std::path::Path;
use crate::linearization::hint_page::{bits_needed, PageOffsetHintTable};
use crate::linearization::hint_shared::SharedObjectHintTable;
use crate::linearization::hint_stream::{
encode_hint_stream_selected, HintStreamMode, OutlineHintTable,
};
use crate::linearization::part1::{Part1Bytes, Part1Placeholders};
use crate::linearization::plan::{
part9_category_order_key, ContainerPart, LinearizationPlan, RoutedObjStmBatch,
};
use crate::linearization::renumber::{ObjStmRelocation, RenumberMap, SecondHalfContainerAnchor};
use crate::pipeline::stdio_file::StdioBuffer;
use crate::qpdf_obj_gen::QpdfObjGen;
use crate::writer::encrypted_strings::EncryptedStringEmitter;
use crate::writer::encryption_state::WriterEncryptionState;
use crate::writer::object::TrailerKind;
use crate::writer::object_streams::{
emit_objstm_body_from_handles_with_writer, planner_config_from_options,
wrap_objstm_body_as_handle,
};
use crate::writer::plain::xref::write_xref_table_from_offsets;
use crate::writer::write_object::WriteObject;
use crate::writer::{
decrement_progress_event, effective_pdf_version_and_ext, effective_stream_policy,
output::{write_decimal_u64, write_object_ref, OutputSink, OutputTarget},
report_progress_event,
serialize::xref_stream,
CompressStreams, NewlineBeforeEndstream, ObjectWriterEmission, WriterOptions, WriterResult,
};
use crate::{ObjectHandle, ObjectRef, Pdf, Result};
const EBADF_ERRNO: i32 = 9;
fn write_indirect_object_header(out: &mut OutputSink<'_>, object_ref: ObjectRef) -> Result<()> {
write_decimal_u64(out, u64::from(object_ref.number))?;
out.write_bytes(b" ")?;
write_decimal_u64(out, u64::from(object_ref.generation))?;
out.write_bytes(b" obj\n")
}
enum Pass1Destination {
Discard,
File {
path: std::path::PathBuf,
writer: std::io::BufWriter<std::fs::File>,
},
}
struct Pass1OutputTarget {
destination: Pass1Destination,
}
impl Pass1OutputTarget {
#[cfg(test)]
fn discard() -> Self {
Self {
destination: Pass1Destination::Discard,
}
}
fn new(path: Option<&Path>) -> Result<Self> {
let destination = match path {
Some(path) => {
let file = std::fs::File::create(path)
.map_err(|source| crate::Error::file_io("open", path, source))?;
Pass1Destination::File {
path: path.to_path_buf(),
writer: std::io::BufWriter::new(file),
}
}
None => Pass1Destination::Discard,
};
Ok(Self { destination })
}
fn write_debug_comments(&mut self, comments: &[u8]) {
if let Pass1Destination::File { writer, .. } = &mut self.destination {
write_pass1_debug_comments(writer, comments);
}
}
}
impl OutputTarget for Pass1OutputTarget {
fn write_chunk(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
match &mut self.destination {
Pass1Destination::Discard => Ok(bytes.len()),
Pass1Destination::File { path, writer } => writer
.write_all(bytes)
.map(|()| bytes.len())
.map_err(|source| {
std::io::Error::new(
source.kind(),
format!("write {}: {source}", path.display()),
)
}),
}
}
fn finish_segment(&mut self) -> Result<()> {
Ok(())
}
fn finish_document(&mut self) -> Result<()> {
match &mut self.destination {
Pass1Destination::Discard => Ok(()),
Pass1Destination::File { writer, .. } => match writer.flush() {
Err(source) if source.raw_os_error() == Some(EBADF_ERRNO) => {
Err(crate::Error::Internal(
"linearization pass1: Pl_StdioFile::finish: stream already closed"
.to_string(),
))
}
Ok(()) | Err(_) => Ok(()),
},
}
}
}
#[derive(Debug, Clone)]
struct ObjStmContainer {
container_new_num: u32,
source_container: Option<ObjectRef>,
members: Vec<(ObjectRef, ObjectRef)>,
}
#[derive(Debug, Clone, Default)]
struct ObjStmLayout {
open_document: Vec<ObjStmContainer>,
part3: Vec<ObjStmContainer>,
part4: Vec<ObjStmContainer>,
member_to_container: BTreeMap<ObjectRef, (u32, u32)>,
source_container_numbers: BTreeSet<u32>,
}
impl ObjStmLayout {
fn is_empty(&self) -> bool {
self.open_document.is_empty() && self.part3.is_empty() && self.part4.is_empty()
}
fn resolve_batches<R: Read + Seek>(
plan: &LinearizationPlan,
pdf: &mut Pdf<R>,
options: &WriterOptions,
) -> Result<crate::linearization::plan::ObjStmBatchPlan> {
let config = planner_config_from_options(options);
let batch_plan = plan.objstm_batches(pdf, &config)?;
let page_dicts: std::collections::BTreeSet<ObjectRef> =
crate::pages::page_refs(pdf)?.into_iter().collect();
let filter_batches = |batches: Vec<RoutedObjStmBatch>| -> Vec<RoutedObjStmBatch> {
batches
.into_iter()
.filter_map(|batch| {
let members: Vec<ObjectRef> = batch
.members
.into_iter()
.filter(|r| !page_dicts.contains(r))
.collect();
(!members.is_empty()).then_some(RoutedObjStmBatch {
members,
route: batch.route,
source_container_number: batch.source_container_number,
})
})
.collect()
};
Ok(crate::linearization::plan::ObjStmBatchPlan {
open_document_batches: filter_batches(batch_plan.open_document_batches),
part3_batches: filter_batches(batch_plan.part3_batches),
part4_batches: filter_batches(batch_plan.part4_batches),
})
}
fn build_from_batches(
batch_plan: &crate::linearization::plan::ObjStmBatchPlan,
container_numbers: &[u32],
renumber: &RenumberMap,
) -> Result<Self> {
if batch_plan.open_document_batches.is_empty()
&& batch_plan.part3_batches.is_empty()
&& batch_plan.part4_batches.is_empty()
{
return Ok(Self::default());
}
let mut member_to_container = BTreeMap::new();
let source_container_numbers: BTreeSet<u32> = batch_plan
.open_document_batches
.iter()
.chain(&batch_plan.part3_batches)
.chain(&batch_plan.part4_batches)
.filter_map(|batch| batch.source_container_number)
.collect();
let take = |batches: &[RoutedObjStmBatch],
out: &mut Vec<ObjStmContainer>,
map: &mut BTreeMap<ObjectRef, (u32, u32)>,
container_iter: &mut std::vec::IntoIter<u32>|
-> Result<()> {
for batch in batches {
if batch.members.is_empty() {
continue;
}
let container_new_num = container_iter.next().ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: ObjStm container-number stream exhausted \
(renumber relocation / batch-plan inconsistency)"
.to_string(),
)
})?;
let source_container = batch
.source_container_number
.map(|number| ObjectRef::new(number, 0));
let mut members = Vec::with_capacity(batch.members.len());
for (idx, &orig) in batch.members.iter().enumerate() {
let new_ref = renumber.new_for_original(orig).ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: ObjStm member {orig} has no renumber \
entry (planner / renumber inconsistency)"
))
})?;
map.insert(orig, (container_new_num, idx as u32));
members.push((orig, new_ref));
}
out.push(ObjStmContainer {
container_new_num,
source_container,
members,
});
}
Ok(())
};
let container_numbers_vec: Vec<u32> = container_numbers.to_vec();
let mut container_iter = container_numbers_vec.into_iter();
let mut open_document = Vec::new();
let mut part3 = Vec::new();
let mut part4 = Vec::new();
take(
&batch_plan.open_document_batches,
&mut open_document,
&mut member_to_container,
&mut container_iter,
)?;
take(
&batch_plan.part3_batches,
&mut part3,
&mut member_to_container,
&mut container_iter,
)?;
take(
&batch_plan.part4_batches,
&mut part4,
&mut member_to_container,
&mut container_iter,
)?;
let _ = container_iter;
Ok(Self {
open_document,
part3,
part4,
member_to_container,
source_container_numbers,
})
}
}
fn preserved_objstm_extends<R: Read + Seek>(
container: &ObjStmContainer,
renumber: &RenumberMap,
pdf: &mut Pdf<R>,
) -> Result<Option<ObjectRef>> {
let Some(source_container) = container.source_container else {
return Ok(None);
};
let source_handle = pdf.get_object_handle(source_container);
source_handle.try_dereference()?;
let Some(source_dict) = source_handle.as_stream_dict() else {
return Ok(None);
};
let extends_handle = source_dict.try_get_key(b"/Extends")?;
let Some(extends) = extends_handle.object_ref() else {
return Ok(None);
};
Ok(Some(
renumber
.new_for_original(extends)
.unwrap_or_else(|| ObjectRef::new(0, 0)),
))
}
fn append_objstm_container_object<R: Read + Seek>(
out: &mut OutputSink<'_>,
container: &ObjStmContainer,
renumber: &RenumberMap,
pdf: &mut Pdf<R>,
removed_refs: &BTreeSet<QpdfObjGen>,
options: &WriterOptions,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
) -> Result<usize> {
let filtered = matches!(effective_stream_policy(options), Some(CompressStreams::Yes));
let map = |object_gen| {
renumber.new_for_raw(object_gen).ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: ObjStm member raw reference {} {} has no renumber entry",
object_gen.get_obj(),
object_gen.get_gen()
))
})
};
let mut members: Vec<(ObjectRef, ObjectHandle)> = Vec::with_capacity(container.members.len());
for &(orig, new_ref) in &container.members {
let handle = pdf.get_object_handle(orig);
handle.try_dereference()?;
let handle = if handle.as_stream_dict().is_some() {
ObjectHandle::null()
} else {
handle
};
members.push((new_ref, handle));
}
let body = emit_objstm_body_from_handles_with_writer(
&members,
&mut |out, _member_index, _object_ref, handle| {
crate::writer::output::with_buffer_sink(out, |out| {
handle.write_object_with_qpdf_obj_gen_map_and_removed(out, &map, removed_refs)
})
},
)?;
let compress = if filtered {
CompressStreams::Yes
} else {
CompressStreams::No
};
let extends = preserved_objstm_extends(container, renumber, pdf)?;
let (stream_handle, data) = wrap_objstm_body_as_handle(body, compress, extends)?;
let stream_dict = stream_handle.as_stream_dict().ok_or_else(|| {
crate::Error::Internal("linearization ObjStm wrapper produced a non-stream handle".into())
})?; let object_ref = ObjectRef::new(container.container_new_num, 0);
if let Some(ctx) = encrypt_ctx {
let mut payload_length = data.len();
crate::writer::adjust_aes_stream_length(&mut payload_length, ctx, true)?;
stream_dict.replace_key(
b"/Length",
ObjectHandle::integer(i64::try_from(payload_length).unwrap_or(i64::MAX)),
)?;
}
let offset = out.position_usize()?;
write_indirect_object_header(out, ObjectRef::new(container.container_new_num, 0))?;
out.write_bytes(b"<< /Type ")?;
stream_dict.try_get_key(b"/Type")?.write_object(out)?;
out.write_bytes(b" /Length ")?;
stream_dict.try_get_key(b"/Length")?.write_object(out)?;
if filtered {
out.write_bytes(b" /Filter ")?;
stream_dict.try_get_key(b"/Filter")?.write_object(out)?;
}
out.write_bytes(b" /N ")?;
stream_dict.try_get_key(b"/N")?.write_object(out)?;
out.write_bytes(b" /First ")?;
stream_dict.try_get_key(b"/First")?.write_object(out)?;
if let Some(extends) = extends {
out.write_bytes(b" /Extends ")?;
write_object_ref(out, extends)?;
}
out.write_bytes(b" >>")?;
if let Some(ctx) = encrypt_ctx {
crate::writer::write_stream_payload_with_pipeline(
out,
data.as_slice(),
options.newline_before_endstream,
object_ref,
ctx,
true,
None,
)?; } else {
crate::writer::serialize::write_stream_payload(
out,
data.as_slice(),
options.newline_before_endstream,
)?; }
out.write_bytes(b"\nendobj\n")?;
Ok(offset)
}
#[derive(Debug, Clone)]
pub struct LinearizedOffsets {
pub file_length: usize,
pub hint_stream_offset: usize,
pub hint_stream_length: usize,
pub first_page_object_new_num: u32,
pub end_of_first_page_offset: usize,
pub last_xref_keyword_offset: usize,
pub last_xref_offset: usize,
pub page_count: u32,
pub part1_placeholders: Part1Placeholders,
pub xref_offsets: BTreeMap<u32, usize>,
pub first_trailer_prev_range: std::ops::Range<usize>,
pub dict_writable_region: std::ops::Range<usize>,
}
#[derive(Debug)]
pub struct LinearizedDocument {
pub bytes: Vec<u8>,
pub offsets: LinearizedOffsets,
}
struct LinearizedWriteResult {
#[allow(dead_code)]
document: Option<LinearizedDocument>,
writer_result: WriterResult,
}
pub(crate) const PREV_PLACEHOLDER_WIDTH: usize = 22;
fn writer_encryption_state_for(
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
) -> WriterEncryptionState {
WriterEncryptionState::new(
encrypt_ctx.is_some(),
encrypt_ctx
.map(|ctx| ctx.file_key.clone())
.unwrap_or_default(),
encrypt_ctx.is_some_and(|ctx| crate::writer::cipher_needs_aes_iv(ctx.cipher)),
encrypt_ctx.map_or(0, |ctx| ctx.encryption_v),
encrypt_ctx.map_or(0, |ctx| ctx.encryption_r),
)
}
struct LinearizedObjectEmitter<'a, 'sink> {
out: &'a mut OutputSink<'sink>,
options: &'a WriterOptions,
encrypt_ctx: Option<&'a crate::writer::EncryptionContext>,
encrypted_string_emitter: Option<&'a mut EncryptedStringEmitter>,
renumber: &'a RenumberMap,
removed_refs: &'a BTreeSet<QpdfObjGen>,
content_normalize_refs: &'a BTreeSet<QpdfObjGen>,
root_source: Option<QpdfObjGen>,
final_pdf_version: &'a str,
final_extension_level: i64,
encryption: WriterEncryptionState,
xref: BTreeMap<u32, (u16, usize)>,
lengths: BTreeMap<u32, usize>,
}
impl WriteObject for LinearizedObjectEmitter<'_, '_> {
type ObjectStreamContainer = ();
fn object_stream_container(&self, _object: QpdfObjGen) -> Option<()> {
None
}
fn write_object_stream(&mut self, _object: &ObjectHandle, (): ()) -> Result<()> {
Err(crate::Error::Internal(
"linearized body-object emitter: unreachable object-stream redirect".to_string(),
))
}
fn indicate_progress(&mut self) -> Result<()> {
report_progress_event(self.options)
}
fn output_number(&self, object: QpdfObjGen) -> Result<u32> {
self.renumber
.new_for_raw(object)
.map(|new_ref| new_ref.number)
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: raw reference {} {} has no renumber entry",
object.get_obj(),
object.get_gen()
))
})
}
fn write_bytes(&mut self, bytes: &[u8]) -> Result<()> {
self.out.write_bytes(bytes)
}
fn output_count(&self) -> Result<usize> {
self.out.position_usize()
}
fn xref(&mut self) -> &mut BTreeMap<u32, (u16, usize)> {
&mut self.xref
}
fn lengths(&mut self) -> &mut BTreeMap<u32, usize> {
&mut self.lengths
}
fn encryption_state(&mut self) -> &mut WriterEncryptionState {
&mut self.encryption
}
fn unparse_object(&mut self, object: &ObjectHandle, _in_object_stream: bool) -> Result<()> {
object.try_dereference()?;
let original_ref = object.qpdf_obj_gen().unwrap_or(QpdfObjGen::new(0, 0));
let root_copy;
let object: &ObjectHandle = if self.root_source == Some(original_ref) {
root_copy = object.output_root_copy_with_adbe(
self.final_pdf_version,
self.final_extension_level,
true,
)?; &root_copy
} else {
object
};
let renumber = self.renumber;
let map = move |object_gen: QpdfObjGen| {
renumber.new_for_raw(object_gen).ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: raw reference {} {} has no renumber entry",
object_gen.get_obj(),
object_gen.get_gen()
))
})
};
let new_ref = ObjectRef::new(self.output_number(original_ref)?, 0);
let removed_refs = self.removed_refs;
let out = &mut *self.out;
if object.as_stream_dict().is_none() {
return match self.encrypted_string_emitter.as_deref_mut() {
Some(emitter) => emitter.write_handle_object_with_qpdf_obj_gen_map_and_mode(
out,
new_ref,
None,
object,
false,
&map,
removed_refs,
), None => {
object.write_object_with_qpdf_obj_gen_map_and_removed(out, &map, removed_refs)
}
};
}
let (stream_dict, data, dictionary_options) =
crate::writer::plain::body::canonical_stream_output_for_linearization(
object,
self.options,
self.options.content_normalization
&& self.content_normalize_refs.contains(&original_ref),
)?; let payload_ctx = self.encrypt_ctx.filter(|ctx| new_ref != ctx.encrypt_ref);
let is_metadata_stream = stream_dict.try_is_dictionary_of_type(b"Metadata", b"")?;
let cleartext_metadata =
payload_ctx.is_some_and(|ctx| !ctx.encrypt_metadata && is_metadata_stream);
let mut payload_length = data.len();
if let Some(ctx) = payload_ctx.filter(|_| !cleartext_metadata) {
crate::writer::adjust_aes_stream_length(&mut payload_length, ctx, true)?;
}
if let Some(emitter) = self.encrypted_string_emitter.as_deref_mut() {
let mut entries = stream_dict.try_as_dictionary()?.unwrap_or_default();
entries.insert(
b"/Length".to_vec(),
ObjectHandle::integer(i64::try_from(payload_length).unwrap_or(i64::MAX)),
);
let dict = ObjectHandle::dictionary(entries.into_iter().collect());
if dict.context().is_none() && object.context().is_some() {
dict.set_child_description(object, b" -> stream dictionary", b"");
}
emitter.write_handle_stream_dict_with_qpdf_obj_gen_map(
out,
new_ref,
None,
&dict,
crate::writer::encrypted_strings::StreamDictOptions::new(
false,
dictionary_options,
true,
),
&map,
removed_refs,
None,
)?; } else {
stream_dict
.write_stream_body_with_qpdf_obj_gen_map_and_removed_with_options_and_length(
out,
dictionary_options,
&map,
removed_refs,
payload_length,
)?; }
if let Some(ctx) = payload_ctx.filter(|_| !cleartext_metadata) {
crate::writer::write_stream_payload_with_pipeline(
out,
&data,
self.options.newline_before_endstream,
new_ref,
ctx,
true,
None,
)?; } else {
crate::writer::serialize::write_stream_payload(
out,
&data,
self.options.newline_before_endstream,
)?; }
Ok(())
}
}
#[allow(clippy::too_many_arguments)]
fn append_body_object_for_ref<R: Read + Seek>(
out: &mut OutputSink<'_>,
pdf: &mut Pdf<R>,
new_ref: ObjectRef,
original_ref: ObjectRef,
options: &WriterOptions,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
encrypted_string_emitter: Option<&mut EncryptedStringEmitter>,
renumber: &RenumberMap,
removed_refs: &BTreeSet<QpdfObjGen>,
content_normalize_refs: &BTreeSet<QpdfObjGen>,
) -> Result<usize> {
let object = pdf.get_object_handle(original_ref);
let mut emitter = LinearizedObjectEmitter {
out,
options,
encrypt_ctx,
encrypted_string_emitter,
renumber,
removed_refs,
content_normalize_refs,
root_source: None,
final_pdf_version: "",
final_extension_level: 0,
encryption: writer_encryption_state_for(encrypt_ctx),
xref: BTreeMap::new(),
lengths: BTreeMap::new(),
};
emitter.write_object(&object, None)?;
Ok(emitter.xref[&new_ref.number].1)
}
#[allow(clippy::too_many_arguments)]
fn append_body_object_for_raw<R: Read + Seek>(
out: &mut OutputSink<'_>,
pdf: &mut Pdf<R>,
new_ref: ObjectRef,
original_gen: QpdfObjGen,
options: &WriterOptions,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
encrypted_string_emitter: Option<&mut EncryptedStringEmitter>,
renumber: &RenumberMap,
removed_refs: &BTreeSet<QpdfObjGen>,
content_normalize_refs: &BTreeSet<QpdfObjGen>,
) -> Result<usize> {
let object =
pdf.get_object_handle_by_raw_identity(original_gen.get_obj(), original_gen.get_gen());
let mut emitter = LinearizedObjectEmitter {
out,
options,
encrypt_ctx,
encrypted_string_emitter,
renumber,
removed_refs,
content_normalize_refs,
root_source: None,
final_pdf_version: "",
final_extension_level: 0,
encryption: writer_encryption_state_for(encrypt_ctx),
xref: BTreeMap::new(),
lengths: BTreeMap::new(),
};
emitter.write_object(&object, None)?;
Ok(emitter.xref[&new_ref.number].1)
}
#[allow(clippy::too_many_arguments)]
fn append_root_object<R: Read + Seek>(
out: &mut OutputSink<'_>,
pdf: &mut Pdf<R>,
new_ref: ObjectRef,
root_ref: ObjectRef,
options: &WriterOptions,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
encrypted_string_emitter: Option<&mut EncryptedStringEmitter>,
renumber: &RenumberMap,
removed_refs: &BTreeSet<QpdfObjGen>,
content_normalize_refs: &BTreeSet<QpdfObjGen>,
final_pdf_version: &str,
final_extension_level: i64,
) -> Result<usize> {
let object = pdf.get_object_handle(root_ref);
let root_source = QpdfObjGen::try_from_object_ref(root_ref)?;
let mut emitter = LinearizedObjectEmitter {
out,
options,
encrypt_ctx,
encrypted_string_emitter,
renumber,
removed_refs,
content_normalize_refs,
root_source: Some(root_source),
final_pdf_version,
final_extension_level,
encryption: writer_encryption_state_for(encrypt_ctx),
xref: BTreeMap::new(),
lengths: BTreeMap::new(),
};
emitter.write_object(&object, None)?;
Ok(emitter.xref[&new_ref.number].1)
}
fn write_repeated_bytes(out: &mut OutputSink<'_>, byte: u8, mut count: usize) -> Result<()> {
const CHUNK_SIZE: usize = 4096;
let chunk = [byte; CHUNK_SIZE];
while count >= CHUNK_SIZE {
out.write_bytes(&chunk)?;
count -= CHUNK_SIZE;
}
if count != 0 {
out.write_bytes(&chunk[..count])?;
} Ok(())
}
const CLASSIC_XREF_ENTRY_WIDTH: usize = 20;
struct Part1XrefPatch {
start_num: u32,
count: u32,
data_range: std::ops::Range<usize>,
}
#[allow(clippy::too_many_arguments)]
fn write_part1_xref_and_trailer(
out: &mut OutputSink<'_>,
param_dict_obj_number: u32,
total_object_count: u32,
first_page_count: u32,
catalog_new_ref: ObjectRef,
info_new_ref: Option<ObjectRef>,
source_trailer: &ObjectHandle,
live_trailer: &ObjectHandle,
canonical_entries: &[(Vec<u8>, Vec<u8>)],
map: &dyn Fn(QpdfObjGen) -> Result<ObjectRef>,
removed_refs: &BTreeSet<QpdfObjGen>,
id_writer: Option<crate::pdf_syntax::ReborrowableIdWriter>,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
pass1: bool,
final_xref_offsets: Option<&BTreeMap<u32, usize>>,
final_prev: Option<usize>,
) -> Result<(usize, std::ops::Range<usize>, Option<Part1XrefPatch>)> {
if out.last_byte() != Some(b'\n') {
out.write_bytes(b"\n")?;
}
let xref_offset = out.position_usize()?;
let first_page_last = first_page_count
.checked_sub(1)
.and_then(|count| param_dict_obj_number.checked_add(count))
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: first-page xref range overflows object numbers".to_string(),
)
})?;
let patch = if pass1 {
let empty_offsets = BTreeMap::new();
write_xref_table_from_offsets(
out,
param_dict_obj_number,
first_page_last,
&empty_offsets,
true,
0,
0,
0,
)?; None
} else if let Some(xref_offsets) = final_xref_offsets {
write_xref_table_from_offsets(
out,
param_dict_obj_number,
first_page_last,
xref_offsets,
false,
0,
0,
0,
)?; None
} else {
let data_len = (first_page_count as usize)
.checked_mul(CLASSIC_XREF_ENTRY_WIDTH)
.ok_or_else(|| {
crate::Error::Unsupported(
"Part-1 xref placeholder length exceeds usize range".to_string(),
)
})?;
let header_end = write_xref_table_from_offsets(
out,
param_dict_obj_number,
first_page_last,
&BTreeMap::new(),
true,
0,
0,
0,
)?; let data_start = header_end.checked_add(1).ok_or_else(|| {
crate::Error::Unsupported("Part-1 xref entry offset overflows usize".to_string())
})?; let data_end = data_start.checked_add(data_len).ok_or_else(|| {
crate::Error::Unsupported("Part-1 xref patch range overflows usize".to_string())
})?;
Some(Part1XrefPatch {
start_num: param_dict_obj_number,
count: first_page_count,
data_range: data_start..data_end,
})
};
out.write_bytes(b"trailer << ")?;
let mut entries = canonical_entries.to_vec();
if let Some(info_ref) = info_new_ref {
entries.push((
b"/Info".to_vec(),
format!("{} {} R", info_ref.number, info_ref.generation).into_bytes(),
));
}
entries.push((
b"/Root".to_vec(),
format!(
"{} {} R",
catalog_new_ref.number, catalog_new_ref.generation
)
.into_bytes(),
));
if source_trailer.try_has_key(b"/Size")? {
entries.push((
b"/Size".to_vec(),
total_object_count.to_string().into_bytes(),
));
}
entries.sort_by(|left, right| left.0.cmp(&right.0));
let mut prev_value_range = None;
for (key, value) in entries {
out.write_bytes(b"/")?;
crate::pdf_syntax::write_name_escaped(out, key.strip_prefix(b"/").unwrap_or(&key))?;
out.write_bytes(b" ")?;
out.write_bytes(&value)?;
if key == b"/Size" {
out.write_bytes(b" /Prev ")?;
let prev_value_start = out.position_usize()?;
let placeholder = format!("{:<PREV_PLACEHOLDER_WIDTH$}", final_prev.unwrap_or(0));
out.write_bytes(placeholder.as_bytes())?;
if final_prev.is_none() {
prev_value_range = Some(prev_value_start..out.position_usize()?);
}
} else {
out.write_bytes(b" ")?;
}
}
out.write_bytes(b"/ID ")?;
let id_value = if pass1 {
linearization_pass1_id_from_live_trailer(live_trailer)?
} else {
source_trailer.try_get_key(b"/ID")?
};
match id_writer {
Some(write_id) => write_id(out),
None => id_value.write_id_value_with_qpdf_obj_gen_map(out, map, removed_refs),
}?;
if let Some(ctx) = encrypt_ctx {
out.write_bytes(
format!(
" /Encrypt {} {} R",
ctx.encrypt_ref.number, ctx.encrypt_ref.generation
)
.as_bytes(),
)?; }
out.write_bytes(b" >>")?;
out.write_bytes(b"\nstartxref\n0\n%%EOF\n")?;
let prev_value_range = prev_value_range.unwrap_or(0..0);
Ok((xref_offset, prev_value_range, patch))
}
fn patch_part1_xref(
out: &mut OutputSink<'_>,
patch: &Part1XrefPatch,
xref_offsets: &BTreeMap<u32, usize>,
) -> Result<()> {
if patch.data_range.end > out.position_usize()? {
return Err(crate::Error::Unsupported(
"Part-1 xref patch range out of bounds".to_string(),
));
}
let mut data = Vec::with_capacity(patch.count as usize * CLASSIC_XREF_ENTRY_WIDTH);
for number in patch.start_num..patch.start_num + patch.count {
let offset = xref_offsets.get(&number).copied().ok_or_else(|| {
crate::Error::Unsupported(format!(
"Part-1 xref: covered object {number} has no offset (planner / writer \
inconsistency — would emit a free entry for a live first-page object)"
))
})?;
data.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
}
if data.len() != patch.data_range.len() {
return Err(crate::Error::Unsupported(format!(
"Part-1 xref payload length drift: encoded {} bytes, reserved {}",
data.len(),
patch.data_range.len()
)));
}
out.patch_bytes(patch.data_range.clone(), &data)
}
#[allow(clippy::too_many_arguments)]
fn write_main_xref_and_trailer(
out: &mut OutputSink<'_>,
xref_offsets: &BTreeMap<u32, usize>,
param_slot: u32, first_page_xref_offset: usize,
source_trailer: &ObjectHandle,
live_trailer: &ObjectHandle,
map: &dyn Fn(QpdfObjGen) -> Result<ObjectRef>,
removed_refs: &BTreeSet<QpdfObjGen>,
pass1: bool,
) -> Result<(usize, usize)> {
let xref_start = out.position_usize()?;
let xref_first_entry_offset = if param_slot == 0 {
out.write_bytes(b"xref\n0 0\n")?;
out.position_usize()?
} else {
let header_end =
write_xref_table_from_offsets(out, 0, param_slot - 1, xref_offsets, false, 0, 0, 0)?;
header_end.checked_add(1).ok_or_else(|| {
crate::Error::Unsupported("main xref entry offset overflows usize".to_string())
})? };
let trailer_map = |object_ref: ObjectRef| map(QpdfObjGen::try_from_object_ref(object_ref)?);
let removed_object_refs: BTreeSet<ObjectRef> = removed_refs
.iter()
.filter_map(|object_gen| object_gen.to_object_ref())
.collect();
let pass1_id_value = if pass1 {
Some(linearization_pass1_id_from_live_trailer(live_trailer)?)
} else {
None
};
let mut pass1_id_writer = |output: &mut OutputSink<'_>| {
let id_value = pass1_id_value.as_ref().ok_or_else(|| {
crate::Error::Internal("missing pass-1 linearization ID value".to_string())
})?; id_value.write_id_value_with_qpdf_obj_gen_map(output, map, removed_refs)
};
source_trailer.write_trailer_with_ref_map_and_kind(
out,
TrailerKind::LinearizedSecond {
size: i64::from(param_slot),
},
false,
false,
pass1.then_some(&mut pass1_id_writer as _),
&trailer_map,
&removed_object_refs,
true,
)?; out.write_bytes(format!("\nstartxref\n{}\n%%EOF\n", first_page_xref_offset).as_bytes())?;
Ok((xref_start, xref_first_entry_offset))
}
struct FirstPageXrefPatch {
first_xref_num: u32,
index_start: u32,
index_count: u32,
region: std::ops::Range<usize>,
catalog_new_ref: ObjectRef,
info_new_ref: Option<ObjectRef>,
size: u32,
id: Option<(Vec<u8>, Vec<u8>)>,
canonical_entries: Vec<(Vec<u8>, Vec<u8>)>,
encrypt: Option<ObjectRef>,
max_id: u32,
max_ostream_index: u64,
filtered: bool,
}
struct FinalFirstPageXref<'a> {
member_new: &'a BTreeMap<u32, (u32, u32)>,
main_xref_offset: usize,
}
fn canonical_linearization_trailer_entries(
trailer: &ObjectHandle,
map: &dyn Fn(QpdfObjGen) -> Result<ObjectRef>,
removed_refs: &BTreeSet<QpdfObjGen>,
) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
let entries = trailer.try_as_dictionary()?.unwrap_or_default();
let mut serialized = Vec::with_capacity(entries.len());
for (key, value) in entries {
if crate::writer::object::TRIMMED_TRAILER_KEYS.contains(&key.as_slice())
|| matches!(key.as_slice(), b"/Info" | b"/Root" | b"/Size")
{
continue;
}
let removed = value.qpdf_obj_gen().is_some_and(|object_gen| {
!object_gen.is_indirect() || removed_refs.contains(&object_gen)
});
if removed || value.try_is_null()? {
continue;
}
let mut value_bytes = Vec::new();
crate::writer::output::with_buffer_sink(&mut value_bytes, |out| {
crate::writer::object::write_child_with_ref_map(&value, out, map, removed_refs)
})?;
serialized.push((key, value_bytes));
}
Ok(serialized)
}
pub(crate) fn first_page_xref_object_count(
total_count: u32,
param_dict_obj_number: u32,
) -> Result<u32> {
total_count
.checked_sub(param_dict_obj_number)
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: param-dict object number ({param_dict_obj_number}) \
exceeds /Size ({total_count}) — cannot size the first-page xref subsection"
))
})
}
fn finalize_linearized_id(
options: &WriterOptions,
source_id0: Option<&[u8]>,
det_id_source_id0: Option<&[u8]>,
copy_encryption: Option<&crate::encryption::CopyEncryptionSource>,
) -> ObjectHandle {
if crate::writer::uses_deterministic_id(options) {
let len0 = det_id_source_id0.map(<[u8]>::len).unwrap_or(16);
ObjectHandle::array(vec![
ObjectHandle::string(vec![0u8; len0]),
ObjectHandle::string(vec![0u8; 16]),
])
} else if let Some(source) = copy_encryption {
let generated = crate::writer::generate_id_handle(None, options.static_id);
let id1 = generated
.as_array()
.and_then(|values| values.get(1).and_then(ObjectHandle::as_string))
.unwrap_or_else(|| source.id0.clone());
ObjectHandle::array(vec![
ObjectHandle::string(source.id0.clone()),
ObjectHandle::string(id1),
])
} else {
crate::writer::generate_id_handle(source_id0, options.static_id)
}
}
fn linearization_pass1_id(source_id0: Option<&[u8]>) -> ObjectHandle {
let first_len = source_id0.map(|id| id.len()).unwrap_or(16);
ObjectHandle::array(vec![
ObjectHandle::string(vec![0u8; first_len]),
ObjectHandle::string(vec![0u8; 16]),
])
}
fn linearization_pass1_id_from_live_trailer(live_trailer: &ObjectHandle) -> Result<ObjectHandle> {
let source_id0 = crate::writer::source_permanent_id_handle(live_trailer)?;
Ok(linearization_pass1_id(source_id0.as_deref()))
}
#[allow(clippy::too_many_arguments)]
fn write_first_page_xref_stream(
out: &mut OutputSink<'_>,
xref_offsets: &mut BTreeMap<u32, usize>,
relocation: &ObjStmRelocation,
total_count: u32, catalog_new_ref: ObjectRef,
info_new_ref: Option<ObjectRef>,
source_trailer: &ObjectHandle,
live_trailer: &ObjectHandle,
canonical_entries: &[(Vec<u8>, Vec<u8>)],
max_ostream_index: u64,
filtered: bool,
encrypt: Option<ObjectRef>,
pass1: bool,
final_layout: Option<FinalFirstPageXref<'_>>,
) -> Result<FirstPageXrefPatch> {
let final_size = total_count;
let first_xref_num = relocation.first_xref_slot;
let index_start = relocation.second_half_count;
let index_count = final_size.checked_sub(index_start).ok_or_else(|| {
crate::Error::Unsupported(
"first-page xref /Index underflow (second-half count exceeds /Size)".to_string(),
)
})?;
let max_id = final_size.saturating_sub(1);
let id_value = if pass1 {
linearization_pass1_id_from_live_trailer(live_trailer)?
} else {
source_trailer.try_get_key(b"/ID")?
};
let id = xref_id_bytes(&id_value)?;
let obj_ref = ObjectRef::new(first_xref_num, 0);
let region_len = {
let dict = xref_stream::XrefStreamDict {
filtered,
widths: xref_stream::first_pass_widths(max_id, max_ostream_index, 0),
index: Some((index_start, index_count)),
info: info_new_ref,
root: Some(catalog_new_ref),
root_value: None,
size: final_size,
prev: Some(0),
canonical_entries: Some(canonical_entries),
id: id.as_ref().map(|(a, b)| (a.as_slice(), b.as_slice())),
encrypt,
};
xref_stream::first_pass_region_len(obj_ref, &dict, index_count as usize)
};
if out.last_byte() != Some(b'\n') {
out.write_bytes(b"\n")?;
}
let obj_offset = out.position_usize()?;
let patch = FirstPageXrefPatch {
first_xref_num,
index_start,
index_count,
region: obj_offset..obj_offset + region_len,
catalog_new_ref,
info_new_ref,
size: final_size,
id,
canonical_entries: canonical_entries.to_vec(),
encrypt,
max_id,
max_ostream_index,
filtered,
};
xref_offsets.insert(first_xref_num, obj_offset);
if pass1 {
let (region, _) =
build_first_page_xref_region(&patch, xref_offsets, &BTreeMap::new(), 0, 0, true)?;
out.write_bytes(®ion)?;
} else if let Some(final_layout) = final_layout {
let (region, _) = build_first_page_xref_region(
&patch,
xref_offsets,
final_layout.member_new,
final_layout.main_xref_offset,
0,
false,
)?; out.write_bytes(®ion)?;
} else {
write_repeated_bytes(out, b' ', region_len)?;
}
out.write_bytes(b"\n")?;
Ok(patch)
}
fn xref_id_bytes(id: &ObjectHandle) -> Result<Option<(Vec<u8>, Vec<u8>)>> {
let Some(values) = id.try_as_array()? else {
return Ok(None);
};
if values.len() != 2 {
return Ok(None);
}
let (first, second) = (&values[0], &values[1]);
Ok(match (first.as_string(), second.as_string()) {
(Some(first), Some(second)) => Some((first, second)),
_ => None,
})
}
fn build_first_page_xref_region(
patch: &FirstPageXrefPatch,
xref_offsets: &mut BTreeMap<u32, usize>,
member_new: &BTreeMap<u32, (u32, u32)>,
main_xref_offset: usize,
_hint_length: usize,
pass1: bool,
) -> Result<(Vec<u8>, Option<std::ops::Range<usize>>)> {
xref_offsets.insert(patch.first_xref_num, patch.region.start);
let (stream_layout, prev) = if pass1 {
let mut pass1_offs: BTreeMap<u32, usize> = xref_offsets
.iter()
.filter(|(_, &off)| off <= patch.region.start)
.map(|(&n, &off)| (n, off))
.collect();
let stream_layout = xref_stream::prepare_xref_stream(
&mut pass1_offs,
&BTreeMap::new(),
patch.index_start,
patch.index_count,
patch.first_xref_num,
patch.region.start,
0,
patch.max_id,
patch.max_ostream_index,
0,
None,
patch.filtered,
true,
true,
)?; (stream_layout, 0u64)
} else {
let stream_layout = xref_stream::prepare_xref_stream_from_physical_offsets(
xref_offsets,
member_new,
patch.index_start,
patch.index_count,
patch.first_xref_num,
patch.region.start,
patch.max_id,
patch.max_ostream_index,
patch.filtered,
)?; (stream_layout, main_xref_offset as u64)
};
let dict = xref_stream::XrefStreamDict {
filtered: patch.filtered,
widths: stream_layout.widths,
index: Some((patch.index_start, patch.index_count)),
info: patch.info_new_ref,
root: Some(patch.catalog_new_ref),
root_value: None,
size: patch.size,
prev: Some(prev),
canonical_entries: Some(&patch.canonical_entries),
id: patch.id.as_ref().map(|(a, b)| (a.as_slice(), b.as_slice())),
encrypt: patch.encrypt,
};
let (region, region_id_range) = xref_stream::write_padded_region(
ObjectRef::new(patch.first_xref_num, 0),
&dict,
&stream_layout,
patch.region.len(),
)?; Ok((
region,
region_id_range.map(|r| patch.region.start + r.start..patch.region.start + r.end),
))
}
fn patch_first_page_xref(
out: &mut OutputSink<'_>,
patch: &FirstPageXrefPatch,
xref_offsets: &mut BTreeMap<u32, usize>,
member_new: &BTreeMap<u32, (u32, u32)>,
main_xref_offset: usize,
hint_length: usize,
) -> Result<Option<std::ops::Range<usize>>> {
let (region, id_range) = build_first_page_xref_region(
patch,
xref_offsets,
member_new,
main_xref_offset,
hint_length,
false,
)?; if patch.region.end > out.position_usize()? {
return Err(crate::Error::Unsupported(
"first-page xref patch range out of bounds".to_string(),
));
}
out.patch_bytes(patch.region.clone(), ®ion)?;
Ok(id_range)
}
#[allow(clippy::too_many_arguments)]
fn write_main_xref_stream_and_trailer(
out: &mut OutputSink<'_>,
xref_offsets: &mut BTreeMap<u32, usize>,
member_new: &BTreeMap<u32, (u32, u32)>,
relocation: &ObjStmRelocation,
total_count: u32, source_trailer: &ObjectHandle,
live_trailer: &ObjectHandle,
first_page_obj_offset: usize,
max_ostream_index: u64,
pass1: bool,
filtered: bool,
) -> Result<(usize, usize, Option<std::ops::Range<usize>>, usize)> {
let final_size = total_count;
let first_xref_num = relocation.first_xref_slot;
let main_xref_num = relocation.main_xref_slot;
let max_id = final_size.saturating_sub(1);
let id_value = if pass1 {
linearization_pass1_id_from_live_trailer(live_trailer)?
} else {
source_trailer.try_get_key(b"/ID")?
};
let id = xref_id_bytes(&id_value)?;
let main_count = relocation.second_half_count;
let main_xref_offset = out.position_usize()?;
xref_offsets.insert(first_xref_num, first_page_obj_offset);
let stream_layout = xref_stream::prepare_xref_stream(
xref_offsets,
member_new,
0,
main_count,
main_xref_num,
main_xref_offset,
main_xref_offset as u64,
max_id,
max_ostream_index,
0,
None,
filtered,
pass1,
false,
)?;
let dict = xref_stream::XrefStreamDict {
filtered,
widths: stream_layout.widths,
index: None,
info: None,
root: None,
root_value: None,
size: main_count,
prev: None,
canonical_entries: None,
id: id.as_ref().map(|(a, b)| (a.as_slice(), b.as_slice())),
encrypt: None,
};
let main_obj_ref = ObjectRef::new(main_xref_num, 0);
let region_len = {
let p1_dict = xref_stream::XrefStreamDict {
filtered,
widths: stream_layout.widths,
index: None,
info: None,
root: None,
root_value: None,
size: main_count,
prev: None,
canonical_entries: None,
id: id.as_ref().map(|(a, b)| (a.as_slice(), b.as_slice())),
encrypt: None,
};
xref_stream::first_pass_region_len(main_obj_ref, &p1_dict, main_count as usize)
};
let (region, region_id_range) =
xref_stream::write_padded_region(main_obj_ref, &dict, &stream_layout, region_len)?;
let region_start = out.position_usize()?;
out.write_bytes(®ion)?;
out.write_bytes(b"\n")?;
let id_range = region_id_range.map(|r| region_start + r.start..region_start + r.end);
let second_xref_end = out.position_usize()?;
out.write_bytes(format!("startxref\n{first_page_obj_offset}\n%%EOF\n").as_bytes())?;
Ok((
main_xref_offset,
main_xref_offset,
id_range,
second_xref_end,
))
}
fn hint_stream_dict_prefix(
shared_section_offset: usize,
outline_section_offset: Option<usize>,
payload_len: usize,
filtered: bool,
) -> String {
let filter_key = if filtered {
"/Filter /FlateDecode "
} else {
""
};
let outline_key = match outline_section_offset {
Some(o) => format!(" /O {o}"),
None => String::new(),
};
format!("<< {filter_key}/S {shared_section_offset}{outline_key} /Length {payload_len} >>")
}
#[cfg(test)]
thread_local! {
static TEST_HINT_STREAM_AES_IVS:
std::cell::RefCell<std::collections::VecDeque<[u8; 16]>> =
const { std::cell::RefCell::new(std::collections::VecDeque::new()) };
}
#[cfg(test)]
fn next_test_hint_stream_aes_iv(default: [u8; 16]) -> [u8; 16] {
TEST_HINT_STREAM_AES_IVS.with(|ivs| ivs.borrow_mut().pop_front().unwrap_or(default))
}
#[allow(clippy::too_many_arguments)]
fn append_hint_stream_object(
bytes: &mut Vec<u8>,
new_ref: ObjectRef,
payload: &[u8],
shared_section_offset: usize,
outline_section_offset: Option<usize>,
filtered: bool,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
hint_stream_aes_iv: [u8; 16],
) -> Result<usize> {
#[cfg(test)]
let hint_stream_aes_iv = next_test_hint_stream_aes_iv(hint_stream_aes_iv);
let mut payload_len = payload.len();
if let Some(ctx) = encrypt_ctx {
crate::writer::adjust_aes_stream_length(&mut payload_len, ctx, true)?;
}
let offset = bytes.len();
crate::writer::output::with_buffer_sink(bytes, |out| {
write_indirect_object_header(out, new_ref)
})?;
bytes.extend_from_slice(
hint_stream_dict_prefix(
shared_section_offset,
outline_section_offset,
payload_len,
filtered,
)
.as_bytes(),
);
if let Some(ctx) = encrypt_ctx {
crate::writer::output::with_buffer_sink(bytes, |out| {
crate::writer::write_stream_payload_with_pipeline_qdf(
out,
payload,
NewlineBeforeEndstream::Never,
true,
new_ref,
ctx,
true,
Some(hint_stream_aes_iv),
)
})?;
} else {
crate::writer::output::with_buffer_sink(bytes, |out| {
crate::writer::serialize::write_stream_payload_with_qdf(
out,
payload,
NewlineBeforeEndstream::Never,
true,
)
})?;
}
bytes.extend_from_slice(b"\nendobj\n");
Ok(offset)
}
struct OutlineHintInfo {
first_object: u32,
nobjects: u32,
}
fn compute_outline_hint_info<R: Read + Seek>(
outlines: &std::collections::BTreeSet<QpdfObjGen>,
pdf: &mut Pdf<R>,
renumber: &RenumberMap,
objstm_layout: &ObjStmLayout,
) -> Result<Option<OutlineHintInfo>> {
if outlines.is_empty() {
return Ok(None);
}
let root = pdf.root_handle()?;
let outlines_ref = if !root.try_is_dictionary()? {
None } else {
let outlines = root.try_get_key(b"/Outlines")?;
outlines
.qpdf_obj_gen()
.filter(|object_gen| object_gen.is_indirect())
};
let Some(outlines_ref) = outlines_ref else {
return Ok(None); };
let unit_of = |object_gen: QpdfObjGen| -> Option<u32> {
if let Some(object_ref) = object_gen.to_object_ref() {
if let Some(&(container_num, _)) = objstm_layout.member_to_container.get(&object_ref) {
return Some(container_num);
}
}
renumber
.new_for_raw(object_gen)
.map(|new_ref| new_ref.number)
};
let units: std::collections::BTreeSet<u32> = outlines
.iter()
.filter_map(|&object_gen| unit_of(object_gen))
.collect();
let Some(first_object) = unit_of(outlines_ref) else {
return Ok(None); };
Ok(Some(OutlineHintInfo {
first_object,
nobjects: units.len() as u32,
}))
}
fn build_outline_hint_table(
info: &OutlineHintInfo,
xref_offsets: &BTreeMap<u32, usize>,
byte_lengths: &BTreeMap<u32, usize>,
) -> Result<OutlineHintTable> {
let first_off = xref_offsets
.get(&info.first_object)
.copied()
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"outline hint: first outline unit (#{}) has no probed offset",
info.first_object
))
})?;
let first_object_offset = u32::try_from(first_off).map_err(|_| {
crate::Error::Unsupported(format!(
"outline hint: pass-1 first unit offset ({first_off}) exceeds the \
32-bit Outlines Hint Table field"
))
})?;
let group_length: u64 = (info.first_object as u64
..info.first_object as u64 + info.nobjects as u64)
.map(|n| byte_lengths.get(&(n as u32)).copied().unwrap_or(0) as u64)
.sum();
Ok(OutlineHintTable {
first_object: info.first_object,
first_object_offset,
nobjects: info.nobjects,
group_length: group_length as u32,
})
}
fn build_pass1_part1(part1: &Part1Bytes) -> Part1Bytes {
const EMPTY_DICT: &[u8] = b"<< >>\nendobj\n";
let mut pass1 = part1.clone();
let region = part1.dict_writable_region.clone();
debug_assert!(region.len() >= EMPTY_DICT.len());
let mut replacement = Vec::with_capacity(region.len());
replacement.extend_from_slice(EMPTY_DICT);
replacement.resize(region.len(), b' ');
pass1.bytes[region].copy_from_slice(&replacement);
pass1
}
struct LinearizedPassOutput {
xref_offsets: BTreeMap<u32, usize>,
first_page_xref_offset: Option<usize>,
hint_stream_offset: usize,
hint_stream_obj_total_len: usize,
end_of_first_page_offset: usize,
last_xref_offset: usize,
last_xref_first_entry_offset: usize,
second_xref_end: usize,
first_trailer_prev_range: std::ops::Range<usize>,
id_ranges: Vec<std::ops::Range<usize>>,
}
struct FinalLinearizedLayout<'a> {
xref_offsets: &'a mut BTreeMap<u32, usize>,
main_xref_offset: usize,
}
#[allow(clippy::too_many_arguments, clippy::type_complexity)]
fn do_write_pass<R: Read + Seek>(
plan: &LinearizationPlan,
renumber: &RenumberMap,
raw_removed_refs: &BTreeSet<QpdfObjGen>,
raw_content_normalize_refs: &BTreeSet<QpdfObjGen>,
pdf: &mut Pdf<R>,
output: &mut OutputSink<'_>,
part1: &Part1Bytes,
catalog_new_ref: ObjectRef,
hint_stream_new_num: u32,
total_count: u32,
info_new_ref: Option<ObjectRef>,
_first_page_object_new_num: u32,
hint_stream_object: Option<&[u8]>,
structural_streams_filtered: bool,
source_trailer: &ObjectHandle,
objstm_layout: &ObjStmLayout,
relocation: &ObjStmRelocation,
options: &WriterOptions,
pass1_digest: bool,
id_writer: Option<crate::pdf_syntax::ReborrowableIdWriter>,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
mut encrypted_string_emitter: Option<&mut EncryptedStringEmitter>,
final_pdf_version: &str,
final_extension_level: i64,
final_layout: Option<FinalLinearizedLayout<'_>>,
) -> Result<LinearizedPassOutput> {
let is_final = final_layout.is_some();
let final_main_xref_offset = final_layout.as_ref().map(|layout| layout.main_xref_offset);
let mut local_xref_offsets = BTreeMap::new();
let xref_offsets: &mut BTreeMap<u32, usize> = match final_layout {
Some(layout) => layout.xref_offsets,
None => &mut local_xref_offsets,
};
let param_dict_obj_number = renumber.param_dict_ref().number;
let param_dict_absolute_offset = part1.obj1_offset;
output.write_bytes(&part1.bytes)?;
if !options.extra_header_text.is_empty() {
output.write_bytes(b"\n")?;
}
output.write_bytes(options.extra_header_text.as_bytes())?;
xref_offsets.insert(param_dict_obj_number, param_dict_absolute_offset);
let mut member_new: BTreeMap<u32, (u32, u32)> = BTreeMap::new();
for container in objstm_layout
.open_document
.iter()
.chain(&objstm_layout.part3)
.chain(&objstm_layout.part4)
{
for (idx, &(_orig, new_ref)) in container.members.iter().enumerate() {
member_new.insert(new_ref.number, (container.container_new_num, idx as u32));
}
}
let max_ostream_index: u64 = member_new
.values()
.map(|&(_, idx)| u64::from(idx))
.max()
.unwrap_or(0);
let trailer_map = |object_gen| {
renumber.new_for_raw(object_gen).ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: trailer raw reference {} {} has no renumber entry",
object_gen.get_obj(),
object_gen.get_gen()
))
})
}; let canonical_entries =
canonical_linearization_trailer_entries(source_trailer, &trailer_map, raw_removed_refs)?;
let mut first_page_xref_patch: Option<FirstPageXrefPatch> = None;
let mut part1_classic_xref_offset: usize = 0;
let mut part1_xref_patch: Option<Part1XrefPatch> = None;
let mut id_ranges: Vec<std::ops::Range<usize>> = Vec::new();
let first_trailer_prev_range = if objstm_layout.is_empty() {
let first_page_count = first_page_xref_object_count(total_count, param_dict_obj_number)?;
let section_start = output.position_usize()?;
let (p1_xref_offset, range, patch) = write_part1_xref_and_trailer(
output,
param_dict_obj_number,
total_count,
first_page_count,
catalog_new_ref,
info_new_ref,
source_trailer,
&pdf.trailer(),
&canonical_entries,
&trailer_map,
raw_removed_refs,
id_writer,
encrypt_ctx,
pass1_digest,
is_final.then_some(&*xref_offsets),
final_main_xref_offset,
)?; part1_classic_xref_offset = p1_xref_offset;
part1_xref_patch = patch;
id_ranges.push(section_start..output.position_usize()?);
range
} else {
let final_first_page_layout = if is_final {
Some(FinalFirstPageXref {
member_new: &member_new,
main_xref_offset: final_main_xref_offset.ok_or_else(|| {
crate::Error::Unsupported(
"final linearization layout has no main xref offset".to_string(),
)
})?, })
} else {
None
};
let patch = write_first_page_xref_stream(
output,
xref_offsets,
relocation,
total_count,
catalog_new_ref,
info_new_ref,
source_trailer,
&pdf.trailer(),
&canonical_entries,
max_ostream_index,
structural_streams_filtered,
encrypt_ctx.map(|ctx| ctx.encrypt_ref),
pass1_digest,
final_first_page_layout,
)?;
first_page_xref_patch = Some(patch);
0..0
};
let mut catalog_emitted_early = false;
if let Some(catalog_orig) = plan.root_ref {
debug_assert!(
!objstm_layout
.member_to_container
.contains_key(&catalog_orig),
"planner invariant: /Catalog is never an ObjStm member"
);
let offset = append_root_object(
output,
pdf,
catalog_new_ref,
catalog_orig,
options,
encrypt_ctx,
encrypted_string_emitter.as_deref_mut(),
renumber,
raw_removed_refs,
raw_content_normalize_refs,
final_pdf_version,
final_extension_level,
)?; xref_offsets.insert(catalog_new_ref.number, offset);
catalog_emitted_early = true;
}
enum OpenDocumentEmit<'a> {
Plain {
original: ObjectRef,
new_ref: ObjectRef,
},
Raw {
original: QpdfObjGen,
new_ref: ObjectRef,
},
Container(&'a ObjStmContainer),
}
let mut open_document_emits: Vec<(u32, OpenDocumentEmit<'_>)> = Vec::new();
for original_ref in &plan.part4_open_document_plain {
if objstm_layout.member_to_container.contains_key(original_ref)
|| objstm_layout
.source_container_numbers
.contains(&original_ref.number)
{
continue;
}
if catalog_emitted_early && plan.root_ref == Some(*original_ref) {
continue;
}
let Some(new_ref) = renumber.new_for_original(*original_ref) else {
return Err(crate::Error::Unsupported(
"part4_open_document_plain ref missing from renumber map".into(),
));
};
open_document_emits.push((
new_ref.number,
OpenDocumentEmit::Plain {
original: *original_ref,
new_ref,
},
));
}
for &original_gen in &plan.raw.part4_open_document_plain {
if original_gen.to_object_ref().is_some() {
continue;
}
let Some(new_ref) = renumber.new_for_raw(original_gen) else {
return Err(crate::Error::Unsupported(format!(
"raw open-document identity {} {} has no renumber entry",
original_gen.get_obj(),
original_gen.get_gen()
)));
};
open_document_emits.push((
new_ref.number,
OpenDocumentEmit::Raw {
original: original_gen,
new_ref,
},
));
}
for container in &objstm_layout.open_document {
open_document_emits.push((
container.container_new_num,
OpenDocumentEmit::Container(container),
));
}
open_document_emits.sort_by_key(|(number, _)| *number);
for (_, emit) in open_document_emits {
match emit {
OpenDocumentEmit::Plain { original, new_ref } => {
let offset = append_body_object_for_ref(
output,
pdf,
new_ref,
original,
options,
encrypt_ctx,
encrypted_string_emitter.as_deref_mut(),
renumber,
raw_removed_refs,
raw_content_normalize_refs,
)?; xref_offsets.insert(new_ref.number, offset);
}
OpenDocumentEmit::Raw { original, new_ref } => {
let offset = append_body_object_for_raw(
output,
pdf,
new_ref,
original,
options,
encrypt_ctx,
encrypted_string_emitter.as_deref_mut(),
renumber,
raw_removed_refs,
raw_content_normalize_refs,
)?; xref_offsets.insert(new_ref.number, offset);
}
OpenDocumentEmit::Container(container) => {
let offset = append_objstm_container_object(
output,
container,
renumber,
pdf,
raw_removed_refs,
options,
encrypt_ctx,
)?; xref_offsets.insert(container.container_new_num, offset);
for _ in &container.members {
if pass1_digest {
decrement_progress_event(options)?;
}
report_progress_event(options)?;
if pass1_digest {
report_progress_event(options)?;
}
}
}
}
}
if let Some(ctx) = encrypt_ctx {
let offset = output.position_usize()?;
output.write_bytes(
format!(
"{} {} obj\n",
ctx.encrypt_ref.number, ctx.encrypt_ref.generation
)
.as_bytes(),
)?; crate::writer::encrypted_strings::write_encryption_dictionary_handle(
output,
&ctx.encrypt_dict,
)?; output.write_bytes(b"\nendobj\n")?;
xref_offsets.insert(ctx.encrypt_ref.number, offset);
}
let hint_stream_offset = output.position_usize()?;
if let Some(hint_stream_object) = hint_stream_object {
output.write_bytes(hint_stream_object)?;
xref_offsets.insert(hint_stream_new_num, hint_stream_offset);
}
let hint_stream_obj_total_len = output
.position_usize()?
.checked_sub(hint_stream_offset)
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization hint stream offset moved backwards".to_string(),
)
})?;
enum FirstPageEmit<'a> {
Plain(ObjectRef),
Raw(QpdfObjGen),
Container(&'a ObjStmContainer),
}
let mut first_page_emits: Vec<(u32, FirstPageEmit<'_>)> = Vec::new();
for original_ref in plan
.part2_objects
.iter()
.chain(&plan.part3_objects)
.chain(&plan.part6_outline_objects)
{
if objstm_layout.member_to_container.contains_key(original_ref)
|| objstm_layout
.source_container_numbers
.contains(&original_ref.number)
|| (catalog_emitted_early && plan.root_ref == Some(*original_ref))
{
continue;
}
let Some(new_ref) = renumber.new_for_original(*original_ref) else {
return Err(crate::Error::Unsupported(format!(
"first-page object {} has no renumber entry",
original_ref
)));
};
first_page_emits.push((new_ref.number, FirstPageEmit::Plain(*original_ref)));
}
for original_gen in plan
.raw
.part2_objects
.iter()
.chain(&plan.raw.part3_objects)
.chain(&plan.raw.part6_outline_objects)
.copied()
.filter(|object_gen| object_gen.to_object_ref().is_none())
{
let Some(new_ref) = renumber.new_for_raw(original_gen) else {
return Err(crate::Error::Unsupported(format!(
"raw first-page identity {} {} has no renumber entry",
original_gen.get_obj(),
original_gen.get_gen()
)));
};
first_page_emits.push((new_ref.number, FirstPageEmit::Raw(original_gen)));
}
for container in &objstm_layout.part3 {
first_page_emits.push((
container.container_new_num,
FirstPageEmit::Container(container),
));
}
first_page_emits.sort_by_key(|(number, _)| *number);
for (_, emit) in first_page_emits {
match emit {
FirstPageEmit::Plain(original_ref) => {
let new_ref = renumber
.new_for_original(original_ref)
.expect("first-page plain object renumber entry checked above");
let offset = append_body_object_for_ref(
output,
pdf,
new_ref,
original_ref,
options,
encrypt_ctx,
encrypted_string_emitter.as_deref_mut(),
renumber,
raw_removed_refs,
raw_content_normalize_refs,
)?; xref_offsets.insert(new_ref.number, offset);
}
FirstPageEmit::Raw(original_gen) => {
let new_ref = renumber
.new_for_raw(original_gen)
.expect("raw first-page object renumber entry checked above");
let offset = append_body_object_for_raw(
output,
pdf,
new_ref,
original_gen,
options,
encrypt_ctx,
encrypted_string_emitter.as_deref_mut(),
renumber,
raw_removed_refs,
raw_content_normalize_refs,
)?; xref_offsets.insert(new_ref.number, offset);
}
FirstPageEmit::Container(container) => {
let offset = append_objstm_container_object(
output,
container,
renumber,
pdf,
raw_removed_refs,
options,
encrypt_ctx,
)?; xref_offsets.insert(container.container_new_num, offset);
for _ in &container.members {
if pass1_digest {
decrement_progress_event(options)?;
}
report_progress_event(options)?;
if pass1_digest {
report_progress_event(options)?;
}
}
}
}
}
let end_of_first_page_offset = output.position_usize()?;
enum Part4Emit<'a> {
Plain(ObjectRef),
Raw(QpdfObjGen),
Container(&'a ObjStmContainer),
}
let mut part4_emits: Vec<(u32, Part4Emit)> = Vec::new();
for original_ref in plan.part4_objects() {
if objstm_layout
.member_to_container
.contains_key(&original_ref)
|| objstm_layout
.source_container_numbers
.contains(&original_ref.number)
{
continue;
}
if catalog_emitted_early && plan.root_ref == Some(original_ref) {
continue;
}
let Some(new_ref) = renumber.new_for_original(original_ref) else {
return Err(crate::Error::Unsupported(format!(
"part4 object {original_ref} has no renumber entry"
)));
};
part4_emits.push((new_ref.number, Part4Emit::Plain(original_ref)));
}
for original_gen in plan
.raw
.part4_other_pages_private
.iter()
.chain(&plan.raw.part4_other_pages_shared)
.chain(&plan.raw.part4_rest)
.chain(&plan.raw.part9_outline_objects)
.copied()
.filter(|object_gen| object_gen.to_object_ref().is_none())
{
let Some(new_ref) = renumber.new_for_raw(original_gen) else {
return Err(crate::Error::Unsupported(format!(
"raw Part-4 identity {} {} has no renumber entry",
original_gen.get_obj(),
original_gen.get_gen()
)));
};
part4_emits.push((new_ref.number, Part4Emit::Raw(original_gen)));
}
for container in &objstm_layout.part4 {
part4_emits.push((container.container_new_num, Part4Emit::Container(container)));
}
part4_emits.sort_by_key(|(number, _)| *number);
for (_, emit) in &part4_emits {
match emit {
Part4Emit::Plain(original_ref) => {
let new_ref = renumber
.new_for_original(*original_ref)
.expect("part4 plain object renumber entry checked above");
let offset = append_body_object_for_ref(
output,
pdf,
new_ref,
*original_ref,
options,
encrypt_ctx,
encrypted_string_emitter.as_deref_mut(),
renumber,
raw_removed_refs,
raw_content_normalize_refs,
)?; xref_offsets.insert(new_ref.number, offset);
}
Part4Emit::Raw(original_gen) => {
let new_ref = renumber
.new_for_raw(*original_gen)
.expect("raw Part-4 object renumber entry checked above");
let offset = append_body_object_for_raw(
output,
pdf,
new_ref,
*original_gen,
options,
encrypt_ctx,
encrypted_string_emitter.as_deref_mut(),
renumber,
raw_removed_refs,
raw_content_normalize_refs,
)?; xref_offsets.insert(new_ref.number, offset);
}
Part4Emit::Container(container) => {
let offset = append_objstm_container_object(
output,
container,
renumber,
pdf,
raw_removed_refs,
options,
encrypt_ctx,
)?; xref_offsets.insert(container.container_new_num, offset);
for _ in &container.members {
if pass1_digest {
decrement_progress_event(options)?;
}
report_progress_event(options)?;
if pass1_digest {
report_progress_event(options)?;
}
}
}
}
}
if let Some(expected) = final_main_xref_offset {
let actual = output.position_usize()?;
if actual != expected {
return Err(crate::Error::Unsupported(format!(
"linearization final main xref offset drifted: expected {}, got {}",
expected, actual
)));
}
}
let (last_xref_offset, last_xref_first_entry_offset, second_xref_end) = if objstm_layout
.is_empty()
{
let main_section_start = output.position_usize()?;
let result = write_main_xref_and_trailer(
output,
&*xref_offsets,
param_dict_obj_number,
part1_classic_xref_offset,
source_trailer,
&pdf.trailer(),
&trailer_map,
raw_removed_refs,
pass1_digest,
)?; id_ranges.push(main_section_start..output.position_usize()?);
if !pass1_digest && !is_final {
let patch = part1_xref_patch
.as_ref()
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: classic path produced no Part-1 xref patch \
(internal invariant violated)"
.to_string(),
)
})?;
patch_part1_xref(output, patch, &*xref_offsets)?; }
(result.0, result.1, 0)
} else {
let patch = first_page_xref_patch.as_ref().ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: ObjStm path produced no first-page xref patch \
(internal invariant violated)"
.to_string(),
)
})?;
let first_page_obj_offset = patch.region.start;
debug_assert!(
first_page_obj_offset < end_of_first_page_offset,
"first-page xref stream offset ({first_page_obj_offset}) must be before \
/E ({end_of_first_page_offset}) — linearization boundary violated"
);
let result = write_main_xref_stream_and_trailer(
output,
xref_offsets,
&member_new,
relocation,
total_count,
source_trailer,
&pdf.trailer(),
first_page_obj_offset,
max_ostream_index,
pass1_digest,
structural_streams_filtered,
)?; let (main_xref_offset, main_first_entry_offset, main_id_range, second_xref_end) = result;
if let Some(id_range) = main_id_range {
id_ranges.push(id_range);
}
if !pass1_digest && !is_final {
let first_page_id_range = patch_first_page_xref(
output,
patch,
xref_offsets, &member_new,
main_xref_offset,
hint_stream_obj_total_len,
)?; if let Some(id_range) = first_page_id_range {
id_ranges.push(id_range);
}
}
(main_xref_offset, main_first_entry_offset, second_xref_end)
};
let pass_xref_offsets = if is_final {
BTreeMap::new()
} else {
std::mem::take(xref_offsets)
};
Ok(LinearizedPassOutput {
xref_offsets: pass_xref_offsets,
first_page_xref_offset: first_page_xref_patch
.as_ref()
.map(|patch| patch.region.start),
hint_stream_offset,
hint_stream_obj_total_len,
end_of_first_page_offset,
last_xref_offset,
last_xref_first_entry_offset,
second_xref_end,
first_trailer_prev_range,
id_ranges,
})
}
fn compute_byte_lengths(
xref_offsets: &BTreeMap<u32, usize>,
last_xref_offset: usize,
hint_stream_new_num: u32,
param_dict_new_num: u32,
) -> BTreeMap<u32, usize> {
let mut sorted: Vec<(usize, u32)> = xref_offsets
.iter()
.filter(|(&num, _)| num != param_dict_new_num)
.map(|(&num, &off)| (off, num))
.collect();
sorted.sort_unstable();
let mut lengths: BTreeMap<u32, usize> = BTreeMap::new();
for (idx, &(off, num)) in sorted.iter().enumerate() {
if num == hint_stream_new_num {
continue;
}
let next_off = if idx + 1 < sorted.len() {
sorted[idx + 1].0
} else {
last_xref_offset
};
lengths.insert(num, next_off.saturating_sub(off));
}
lengths
}
fn part7_owner_for_plan(
plan: &LinearizationPlan,
member: ObjectRef,
source_container_by_member: &BTreeMap<u32, u32>,
) -> Option<usize> {
if let Some(optimization) = plan.optimization.as_ref() {
let owner = source_container_by_member
.get(&member.number)
.copied()
.map(|source| ObjectRef::new(source, 0))
.unwrap_or(member);
return optimization
.other_page_private_owner(owner)
.map(|page| page as usize);
}
plan.per_page_private_objects
.iter()
.enumerate()
.skip(1)
.find_map(|(page, members)| members.contains(&member).then_some(page))
}
fn qpdf_object_number(object_gen: QpdfObjGen) -> u32 {
u32::try_from(object_gen.get_obj()).unwrap_or(u32::MAX)
}
type SecondHalfPlainRank = (u8, usize, u8, u32);
type SecondHalfPlainObject = (QpdfObjGen, SecondHalfPlainRank);
fn preserve_part9_category_with_users(
object_gen: QpdfObjGen,
part9_pages: &BTreeSet<ObjectRef>,
optimization: Option<&crate::optimization::Optimization>,
) -> (u8, u32) {
match (object_gen.to_object_ref(), optimization) {
(Some(object_ref), Some(optimization)) => {
part9_category_order_key(optimization, part9_pages, [&object_ref])
}
_ => preserve_part9_category(object_gen, part9_pages),
}
}
fn preserve_part9_category(object_gen: QpdfObjGen, part9_pages: &BTreeSet<ObjectRef>) -> (u8, u32) {
if object_gen
.to_object_ref()
.is_some_and(|object_ref| part9_pages.contains(&object_ref))
{
(0, 0)
} else {
(4, 0)
}
}
fn part9_plain_rank(
object_gen: QpdfObjGen,
part9_pages: &BTreeSet<ObjectRef>,
category: u8,
page: u32,
) -> SecondHalfPlainRank {
if category == 0
&& object_gen
.to_object_ref()
.is_some_and(|object_ref| part9_pages.contains(&object_ref))
{
return (2, 0, 0, qpdf_object_number(object_gen));
}
(
2 + category,
page as usize,
0,
qpdf_object_number(object_gen),
)
}
fn second_half_container_anchors(
plan: &LinearizationPlan,
open_document_batches: &[RoutedObjStmBatch],
first_half_batches: &[RoutedObjStmBatch],
part4_batches: &[RoutedObjStmBatch],
source_container_by_member: &BTreeMap<u32, u32>,
) -> Vec<SecondHalfContainerAnchor> {
let fallback_source_membership = if source_container_by_member.is_empty() {
plan.preserve_source_membership().unwrap_or_default()
} else {
BTreeMap::new()
};
let source_container_by_member = if source_container_by_member.is_empty() {
&fallback_source_membership
} else {
source_container_by_member
};
let member_set: BTreeSet<QpdfObjGen> = open_document_batches
.iter()
.chain(first_half_batches)
.chain(part4_batches)
.flat_map(|batch| batch.members.iter().copied())
.filter_map(|object_ref| QpdfObjGen::try_from_object_ref(object_ref).ok())
.collect();
let source_container_set: BTreeSet<QpdfObjGen> = part4_batches
.iter()
.filter_map(|batch| batch.source_container_number)
.map(|number| QpdfObjGen::new(number as i32, 0))
.collect();
let generate_batches = part4_batches
.iter()
.all(|batch| batch.source_container_number.is_none());
let part9_pages: BTreeSet<ObjectRef> = plan
.optimization
.as_ref()
.map(|optimization| {
optimization
.objects_for_root_key(b"Pages")
.collect::<BTreeSet<ObjectRef>>()
})
.filter(|pages| !pages.is_empty())
.unwrap_or_else(|| plan.pages_tree_ref.into_iter().collect());
let mut plain_ranked: Vec<SecondHalfPlainObject> = Vec::new();
let push_page_private = |plain_ranked: &mut Vec<SecondHalfPlainObject>,
page: usize,
page_head: Option<QpdfObjGen>,
objects: &[QpdfObjGen]| {
for &object_gen in objects {
if !member_set.contains(&object_gen) && !source_container_set.contains(&object_gen) {
let page_head_rank = u8::from(Some(object_gen) != page_head);
plain_ranked.push((
object_gen,
(0, page, page_head_rank, qpdf_object_number(object_gen)),
));
}
}
};
if plan.has_raw_projection_gap() {
for (page, objects) in plan.raw.per_page_private_objects.iter().enumerate().skip(1) {
let page_head = plan
.page_hints
.get(page)
.and_then(|hint| QpdfObjGen::try_from_object_ref(hint.page_ref).ok());
push_page_private(&mut plain_ranked, page, page_head, objects);
}
for &object_gen in &plan.raw.part4_other_pages_shared {
if !member_set.contains(&object_gen) && !source_container_set.contains(&object_gen) {
plain_ranked.push((object_gen, (1, 0, 0, qpdf_object_number(object_gen))));
}
}
for &object_gen in &plan.raw.part4_rest {
if !member_set.contains(&object_gen) && !source_container_set.contains(&object_gen) {
let (category, page) = if generate_batches {
object_gen
.to_object_ref()
.zip(plan.optimization.as_ref())
.map(|(object_ref, optimization)| {
part9_category_order_key(optimization, &part9_pages, [&object_ref])
})
.unwrap_or((0, 0))
} else {
preserve_part9_category_with_users(
object_gen,
&part9_pages,
plan.optimization.as_ref(),
)
};
plain_ranked.push((
object_gen,
part9_plain_rank(object_gen, &part9_pages, category, page),
));
}
}
} else {
for (i, privates) in plan.per_page_private_objects.iter().enumerate().skip(1) {
let page_head = plan
.page_hints
.get(i)
.and_then(|hint| QpdfObjGen::try_from_object_ref(hint.page_ref).ok());
let objects: Vec<QpdfObjGen> = privates
.iter()
.filter_map(|object_ref| QpdfObjGen::try_from_object_ref(*object_ref).ok())
.collect();
push_page_private(&mut plain_ranked, i, page_head, &objects);
}
for &object_ref in &plan.part4_other_pages_shared {
let object_gen = QpdfObjGen::try_from_object_ref(object_ref)
.expect("checked Part-8 object must fit qpdf raw identity");
if !member_set.contains(&object_gen) && !source_container_set.contains(&object_gen) {
plain_ranked.push((object_gen, (1, 0, 0, qpdf_object_number(object_gen))));
}
}
for &object_ref in &plan.part4_rest {
let object_gen = QpdfObjGen::try_from_object_ref(object_ref)
.expect("checked Part-9 object must fit qpdf raw identity");
if !member_set.contains(&object_gen) && !source_container_set.contains(&object_gen) {
let (category, page) = if generate_batches {
let optimization = plan
.optimization
.as_ref()
.expect("generated ObjStm batches require optimization users");
part9_category_order_key(optimization, &part9_pages, [&object_ref])
} else {
preserve_part9_category_with_users(
object_gen,
&part9_pages,
plan.optimization.as_ref(),
)
};
plain_ranked.push((
object_gen,
part9_plain_rank(object_gen, &part9_pages, category, page),
));
}
}
}
let member_is_in_route = |member: ObjectRef, route: ContainerPart| -> bool {
let object_gen = QpdfObjGen::try_from_object_ref(member)
.expect("ObjStm members must fit qpdf raw identity");
match route {
ContainerPart::OtherPagePrivate => plan
.raw
.per_page_private_objects
.iter()
.skip(1)
.any(|objects| objects.contains(&object_gen)),
ContainerPart::OtherPageShared => {
plan.raw.part4_other_pages_shared.contains(&object_gen)
}
ContainerPart::Rest => {
plan.raw.part4_rest.contains(&object_gen)
|| plan.raw.part9_outline_objects.contains(&object_gen)
}
_ => false, }
};
let anchors: Vec<_> = part4_batches
.iter()
.map(|batch| {
if batch.members.is_empty() {
return SecondHalfContainerAnchor::AfterLast; }
let object_number = if let Some(source_container_number) = batch.source_container_number
{
source_container_number
} else if plan.has_raw_projection_gap() {
batch
.members
.iter()
.filter_map(|member| QpdfObjGen::try_from_object_ref(*member).ok())
.filter(|object_gen| {
batch.members.iter().any(|member| {
QpdfObjGen::try_from_object_ref(*member).ok() == Some(*object_gen)
&& member_is_in_route(*member, batch.route)
})
})
.map(qpdf_object_number)
.min()
.expect("raw ObjStm batch must have a member in its second-half route")
} else {
u32::MAX
};
let batch_rank: (u8, usize, u8, u32) = match batch.route {
ContainerPart::OtherPagePrivate => {
let owner = batch
.members
.iter()
.find_map(|member| {
part7_owner_for_plan(plan, *member, source_container_by_member)
})
.expect("Part-7 ObjStm route must have one non-first-page owner");
(0, owner, 1, object_number)
}
ContainerPart::OtherPageShared => (1, 0, 0, object_number),
ContainerPart::Rest if batch.source_container_number.is_some() => {
let (category, page) = plan
.optimization
.as_ref()
.map(|optimization| {
let source_container = ObjectRef::new(
batch
.source_container_number
.expect("Preserve Rest batch has a source container"),
0,
);
part9_category_order_key(
optimization,
&part9_pages,
[&source_container],
)
})
.unwrap_or((4, 0));
(2 + category, page as usize, 0, object_number)
}
ContainerPart::Rest if generate_batches => {
let optimization = plan
.optimization
.as_ref()
.expect("generated ObjStm batches require optimization users");
let (category, page) =
part9_category_order_key(optimization, &part9_pages, batch.members.iter());
(2 + category, page as usize, 1, object_number)
}
ContainerPart::Rest => (2 + 4, 0, 0, object_number),
ContainerPart::OpenDocument
| ContainerPart::FirstPagePrivate
| ContainerPart::FirstPageShared
| ContainerPart::FirstPageOutlines => {
unreachable!("first-half route in second-half ObjStm batches")
} };
let previous = plain_ranked
.iter()
.filter(|(_, rank)| *rank <= batch_rank)
.max_by_key(|(_, rank)| *rank)
.map(|(r, _)| *r);
match previous {
Some(r) => SecondHalfContainerAnchor::After(r),
None if plain_ranked.is_empty() => SecondHalfContainerAnchor::AfterLast,
None => SecondHalfContainerAnchor::BeforeFirst,
}
})
.collect();
anchors
}
fn preserved_source_container_number(
container: &ObjStmContainer,
source_container_by_member: &BTreeMap<u32, u32>,
) -> Result<u32> {
let source_container_number = container
.members
.first()
.and_then(|(original_ref, _)| {
(original_ref.generation == 0)
.then(|| {
source_container_by_member
.get(&original_ref.number)
.copied()
})
.flatten()
})
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"preserved ObjStm container {} has no source container",
container.container_new_num
))
})?;
if container.members.iter().any(|(original_ref, _)| {
original_ref.generation != 0
|| source_container_by_member
.get(&original_ref.number)
.copied()
!= Some(source_container_number)
}) {
return Err(crate::Error::Unsupported(format!(
"preserved ObjStm container {} combines multiple source containers",
container.container_new_num
)));
}
Ok(source_container_number)
}
fn reject_multiple_generations(plan: &LinearizationPlan) -> Result<()> {
let mut previous_number = None;
for object_gen in plan.renumber_assigned_raw() {
let object_number = object_gen.get_obj();
if previous_number == Some(object_number) {
return Err(crate::Error::System(
"QPDF cannot currently linearize files that contain multiple objects with the \
same object ID and different generations. If you see this error message, \
please file a bug report and attach the file if possible. As a workaround, \
first convert the file with qpdf without linearizing, and then linearize the \
result of that conversion."
.to_string(),
));
}
previous_number = Some(object_number);
}
Ok(())
}
#[cfg(test)]
pub(crate) fn write_linearized<R: Read + Seek>(
plan: &LinearizationPlan,
renumber: &RenumberMap,
pdf: &mut Pdf<R>,
options: &WriterOptions,
) -> Result<LinearizedDocument> {
let setup = crate::writer::build_writer_setup(pdf, options)?;
let output = write_linearized_impl(plan, renumber.clone(), pdf, options, None, setup, None)?;
output.document.ok_or_else(|| {
crate::Error::Internal(
"linearization memory helper returned no in-memory document".to_string(),
)
})
}
pub(crate) fn write_linearized_for_pdf_writer<R: Read + Seek>(
pdf: &mut Pdf<R>,
options: &WriterOptions,
pass1_path: Option<&Path>,
setup: crate::writer::WriterSetupState,
special_streams: Option<&crate::writer::SpecialStreams>,
output: &mut dyn OutputTarget,
) -> Result<WriterResult> {
let plan_result = (|| {
let mode = if crate::writer::force_version_below_1_5(options) {
crate::writer::ObjectStreamMode::Disable
} else {
options.object_streams
};
let mut plan_options = options.clone();
plan_options.object_streams = mode;
let plan = LinearizationPlan::from_pdf_with_writer_options_and_source_membership(
pdf,
&plan_options,
Some(&setup.source_object_stream_data),
setup.generated_compressible.as_ref(),
special_streams.map(crate::writer::SpecialStreams::normalized_streams_raw),
)?;
let generated_object_stream_count = if mode == crate::writer::ObjectStreamMode::Generate {
match setup.generated_compressible.as_ref() {
Some(compressible) => {
crate::writer::object_streams::even_split_into_streams(&compressible.eligible)
.len()
}
None => {
let compressible =
crate::writer::object_streams::compressible_objgens_qpdf_plan(pdf)?;
crate::writer::object_streams::even_split_into_streams(&compressible.eligible)
.len()
}
}
} else {
0
};
crate::writer::configure_progress_for_pdf(
pdf,
options,
generated_object_stream_count,
true,
)?;
let renumber = RenumberMap::from_plan(&plan);
Ok((plan, renumber))
})();
plan_result.and_then(|(plan, renumber)| {
let output = write_linearized_impl(
&plan,
renumber,
pdf,
options,
pass1_path,
setup,
Some(output),
)?;
Ok(output.writer_result)
})
}
fn write_pass1_debug_comments(writer: &mut dyn Write, comments: &[u8]) {
debug_assert!(comments.len() < 4096);
let mut buffered = StdioBuffer::new(writer);
let _ = buffered.write(comments);
}
fn validate_per_page_private_objects(plan: &LinearizationPlan) -> Result<()> {
if plan.per_page_private_objects.len() != plan.page_hints.len() {
return Err(crate::Error::Unsupported(format!(
"linearization writer: per_page_private_objects length ({}) does not \
match page_hints length ({}) — plan invariant violated",
plan.per_page_private_objects.len(),
plan.page_hints.len()
)));
}
Ok(())
}
#[inline(never)]
fn write_linearized_impl<R: Read + Seek>(
plan: &LinearizationPlan,
renumber: RenumberMap,
pdf: &mut Pdf<R>,
options: &WriterOptions,
pass1_path: Option<&Path>,
setup: crate::writer::WriterSetupState,
final_output: Option<&mut dyn OutputTarget>,
) -> Result<LinearizedWriteResult> {
let crate::writer::WriterSetupState {
generated_id,
encryption_parameters,
source_object_stream_data,
generated_compressible: _,
generated_object_stream_sources: _,
} = setup;
let source_container_by_member = source_object_stream_data;
let deterministic_id = crate::writer::uses_deterministic_id(options);
let source_id0 = if deterministic_id || options.copy_encryption.is_some() {
crate::writer::source_permanent_id_handle(&pdf.trailer())?
} else {
generated_id
.as_ref()
.and_then(ObjectHandle::as_array)
.and_then(|values| values.first().and_then(ObjectHandle::as_string))
.filter(|value| !value.is_empty())
};
let source_trailer_handle = pdf.trailer().shallow_copy()?;
let (det_id_source_id0, det_id_info_suffix): (Option<Vec<u8>>, Vec<u8>) = if deterministic_id {
let suffix = crate::writer::deterministic_id_info_suffix(pdf);
(source_id0.clone(), suffix)
} else {
(None, Vec::new())
};
let pass1_id = linearization_pass1_id(source_id0.as_deref());
let finalized_id = generated_id.unwrap_or_else(|| {
finalize_linearized_id(
options,
source_id0.as_deref(),
det_id_source_id0.as_deref(),
options.copy_encryption.as_ref(),
)
});
source_trailer_handle.replace_key(b"/ID", finalized_id)?;
let pass1_source_trailer = source_trailer_handle.shallow_copy()?;
pass1_source_trailer.replace_key(b"/ID", pass1_id)?;
if options.deterministic_id && (options.encrypt.is_some() || options.copy_encryption.is_some())
{
return Err(crate::writer::deterministic_id_encryption_error(options));
}
crate::optimization::Optimization::prepare_for_linearized_write(pdf)?;
let effective_object_stream_mode = if crate::writer::force_version_below_1_5(options) {
crate::writer::ObjectStreamMode::Disable
} else {
options.object_streams
};
let must_rebuild_plan = plan.object_stream_mode != effective_object_stream_mode;
let rebuilt = if must_rebuild_plan {
let rebuilt_plan =
LinearizationPlan::from_pdf_with_object_stream_mode(pdf, effective_object_stream_mode)?;
let rebuilt_renumber = RenumberMap::from_plan(&rebuilt_plan);
Some((rebuilt_plan, rebuilt_renumber))
} else {
None
};
let must_normalize_options = options.object_streams != effective_object_stream_mode;
let normalized_options = if must_normalize_options {
Some(WriterOptions {
object_streams: effective_object_stream_mode,
..options.clone()
})
} else {
None
};
let (rebuilt_plan, renumber) = match rebuilt {
Some((rebuilt_plan, rebuilt_renumber)) => (Some(rebuilt_plan), rebuilt_renumber),
None => (None, renumber),
};
let plan = rebuilt_plan.as_ref().unwrap_or(plan);
let options = normalized_options.as_ref().unwrap_or(options);
reject_multiple_generations(plan)?;
let resolved_batch_plan = ObjStmLayout::resolve_batches(plan, pdf, options)?;
let emits_object_streams = !resolved_batch_plan.open_document_batches.is_empty()
|| !resolved_batch_plan.part3_batches.is_empty()
|| !resolved_batch_plan.part4_batches.is_empty();
let mut local_renumber = renumber;
let second_half_anchors = second_half_container_anchors(
plan,
&resolved_batch_plan.open_document_batches,
&resolved_batch_plan.part3_batches,
&resolved_batch_plan.part4_batches,
&source_container_by_member,
);
let part4_members: Vec<Vec<ObjectRef>> = resolved_batch_plan
.part4_batches
.iter()
.map(|batch| batch.members.clone())
.collect();
let part4_member_set: BTreeSet<ObjectRef> = part4_members.iter().flatten().copied().collect();
let part9_pages: BTreeSet<ObjectRef> = plan
.optimization
.as_ref()
.map(|optimization| {
optimization
.objects_for_root_key(b"Pages")
.collect::<BTreeSet<ObjectRef>>()
})
.filter(|pages| !pages.is_empty())
.unwrap_or_else(|| plan.pages_tree_ref.into_iter().collect());
let part9_thumbnail_objects: BTreeSet<ObjectRef> = plan
.optimization
.as_ref()
.map(|optimization| {
optimization
.thumbnail_objects()
.collect::<BTreeSet<ObjectRef>>()
})
.unwrap_or_default();
let part9_thumbnail_pre_plain: BTreeSet<ObjectRef> =
if options.object_streams == crate::writer::ObjectStreamMode::Generate {
let optimization = plan
.optimization
.as_ref()
.expect("Generate linearization requires optimization users");
plan.part4_rest
.iter()
.copied()
.filter(|object| {
part9_thumbnail_objects.contains(object)
&& matches!(
part9_category_order_key(optimization, &part9_pages, [object]).0,
1 | 2
)
})
.collect()
} else {
BTreeSet::new()
};
let second_half_post_plain: BTreeSet<ObjectRef> =
if options.object_streams == crate::writer::ObjectStreamMode::Preserve {
BTreeSet::new()
} else {
plan.part4_rest
.iter()
.chain(&plan.part9_outline_objects)
.copied()
.filter(|r| {
!part4_member_set.contains(r)
&& !part9_pages.contains(r)
&& !part9_thumbnail_pre_plain.contains(r)
&& Some(*r) != plan.info_ref
})
.collect()
};
let first_half_member_set: BTreeSet<ObjectRef> = resolved_batch_plan
.open_document_batches
.iter()
.chain(&resolved_batch_plan.part3_batches)
.flat_map(|batch| batch.members.iter().copied())
.collect();
let mut first_half_post_plain: BTreeSet<ObjectRef> = plan
.part6_outline_objects
.iter()
.copied()
.filter(|object| !first_half_member_set.contains(object))
.collect();
let first_half_outline_root = plan
.part6_outline_objects
.first()
.copied()
.filter(|root| !first_half_member_set.contains(root));
if let Some(root) = first_half_outline_root {
first_half_post_plain.remove(&root);
}
let first_half_outline_batch_count = resolved_batch_plan
.part3_batches
.iter()
.filter(|batch| {
batch
.members
.iter()
.any(|member| plan.part6_outline_objects.contains(member))
})
.count();
if let Some(pre_objects) = plan
.optimization
.as_ref()
.and_then(|optimization| optimization.pre_optimization_object_refs())
{
first_half_post_plain.extend(
plan.part3_objects
.iter()
.chain(&plan.part4_open_document_plain)
.chain(&plan.part6_outline_objects)
.copied()
.filter(|object| {
!pre_objects.contains(object) && !first_half_member_set.contains(object)
}),
);
}
let relocation = if emits_object_streams {
local_renumber.place_objstm_members_per_half(
&resolved_batch_plan.open_document_batches,
&resolved_batch_plan.part3_batches,
&resolved_batch_plan.part4_batches,
&second_half_anchors,
&second_half_post_plain,
&first_half_post_plain,
first_half_outline_batch_count,
first_half_outline_root,
)
} else {
ObjStmRelocation::default()
};
let mut container_numbers = relocation.container_numbers.clone();
let encrypt_ctx = if let Some(parameters) = encryption_parameters {
let encrypt_ref = local_renumber.reserve_encrypt_dict_slot();
for container_number in &mut container_numbers {
if *container_number >= encrypt_ref.number {
*container_number += 1;
}
}
Some(parameters.into_context(encrypt_ref))
} else {
None
};
let mut encrypted_string_emitter = encrypt_ctx
.as_ref()
.map(EncryptedStringEmitter::from_context);
let hint_stream_aes_iv: [u8; 16] = match &encrypt_ctx {
Some(ctx) if ctx.static_aes_iv => crate::pipeline::aes::static_initialization_vector(),
Some(ctx) if crate::writer::cipher_needs_aes_iv(ctx.cipher) => {
let mut iv = [0u8; 16];
getrandom::fill(&mut iv).map_err(|e| {
crate::Error::Unsupported(format!(
"OS CSPRNG (getrandom) unavailable for AES IV generation: {e}"
))
})?;
iv
}
_ => [0u8; 16],
};
let renumber: &RenumberMap = &local_renumber;
let source_ver = pdf.version().to_string();
let source_ext = pdf.adobe_extension_level()?.unwrap_or(0);
let (eff_version, eff_ext) =
effective_pdf_version_and_ext(&source_ver, source_ext, options, emits_object_streams);
let part1 = Part1Bytes::build(plan, renumber, eff_version);
let part1_placeholders = part1.placeholders.clone();
let part1_dict_region = part1.dict_writable_region.clone();
let root = pdf.root_handle()?;
if root.is_direct() {
return Err(crate::Error::System(
"error encountered after writing part 4 of linearized data".to_string(),
));
}
let catalog_orig = plan.root_ref.ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: plan.root_ref is None — \
cannot determine catalog reference for the trailer"
.to_string(),
)
})?;
let catalog_new_ref: ObjectRef = renumber.new_for_original(catalog_orig).ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: catalog {catalog_orig} is not in RenumberMap \
(planner / renumber inconsistency)"
))
})?;
let hint_stream_new_num: u32 = renumber.hint_stream_slot();
let objstm_layout =
ObjStmLayout::build_from_batches(&resolved_batch_plan, &container_numbers, renumber)?;
let container_shared_sort_key: std::collections::BTreeMap<u32, (u8, u32)> = match options
.object_streams
{
crate::writer::ObjectStreamMode::Preserve => {
let mut keys = std::collections::BTreeMap::new();
for container in objstm_layout
.open_document
.iter()
.chain(&objstm_layout.part3)
.chain(&objstm_layout.part4)
{
let source_container_number =
preserved_source_container_number(container, &source_container_by_member)?;
keys.insert(container.container_new_num, (0, source_container_number));
}
keys
}
_ => {
use crate::linearization::plan::objstm_membership_linearized_with_eligibility;
let assigned = plan.renumber_assigned_refs();
let membership = objstm_membership_linearized_with_eligibility(
pdf,
&assigned,
plan.optimization
.as_ref()
.and_then(|optimization| optimization.generate_objstm_eligible()),
)?; let mut rank = std::collections::BTreeMap::new();
for (split_index, members) in membership.iter().enumerate() {
let first = *members
.first()
.expect("objstm_membership_linearized never yields an empty container");
if let Some(&(container_num, _)) = objstm_layout.member_to_container.get(&first) {
rank.insert(container_num, (1, split_index as u32));
}
}
rank
}
};
let total_count: u32 = renumber
.len()
.checked_add(1)
.and_then(|n| u32::try_from(n).ok())
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: /Size overflows u32 (too many objects / \
ObjStm containers)"
.to_string(),
)
})?;
let info_handle = source_trailer_handle.try_get_key(b"/Info")?;
let info_new_ref: Option<ObjectRef> = plan
.raw
.info
.and_then(|object_gen| renumber.new_for_raw(object_gen))
.or_else(|| {
info_handle
.qpdf_obj_gen()
.and_then(|object_gen| renumber.new_for_raw(object_gen))
});
let first_page_object_new_num: u32 = {
let first_page_hint = plan.page_hints.first().ok_or_else(|| {
crate::Error::Unsupported(
"linearization plan has no page hints (empty document?)".to_string(),
)
})?;
renumber
.new_for_original(first_page_hint.page_ref)
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"first-page page_ref {} has no renumber entry",
first_page_hint.page_ref,
))
})?
.number
};
let second_half_container_nums: std::collections::BTreeSet<u32> = objstm_layout
.part4
.iter()
.map(|c| c.container_new_num)
.collect();
let open_document_container_nums: std::collections::BTreeSet<u32> = objstm_layout
.open_document
.iter()
.map(|c| c.container_new_num)
.collect();
let outlines = plan
.optimization
.as_ref()
.map(|optimization| optimization.raw_objects_for_root_key(b"Outlines"))
.unwrap_or_default();
let outline_info = compute_outline_hint_info(&outlines, pdf, renumber, &objstm_layout)?;
let structural_streams_filtered =
matches!(effective_stream_policy(options), Some(CompressStreams::Yes));
let raw_removed_refs: BTreeSet<QpdfObjGen> = plan
.removed_refs
.iter()
.filter_map(|object_ref| QpdfObjGen::try_from_object_ref(*object_ref).ok())
.collect();
let raw_content_normalize_refs = &plan.raw.content_normalize_refs;
let pass1_part1 = build_pass1_part1(&part1);
let mut pass1_target = Pass1OutputTarget::new(pass1_path)?;
let mut pass1_sink = OutputSink::new(&mut pass1_target);
if deterministic_id {
pass1_sink.begin_digest();
}
let pass1_result = do_write_pass(
plan,
renumber,
&raw_removed_refs,
raw_content_normalize_refs,
pdf,
&mut pass1_sink,
&pass1_part1,
catalog_new_ref,
hint_stream_new_num,
total_count,
info_new_ref,
first_page_object_new_num,
None,
structural_streams_filtered,
&pass1_source_trailer,
&objstm_layout,
&relocation,
options,
true,
None,
encrypt_ctx.as_ref(),
encrypted_string_emitter.as_mut(),
eff_version,
eff_ext,
None,
);
let pass1_finish = pass1_sink.finish_document();
let pass1_file_length = pass1_sink.position_usize()?;
let pass1_digest = if deterministic_id {
Some(pass1_sink.take_digest())
} else {
None
};
drop(pass1_sink);
let mut pass1_output = pass1_result?; pass1_finish?;
let pass1_digest = pass1_digest.transpose()?;
let classic_det_id: Option<(Vec<u8>, [u8; 16])> = if deterministic_id {
Some(crate::writer::compute_deterministic_id_from_digest(
pass1_digest.expect("deterministic pass-1 digest is enabled"),
&det_id_info_suffix,
det_id_source_id0.as_deref(),
))
} else {
None
};
let hint_stream_object = build_hint_stream_from_pass1(&HintStreamBuildInput {
plan,
renumber,
objstm_layout: &objstm_layout,
source_container_by_member: &source_container_by_member,
container_shared_sort_key: &container_shared_sort_key,
pass1_output: &pass1_output,
hint_stream_new_num,
second_half_container_nums: &second_half_container_nums,
open_document_container_nums: &open_document_container_nums,
outline_info: outline_info.as_ref(),
structural_streams_filtered,
encrypt_ctx: encrypt_ctx.as_ref(),
hint_stream_aes_iv,
})?;
if pass1_path.is_some() {
let debug_comments = format!(
"% hint_offset={}\n\
% hint_length={}\n\
% second_xref_offset={}\n\
% second_xref_end={}\n",
pass1_output.hint_stream_offset,
hint_stream_object.len(),
pass1_output.last_xref_offset,
pass1_output.second_xref_end
);
pass1_target.write_debug_comments(debug_comments.as_bytes());
}
let hint_stream_length = hint_stream_object.len();
let hint_stream_offset = pass1_output.hint_stream_offset;
for (&number, offset) in pass1_output.xref_offsets.iter_mut() {
if number != hint_stream_new_num && *offset >= hint_stream_offset {
let adjusted = (*offset).checked_add(hint_stream_length).ok_or_else(|| {
crate::Error::Unsupported(
"linearization final xref offset exceeds usize range".to_string(),
)
})?; *offset = adjusted;
}
}
pass1_output
.xref_offsets
.insert(hint_stream_new_num, hint_stream_offset);
let final_main_xref_offset = pass1_output
.last_xref_offset
.checked_add(hint_stream_length)
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization final main xref offset exceeds usize range".to_string(),
)
})?; let final_end_of_first_page_offset = pass1_output
.end_of_first_page_offset
.checked_add(hint_stream_length)
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization final first-page end exceeds usize range".to_string(),
)
})?; let final_last_xref_first_entry_offset = pass1_output
.last_xref_first_entry_offset
.checked_add(hint_stream_length)
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization final xref entry offset exceeds usize range".to_string(),
)
})?; let final_file_length = pass1_file_length
.checked_add(hint_stream_length)
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization final file length exceeds usize range".to_string(),
)
})?; let page_count = plan.page_hints.len() as u32;
let mut final_part1_offsets = LinearizedOffsets {
file_length: final_file_length,
hint_stream_offset: pass1_output.hint_stream_offset,
hint_stream_length,
first_page_object_new_num,
end_of_first_page_offset: final_end_of_first_page_offset,
last_xref_keyword_offset: final_main_xref_offset,
last_xref_offset: final_last_xref_first_entry_offset.saturating_sub(1),
page_count,
part1_placeholders: part1_placeholders.clone(),
xref_offsets: BTreeMap::new(),
first_trailer_prev_range: 0..0,
dict_writable_region: part1_dict_region.clone(),
};
let mut final_part1 = part1.clone();
crate::linearization::back_patch::back_patch_param_dict(
&mut final_part1.bytes,
&mut final_part1_offsets,
)?; let final_layout = FinalLinearizedLayout {
xref_offsets: &mut pass1_output.xref_offsets,
main_xref_offset: final_main_xref_offset,
};
let final_source_trailer = if let Some((id0, id1)) = &classic_det_id {
let trailer = source_trailer_handle.shallow_copy()?;
trailer.replace_key(
b"/ID",
ObjectHandle::array(vec![
ObjectHandle::string(id0.clone()),
ObjectHandle::string(id1.to_vec()),
]),
)?; trailer
} else {
source_trailer_handle.shallow_copy()?
};
let streaming_output = final_output.is_some();
let mut final_bytes = Vec::new();
let target: &mut dyn OutputTarget = match final_output {
Some(target) => target,
None => &mut final_bytes,
};
let mut final_sink = OutputSink::new(target);
let final_result = do_write_pass(
plan,
renumber,
&raw_removed_refs,
raw_content_normalize_refs,
pdf,
&mut final_sink,
&final_part1,
catalog_new_ref,
hint_stream_new_num,
total_count,
info_new_ref,
first_page_object_new_num,
Some(hint_stream_object.as_slice()),
structural_streams_filtered,
&final_source_trailer,
&objstm_layout,
&relocation,
options,
false,
None,
encrypt_ctx.as_ref(),
encrypted_string_emitter.as_mut(),
eff_version,
eff_ext,
Some(final_layout),
);
let final_finish = final_sink.finish_document();
let final_output_length = final_sink.position_usize()?;
drop(final_sink);
let final_output = final_result?; final_finish?;
let LinearizedPassOutput {
xref_offsets: _final_xref_offsets,
first_page_xref_offset: final_first_page_xref_offset,
hint_stream_offset: final_hint_stream_offset,
hint_stream_obj_total_len: final_hint_stream_obj_total_len,
end_of_first_page_offset: final_end_of_first_page_offset,
last_xref_offset: final_last_xref_keyword_offset,
last_xref_first_entry_offset: final_last_xref_first_entry_offset,
second_xref_end: _final_second_xref_end,
first_trailer_prev_range: final_first_trailer_prev_range,
id_ranges: _final_id_ranges,
} = final_output;
let mut old_to_new: BTreeMap<ObjectRef, ObjectRef> = renumber
.iter_in_layout_order()
.map(|(new_ref, old_ref)| (old_ref, new_ref))
.collect();
if options.object_streams == crate::writer::ObjectStreamMode::Preserve
&& !objstm_layout.is_empty()
{
for container in objstm_layout
.open_document
.iter()
.chain(&objstm_layout.part3)
.chain(&objstm_layout.part4)
{
let source_container_number =
preserved_source_container_number(container, &source_container_by_member)?;
old_to_new.insert(
ObjectRef::new(source_container_number, 0),
ObjectRef::new(container.container_new_num, 0),
);
}
}
let mut written_xref = pass1_output
.xref_offsets
.iter()
.map(|(&number, &offset)| {
(
ObjectRef::new(number, 0),
crate::XrefEntry::Uncompressed {
offset: offset as u64,
},
)
})
.collect::<BTreeMap<_, _>>();
if !objstm_layout.is_empty() {
let first_xref_offset = final_first_page_xref_offset.ok_or_else(|| {
crate::Error::Unsupported(
"linearization result: missing first-page xref offset".to_string(),
)
})?;
written_xref.insert(
ObjectRef::new(relocation.first_xref_slot, 0),
crate::XrefEntry::Uncompressed {
offset: first_xref_offset as u64,
},
);
written_xref.insert(
ObjectRef::new(relocation.main_xref_slot, 0),
crate::XrefEntry::Uncompressed {
offset: final_last_xref_keyword_offset as u64,
},
);
for container in objstm_layout
.open_document
.iter()
.chain(&objstm_layout.part3)
.chain(&objstm_layout.part4)
{
for (index, &(_original, new_ref)) in container.members.iter().enumerate() {
written_xref.insert(
new_ref,
crate::XrefEntry::Compressed {
stream: container.container_new_num,
index: index as u32,
},
);
}
}
}
let offsets = LinearizedOffsets {
file_length: final_output_length,
hint_stream_offset: final_hint_stream_offset,
hint_stream_length: final_hint_stream_obj_total_len,
first_page_object_new_num,
end_of_first_page_offset: final_end_of_first_page_offset,
last_xref_keyword_offset: final_last_xref_keyword_offset,
last_xref_offset: final_last_xref_first_entry_offset.saturating_sub(1),
page_count,
part1_placeholders: final_part1_offsets.part1_placeholders.clone(),
xref_offsets: pass1_output.xref_offsets,
first_trailer_prev_range: final_first_trailer_prev_range,
dict_writable_region: part1_dict_region,
};
let writer_result = WriterResult::new(old_to_new, written_xref);
let document = if streaming_output {
None
} else {
Some(LinearizedDocument {
bytes: final_bytes,
offsets: offsets.clone(),
})
};
Ok(LinearizedWriteResult {
document,
writer_result,
})
}
struct HintStreamBuildInput<'a> {
plan: &'a LinearizationPlan,
renumber: &'a RenumberMap,
objstm_layout: &'a ObjStmLayout,
source_container_by_member: &'a BTreeMap<u32, u32>,
container_shared_sort_key: &'a BTreeMap<u32, (u8, u32)>,
pass1_output: &'a LinearizedPassOutput,
hint_stream_new_num: u32,
second_half_container_nums: &'a BTreeSet<u32>,
open_document_container_nums: &'a BTreeSet<u32>,
outline_info: Option<&'a OutlineHintInfo>,
structural_streams_filtered: bool,
encrypt_ctx: Option<&'a crate::writer::EncryptionContext>,
hint_stream_aes_iv: [u8; 16],
}
#[inline(never)]
fn build_hint_stream_from_pass1(input: &HintStreamBuildInput<'_>) -> Result<Vec<u8>> {
let plan = input.plan;
let renumber = input.renumber;
let objstm_layout = input.objstm_layout;
let source_container_by_member = input.source_container_by_member;
let container_shared_sort_key = input.container_shared_sort_key;
let pass1_output = input.pass1_output;
let hint_stream_new_num = input.hint_stream_new_num;
let second_half_container_nums = input.second_half_container_nums;
let open_document_container_nums = input.open_document_container_nums;
let outline_info = input.outline_info;
let structural_streams_filtered = input.structural_streams_filtered;
let encrypt_ctx = input.encrypt_ctx;
let hint_stream_aes_iv = input.hint_stream_aes_iv;
let xref_offsets = &pass1_output.xref_offsets;
let hint_stream_offset = pass1_output.hint_stream_offset;
let last_xref_offset = pass1_output.last_xref_offset;
let byte_lengths = compute_byte_lengths(
xref_offsets,
last_xref_offset,
hint_stream_new_num,
renumber.param_dict_ref().number,
);
let part3_plain_len: u64 = plan
.part3_objects
.iter()
.filter(|orig| {
!objstm_layout.member_to_container.contains_key(orig)
&& !objstm_layout
.source_container_numbers
.contains(&orig.number)
})
.map(|orig| {
renumber
.new_for_original(*orig)
.and_then(|new_ref| byte_lengths.get(&new_ref.number).copied())
.unwrap_or(0) as u64
})
.sum();
let raw_plain_byte_len = |orig: &QpdfObjGen| -> u64 {
renumber
.new_for_raw(*orig)
.and_then(|new_ref| byte_lengths.get(&new_ref.number).copied())
.unwrap_or(0) as u64
};
let raw_part3_plain_len: u64 = plan
.raw
.part3_objects
.iter()
.filter(|object_gen| object_gen.to_object_ref().is_none())
.map(raw_plain_byte_len)
.sum();
let part3_container_len: u64 = objstm_layout
.part3
.iter()
.map(|c| byte_lengths.get(&c.container_new_num).copied().unwrap_or(0) as u64)
.sum();
let part3_byte_len: u64 = part3_plain_len + raw_part3_plain_len + part3_container_len;
validate_per_page_private_objects(plan)?;
let non_page_owned = crate::linearization::hint_page::non_page_owned_containers(
plan,
&objstm_layout.member_to_container,
source_container_by_member,
);
let plain_byte_len = |orig: &ObjectRef| -> u64 {
renumber
.new_for_original(*orig)
.and_then(|new_ref| byte_lengths.get(&new_ref.number).copied())
.unwrap_or(0) as u64
};
let per_page_byte_lengths: Vec<u64> = plan
.per_page_private_objects
.iter()
.enumerate()
.map(|(page_idx, privates)| {
if page_idx == 0 {
let part2_len: u64 = privates
.iter()
.filter(|orig| {
!objstm_layout.member_to_container.contains_key(*orig)
&& !objstm_layout
.source_container_numbers
.contains(&orig.number)
})
.map(plain_byte_len)
.sum::<u64>()
+ plan
.raw
.part2_objects
.iter()
.filter(|object_gen| object_gen.to_object_ref().is_none())
.map(raw_plain_byte_len)
.sum::<u64>();
let part6_plain_len: u64 = plan
.raw
.outline_first_page_members
.iter()
.filter(|object_gen| object_gen.to_object_ref().is_none())
.map(raw_plain_byte_len)
.sum();
let part6_public_plain_len: u64 = plan
.part6_outline_objects
.iter()
.filter(|orig| {
!objstm_layout.member_to_container.contains_key(*orig)
&& !objstm_layout
.source_container_numbers
.contains(&orig.number)
})
.map(plain_byte_len)
.sum();
part2_len + part3_byte_len + part6_public_plain_len + part6_plain_len
} else {
let mut len = 0u64;
let mut containers: std::collections::BTreeSet<u32> =
std::collections::BTreeSet::new();
for orig in privates {
match objstm_layout.member_to_container.get(orig) {
Some(&(container_num, _)) => {
if !non_page_owned.contains(&container_num) {
containers.insert(container_num);
}
}
None if objstm_layout
.source_container_numbers
.contains(&orig.number) =>
{
if let Some(new_ref) = renumber.new_for_original(*orig) {
if !non_page_owned.contains(&new_ref.number) {
containers.insert(new_ref.number);
}
}
}
None => len += plain_byte_len(orig),
}
}
len += plan
.raw
.per_page_private_objects
.get(page_idx)
.into_iter()
.flatten()
.filter(|object_gen| object_gen.to_object_ref().is_none())
.map(raw_plain_byte_len)
.sum::<u64>();
len + containers
.iter()
.map(|c| byte_lengths.get(c).copied().unwrap_or(0) as u64)
.sum::<u64>()
}
})
.collect();
let mut po_table = PageOffsetHintTable::from_plan_with_source_membership(
plan,
renumber,
&objstm_layout.member_to_container,
container_shared_sort_key,
second_half_container_nums,
open_document_container_nums,
source_container_by_member,
);
let mut so_table = SharedObjectHintTable::from_plan(
plan,
renumber,
&objstm_layout.member_to_container,
second_half_container_nums,
open_document_container_nums,
);
po_table.header.location_of_first_page = hint_stream_offset as u64;
let least_pl = per_page_byte_lengths.iter().copied().min().unwrap_or(0);
let max_pl = per_page_byte_lengths.iter().copied().max().unwrap_or(0);
let bits_delta_pl = bits_needed(max_pl.saturating_sub(least_pl));
po_table.header.least_page_length = least_pl;
po_table.header.bits_page_length_delta = bits_delta_pl;
po_table.header.least_content_length = least_pl;
po_table.header.bits_content_length_delta = bits_delta_pl;
for (entry, &bl) in po_table
.entries
.iter_mut()
.zip(per_page_byte_lengths.iter())
{
let delta = bl.saturating_sub(least_pl);
entry.page_length_minus_least = delta;
entry.content_stream_length = delta;
}
let folded_shared = plan.canonical_raw_shared_hints(
&objstm_layout.member_to_container,
renumber,
second_half_container_nums,
open_document_container_nums,
);
let shared_section_lens: Vec<u64> = folded_shared
.iter()
.map(|h| -> Result<u64> {
if let Some(container) = h.container {
let len = byte_lengths.get(&container).copied().ok_or_else(|| {
crate::Error::Unsupported(format!(
"shared hint container (new #{container}) has no probed byte length",
))
})?;
return Ok(len as u64);
}
let object = h.object.ok_or_else(|| {
crate::Error::Unsupported(
"shared hint entry has neither a raw object nor a container".to_string(),
)
})?;
let new_ref = renumber.new_for_raw(object).ok_or_else(|| {
crate::Error::Unsupported(format!(
"shared hint raw object {} {} has no renumber entry",
object.get_obj(),
object.get_gen()
))
})?;
let len = byte_lengths.get(&new_ref.number).copied().ok_or_else(|| {
crate::Error::Unsupported(format!(
"shared hint raw object {} {} (new #{}) has no probed byte length",
object.get_obj(),
object.get_gen(),
new_ref.number
))
})?;
Ok(len as u64)
})
.collect::<Result<Vec<_>>>()?;
let least = shared_section_lens.iter().copied().min().unwrap_or(0);
let max = shared_section_lens.iter().copied().max().unwrap_or(0);
so_table.header.least_length = least;
so_table.header.bits_length_delta = bits_needed(max.saturating_sub(least));
let first_part8_lookup_num = so_table.header.first_object_number;
if first_part8_lookup_num != 0 {
let first_part8_off = xref_offsets
.get(&first_part8_lookup_num)
.copied()
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"first Part-8 shared object (lookup #{first_part8_lookup_num}) \
has no probed offset"
))
})?;
so_table.header.location = first_part8_off as u64;
}
for (obj, &len) in so_table.objects.iter_mut().zip(&shared_section_lens) {
obj.length_minus_least = (len.saturating_sub(least)) as u32;
}
let outline_table = outline_info
.as_ref()
.map(|info| build_outline_hint_table(info, xref_offsets, &byte_lengths))
.transpose()?;
let hint_mode = if structural_streams_filtered {
HintStreamMode::Compressed
} else {
HintStreamMode::Uncompressed
};
let new_hint =
encode_hint_stream_selected(&po_table, &so_table, outline_table.as_ref(), hint_mode)?;
let new_shared_s = new_hint.shared_section_offset_in_uncompressed;
let new_outline_o = new_hint.outline_section_offset_in_uncompressed;
let mut hint_stream_object = Vec::new();
append_hint_stream_object(
&mut hint_stream_object,
ObjectRef::new(hint_stream_new_num, 0),
&new_hint.payload,
new_shared_s,
new_outline_o,
structural_streams_filtered,
encrypt_ctx,
hint_stream_aes_iv,
)?; Ok(hint_stream_object)
}
#[cfg(test)]
mod tests {
use super::*;
use md5::Digest as _;
use std::io::Cursor;
use std::rc::Rc;
use crate::writer::{output::OutputSink, ProgressReporter, WriteCipher};
fn test_encryption_context() -> crate::writer::EncryptionContext {
crate::writer::EncryptionContext {
encrypt_dict: ObjectHandle::dictionary(Vec::new()),
file_key: vec![1; 5],
cipher: WriteCipher::PerObject(crate::encryption::standard::ObjectKeyAlg::Rc4),
encryption_v: 2,
encryption_r: 3,
encrypt_ref: ObjectRef::new(99, 0),
id0: b"id".to_vec(),
static_aes_iv: true,
encrypt_metadata: true,
metadata_ref: None,
}
}
#[test]
fn part7_owner_uses_the_manual_plan_fallback_without_a_set_table() {
let member = ObjectRef::new(7, 0);
let other = ObjectRef::new(8, 0);
let plan = LinearizationPlan {
per_page_private_objects: vec![vec![], vec![member]],
..LinearizationPlan::default()
};
let source_membership = BTreeMap::new();
assert_eq!(
part7_owner_for_plan(&plan, member, &source_membership),
Some(1)
);
assert_eq!(part7_owner_for_plan(&plan, other, &source_membership), None);
}
#[test]
fn part7_owner_prefers_setup_membership_over_plan_reconstruction() {
let member = ObjectRef::new(7, 0);
let setup_source = ObjectRef::new(70, 0);
let plan_source = ObjectRef::new(71, 0);
let mut optimization = crate::optimization::Optimization::default();
optimization.record_for_test(crate::optimization::ObjectUser::Page(2), setup_source);
let plan = LinearizationPlan {
optimization: Some(optimization),
preserve_objstm_plan: Some(crate::writer::object_streams::ObjectStreamPlan {
groups: vec![
crate::writer::object_streams::ObjectStreamGroup::SourceBacked {
source: plan_source,
members: vec![member],
},
],
removed_refs: BTreeSet::new(),
source_membership_present: true,
}),
..LinearizationPlan::default()
};
let source_membership = BTreeMap::from([(member.number, setup_source.number)]);
assert_eq!(
part7_owner_for_plan(&plan, member, &source_membership),
Some(2)
);
}
#[test]
fn preserved_objstm_extends_ignores_a_non_stream_source() {
let mut pdf = Pdf::empty().expect("empty PDF for source inspection");
let source = ObjectRef::new(1, 0);
pdf.replace_object(source, ObjectHandle::integer(7))
.expect("replace source with a direct non-stream value");
let container = ObjStmContainer {
container_new_num: 2,
source_container: Some(source),
members: Vec::new(),
};
let renumber = RenumberMap::from_plan(&LinearizationPlan::default());
assert_eq!(
preserved_objstm_extends(&container, &renumber, &mut pdf)
.expect("non-stream source is ignored"),
None
);
}
#[test]
fn preserved_objstm_extends_maps_an_unmapped_indirect_target_to_object_zero() {
let mut pdf = Pdf::empty().expect("empty PDF for source inspection");
let source = ObjectRef::new(1, 0);
let extends_target = pdf.get_object_handle(ObjectRef::new(9, 0));
let source_stream = ObjectHandle::stream(
ObjectHandle::dictionary(vec![
(b"Type".to_vec(), ObjectHandle::name(b"ObjStm".to_vec())),
(b"Extends".to_vec(), extends_target),
]),
Rc::new(Vec::new()),
);
pdf.replace_object(source, source_stream)
.expect("replace source with an ObjStm stream");
let container = ObjStmContainer {
container_new_num: 2,
source_container: Some(source),
members: Vec::new(),
};
let renumber = RenumberMap::from_plan(&LinearizationPlan::default());
let extends = preserved_objstm_extends(&container, &renumber, &mut pdf)
.expect("an unmapped /Extends target follows qpdf's zero default");
assert_eq!(extends, Some(ObjectRef::new(0, 0)));
}
#[test]
fn preserve_setup_membership_covers_filtered_plan_members() {
let mut pdf = Pdf::open(Cursor::new(include_bytes!(
"../../../../tests/fixtures/compat/objstm-lin-cap-boundary-199-bearing.pdf"
)))
.expect("open Preserve ObjStm fixture");
let options = WriterOptions {
object_streams: crate::writer::ObjectStreamMode::Preserve,
..WriterOptions::default()
};
let setup =
crate::writer::build_writer_setup(&mut pdf, &options).expect("build writer setup");
let plan = LinearizationPlan::from_pdf_with_writer_options_and_source_membership(
&mut pdf,
&options,
Some(&setup.source_object_stream_data),
None,
None,
)
.expect("build Preserve linearization plan");
let preserve_plan = plan
.preserve_objstm_plan
.as_ref()
.expect("fixture must retain source ObjStm groups");
for group in &preserve_plan.groups {
if let crate::writer::object_streams::ObjectStreamGroup::SourceBacked {
members, ..
} = group
{
assert!(members
.iter()
.all(|member| setup.source_object_stream_data.contains_key(&member.number)));
} }
}
#[test]
fn raw_second_half_anchor_sees_raw_part7_plain_peers() {
let member = ObjectRef::new(7, 0);
let page = ObjectRef::new(2, 0);
let plan = LinearizationPlan {
page_hints: vec![
crate::linearization::plan::PageHintEntry::placeholder(ObjectRef::new(1, 0)),
crate::linearization::plan::PageHintEntry::placeholder(page),
],
per_page_private_objects: vec![vec![], vec![page, member]],
raw: crate::linearization::plan::RawLinearizationPlan {
per_page_private_objects: vec![
vec![],
vec![
QpdfObjGen::new(2, 0),
QpdfObjGen::new(7, 0),
QpdfObjGen::new(8, 65_536),
],
],
part4_other_pages_private: vec![
QpdfObjGen::new(2, 0),
QpdfObjGen::new(7, 0),
QpdfObjGen::new(8, 65_536),
],
projection_gap: true,
..Default::default()
},
..LinearizationPlan::default()
};
let batches = vec![RoutedObjStmBatch {
members: vec![member],
route: ContainerPart::OtherPagePrivate,
source_container_number: None,
}];
let anchors = second_half_container_anchors(&plan, &[], &[], &batches, &BTreeMap::new());
assert_eq!(
anchors,
vec![SecondHalfContainerAnchor::After(QpdfObjGen::new(2, 0))]
);
}
#[test]
fn second_half_anchor_ignores_members_routed_to_the_first_half() {
let page = ObjectRef::new(2, 0);
let plain = ObjectRef::new(3, 0);
let first_half_member = ObjectRef::new(4, 0);
let second_half_member = ObjectRef::new(5, 0);
let plan = LinearizationPlan {
page_hints: vec![
crate::linearization::plan::PageHintEntry::placeholder(ObjectRef::new(1, 0)),
crate::linearization::plan::PageHintEntry::placeholder(page),
],
per_page_private_objects: vec![
vec![],
vec![page, plain, first_half_member, second_half_member],
],
..LinearizationPlan::default()
};
let first_half_batches = vec![RoutedObjStmBatch {
members: vec![first_half_member],
route: ContainerPart::FirstPagePrivate,
source_container_number: None,
}];
let second_half_batches = vec![RoutedObjStmBatch {
members: vec![second_half_member],
route: ContainerPart::OtherPagePrivate,
source_container_number: None,
}];
let anchors = second_half_container_anchors(
&plan,
&[],
&first_half_batches,
&second_half_batches,
&BTreeMap::new(),
);
assert_eq!(
anchors,
vec![SecondHalfContainerAnchor::After(QpdfObjGen::new(3, 0))]
);
}
#[test]
fn raw_second_half_anchor_covers_part9_outline_and_preserve_modes() {
let plan = LinearizationPlan {
raw: crate::linearization::plan::RawLinearizationPlan {
part4_rest: vec![QpdfObjGen::new(8, 65_536)],
part9_outline_objects: vec![QpdfObjGen::new(7, 0)],
projection_gap: true,
..Default::default()
},
..LinearizationPlan::default()
};
let batches = vec![
RoutedObjStmBatch {
members: vec![ObjectRef::new(7, 0)],
route: ContainerPart::Rest,
source_container_number: None,
},
RoutedObjStmBatch {
members: vec![ObjectRef::new(9, 0)],
route: ContainerPart::Rest,
source_container_number: Some(99),
},
];
let anchors = second_half_container_anchors(&plan, &[], &[], &batches, &BTreeMap::new());
assert_eq!(
anchors,
vec![
SecondHalfContainerAnchor::BeforeFirst,
SecondHalfContainerAnchor::After(QpdfObjGen::new(8, 65_536)),
]
);
}
#[test]
fn hint_generation_boundary_preserves_plan_validation() {
let plan = LinearizationPlan {
per_page_private_objects: vec![Vec::new()],
..LinearizationPlan::default()
};
let renumber = RenumberMap::from_plan(&plan);
let container_shared_sort_key = BTreeMap::new();
let pass1_output = LinearizedPassOutput {
xref_offsets: BTreeMap::new(),
first_page_xref_offset: None,
hint_stream_offset: 0,
hint_stream_obj_total_len: 0,
end_of_first_page_offset: 0,
last_xref_offset: 0,
last_xref_first_entry_offset: 0,
second_xref_end: 0,
first_trailer_prev_range: 0..0,
id_ranges: Vec::new(),
};
let input = HintStreamBuildInput {
plan: &plan,
renumber: &renumber,
objstm_layout: &ObjStmLayout::default(),
source_container_by_member: &BTreeMap::new(),
container_shared_sort_key: &container_shared_sort_key,
pass1_output: &pass1_output,
hint_stream_new_num: 2,
second_half_container_nums: &BTreeSet::new(),
open_document_container_nums: &BTreeSet::new(),
outline_info: None,
structural_streams_filtered: false,
encrypt_ctx: None,
hint_stream_aes_iv: [0; 16],
};
let error = build_hint_stream_from_pass1(&input).expect_err("invalid plan must fail");
assert!(error
.to_string()
.contains("per_page_private_objects length"));
}
#[test]
fn pass1_target_is_forward_only_and_counted() {
let mut target = Pass1OutputTarget::discard();
let mut sink = OutputSink::new(&mut target);
sink.begin_digest();
sink.write_bytes(b"pass-1")
.expect("pass-1 bytes are accepted");
let digest = sink.take_digest().expect("pass-1 digest is enabled");
let expected: [u8; 16] = md5::Md5::digest(b"pass-1").into();
assert_eq!(sink.position(), 6);
assert_eq!(digest, expected);
assert!(
target.patch_bytes(0..1, b"x").is_err(),
"pass-1 target must remain forward-only"
);
}
#[test]
fn indirect_object_header_writer_preserves_qpdf_line_shape() {
let mut bytes = Vec::new();
let mut sink = OutputSink::new(&mut bytes);
write_indirect_object_header(&mut sink, ObjectRef::new(123, 7))
.expect("write object header");
drop(sink);
assert_eq!(bytes, b"123 7 obj\n");
}
#[test]
fn repeated_padding_writes_large_and_partial_chunks() {
let mut bytes = Vec::new();
let mut sink = OutputSink::new(&mut bytes);
write_repeated_bytes(&mut sink, b' ', 4096 * 2 + 7).expect("padding writes");
drop(sink);
assert_eq!(bytes.len(), 4096 * 2 + 7);
assert!(bytes.iter().all(|&byte| byte == b' '));
}
#[test]
fn linearization_body_stream_and_objstm_paths_write_through_output_sink() {
let mut pdf = Pdf::empty().expect("empty PDF for stream/ObjStm output");
let stream_handle = pdf
.make_indirect_from_object_handle(ObjectHandle::stream(
ObjectHandle::dictionary(Vec::new()),
Rc::new(b"stream-data".to_vec()),
))
.expect("indirect stream handle");
let stream_ref = stream_handle.object_ref().expect("stream is indirect");
let plan = LinearizationPlan {
part4_rest: vec![stream_ref],
total_object_count: 1,
..Default::default()
};
let renumber = RenumberMap::from_plan(&plan);
let new_ref = renumber
.new_for_original(stream_ref)
.expect("stream ref is in the renumber map");
let mut plain_bytes = Vec::new();
let mut plain_sink = OutputSink::new(&mut plain_bytes);
append_body_object_for_ref(
&mut plain_sink,
&mut pdf,
new_ref,
stream_ref,
&WriterOptions::default(),
None,
None,
&renumber,
&BTreeSet::new(),
&BTreeSet::new(),
)
.expect("plain stream body writes");
drop(plain_sink);
assert!(plain_bytes
.windows(b"endstream".len())
.any(|window| window == b"endstream"));
let container = ObjStmContainer {
container_new_num: 2,
source_container: None,
members: Vec::new(),
};
let mut plain_objstm_bytes = Vec::new();
let mut plain_objstm_sink = OutputSink::new(&mut plain_objstm_bytes);
append_objstm_container_object(
&mut plain_objstm_sink,
&container,
&renumber,
&mut pdf,
&BTreeSet::new(),
&WriterOptions::default(),
None,
)
.expect("plain ObjStm body writes");
drop(plain_objstm_sink);
let mut encrypted_pdf = Pdf::empty().expect("empty PDF for encrypted ObjStm output");
let encryption = test_encryption_context();
let mut encrypted_bytes = Vec::new();
let mut encrypted_sink = OutputSink::new(&mut encrypted_bytes);
append_objstm_container_object(
&mut encrypted_sink,
&container,
&renumber,
&mut encrypted_pdf,
&BTreeSet::new(),
&WriterOptions::default(),
Some(&encryption),
)
.expect("encrypted ObjStm body writes");
drop(encrypted_sink);
assert!(plain_objstm_bytes.starts_with(b"2 0 obj\n"));
assert!(encrypted_bytes.starts_with(b"2 0 obj\n"));
}
#[test]
fn classic_main_xref_rejects_missing_offsets_like_qpdf() {
let trailer = ObjectHandle::dictionary(vec![(
b"/ID".to_vec(),
ObjectHandle::array(vec![
ObjectHandle::new_indirect_unresolved(ObjectRef::new(8, 0), -1),
ObjectHandle::string(vec![1; 16]),
]),
)]);
let mut bytes = Vec::new();
let mut sink = OutputSink::new(&mut bytes);
let error = write_main_xref_and_trailer(
&mut sink,
&BTreeMap::new(),
3,
0,
&trailer,
&trailer,
&|object_gen| Ok(object_gen.to_object_ref().expect("test ID ref is valid")),
&BTreeSet::new(),
false,
)
.expect_err("qpdf rejects a nonzero xref row without an offset");
assert!(matches!(
error,
crate::Error::Internal(message)
if message == "getOffset called for xref entry of type != 1"
));
}
#[test]
fn classic_main_xref_zero_sized_subsection_keeps_qpdf_header() {
let trailer = ObjectHandle::dictionary(vec![(
b"/ID".to_vec(),
ObjectHandle::array(vec![
ObjectHandle::new_indirect_unresolved(ObjectRef::new(8, 0), -1),
ObjectHandle::string(vec![1; 16]),
]),
)]);
let mut bytes = Vec::new();
let mut sink = OutputSink::new(&mut bytes);
write_main_xref_and_trailer(
&mut sink,
&BTreeMap::new(),
0,
0,
&trailer,
&trailer,
&|object_gen| Ok(object_gen.to_object_ref().expect("test ID ref is valid")),
&BTreeSet::new(),
false,
)
.expect("an empty classic xref subsection still writes its trailer");
drop(sink);
assert!(bytes.starts_with(b"xref\n0 0\n"));
}
fn one_page_pdf_with_malformed_trailer_id() -> Vec<u8> {
let mut pdf = b"%PDF-1.4\n".to_vec();
let off1 = pdf.len();
pdf.extend_from_slice(b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n");
let off2 = pdf.len();
pdf.extend_from_slice(b"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n");
let off3 = pdf.len();
pdf.extend_from_slice(
b"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>\nendobj\n",
);
let xref = pdf.len();
pdf.extend_from_slice(
format!(
"xref\n0 4\n0000000000 65535 f \n{off1:010} 00000 n \n{off2:010} 00000 n \n{off3:010} 00000 n \ntrailer\n<< /Size 4 /Root 1 0 R /ID 7 >>\nstartxref\n{xref}\n%%EOF\n"
)
.as_bytes(),
);
pdf
}
#[test]
fn linearized_writer_reads_the_permanent_id_through_the_live_trailer() {
let mut pdf = Pdf::open(Cursor::new(one_page_pdf_with_malformed_trailer_id()))
.expect("source parses");
let options = WriterOptions {
static_id: true,
..WriterOptions::default()
};
let setup = crate::writer::build_writer_setup(&mut pdf, &options).unwrap();
let mut bytes = Vec::new();
write_linearized_for_pdf_writer(&mut pdf, &options, None, setup, None, &mut bytes)
.expect("a malformed present /ID warns instead of aborting the write");
assert!(
!bytes.is_empty(),
"qpdf emits a complete linearized file for this input"
);
let messages: Vec<String> = pdf
.repair_diagnostics()
.entries()
.iter()
.map(|entry| String::from_utf8_lossy(entry.what_bytes()).into_owned())
.collect();
assert!(
messages.iter().any(|message| message.contains(
"operation for array attempted on object of type integer: returning null"
)),
"expected the array accessor warning, got {messages:?}"
);
assert!(
messages.iter().any(|message| message.contains(
"operation for string attempted on object of type null: returning empty string"
)),
"expected the string accessor warning, got {messages:?}"
);
assert_eq!(
messages
.iter()
.filter(|message| message.contains(
"operation for array attempted on object of type integer: returning null"
))
.count(),
3,
"qpdf rereads /ID[0] at generateID and both pass-1 trailers: {messages:?}"
);
assert_eq!(
messages
.iter()
.filter(|message| message.contains(
"operation for string attempted on object of type null: returning empty string"
))
.count(),
3,
"qpdf rereads the string value at generateID and both pass-1 trailers: {messages:?}"
);
}
fn one_page_pdf_with_direct_outlines() -> Vec<u8> {
let mut pdf = b"%PDF-1.4\n".to_vec();
let off1 = pdf.len();
pdf.extend_from_slice(
b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R /Outlines << /Count 0 >> >>\nendobj\n",
);
let off2 = pdf.len();
pdf.extend_from_slice(b"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n");
let off3 = pdf.len();
pdf.extend_from_slice(
b"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>\nendobj\n",
);
let xref = pdf.len();
pdf.extend_from_slice(
format!(
"xref\n0 4\n0000000000 65535 f \n{off1:010} 00000 n \n{off2:010} 00000 n \n{off3:010} 00000 n \ntrailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n"
)
.as_bytes(),
);
pdf
}
#[test]
fn linearized_generate_falls_back_without_a_setup_snapshot() {
let mut pdf = Pdf::open(std::io::Cursor::new(
include_bytes!("../../../../tests/fixtures/compat/one-page.pdf").to_vec(),
))
.expect("open linearized Generate fixture");
let options = WriterOptions {
object_streams: crate::writer::ObjectStreamMode::Generate,
static_id: true,
..WriterOptions::default()
};
let setup = crate::writer::build_writer_setup(&mut pdf, &options).unwrap();
let mut bytes = Vec::new();
write_linearized_for_pdf_writer(&mut pdf, &options, None, setup, None, &mut bytes)
.expect("linearized Generate fallback succeeds");
assert!(bytes
.windows(b"/Type /ObjStm".len())
.any(|window| { window == b"/Type /ObjStm" }));
}
#[test]
fn linearized_writer_keeps_catalog_planning_mutation_after_progress_failure() {
let mut pdf =
Pdf::open(Cursor::new(one_page_pdf_with_direct_outlines())).expect("source parses");
let root_ref = pdf.root_ref().expect("Catalog present");
let options = WriterOptions {
progress_reporter: Some(ProgressReporter::new(Box::new(|_| {
Err(crate::Error::System("test progress failure".to_owned()))
}))),
..WriterOptions::default()
};
let setup = crate::writer::build_writer_setup(&mut pdf, &options).unwrap();
let mut bytes = Vec::new();
let error =
write_linearized_for_pdf_writer(&mut pdf, &options, None, setup, None, &mut bytes)
.expect_err("canonical progress reporter failure must abort writing");
assert!(
error.to_string().contains("test progress failure"),
"unexpected writer error: {error}"
);
let root = pdf.get_object_handle(root_ref);
root.try_is_scalar().expect("Catalog resolves");
assert!(
root.try_get_key(b"/Outlines")
.expect("Outlines lookup")
.is_indirect(),
"planning must keep qpdf's direct /Outlines promotion"
);
}
#[test]
fn body_object_append_propagates_member_resolution_errors() {
let unresolved = ObjectHandle::new_indirect_unresolved(ObjectRef::new(91, 0), -1);
let plan = LinearizationPlan {
part4_rest: vec![ObjectRef::new(91, 0)],
total_object_count: 1,
..Default::default()
};
let renumber = RenumberMap::from_plan(&plan);
let mut bytes = Vec::new();
let mut sink = OutputSink::new(&mut bytes);
let options = WriterOptions::default();
let removed_refs = BTreeSet::new();
let content_normalize_refs = BTreeSet::new();
let mut emitter = LinearizedObjectEmitter {
out: &mut sink,
options: &options,
encrypt_ctx: None,
encrypted_string_emitter: None,
renumber: &renumber,
removed_refs: &removed_refs,
content_normalize_refs: &content_normalize_refs,
root_source: None,
final_pdf_version: "",
final_extension_level: 0,
encryption: writer_encryption_state_for(None),
xref: BTreeMap::new(),
lengths: BTreeMap::new(),
};
let error = emitter
.write_object(&unresolved, None)
.expect_err("body-object emission must propagate member resolution errors");
assert!(matches!(
error,
crate::Error::Internal(message) if message == "object 91 0 belongs to a dropped PDF"
));
}
#[test]
fn linearization_trailer_serializes_live_direct_custom_values() {
let pdf = Pdf::empty().expect("create trailer reference owner");
let indirect = pdf
.make_indirect_from_object_handle(ObjectHandle::integer(11))
.expect("create indirect custom value");
let indirect_ref = indirect.object_ref().unwrap();
let removed = pdf
.make_indirect_from_object_handle(ObjectHandle::integer(12))
.expect("create removed custom value");
let removed_ref = removed.object_ref().unwrap();
let trailer = ObjectHandle::dictionary(vec![
(
b"/Direct".to_vec(),
ObjectHandle::dictionary(vec![(
b"/Name".to_vec(),
ObjectHandle::name(b"Value".to_vec()),
)]),
),
(b"/Indirect".to_vec(), indirect),
(b"/Removed".to_vec(), removed),
(b"/Null".to_vec(), ObjectHandle::null()),
(b"/Size".to_vec(), ObjectHandle::integer(99)),
]);
let map = |object_gen: QpdfObjGen| {
assert_eq!(object_gen.to_object_ref(), Some(indirect_ref));
Ok(ObjectRef::new(21, 0))
};
let entries = canonical_linearization_trailer_entries(
&trailer,
&map,
&[QpdfObjGen::try_from_object_ref(removed_ref).unwrap()]
.into_iter()
.collect(),
)
.expect("serialize canonical linearization trailer entries");
assert_eq!(
entries,
vec![
(b"/Direct".to_vec(), b"<< /Name /Value >>".to_vec()),
(b"/Indirect".to_vec(), b"21 0 R".to_vec()),
]
);
}
}