use std::collections::{BTreeMap, BTreeSet};
use std::io::{Read, Seek, Write};
use std::path::Path;
use crate::linearization::hint_page::{bits_needed, PageOffsetHintTable};
use crate::linearization::hint_shared::SharedObjectHintTable;
use crate::linearization::hint_stream::{encode_hint_stream, OutlineHintTable};
use crate::linearization::part1::{Part1Bytes, Part1Placeholders};
use crate::linearization::plan::{ContainerPart, LinearizationPlan, RoutedObjStmBatch};
use crate::linearization::renumber::{ObjStmRelocation, RenumberMap, SecondHalfContainerAnchor};
use crate::pipeline::stdio_file::StdioBuffer;
use crate::writer::encrypted_strings::EncryptedStringEmitter;
use crate::writer::object_streams::{
emit_objstm_body_from_handles_with_writer, planner_config_from_options,
wrap_objstm_body_as_handle,
};
use crate::writer::{
decrement_progress_event, effective_pdf_version_and_ext, effective_stream_policy,
inject_adbe_extension, report_progress_event, serialize::xref_stream, strip_adbe_extension,
CompressStreams, NewlineBeforeEndstream, ObjectWriterEmission, WriterOptions, WriterResult,
};
use crate::{ObjectHandle, ObjectRef, Pdf, Result};
const EBADF_ERRNO: i32 = 9;
#[derive(Debug, Clone)]
struct ObjStmContainer {
container_new_num: u32,
members: Vec<(ObjectRef, ObjectRef)>,
}
#[derive(Debug, Clone, Default)]
struct ObjStmLayout {
open_document: Vec<ObjStmContainer>,
part3: Vec<ObjStmContainer>,
part4: Vec<ObjStmContainer>,
member_to_container: BTreeMap<ObjectRef, (u32, u32)>,
}
impl ObjStmLayout {
fn is_empty(&self) -> bool {
self.open_document.is_empty() && self.part3.is_empty() && self.part4.is_empty()
}
fn resolve_batches<R: Read + Seek>(
plan: &LinearizationPlan,
pdf: &mut Pdf<R>,
options: &WriterOptions,
) -> Result<crate::linearization::plan::ObjStmBatchPlan> {
let config = planner_config_from_options(options);
let batch_plan = plan.objstm_batches(pdf, &config)?;
let page_dicts: std::collections::BTreeSet<ObjectRef> =
crate::pages::page_refs(pdf)?.into_iter().collect();
let filter_batches = |batches: Vec<Vec<ObjectRef>>| -> Vec<Vec<ObjectRef>> {
batches
.into_iter()
.filter_map(|batch| {
let kept: Vec<ObjectRef> = batch
.into_iter()
.filter(|r| !page_dicts.contains(r))
.collect();
if kept.is_empty() {
None
} else {
Some(kept)
}
})
.collect()
};
let filter_routed_batches = |batches: Vec<RoutedObjStmBatch>| -> Vec<RoutedObjStmBatch> {
batches
.into_iter()
.filter_map(|batch| {
let members: Vec<ObjectRef> = batch
.members
.into_iter()
.filter(|r| !page_dicts.contains(r))
.collect();
(!members.is_empty()).then_some(RoutedObjStmBatch {
members,
route: batch.route,
source_container_number: batch.source_container_number,
})
})
.collect()
};
Ok(crate::linearization::plan::ObjStmBatchPlan {
open_document_batches: filter_batches(batch_plan.open_document_batches),
part3_batches: filter_batches(batch_plan.part3_batches),
part4_batches: filter_routed_batches(batch_plan.part4_batches),
})
}
fn build_from_batches(
batch_plan: &crate::linearization::plan::ObjStmBatchPlan,
container_numbers: &[u32],
renumber: &RenumberMap,
) -> Result<Self> {
if batch_plan.open_document_batches.is_empty()
&& batch_plan.part3_batches.is_empty()
&& batch_plan.part4_batches.is_empty()
{
return Ok(Self::default());
}
let mut member_to_container = BTreeMap::new();
let take = |batches: &[Vec<ObjectRef>],
out: &mut Vec<ObjStmContainer>,
map: &mut BTreeMap<ObjectRef, (u32, u32)>,
container_iter: &mut std::vec::IntoIter<u32>|
-> Result<()> {
for batch in batches {
if batch.is_empty() {
continue;
}
let container_new_num = container_iter.next().ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: ObjStm container-number stream exhausted \
(renumber relocation / batch-plan inconsistency)"
.to_string(),
)
})?;
let mut members = Vec::with_capacity(batch.len());
for (idx, &orig) in batch.iter().enumerate() {
let new_ref = renumber.new_for_original(orig).ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: ObjStm member {orig} has no renumber \
entry (planner / renumber inconsistency)"
))
})?;
map.insert(orig, (container_new_num, idx as u32));
members.push((orig, new_ref));
}
out.push(ObjStmContainer {
container_new_num,
members,
});
}
Ok(())
};
let container_numbers_vec: Vec<u32> = container_numbers.to_vec();
let mut container_iter = container_numbers_vec.into_iter();
let mut open_document = Vec::new();
let mut part3 = Vec::new();
let mut part4 = Vec::new();
let part4_members: Vec<Vec<ObjectRef>> = batch_plan
.part4_batches
.iter()
.map(|batch| batch.members.clone())
.collect();
take(
&batch_plan.open_document_batches,
&mut open_document,
&mut member_to_container,
&mut container_iter,
)?;
take(
&batch_plan.part3_batches,
&mut part3,
&mut member_to_container,
&mut container_iter,
)?;
take(
&part4_members,
&mut part4,
&mut member_to_container,
&mut container_iter,
)?;
let _ = container_iter;
Ok(Self {
open_document,
part3,
part4,
member_to_container,
})
}
}
fn append_objstm_container_object<R: Read + Seek>(
bytes: &mut Vec<u8>,
container: &ObjStmContainer,
renumber: &RenumberMap,
pdf: &mut Pdf<R>,
removed_refs: &BTreeSet<ObjectRef>,
options: &WriterOptions,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
) -> Result<usize> {
let filtered = matches!(effective_stream_policy(options), Some(CompressStreams::Yes));
let map = |object_ref| {
renumber.new_for_original(object_ref).ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: ObjStm member reference {object_ref} has no renumber entry"
))
})
};
let mut members: Vec<(ObjectRef, ObjectHandle)> = Vec::with_capacity(container.members.len());
for &(orig, new_ref) in &container.members {
let handle = pdf.get_object_handle(orig);
pdf.resolve(&handle)?;
let handle = if handle.as_stream_dict().is_some() {
ObjectHandle::null()
} else {
handle
};
members.push((new_ref, handle));
}
let body = emit_objstm_body_from_handles_with_writer(
&members,
&mut |out, _member_index, _object_ref, handle| {
handle.write_object_with_ref_map_and_removed(out, &map, removed_refs)
},
)?;
let compress = if filtered {
CompressStreams::Yes
} else {
CompressStreams::No
};
let (stream_handle, data) = wrap_objstm_body_as_handle(&body, compress, None)?;
let stream_dict = stream_handle.as_stream_dict().ok_or_else(|| {
crate::Error::Internal("linearization ObjStm wrapper produced a non-stream handle".into())
})?; let object_ref = ObjectRef::new(container.container_new_num, 0);
if let Some(ctx) = encrypt_ctx {
let mut payload_length = data.len();
crate::writer::adjust_aes_stream_length(&mut payload_length, ctx, true)?;
stream_dict.replace_key(
b"/Length",
ObjectHandle::integer(i64::try_from(payload_length).unwrap_or(i64::MAX)),
)?;
}
let offset = bytes.len();
bytes.extend_from_slice(format!("{} 0 obj\n", container.container_new_num).as_bytes());
bytes.extend_from_slice(b"<< /Type ");
stream_dict.get_key(b"/Type").write_object(bytes)?;
bytes.extend_from_slice(b" /Length ");
stream_dict.get_key(b"/Length").write_object(bytes)?;
if filtered {
bytes.extend_from_slice(b" /Filter ");
stream_dict.get_key(b"/Filter").write_object(bytes)?;
}
bytes.extend_from_slice(b" /N ");
stream_dict.get_key(b"/N").write_object(bytes)?;
bytes.extend_from_slice(b" /First ");
stream_dict.get_key(b"/First").write_object(bytes)?;
bytes.extend_from_slice(b" >>");
if let Some(ctx) = encrypt_ctx {
crate::writer::write_stream_payload_with_pipeline(
bytes,
&data,
options.newline_before_endstream,
object_ref,
ctx,
true,
None,
)?;
} else {
crate::writer::serialize::write_stream_payload(
bytes,
&data,
options.newline_before_endstream,
);
}
bytes.extend_from_slice(b"\nendobj\n");
Ok(offset)
}
#[derive(Debug, Clone)]
pub struct LinearizedOffsets {
pub file_length: usize,
pub hint_stream_offset: usize,
pub hint_stream_length: usize,
pub first_page_object_new_num: u32,
pub end_of_first_page_offset: usize,
pub last_xref_keyword_offset: usize,
pub last_xref_offset: usize,
pub page_count: u32,
pub part1_placeholders: Part1Placeholders,
pub xref_offsets: BTreeMap<u32, usize>,
pub first_trailer_prev_range: std::ops::Range<usize>,
pub dict_writable_region: std::ops::Range<usize>,
}
#[derive(Debug)]
pub struct LinearizedDocument {
pub bytes: Vec<u8>,
pub offsets: LinearizedOffsets,
}
pub(crate) const PREV_PLACEHOLDER_WIDTH: usize = 22;
fn append_object(
bytes: &mut Vec<u8>,
new_ref: ObjectRef,
object: &ObjectHandle,
map: &dyn Fn(ObjectRef) -> Result<ObjectRef>,
removed_refs: &BTreeSet<ObjectRef>,
encrypted_string_emitter: Option<&mut EncryptedStringEmitter>,
) -> Result<usize> {
let offset = bytes.len();
bytes.extend_from_slice(format!("{} {} obj\n", new_ref.number, new_ref.generation).as_bytes());
if let Some(emitter) = encrypted_string_emitter {
emitter.write_handle_object_with_ref_map(
bytes,
new_ref,
None,
object,
false,
map,
removed_refs,
)?; } else {
object.write_object_with_ref_map_and_removed(bytes, map, removed_refs)?;
}
bytes.extend_from_slice(b"\nendobj\n");
Ok(offset)
}
fn prepend_crypt_filter_to_handle_entries(
entries: &mut BTreeMap<Vec<u8>, ObjectHandle>,
cf_name: &[u8],
) -> Result<()> {
let crypt_decode_parms = ObjectHandle::dictionary(vec![
(
b"/Type".to_vec(),
ObjectHandle::name(b"CryptFilterDecodeParms".to_vec()),
),
(b"/Name".to_vec(), ObjectHandle::name(cf_name.to_vec())),
]);
let existing_filter = entries.remove(b"/Filter".as_slice());
let existing_decode_parms = entries.remove(b"/DecodeParms".as_slice());
match existing_filter {
None => {
entries.insert(b"/Filter".to_vec(), ObjectHandle::name(b"Crypt".to_vec()));
entries.insert(b"/DecodeParms".to_vec(), crypt_decode_parms);
}
Some(filter) => {
if let Some(name) = filter.try_as_name()? {
entries.insert(
b"/Filter".to_vec(),
ObjectHandle::array(vec![
ObjectHandle::name(b"Crypt".to_vec()),
ObjectHandle::name(name),
]),
);
entries.insert(
b"/DecodeParms".to_vec(),
ObjectHandle::array(vec![
crypt_decode_parms,
existing_decode_parms.unwrap_or_else(ObjectHandle::null),
]),
);
} else if let Some(mut filters) = filter.try_as_array()? {
let chain_len = filters.len();
let mut new_filters = Vec::with_capacity(chain_len + 1);
new_filters.push(ObjectHandle::name(b"Crypt".to_vec()));
new_filters.append(&mut filters);
let mut new_decode_parms = Vec::with_capacity(chain_len + 1);
new_decode_parms.push(crypt_decode_parms);
match existing_decode_parms {
None => new_decode_parms.extend((0..chain_len).map(|_| ObjectHandle::null())),
Some(params) => {
if let Some(params) = params.try_as_array()? {
new_decode_parms.extend(
params
.into_iter()
.chain(std::iter::repeat_with(ObjectHandle::null))
.take(chain_len),
);
} else {
new_decode_parms.push(params);
new_decode_parms.extend((1..chain_len).map(|_| ObjectHandle::null()));
}
}
}
entries.insert(b"/Filter".to_vec(), ObjectHandle::array(new_filters));
entries.insert(
b"/DecodeParms".to_vec(),
ObjectHandle::array(new_decode_parms),
);
} else {
entries.insert(b"/Filter".to_vec(), ObjectHandle::name(b"Crypt".to_vec()));
entries.insert(b"/DecodeParms".to_vec(), crypt_decode_parms);
}
}
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn append_body_object(
bytes: &mut Vec<u8>,
new_ref: ObjectRef,
original_ref: ObjectRef,
object: &ObjectHandle,
options: &WriterOptions,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
encrypted_string_emitter: Option<&mut EncryptedStringEmitter>,
renumber: &RenumberMap,
removed_refs: &BTreeSet<ObjectRef>,
content_normalize_refs: &BTreeSet<ObjectRef>,
) -> Result<usize> {
let map = |object_ref| {
renumber.new_for_original(object_ref).ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: reference {object_ref} has no renumber entry"
))
})
};
if object.as_stream_dict().is_none() {
return append_object(
bytes,
new_ref,
object,
&map,
removed_refs,
encrypted_string_emitter,
);
}
let (dict, data, mut refiltered) =
crate::writer::plain::body::canonical_stream_output_for_linearization(
object,
options,
options.content_normalization && content_normalize_refs.contains(&original_ref),
)?; let mut entries = dict.try_as_dictionary()?.unwrap_or_default();
let payload_ctx = encrypt_ctx.filter(|ctx| new_ref != ctx.encrypt_ref);
let cleartext_metadata = payload_ctx
.is_some_and(|ctx| !ctx.encrypt_metadata && ctx.metadata_ref == Some(original_ref));
if cleartext_metadata {
prepend_crypt_filter_to_handle_entries(&mut entries, b"Identity")?;
refiltered = false;
}
let mut payload_length = data.len();
if let Some(ctx) = payload_ctx.filter(|_| !cleartext_metadata) {
crate::writer::adjust_aes_stream_length(&mut payload_length, ctx, true)?;
}
entries.insert(
b"/Length".to_vec(),
ObjectHandle::integer(i64::try_from(payload_length).unwrap_or(i64::MAX)),
);
let dict = ObjectHandle::dictionary(entries.into_iter().collect());
let offset = bytes.len();
bytes.extend_from_slice(format!("{} {} obj\n", new_ref.number, new_ref.generation).as_bytes());
if let Some(emitter) = encrypted_string_emitter {
emitter.write_handle_stream_dict_with_ref_map(
bytes,
new_ref,
None,
&dict,
crate::writer::encrypted_strings::StreamDictOptions::new(false, refiltered, true),
&map,
removed_refs,
None,
)?; } else {
dict.write_stream_body_with_ref_map_and_removed(bytes, refiltered, &map, removed_refs)?;
}
if let Some(ctx) = payload_ctx.filter(|_| !cleartext_metadata) {
crate::writer::write_stream_payload_with_pipeline(
bytes,
&data,
options.newline_before_endstream,
new_ref,
ctx,
true,
None,
)?; } else {
crate::writer::serialize::write_stream_payload(
bytes,
&data,
options.newline_before_endstream,
);
}
bytes.extend_from_slice(b"\nendobj\n");
Ok(offset)
}
#[allow(clippy::too_many_arguments)]
fn append_body_object_for_ref<R: Read + Seek>(
bytes: &mut Vec<u8>,
pdf: &mut Pdf<R>,
new_ref: ObjectRef,
original_ref: ObjectRef,
options: &WriterOptions,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
encrypted_string_emitter: Option<&mut EncryptedStringEmitter>,
renumber: &RenumberMap,
removed_refs: &BTreeSet<ObjectRef>,
content_normalize_refs: &BTreeSet<ObjectRef>,
) -> Result<usize> {
let object = pdf.get_object_handle(original_ref);
pdf.resolve(&object)?;
append_body_object(
bytes,
new_ref,
original_ref,
&object,
options,
encrypt_ctx,
encrypted_string_emitter,
renumber,
removed_refs,
content_normalize_refs,
)
}
const CLASSIC_XREF_ENTRY_WIDTH: usize = 20;
struct Part1XrefPatch {
start_num: u32,
count: u32,
data_range: std::ops::Range<usize>,
}
#[allow(clippy::too_many_arguments)]
fn write_part1_xref_and_trailer(
bytes: &mut Vec<u8>,
param_dict_obj_number: u32,
total_object_count: u32,
first_page_count: u32,
catalog_new_ref: ObjectRef,
info_new_ref: Option<ObjectRef>,
source_trailer: &ObjectHandle,
canonical_entries: &[(Vec<u8>, Vec<u8>)],
map: &dyn Fn(ObjectRef) -> Result<ObjectRef>,
removed_refs: &BTreeSet<ObjectRef>,
id_writer: Option<crate::pdf_syntax::ReborrowableIdWriter>,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
) -> Result<(usize, std::ops::Range<usize>, Part1XrefPatch)> {
bytes.push(b'\n');
let xref_offset = bytes.len();
bytes.extend_from_slice(
format!("xref\n{param_dict_obj_number} {first_page_count}\n").as_bytes(),
);
let data_start = bytes.len();
bytes.resize(
data_start + (first_page_count as usize) * CLASSIC_XREF_ENTRY_WIDTH,
b' ',
);
let data_end = bytes.len();
let patch = Part1XrefPatch {
start_num: param_dict_obj_number,
count: first_page_count,
data_range: data_start..data_end,
};
bytes.extend_from_slice(b"trailer << ");
let mut entries = canonical_entries.to_vec();
if let Some(info_ref) = info_new_ref {
entries.push((
b"/Info".to_vec(),
format!("{} {} R", info_ref.number, info_ref.generation).into_bytes(),
));
}
entries.push((
b"/Root".to_vec(),
format!(
"{} {} R",
catalog_new_ref.number, catalog_new_ref.generation
)
.into_bytes(),
));
entries.push((
b"/Size".to_vec(),
total_object_count.to_string().into_bytes(),
));
entries.sort_by(|left, right| left.0.cmp(&right.0));
let mut prev_value_range = None;
for (key, value) in entries {
bytes.push(b'/');
crate::pdf_syntax::write_name_escaped(bytes, key.strip_prefix(b"/").unwrap_or(&key));
bytes.push(b' ');
bytes.extend_from_slice(&value);
if key == b"/Size" {
bytes.extend_from_slice(b" /Prev ");
let prev_value_start = bytes.len();
let placeholder = format!("{:<PREV_PLACEHOLDER_WIDTH$}", 0);
bytes.extend_from_slice(placeholder.as_bytes());
prev_value_range = Some(prev_value_start..bytes.len());
} else {
bytes.push(b' ');
}
}
bytes.extend_from_slice(b"/ID ");
let id_value = source_trailer.try_get_key(b"/ID")?;
match id_writer {
Some(write_id) => write_id(bytes),
None => id_value.write_id_value_with_ref_map(bytes, map, removed_refs)?,
}
if let Some(ctx) = encrypt_ctx {
bytes.extend_from_slice(
format!(
" /Encrypt {} {} R",
ctx.encrypt_ref.number, ctx.encrypt_ref.generation
)
.as_bytes(),
);
}
bytes.extend_from_slice(b" >>");
bytes.extend_from_slice(b"\nstartxref\n0\n%%EOF\n");
let prev_value_range = prev_value_range.ok_or_else(|| {
crate::Error::Unsupported(
"linearization first trailer has no /Size entry for /Prev patch".to_string(),
)
})?; Ok((xref_offset, prev_value_range, patch))
}
fn patch_part1_xref(
bytes: &mut [u8],
patch: &Part1XrefPatch,
xref_offsets: &BTreeMap<u32, usize>,
) -> Result<()> {
if patch.data_range.end > bytes.len() {
return Err(crate::Error::Unsupported(
"Part-1 xref patch range out of bounds".to_string(),
));
}
let mut data = Vec::with_capacity(patch.count as usize * CLASSIC_XREF_ENTRY_WIDTH);
for number in patch.start_num..patch.start_num + patch.count {
let offset = xref_offsets.get(&number).copied().ok_or_else(|| {
crate::Error::Unsupported(format!(
"Part-1 xref: covered object {number} has no offset (planner / writer \
inconsistency — would emit a free entry for a live first-page object)"
))
})?;
data.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());
}
if data.len() != patch.data_range.len() {
return Err(crate::Error::Unsupported(format!(
"Part-1 xref payload length drift: encoded {} bytes, reserved {}",
data.len(),
patch.data_range.len()
)));
}
bytes[patch.data_range.clone()].copy_from_slice(&data);
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn write_main_xref_and_trailer(
bytes: &mut Vec<u8>,
xref_offsets: &BTreeMap<u32, usize>,
param_slot: u32, first_page_xref_offset: usize,
source_trailer: &ObjectHandle,
map: &dyn Fn(ObjectRef) -> Result<ObjectRef>,
removed_refs: &BTreeSet<ObjectRef>,
id_writer: Option<crate::pdf_syntax::ReborrowableIdWriter>,
) -> Result<(usize, usize)> {
let xref_start = bytes.len();
let xref_header = format!("xref\n0 {}\n", param_slot);
bytes.extend_from_slice(xref_header.as_bytes());
let xref_first_entry_offset = bytes.len();
bytes.extend_from_slice(b"0000000000 65535 f \n");
for number in 1..param_slot {
match xref_offsets.get(&number) {
Some(offset) => {
bytes.extend_from_slice(format!("{:010} 00000 n \n", offset).as_bytes())
}
None => bytes.extend_from_slice(b"0000000000 65535 f \n"),
}
}
bytes.extend_from_slice(b"trailer << ");
bytes.extend_from_slice(format!("/Size {} ", param_slot).as_bytes());
bytes.extend_from_slice(b"/ID ");
let id_value = source_trailer.try_get_key(b"/ID")?;
match id_writer {
Some(write_id) => write_id(bytes),
None => id_value.write_id_value_with_ref_map(bytes, map, removed_refs)?,
}
bytes.extend_from_slice(b" ");
bytes.extend_from_slice(b">>");
bytes.extend_from_slice(format!("\nstartxref\n{}\n%%EOF\n", first_page_xref_offset).as_bytes());
Ok((xref_start, xref_first_entry_offset))
}
struct FirstPageXrefPatch {
first_xref_num: u32,
index_start: u32,
index_count: u32,
region: std::ops::Range<usize>,
catalog_new_ref: ObjectRef,
info_new_ref: Option<ObjectRef>,
size: u32,
id: Option<(Vec<u8>, Vec<u8>)>,
canonical_entries: Vec<(Vec<u8>, Vec<u8>)>,
encrypt: Option<ObjectRef>,
max_id: u32,
max_ostream_index: u64,
filtered: bool,
}
fn canonical_linearization_trailer_entries(
trailer: &ObjectHandle,
map: &dyn Fn(ObjectRef) -> Result<ObjectRef>,
removed_refs: &BTreeSet<ObjectRef>,
) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
let entries = trailer.try_as_dictionary()?.unwrap_or_default();
let mut serialized = Vec::with_capacity(entries.len());
for (key, value) in entries {
if matches!(
key.as_slice(),
b"/ID"
| b"/Encrypt"
| b"/Info"
| b"/Prev"
| b"/Root"
| b"/Size"
| b"/Type"
| b"/F"
| b"/FFilter"
| b"/FDecodeParms"
| b"/W"
| b"/Index"
| b"/Length"
| b"/Filter"
| b"/DecodeParms"
| b"/XRefStm"
) {
continue;
}
let removed = value
.object_ref()
.is_some_and(|object_ref| object_ref.number == 0 || removed_refs.contains(&object_ref))
|| value.object_ref().is_some_and(|object_ref| {
object_ref.number == 0 || removed_refs.contains(&object_ref)
});
if removed || value.try_is_null()? {
continue;
}
let mut value_bytes = Vec::new();
if let Some(object_ref) = value.object_ref() {
let mapped = map(object_ref)?;
value_bytes.extend_from_slice(mapped.to_string().as_bytes());
} else {
value.write_object_with_ref_map_and_removed(&mut value_bytes, map, removed_refs)?;
}
serialized.push((key, value_bytes));
}
Ok(serialized)
}
pub(crate) fn first_page_xref_object_count(
total_count: u32,
param_dict_obj_number: u32,
) -> Result<u32> {
total_count
.checked_sub(param_dict_obj_number)
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: param-dict object number ({param_dict_obj_number}) \
exceeds /Size ({total_count}) — cannot size the first-page xref subsection"
))
})
}
fn finalize_linearized_id(
options: &WriterOptions,
source_id0: Option<&[u8]>,
det_id_source_id0: Option<&[u8]>,
copy_encryption: Option<&crate::encryption::CopyEncryptionSource>,
) -> ObjectHandle {
if crate::writer::uses_deterministic_id(options) {
let len0 = det_id_source_id0.map(<[u8]>::len).unwrap_or(16);
ObjectHandle::array(vec![
ObjectHandle::string(vec![0u8; len0]),
ObjectHandle::string(vec![0u8; 16]),
])
} else if let Some(source) = copy_encryption {
let generated = crate::writer::generate_id_handle(None, options.static_id);
let id1 = generated
.as_array()
.and_then(|values| values.get(1).and_then(ObjectHandle::as_string))
.unwrap_or_else(|| source.id0.clone());
ObjectHandle::array(vec![
ObjectHandle::string(source.id0.clone()),
ObjectHandle::string(id1),
])
} else {
crate::writer::generate_id_handle(source_id0, options.static_id)
}
}
fn linearization_pass1_id(source_id0: Option<&[u8]>) -> ObjectHandle {
let first_len = source_id0.map(|id| id.len()).unwrap_or(16);
ObjectHandle::array(vec![
ObjectHandle::string(vec![0u8; first_len]),
ObjectHandle::string(vec![0u8; 16]),
])
}
fn patch_linearized_deterministic_id(
bytes: &mut [u8],
id_ranges: &[std::ops::Range<usize>],
id0: &[u8],
id1: &[u8; 16],
) {
use crate::writer::{deterministic_id_array_len, write_deterministic_id_array};
let len = deterministic_id_array_len(id0.len());
let mut placeholder = Vec::with_capacity(len);
write_deterministic_id_array(&mut placeholder, &vec![0u8; id0.len()], &[0u8; 16]);
let mut final_id = Vec::with_capacity(len);
write_deterministic_id_array(&mut final_id, id0, id1);
for range in id_ranges {
let start = range.start.min(bytes.len());
let end = range.end.min(bytes.len());
let mut patched = 0usize;
let mut i = start;
while i + len <= end {
if &bytes[i..i + len] == placeholder.as_slice() {
bytes[i..i + len].copy_from_slice(&final_id);
patched += 1;
i += len;
} else {
i += 1;
}
}
debug_assert_eq!(
patched, 1,
"each /ID section must contain exactly one deterministic /ID placeholder \
(0 or >1 indicates a linearization writer bug)"
);
}
}
#[allow(clippy::too_many_arguments)]
fn write_first_page_xref_stream(
bytes: &mut Vec<u8>,
relocation: &ObjStmRelocation,
total_count: u32, catalog_new_ref: ObjectRef,
info_new_ref: Option<ObjectRef>,
source_trailer: &ObjectHandle,
canonical_entries: &[(Vec<u8>, Vec<u8>)],
max_ostream_index: u64,
filtered: bool,
encrypt: Option<ObjectRef>,
) -> Result<FirstPageXrefPatch> {
let final_size = total_count;
let first_xref_num = relocation.first_xref_slot;
let index_start = relocation.second_half_count;
let index_count = final_size.checked_sub(index_start).ok_or_else(|| {
crate::Error::Unsupported(
"first-page xref /Index underflow (second-half count exceeds /Size)".to_string(),
)
})?;
let max_id = final_size.saturating_sub(1);
let id = xref_id_bytes(source_trailer)?;
let obj_ref = ObjectRef::new(first_xref_num, 0);
let region_len = {
let dict = xref_stream::XrefStreamDict {
filtered,
widths: xref_stream::first_pass_widths(max_id, max_ostream_index, 0),
index: Some((index_start, index_count)),
info: info_new_ref,
root: Some(catalog_new_ref),
root_value: None,
size: final_size,
prev: Some(0),
canonical_entries: Some(canonical_entries),
id: id.as_ref().map(|(a, b)| (a.as_slice(), b.as_slice())),
encrypt,
};
xref_stream::first_pass_region_len(obj_ref, &dict, index_count as usize)
};
bytes.push(b'\n');
let obj_offset = bytes.len();
bytes.resize(obj_offset + region_len, b' ');
bytes.push(b'\n');
Ok(FirstPageXrefPatch {
first_xref_num,
index_start,
index_count,
region: obj_offset..obj_offset + region_len,
catalog_new_ref,
info_new_ref,
size: final_size,
id,
canonical_entries: canonical_entries.to_vec(),
encrypt,
max_id,
max_ostream_index,
filtered,
})
}
fn xref_id_bytes(source_trailer: &ObjectHandle) -> Result<Option<(Vec<u8>, Vec<u8>)>> {
let id = source_trailer.try_get_key(b"/ID")?;
let Some(values) = id.try_as_array()? else {
return Ok(None);
};
if values.len() != 2 {
return Ok(None);
}
let (first, second) = (&values[0], &values[1]);
Ok(match (first.as_string(), second.as_string()) {
(Some(first), Some(second)) => Some((first, second)),
_ => None,
})
}
fn patch_first_page_xref(
bytes: &mut [u8],
patch: &FirstPageXrefPatch,
xref_offsets: &BTreeMap<u32, usize>,
member_new: &BTreeMap<u32, (u32, u32)>,
main_xref_offset: usize,
hint_length: usize,
pass1: bool,
) -> Result<Option<std::ops::Range<usize>>> {
let mut offs = xref_offsets.clone();
offs.insert(patch.first_xref_num, patch.region.start);
let (widths, payload, prev) = if pass1 {
let pass1_offs: BTreeMap<u32, usize> = offs
.iter()
.filter(|(_, &off)| off <= patch.region.start)
.map(|(&n, &off)| (n, off))
.collect();
let entries = xref_stream::build_entries(
&pass1_offs,
&BTreeMap::new(),
patch.index_start,
patch.index_count,
);
let widths = xref_stream::first_pass_widths(patch.max_id, patch.max_ostream_index, 0);
let payload = if patch.filtered {
xref_stream::encode_payload_uncompressed(&entries, widths)?
} else {
xref_stream::encode_payload_raw(&entries, widths)?
};
(widths, payload, 0u64)
} else {
let entries =
xref_stream::build_entries(&offs, member_new, patch.index_start, patch.index_count);
let widths = xref_stream::second_pass_widths(
xref_stream::max_entry_offset(&entries),
hint_length as u64,
patch.max_id,
patch.max_ostream_index,
);
let payload = if patch.filtered {
xref_stream::encode_payload(&entries, widths)?
} else {
xref_stream::encode_payload_raw(&entries, widths)?
};
(widths, payload, main_xref_offset as u64)
};
let dict = xref_stream::XrefStreamDict {
filtered: patch.filtered,
widths,
index: Some((patch.index_start, patch.index_count)),
info: patch.info_new_ref,
root: Some(patch.catalog_new_ref),
root_value: None,
size: patch.size,
prev: Some(prev),
canonical_entries: Some(&patch.canonical_entries),
id: patch.id.as_ref().map(|(a, b)| (a.as_slice(), b.as_slice())),
encrypt: patch.encrypt,
};
let (region, region_id_range) = xref_stream::write_padded_region(
ObjectRef::new(patch.first_xref_num, 0),
&dict,
&payload,
patch.region.len(),
)?; if patch.region.end > bytes.len() {
return Err(crate::Error::Unsupported(
"first-page xref patch range out of bounds".to_string(),
));
}
bytes[patch.region.clone()].copy_from_slice(®ion);
Ok(region_id_range.map(|r| patch.region.start + r.start..patch.region.start + r.end))
}
#[allow(clippy::too_many_arguments)]
fn write_main_xref_stream_and_trailer(
bytes: &mut Vec<u8>,
xref_offsets: &BTreeMap<u32, usize>,
member_new: &BTreeMap<u32, (u32, u32)>,
relocation: &ObjStmRelocation,
total_count: u32, source_trailer: &ObjectHandle,
first_page_obj_offset: usize,
max_ostream_index: u64,
pass1: bool,
filtered: bool,
) -> Result<(usize, usize, Option<std::ops::Range<usize>>)> {
let final_size = total_count;
let first_xref_num = relocation.first_xref_slot;
let main_xref_num = relocation.main_xref_slot;
let max_id = final_size.saturating_sub(1);
let id = xref_id_bytes(source_trailer)?;
let main_count = relocation.second_half_count;
let main_xref_offset = bytes.len();
let mut offs2 = xref_offsets.clone();
offs2.insert(first_xref_num, first_page_obj_offset);
offs2.insert(main_xref_num, main_xref_offset);
let entries = xref_stream::build_entries(&offs2, member_new, 0, main_count);
let widths = xref_stream::second_pass_widths(
xref_stream::max_entry_offset(&entries),
0,
max_id,
max_ostream_index,
);
let payload = if !filtered {
xref_stream::encode_payload_raw(&entries, widths)?
} else if pass1 {
xref_stream::encode_payload_uncompressed(&entries, widths)?
} else {
xref_stream::encode_payload(&entries, widths)?
};
let dict = xref_stream::XrefStreamDict {
filtered,
widths,
index: None,
info: None,
root: None,
root_value: None,
size: main_count,
prev: None,
canonical_entries: None,
id: id.as_ref().map(|(a, b)| (a.as_slice(), b.as_slice())),
encrypt: None,
};
let main_obj_ref = ObjectRef::new(main_xref_num, 0);
let region_len = {
let p1_dict = xref_stream::XrefStreamDict {
filtered,
widths,
index: None,
info: None,
root: None,
root_value: None,
size: main_count,
prev: None,
canonical_entries: None,
id: id.as_ref().map(|(a, b)| (a.as_slice(), b.as_slice())),
encrypt: None,
};
xref_stream::first_pass_region_len(main_obj_ref, &p1_dict, main_count as usize)
};
let (region, region_id_range) =
xref_stream::write_padded_region(main_obj_ref, &dict, &payload, region_len)?;
let region_start = bytes.len();
bytes.extend_from_slice(®ion);
bytes.push(b'\n');
let id_range = region_id_range.map(|r| region_start + r.start..region_start + r.end);
bytes.extend_from_slice(format!("startxref\n{first_page_obj_offset}\n%%EOF\n").as_bytes());
Ok((main_xref_offset, main_xref_offset, id_range))
}
fn hint_stream_dict_prefix(
shared_section_offset: usize,
outline_section_offset: Option<usize>,
payload_len: usize,
filtered: bool,
) -> String {
let filter_key = if filtered {
"/Filter /FlateDecode "
} else {
""
};
let outline_key = match outline_section_offset {
Some(o) => format!(" /O {o}"),
None => String::new(),
};
format!("<< {filter_key}/S {shared_section_offset}{outline_key} /Length {payload_len} >>")
}
#[cfg(test)]
thread_local! {
static TEST_HINT_STREAM_AES_IVS:
std::cell::RefCell<std::collections::VecDeque<[u8; 16]>> =
const { std::cell::RefCell::new(std::collections::VecDeque::new()) };
}
#[cfg(test)]
fn next_test_hint_stream_aes_iv(default: [u8; 16]) -> [u8; 16] {
TEST_HINT_STREAM_AES_IVS.with(|ivs| ivs.borrow_mut().pop_front().unwrap_or(default))
}
#[allow(clippy::too_many_arguments)]
fn append_hint_stream_object(
bytes: &mut Vec<u8>,
new_ref: ObjectRef,
payload: &[u8],
shared_section_offset: usize,
outline_section_offset: Option<usize>,
filtered: bool,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
hint_stream_aes_iv: [u8; 16],
) -> Result<usize> {
#[cfg(test)]
let hint_stream_aes_iv = next_test_hint_stream_aes_iv(hint_stream_aes_iv);
let mut payload_len = payload.len();
if let Some(ctx) = encrypt_ctx {
crate::writer::adjust_aes_stream_length(&mut payload_len, ctx, true)?;
}
let offset = bytes.len();
bytes.extend_from_slice(format!("{} {} obj\n", new_ref.number, new_ref.generation).as_bytes());
bytes.extend_from_slice(
hint_stream_dict_prefix(
shared_section_offset,
outline_section_offset,
payload_len,
filtered,
)
.as_bytes(),
);
if let Some(ctx) = encrypt_ctx {
crate::writer::write_stream_payload_with_pipeline_qdf(
bytes,
payload,
NewlineBeforeEndstream::Never,
true,
new_ref,
ctx,
true,
Some(hint_stream_aes_iv),
)?;
} else {
crate::writer::serialize::write_stream_payload_with_qdf(
bytes,
payload,
NewlineBeforeEndstream::Never,
true,
);
}
bytes.extend_from_slice(b"\nendobj\n");
Ok(offset)
}
struct OutlineHintInfo {
first_object: u32,
nobjects: u32,
}
fn compute_outline_hint_info<R: Read + Seek>(
outlines: &std::collections::BTreeSet<ObjectRef>,
pdf: &mut Pdf<R>,
renumber: &RenumberMap,
objstm_layout: &ObjStmLayout,
) -> Result<Option<OutlineHintInfo>> {
if outlines.is_empty() {
return Ok(None);
}
let outlines_ref = if let Some(root_ref) = pdf.root_ref() {
let root = pdf.get_object_handle(root_ref);
pdf.resolve(&root)?;
if root.try_as_dictionary()?.is_none() {
None } else {
let outlines = root.try_get_key(b"/Outlines")?;
outlines.object_ref()
}
} else {
None };
let Some(outlines_ref) = outlines_ref else {
return Ok(None); };
let unit_of = |r: ObjectRef| -> Option<u32> {
match objstm_layout.member_to_container.get(&r) {
Some(&(container_num, _)) => Some(container_num),
None => renumber.new_for_original(r).map(|nr| nr.number),
}
};
let units: std::collections::BTreeSet<u32> =
outlines.iter().filter_map(|&r| unit_of(r)).collect();
let Some(first_object) = unit_of(outlines_ref) else {
return Ok(None); };
Ok(Some(OutlineHintInfo {
first_object,
nobjects: units.len() as u32,
}))
}
fn build_outline_hint_table(
info: &OutlineHintInfo,
xref_offsets: &BTreeMap<u32, usize>,
byte_lengths: &BTreeMap<u32, usize>,
) -> Result<OutlineHintTable> {
let first_off = xref_offsets
.get(&info.first_object)
.copied()
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"outline hint: first outline unit (#{}) has no probed offset",
info.first_object
))
})?;
let first_object_offset = u32::try_from(first_off).map_err(|_| {
crate::Error::Unsupported(format!(
"outline hint: pass-1 first unit offset ({first_off}) exceeds the \
32-bit Outlines Hint Table field"
))
})?;
let group_length: u64 = (info.first_object as u64
..info.first_object as u64 + info.nobjects as u64)
.map(|n| byte_lengths.get(&(n as u32)).copied().unwrap_or(0) as u64)
.sum();
Ok(OutlineHintTable {
first_object: info.first_object,
first_object_offset,
nobjects: info.nobjects,
group_length: group_length as u32,
})
}
fn build_pass1_part1(part1: &Part1Bytes) -> Part1Bytes {
const EMPTY_DICT: &[u8] = b"<< >>\nendobj\n";
let mut pass1 = part1.clone();
let region = part1.dict_writable_region.clone();
debug_assert!(region.len() >= EMPTY_DICT.len());
let mut replacement = Vec::with_capacity(region.len());
replacement.extend_from_slice(EMPTY_DICT);
replacement.resize(region.len(), b' ');
pass1.bytes[region].copy_from_slice(&replacement);
pass1
}
struct LinearizedPassOutput {
bytes: Vec<u8>,
xref_offsets: BTreeMap<u32, usize>,
first_page_xref_offset: Option<usize>,
hint_stream_offset: usize,
hint_stream_obj_total_len: usize,
end_of_first_page_offset: usize,
last_xref_offset: usize,
last_xref_first_entry_offset: usize,
first_trailer_prev_range: std::ops::Range<usize>,
id_ranges: Vec<std::ops::Range<usize>>,
}
#[allow(clippy::too_many_arguments, clippy::type_complexity)]
fn do_write_pass<R: Read + Seek>(
plan: &LinearizationPlan,
renumber: &RenumberMap,
pdf: &mut Pdf<R>,
part1: &Part1Bytes,
catalog_new_ref: ObjectRef,
hint_stream_new_num: u32,
total_count: u32,
info_new_ref: Option<ObjectRef>,
_first_page_object_new_num: u32,
hint_stream_object: Option<&[u8]>,
structural_streams_filtered: bool,
source_trailer: &ObjectHandle,
objstm_layout: &ObjStmLayout,
relocation: &ObjStmRelocation,
options: &WriterOptions,
pass1_digest: bool,
mut id_writer: Option<crate::pdf_syntax::ReborrowableIdWriter>,
encrypt_ctx: Option<&crate::writer::EncryptionContext>,
mut encrypted_string_emitter: Option<&mut EncryptedStringEmitter>,
) -> Result<LinearizedPassOutput> {
let mut bytes: Vec<u8> = Vec::new();
let mut xref_offsets: BTreeMap<u32, usize> = BTreeMap::new();
let param_dict_obj_number = renumber.param_dict_ref().number;
let param_dict_absolute_offset = part1.obj1_offset;
bytes.extend_from_slice(&part1.bytes);
bytes.extend_from_slice(options.extra_header_text.as_bytes());
xref_offsets.insert(param_dict_obj_number, param_dict_absolute_offset);
let mut member_new: BTreeMap<u32, (u32, u32)> = BTreeMap::new();
for container in objstm_layout
.open_document
.iter()
.chain(&objstm_layout.part3)
.chain(&objstm_layout.part4)
{
for (idx, &(_orig, new_ref)) in container.members.iter().enumerate() {
member_new.insert(new_ref.number, (container.container_new_num, idx as u32));
}
}
let max_ostream_index: u64 = member_new
.values()
.map(|&(_, idx)| u64::from(idx))
.max()
.unwrap_or(0);
let trailer_map = |object_ref| {
renumber.new_for_original(object_ref).ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: trailer reference {object_ref} has no renumber entry"
))
})
}; let canonical_entries =
canonical_linearization_trailer_entries(source_trailer, &trailer_map, &plan.removed_refs)?;
let mut first_page_xref_patch: Option<FirstPageXrefPatch> = None;
let mut part1_classic_xref_offset: usize = 0;
let mut part1_xref_patch: Option<Part1XrefPatch> = None;
let mut id_ranges: Vec<std::ops::Range<usize>> = Vec::new();
let first_trailer_prev_range = if objstm_layout.is_empty() {
let first_page_count = first_page_xref_object_count(total_count, param_dict_obj_number)?;
let section_start = bytes.len();
let (p1_xref_offset, range, patch) = write_part1_xref_and_trailer(
&mut bytes,
param_dict_obj_number,
total_count,
first_page_count,
catalog_new_ref,
info_new_ref,
source_trailer,
&canonical_entries,
&trailer_map,
&plan.removed_refs,
id_writer.as_deref_mut(),
encrypt_ctx,
)?; part1_classic_xref_offset = p1_xref_offset;
part1_xref_patch = Some(patch);
id_ranges.push(section_start..bytes.len());
range
} else {
let patch = write_first_page_xref_stream(
&mut bytes,
relocation,
total_count,
catalog_new_ref,
info_new_ref,
source_trailer,
&canonical_entries,
max_ostream_index,
structural_streams_filtered,
encrypt_ctx.map(|ctx| ctx.encrypt_ref),
)?;
first_page_xref_patch = Some(patch);
0..0
};
let mut catalog_emitted_early = false;
if let Some(catalog_orig) = plan.root_ref {
debug_assert!(
!objstm_layout
.member_to_container
.contains_key(&catalog_orig),
"planner invariant: /Catalog is never an ObjStm member"
);
let offset = append_body_object_for_ref(
&mut bytes,
pdf,
catalog_new_ref,
catalog_orig,
options,
encrypt_ctx,
encrypted_string_emitter.as_deref_mut(),
renumber,
&plan.removed_refs,
&plan.content_normalize_refs,
)?; xref_offsets.insert(catalog_new_ref.number, offset);
report_progress_event(options)?;
catalog_emitted_early = true;
}
for original_ref in &plan.part4_open_document_plain {
if objstm_layout.member_to_container.contains_key(original_ref) {
continue;
}
if catalog_emitted_early && plan.root_ref == Some(*original_ref) {
continue;
}
let Some(new_ref) = renumber.new_for_original(*original_ref) else {
return Err(crate::Error::Unsupported(
"part4_open_document_plain ref missing from renumber map".into(),
));
};
let offset = append_body_object_for_ref(
&mut bytes,
pdf,
new_ref,
*original_ref,
options,
encrypt_ctx,
encrypted_string_emitter.as_deref_mut(),
renumber,
&plan.removed_refs,
&plan.content_normalize_refs,
)?; xref_offsets.insert(new_ref.number, offset);
report_progress_event(options)?;
}
for container in &objstm_layout.open_document {
let offset = append_objstm_container_object(
&mut bytes,
container,
renumber,
pdf,
&plan.removed_refs,
options,
encrypt_ctx,
)?; xref_offsets.insert(container.container_new_num, offset);
for _ in &container.members {
if pass1_digest {
decrement_progress_event(options)?;
}
report_progress_event(options)?;
if pass1_digest {
report_progress_event(options)?;
}
}
}
if let Some(ctx) = encrypt_ctx {
let offset = bytes.len();
bytes.extend_from_slice(
format!(
"{} {} obj\n",
ctx.encrypt_ref.number, ctx.encrypt_ref.generation
)
.as_bytes(),
);
crate::writer::encrypted_strings::write_encryption_dictionary_handle(
&mut bytes,
&ctx.encrypt_dict,
)?; bytes.extend_from_slice(b"\nendobj\n");
xref_offsets.insert(ctx.encrypt_ref.number, offset);
}
let hint_stream_offset = bytes.len();
if let Some(hint_stream_object) = hint_stream_object {
bytes.extend_from_slice(hint_stream_object);
xref_offsets.insert(hint_stream_new_num, hint_stream_offset);
}
let hint_stream_obj_total_len = bytes.len() - hint_stream_offset;
enum FirstPageEmit<'a> {
Plain(ObjectRef),
Container(&'a ObjStmContainer),
}
let mut first_page_emits: Vec<(u32, FirstPageEmit<'_>)> = Vec::new();
for original_ref in plan
.part2_objects
.iter()
.chain(&plan.part3_objects)
.chain(&plan.part6_outline_objects)
{
if objstm_layout.member_to_container.contains_key(original_ref)
|| (catalog_emitted_early && plan.root_ref == Some(*original_ref))
{
continue;
}
let Some(new_ref) = renumber.new_for_original(*original_ref) else {
return Err(crate::Error::Unsupported(format!(
"first-page object {} has no renumber entry",
original_ref
)));
};
first_page_emits.push((new_ref.number, FirstPageEmit::Plain(*original_ref)));
}
for container in &objstm_layout.part3 {
first_page_emits.push((
container.container_new_num,
FirstPageEmit::Container(container),
));
}
first_page_emits.sort_by_key(|(number, _)| *number);
for (_, emit) in first_page_emits {
match emit {
FirstPageEmit::Plain(original_ref) => {
let new_ref = renumber
.new_for_original(original_ref)
.expect("first-page plain object renumber entry checked above");
let offset = append_body_object_for_ref(
&mut bytes,
pdf,
new_ref,
original_ref,
options,
encrypt_ctx,
encrypted_string_emitter.as_deref_mut(),
renumber,
&plan.removed_refs,
&plan.content_normalize_refs,
)?; xref_offsets.insert(new_ref.number, offset);
report_progress_event(options)?;
}
FirstPageEmit::Container(container) => {
let offset = append_objstm_container_object(
&mut bytes,
container,
renumber,
pdf,
&plan.removed_refs,
options,
encrypt_ctx,
)?; xref_offsets.insert(container.container_new_num, offset);
for _ in &container.members {
if pass1_digest {
decrement_progress_event(options)?;
}
report_progress_event(options)?;
if pass1_digest {
report_progress_event(options)?;
}
}
}
}
}
let end_of_first_page_offset = bytes.len();
enum Part4Emit<'a> {
Plain(ObjectRef),
Container(&'a ObjStmContainer),
}
let mut part4_emits: Vec<(u32, Part4Emit)> = Vec::new();
for original_ref in plan.part4_objects() {
if objstm_layout
.member_to_container
.contains_key(&original_ref)
{
continue;
}
if catalog_emitted_early && plan.root_ref == Some(original_ref) {
continue;
}
let Some(new_ref) = renumber.new_for_original(original_ref) else {
return Err(crate::Error::Unsupported(format!(
"part4 object {original_ref} has no renumber entry"
)));
};
part4_emits.push((new_ref.number, Part4Emit::Plain(original_ref)));
}
for container in &objstm_layout.part4 {
part4_emits.push((container.container_new_num, Part4Emit::Container(container)));
}
part4_emits.sort_by_key(|(number, _)| *number);
for (_, emit) in &part4_emits {
match emit {
Part4Emit::Plain(original_ref) => {
let new_ref = renumber
.new_for_original(*original_ref)
.expect("part4 plain object renumber entry checked above");
let offset = append_body_object_for_ref(
&mut bytes,
pdf,
new_ref,
*original_ref,
options,
encrypt_ctx,
encrypted_string_emitter.as_deref_mut(),
renumber,
&plan.removed_refs,
&plan.content_normalize_refs,
)?; xref_offsets.insert(new_ref.number, offset);
report_progress_event(options)?;
}
Part4Emit::Container(container) => {
let offset = append_objstm_container_object(
&mut bytes,
container,
renumber,
pdf,
&plan.removed_refs,
options,
encrypt_ctx,
)?; xref_offsets.insert(container.container_new_num, offset);
for _ in &container.members {
if pass1_digest {
decrement_progress_event(options)?;
}
report_progress_event(options)?;
if pass1_digest {
report_progress_event(options)?;
}
}
}
}
}
let (last_xref_offset, last_xref_first_entry_offset) = if objstm_layout.is_empty() {
let main_section_start = bytes.len();
let result = write_main_xref_and_trailer(
&mut bytes,
&xref_offsets,
param_dict_obj_number,
part1_classic_xref_offset,
source_trailer,
&trailer_map,
&plan.removed_refs,
id_writer,
)?; id_ranges.push(main_section_start..bytes.len());
let patch = part1_xref_patch
.as_ref()
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: classic path produced no Part-1 xref patch \
(internal invariant violated)"
.to_string(),
)
})?;
if pass1_digest {
let zero_offsets: BTreeMap<u32, usize> = (patch.start_num
..patch.start_num + patch.count)
.map(|n| (n, 0usize))
.collect();
patch_part1_xref(&mut bytes, patch, &zero_offsets)?;
} else {
patch_part1_xref(&mut bytes, patch, &xref_offsets)?;
}
result
} else {
let patch = first_page_xref_patch.as_ref().ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: ObjStm path produced no first-page xref patch \
(internal invariant violated)"
.to_string(),
)
})?;
let first_page_obj_offset = patch.region.start;
debug_assert!(
first_page_obj_offset < end_of_first_page_offset,
"first-page xref stream offset ({first_page_obj_offset}) must be before \
/E ({end_of_first_page_offset}) — linearization boundary violated"
);
let result = write_main_xref_stream_and_trailer(
&mut bytes,
&xref_offsets,
&member_new,
relocation,
total_count,
source_trailer,
first_page_obj_offset,
max_ostream_index,
pass1_digest,
structural_streams_filtered,
)?; let (main_xref_offset, main_first_entry_offset, main_id_range) = result;
if let Some(id_range) = main_id_range {
id_ranges.push(id_range);
}
let first_page_id_range = patch_first_page_xref(
&mut bytes,
patch,
&xref_offsets,
&member_new,
main_xref_offset,
hint_stream_obj_total_len,
pass1_digest,
)?; if let Some(id_range) = first_page_id_range {
id_ranges.push(id_range);
}
(main_xref_offset, main_first_entry_offset)
};
Ok(LinearizedPassOutput {
bytes,
xref_offsets,
first_page_xref_offset: first_page_xref_patch
.as_ref()
.map(|patch| patch.region.start),
hint_stream_offset,
hint_stream_obj_total_len,
end_of_first_page_offset,
last_xref_offset,
last_xref_first_entry_offset,
first_trailer_prev_range,
id_ranges,
})
}
fn compute_byte_lengths(
xref_offsets: &BTreeMap<u32, usize>,
last_xref_offset: usize,
hint_stream_new_num: u32,
param_dict_new_num: u32,
) -> BTreeMap<u32, usize> {
let mut sorted: Vec<(usize, u32)> = xref_offsets
.iter()
.filter(|(&num, _)| num != param_dict_new_num)
.map(|(&num, &off)| (off, num))
.collect();
sorted.sort_unstable();
let mut lengths: BTreeMap<u32, usize> = BTreeMap::new();
for (idx, &(off, num)) in sorted.iter().enumerate() {
if num == hint_stream_new_num {
continue;
}
let next_off = if idx + 1 < sorted.len() {
sorted[idx + 1].0
} else {
last_xref_offset
};
lengths.insert(num, next_off.saturating_sub(off));
}
lengths
}
fn second_half_container_anchors(
plan: &LinearizationPlan,
part4_batches: &[RoutedObjStmBatch],
) -> Vec<SecondHalfContainerAnchor> {
let member_set: BTreeSet<ObjectRef> = part4_batches
.iter()
.flat_map(|batch| batch.members.iter().copied())
.collect();
let mut plain_ranked: Vec<(ObjectRef, (u8, usize, u8, u32))> = Vec::new();
for (i, privates) in plan.per_page_private_objects.iter().enumerate().skip(1) {
let page_ref = plan.page_hints.get(i).map(|hint| hint.page_ref);
for &r in privates {
if !member_set.contains(&r) {
let page_head_rank = u8::from(Some(r) != page_ref);
plain_ranked.push((r, (0, i, page_head_rank, r.number)));
}
}
}
for &r in &plan.part4_other_pages_shared {
if !member_set.contains(&r) {
plain_ranked.push((r, (1, 0, 0, r.number)));
}
}
for &r in &plan.part4_rest {
if !member_set.contains(&r) {
plain_ranked.push((r, (2, 0, 0, r.number)));
}
}
let part9_pages: BTreeSet<ObjectRef> = plan
.optimization
.as_ref()
.map(|optimization| optimization.objects_for_root_key(b"Pages"))
.filter(|pages| !pages.is_empty())
.unwrap_or_else(|| plan.pages_tree_ref.into_iter().collect());
let page_private_sets: Vec<BTreeSet<ObjectRef>> = plan
.per_page_private_objects
.iter()
.map(|v| v.iter().copied().collect())
.collect();
part4_batches
.iter()
.map(|batch| {
if batch.members.is_empty() {
return SecondHalfContainerAnchor::AfterLast; }
let object_number = batch.source_container_number.unwrap_or(u32::MAX);
let batch_rank: (u8, usize, u8, u32) = match batch.route {
ContainerPart::OtherPagePrivate => {
let owner = (1..page_private_sets.len())
.find(|&i| {
batch
.members
.iter()
.any(|m| page_private_sets[i].contains(m))
})
.expect("Part-7 ObjStm route must have one non-first-page owner");
(0, owner, 1, object_number)
}
ContainerPart::OtherPageShared => (1, 0, 0, object_number),
ContainerPart::Rest
if batch.source_container_number.is_some()
&& batch.members.iter().any(|m| part9_pages.contains(m)) =>
{
(2, 0, 0, 0)
}
ContainerPart::Rest => (2, 0, 0, object_number),
ContainerPart::OpenDocument
| ContainerPart::FirstPagePrivate
| ContainerPart::FirstPageShared
| ContainerPart::FirstPageOutlines => {
unreachable!("first-half route in second-half ObjStm batches")
} };
let previous = plain_ranked
.iter()
.rfind(|(_, rank)| *rank <= batch_rank)
.map(|(r, _)| *r);
match previous {
Some(r) => SecondHalfContainerAnchor::After(r),
None if plain_ranked.is_empty() => SecondHalfContainerAnchor::AfterLast,
None => SecondHalfContainerAnchor::BeforeFirst,
}
})
.collect()
}
fn preserved_source_container_number(
container: &ObjStmContainer,
source_container_by_member: &BTreeMap<ObjectRef, u32>,
) -> Result<u32> {
let source_container_number = container
.members
.first()
.and_then(|(original_ref, _)| source_container_by_member.get(original_ref).copied())
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"preserved ObjStm container {} has no source container",
container.container_new_num
))
})?;
if container.members.iter().any(|(original_ref, _)| {
source_container_by_member.get(original_ref).copied() != Some(source_container_number)
}) {
return Err(crate::Error::Unsupported(format!(
"preserved ObjStm container {} combines multiple source containers",
container.container_new_num
)));
}
Ok(source_container_number)
}
fn reject_multiple_generations(plan: &LinearizationPlan) -> Result<()> {
let mut previous_number = None;
for object_ref in plan.renumber_assigned_refs() {
if previous_number == Some(object_ref.number) {
return Err(crate::Error::Unsupported(
"QPDF cannot currently linearize files that contain multiple objects with the \
same object ID and different generations. If you see this error message, \
please file a bug report and attach the file if possible. As a workaround, \
first convert the file with qpdf without linearizing, and then linearize the \
result of that conversion."
.to_string(),
));
}
previous_number = Some(object_ref.number);
}
Ok(())
}
struct CatalogAdbeStatus {
has_adbe: bool,
}
fn prepare_linearization_catalog<R: Read + Seek>(pdf: &mut Pdf<R>) -> Result<bool> {
let Some(root_ref) = pdf.root_ref() else {
return Ok(false);
};
let root = pdf.get_object_handle(root_ref);
pdf.resolve(&root)?;
if root.try_as_dictionary()?.is_none() {
return Ok(false);
}
let extensions = root.try_get_key(b"/Extensions")?;
if extensions.try_as_dictionary()?.is_none() {
return Ok(false);
}
let mut changed = false;
let extensions = if extensions.is_indirect() {
let direct = extensions.shallow_copy()?;
root.replace_key(b"/Extensions", direct.clone())?;
changed = true;
direct
} else {
extensions
};
if extensions.try_has_key(b"/ADBE")? {
let mut adbe = extensions.try_get_key(b"/ADBE")?;
if adbe.is_indirect() {
adbe.make_direct(false)?;
let mut entries = extensions
.try_as_dictionary()?
.expect("checked is_some above");
entries.insert(b"/ADBE".to_vec(), adbe);
let extensions = ObjectHandle::dictionary(entries.into_iter().collect());
root.replace_key(b"/Extensions", extensions)?;
changed = true;
}
}
if changed {
pdf.mark_object_handle_dirty(&root)?;
}
Ok(changed)
}
fn finish_linearization_write<T>(result: Result<T>, restore: Result<()>) -> Result<T> {
match (result, restore) {
(Err(error), _) => Err(error),
(Ok(value), Ok(())) => Ok(value),
(Ok(_), Err(error)) => Err(error),
}
}
fn resolve_catalog_adbe_status<R: Read + Seek>(pdf: &mut Pdf<R>) -> Result<CatalogAdbeStatus> {
const NONE: CatalogAdbeStatus = CatalogAdbeStatus { has_adbe: false };
let Some(root_ref) = pdf.root_ref() else {
return Ok(NONE);
};
let catalog = pdf.get_object_handle(root_ref);
pdf.resolve(&catalog)?;
if catalog.try_as_dictionary()?.is_none() {
return Ok(NONE);
}
let extensions = catalog.try_get_key(b"/Extensions")?;
if extensions.try_as_dictionary()?.is_none() {
return Ok(NONE);
}
Ok(CatalogAdbeStatus {
has_adbe: extensions.try_has_key(b"/ADBE")?,
})
}
#[cfg(test)]
pub(crate) fn write_linearized<R: Read + Seek>(
plan: &LinearizationPlan,
renumber: &RenumberMap,
pdf: &mut Pdf<R>,
options: &WriterOptions,
) -> Result<LinearizedDocument> {
Ok(write_linearized_impl(plan, renumber, pdf, options, None)?.0)
}
pub(crate) fn write_linearized_for_pdf_writer<R: Read + Seek>(
pdf: &mut Pdf<R>,
options: &WriterOptions,
pass1_path: Option<&Path>,
) -> Result<(LinearizedDocument, WriterResult)> {
let mut catalog_snapshot = crate::writer::snapshot_catalog_extensions(pdf)?;
let root_ref_before = pdf.root_ref();
let root_was_dirty_before = root_ref_before.is_some_and(|root_ref| pdf.is_dirty(root_ref));
let plan_result = (|| {
let catalog_prepared = prepare_linearization_catalog(pdf)?;
if catalog_prepared && !root_was_dirty_before {
if let Some(root_ref) = root_ref_before {
pdf.clear_dirty(root_ref);
}
}
let mode = if crate::writer::force_version_below_1_5(options) {
crate::writer::ObjectStreamMode::Disable
} else {
options.object_streams
};
let mut plan_options = options.clone();
plan_options.object_streams = mode;
let plan = LinearizationPlan::from_pdf_with_writer_options(pdf, &plan_options)?;
let generated_object_stream_count = if mode == crate::writer::ObjectStreamMode::Generate {
let compressible = crate::writer::object_streams::compressible_objgens_qpdf_plan(pdf)?;
crate::writer::object_streams::even_split_into_streams(&compressible.eligible).len()
} else {
0
};
crate::writer::configure_progress_for_pdf(
pdf,
options,
generated_object_stream_count,
true,
)?;
let renumber = RenumberMap::from_plan(&plan);
Ok((plan, renumber))
})();
crate::writer::record_catalog_snapshot_dirty_baseline(pdf, &mut catalog_snapshot);
let result = plan_result.and_then(|(plan, renumber)| {
write_linearized_impl(&plan, &renumber, pdf, options, pass1_path)
});
let restore = crate::writer::restore_catalog_extensions(pdf, catalog_snapshot);
finish_linearization_write(result, restore)
}
fn write_pass1_stdio_body(
writer: &mut dyn Write,
mut body: &[u8],
pass1_path: &Path,
) -> Result<()> {
let mut buffered = StdioBuffer::new(writer);
while !body.is_empty() {
match buffered.write(body) {
Ok(0) => {
return Err(crate::Error::file_io(
"write",
pass1_path,
std::io::Error::new(
std::io::ErrorKind::WriteZero,
"failed to write buffered data",
),
));
}
Ok(written) => body = &body[written..],
Err(source) => {
return Err(crate::Error::file_io("write", pass1_path, source));
}
}
}
match buffered.flush() {
Err(source) if source.raw_os_error() == Some(EBADF_ERRNO) => Err(crate::Error::Internal(
"linearization pass1: Pl_StdioFile::finish: stream already closed".to_string(),
)),
Ok(()) | Err(_) => Ok(()),
}
}
fn write_pass1_debug_comments(writer: &mut dyn Write, comments: &[u8]) {
debug_assert!(comments.len() < 4096);
let mut buffered = StdioBuffer::new(writer);
let _ = buffered.write(comments);
}
fn validate_per_page_private_objects(plan: &LinearizationPlan) -> Result<()> {
if plan.per_page_private_objects.len() != plan.page_hints.len() {
return Err(crate::Error::Unsupported(format!(
"linearization writer: per_page_private_objects length ({}) does not \
match page_hints length ({}) — plan invariant violated",
plan.per_page_private_objects.len(),
plan.page_hints.len()
)));
}
Ok(())
}
fn write_linearized_impl<R: Read + Seek>(
plan: &LinearizationPlan,
renumber: &RenumberMap,
pdf: &mut Pdf<R>,
options: &WriterOptions,
pass1_path: Option<&Path>,
) -> Result<(LinearizedDocument, WriterResult)> {
let deterministic_id = crate::writer::uses_deterministic_id(options);
let source_trailer_handle = pdf.trailer().shallow_copy()?;
let source_id0 = crate::writer::source_permanent_id_handle(&source_trailer_handle);
let (det_id_source_id0, det_id_info_suffix): (Option<Vec<u8>>, Vec<u8>) = if deterministic_id {
let suffix = crate::writer::deterministic_id_info_suffix(pdf);
(source_id0.clone(), suffix)
} else {
(None, Vec::new())
};
let pass1_id = linearization_pass1_id(source_id0.as_deref());
let finalized_id = finalize_linearized_id(
options,
source_id0.as_deref(),
det_id_source_id0.as_deref(),
options.copy_encryption.as_ref(),
);
let id0: Vec<u8> = finalized_id
.as_array()
.and_then(|values| values.first().and_then(ObjectHandle::as_string))
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: finalize_linearized_id did not return a \
well-formed /ID array"
.to_string(),
)
})?; source_trailer_handle.replace_key(b"/ID", finalized_id)?;
let pass1_source_trailer = source_trailer_handle.shallow_copy()?;
pass1_source_trailer.replace_key(b"/ID", pass1_id)?;
if options.deterministic_id && (options.encrypt.is_some() || options.copy_encryption.is_some())
{
return Err(crate::writer::deterministic_id_encryption_error(options));
}
crate::optimization::Optimization::prepare_for_linearized_write(pdf)?;
let effective_object_stream_mode = if crate::writer::force_version_below_1_5(options) {
crate::writer::ObjectStreamMode::Disable
} else {
options.object_streams
};
let must_rebuild_plan = plan.object_stream_mode != effective_object_stream_mode;
let rebuilt = if must_rebuild_plan {
let rebuilt_plan =
LinearizationPlan::from_pdf_with_object_stream_mode(pdf, effective_object_stream_mode)?;
let rebuilt_renumber = RenumberMap::from_plan(&rebuilt_plan);
Some((rebuilt_plan, rebuilt_renumber))
} else {
None
};
let must_normalize_options = options.object_streams != effective_object_stream_mode;
let normalized_options = if must_normalize_options {
Some(WriterOptions {
object_streams: effective_object_stream_mode,
..options.clone()
})
} else {
None
};
let (plan, renumber) = match rebuilt.as_ref() {
Some((rebuilt_plan, rebuilt_renumber)) => (rebuilt_plan, rebuilt_renumber),
None => (plan, renumber),
};
let options = normalized_options.as_ref().unwrap_or(options);
reject_multiple_generations(plan)?;
let resolved_batch_plan = ObjStmLayout::resolve_batches(plan, pdf, options)?;
let emits_object_streams = !resolved_batch_plan.open_document_batches.is_empty()
|| !resolved_batch_plan.part3_batches.is_empty()
|| !resolved_batch_plan.part4_batches.is_empty();
let mut local_renumber = renumber.clone();
let second_half_anchors =
second_half_container_anchors(plan, &resolved_batch_plan.part4_batches);
let part4_members: Vec<Vec<ObjectRef>> = resolved_batch_plan
.part4_batches
.iter()
.map(|batch| batch.members.clone())
.collect();
let part4_member_set: BTreeSet<ObjectRef> = part4_members.iter().flatten().copied().collect();
let part9_pages: BTreeSet<ObjectRef> = plan
.optimization
.as_ref()
.map(|optimization| optimization.objects_for_root_key(b"Pages"))
.filter(|pages| !pages.is_empty())
.unwrap_or_else(|| plan.pages_tree_ref.into_iter().collect());
let second_half_post_plain: BTreeSet<ObjectRef> =
if options.object_streams == crate::writer::ObjectStreamMode::Preserve {
BTreeSet::new()
} else {
plan.part4_rest
.iter()
.chain(&plan.part9_outline_objects)
.copied()
.filter(|r| {
!part4_member_set.contains(r)
&& !part9_pages.contains(r)
&& Some(*r) != plan.info_ref
})
.collect()
};
let first_half_member_set: BTreeSet<ObjectRef> = resolved_batch_plan
.open_document_batches
.iter()
.chain(&resolved_batch_plan.part3_batches)
.flatten()
.copied()
.collect();
let mut first_half_post_plain: BTreeSet<ObjectRef> = plan
.part6_outline_objects
.iter()
.copied()
.filter(|object| !first_half_member_set.contains(object))
.collect();
if let Some(pre_objects) = plan
.optimization
.as_ref()
.and_then(|optimization| optimization.pre_optimization_object_refs())
{
first_half_post_plain.extend(
plan.part3_objects
.iter()
.chain(&plan.part4_open_document_plain)
.chain(&plan.part6_outline_objects)
.copied()
.filter(|object| {
!pre_objects.contains(object) && !first_half_member_set.contains(object)
}),
);
}
let open_document_source_container_numbers: Vec<Option<u32>> =
if options.object_streams == crate::writer::ObjectStreamMode::Preserve {
let source_container_by_member: BTreeMap<ObjectRef, u32> = pdf
.source_xref_entries()
.into_iter()
.filter_map(|(object_ref, entry)| match entry {
crate::XrefEntry::Compressed { stream, .. } => Some((object_ref, stream)),
_ => None,
})
.collect();
resolved_batch_plan
.open_document_batches
.iter()
.map(|members| {
let source = members
.first()
.and_then(|member| source_container_by_member.get(member).copied())
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: preserved open-document ObjStm batch \
has no source container"
.to_string(),
)
})?; if members.iter().any(|member| {
source_container_by_member.get(member).copied() != Some(source)
}) {
return Err(crate::Error::Unsupported(
"linearization writer: preserved open-document ObjStm batch \
combines multiple source containers"
.to_string(),
));
}
Ok(Some(source))
})
.collect::<Result<Vec<_>>>()?
} else {
vec![None; resolved_batch_plan.open_document_batches.len()]
};
let relocation = if emits_object_streams {
local_renumber.place_objstm_members_per_half(
&resolved_batch_plan.open_document_batches,
&open_document_source_container_numbers,
&resolved_batch_plan.part3_batches,
&part4_members,
&second_half_anchors,
&second_half_post_plain,
&first_half_post_plain,
)
} else {
ObjStmRelocation::default()
};
let mut container_numbers = relocation.container_numbers.clone();
let encrypt_ctx: Option<crate::writer::EncryptionContext> =
if let Some(params) = options.encrypt.as_ref() {
debug_assert!(
!options.deterministic_id,
"deterministic_id && encrypting must have already been rejected \
by the guard above `write_linearized`'s /ID finalization — \
reaching here with deterministic_id set would derive the file \
encryption key from an all-zero /ID[0] placeholder"
);
let existing_max: u32 = local_renumber.len().try_into().map_err(|_| {
crate::Error::Unsupported(
"linearization writer: object count overflows u32 for /Encrypt slot \
reservation"
.to_string(),
)
})?; let metadata_ref = if params.encrypt_metadata {
None
} else {
crate::writer::resolve_metadata_stream_ref(pdf)?
};
let ctx_result = crate::writer::build_encryption_context(
options,
params,
existing_max,
metadata_ref,
&id0,
);
let mut ctx = ctx_result?;
ctx.encrypt_ref = local_renumber.reserve_encrypt_dict_slot();
for container_number in &mut container_numbers {
if *container_number >= ctx.encrypt_ref.number {
*container_number += 1;
}
}
Some(ctx)
} else if let Some(source) = options.copy_encryption.as_ref() {
let existing_max: u32 = local_renumber.len().try_into().map_err(|_| {
crate::Error::Unsupported(
"linearization writer: object count overflows u32 for /Encrypt slot \
reservation"
.to_string(),
)
})?;
let encrypt_metadata = source
.encrypt_dict
.try_get_key(b"/EncryptMetadata")?
.as_boolean()
.unwrap_or(true);
let metadata_ref = if encrypt_metadata {
None
} else {
crate::writer::resolve_metadata_stream_ref(pdf)?
};
let mut ctx = crate::writer::build_copy_encryption_context(
source,
options,
existing_max,
metadata_ref,
)?;
ctx.encrypt_ref = local_renumber.reserve_encrypt_dict_slot();
for container_number in &mut container_numbers {
if *container_number >= ctx.encrypt_ref.number {
*container_number += 1;
}
}
Some(ctx)
} else {
None
};
let mut encrypted_string_emitter = encrypt_ctx
.as_ref()
.map(EncryptedStringEmitter::from_context);
let hint_stream_aes_iv: [u8; 16] = match &encrypt_ctx {
Some(ctx) if ctx.static_aes_iv => crate::pipeline::aes::static_initialization_vector(),
Some(ctx) if crate::writer::cipher_needs_aes_iv(ctx.cipher) => {
let mut iv = [0u8; 16];
getrandom::fill(&mut iv).map_err(|e| {
crate::Error::Unsupported(format!(
"OS CSPRNG (getrandom) unavailable for AES IV generation: {e}"
))
})?;
iv
}
_ => [0u8; 16],
};
let renumber: &RenumberMap = &local_renumber;
let source_ver = pdf.version().to_string();
let source_ext = pdf.adobe_extension_level()?.unwrap_or(0);
let (eff_version, eff_ext) =
effective_pdf_version_and_ext(&source_ver, source_ext, options, true, emits_object_streams);
let adbe_status = resolve_catalog_adbe_status(pdf)?;
if eff_ext > 0 || adbe_status.has_adbe {
if eff_ext > 0 {
inject_adbe_extension(pdf, eff_version, eff_ext)?;
} else {
strip_adbe_extension(pdf, eff_version, eff_ext)?;
}
}
let part1 = Part1Bytes::build(plan, renumber, eff_version);
let part1_placeholders = part1.placeholders.clone();
let part1_dict_region = part1.dict_writable_region.clone();
let catalog_orig = plan.root_ref.ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: plan.root_ref is None — \
cannot determine catalog reference for the trailer"
.to_string(),
)
})?;
let catalog_new_ref: ObjectRef = renumber.new_for_original(catalog_orig).ok_or_else(|| {
crate::Error::Unsupported(format!(
"linearization writer: catalog {catalog_orig} is not in RenumberMap \
(planner / renumber inconsistency)"
))
})?;
let hint_stream_new_num: u32 = renumber.hint_stream_slot();
let objstm_layout =
ObjStmLayout::build_from_batches(&resolved_batch_plan, &container_numbers, renumber)?;
let container_shared_sort_key: std::collections::BTreeMap<u32, (u8, u32)> = match options
.object_streams
{
crate::writer::ObjectStreamMode::Preserve => {
let source_container_by_member: std::collections::BTreeMap<ObjectRef, u32> = pdf
.source_xref_entries()
.into_iter()
.filter_map(|(object_ref, offset)| match offset {
crate::XrefEntry::Compressed { stream, .. } => Some((object_ref, stream)),
_ => None,
})
.collect();
let mut keys = std::collections::BTreeMap::new();
for container in objstm_layout
.open_document
.iter()
.chain(&objstm_layout.part3)
.chain(&objstm_layout.part4)
{
let source_container_number =
preserved_source_container_number(container, &source_container_by_member)?;
keys.insert(container.container_new_num, (0, source_container_number));
}
keys
}
_ => {
use crate::linearization::plan::objstm_membership_linearized_with_eligibility;
let assigned = plan.renumber_assigned_refs();
let membership = objstm_membership_linearized_with_eligibility(
pdf,
&assigned,
plan.optimization
.as_ref()
.and_then(|optimization| optimization.generate_objstm_eligible()),
)?; let mut rank = std::collections::BTreeMap::new();
for (split_index, members) in membership.iter().enumerate() {
let first = *members
.first()
.expect("objstm_membership_linearized never yields an empty container");
if let Some(&(container_num, _)) = objstm_layout.member_to_container.get(&first) {
rank.insert(container_num, (1, split_index as u32));
}
}
rank
}
};
let total_count: u32 = renumber
.len()
.checked_add(1)
.and_then(|n| u32::try_from(n).ok())
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: /Size overflows u32 (too many objects / \
ObjStm containers)"
.to_string(),
)
})?;
let info_new_ref: Option<ObjectRef> = source_trailer_handle
.try_get_key(b"/Info")?
.object_ref()
.and_then(|orig| renumber.new_for_original(orig));
let first_page_object_new_num: u32 = {
let first_page_hint = plan.page_hints.first().ok_or_else(|| {
crate::Error::Unsupported(
"linearization plan has no page hints (empty document?)".to_string(),
)
})?;
renumber
.new_for_original(first_page_hint.page_ref)
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"first-page page_ref {} has no renumber entry",
first_page_hint.page_ref,
))
})?
.number
};
let second_half_container_nums: std::collections::BTreeSet<u32> = objstm_layout
.part4
.iter()
.map(|c| c.container_new_num)
.collect();
let open_document_container_nums: std::collections::BTreeSet<u32> = objstm_layout
.open_document
.iter()
.map(|c| c.container_new_num)
.collect();
let outlines = plan
.optimization
.as_ref()
.map(|optimization| optimization.objects_for_root_key(b"Outlines"))
.unwrap_or_default();
let outline_info = compute_outline_hint_info(&outlines, pdf, renumber, &objstm_layout)?;
let structural_streams_filtered =
matches!(effective_stream_policy(options), Some(CompressStreams::Yes));
let pass1_part1 = build_pass1_part1(&part1);
let pass1_output = do_write_pass(
plan,
renumber,
pdf,
&pass1_part1,
catalog_new_ref,
hint_stream_new_num,
total_count,
info_new_ref,
first_page_object_new_num,
None,
structural_streams_filtered,
&pass1_source_trailer,
&objstm_layout,
&relocation,
options,
true,
None,
encrypt_ctx.as_ref(),
encrypted_string_emitter.as_mut(),
)?;
let classic_det_id: Option<(Vec<u8>, [u8; 16])> = if deterministic_id {
let pass1_bytes = &pass1_output.bytes;
Some(crate::writer::compute_deterministic_id(
pass1_bytes,
pass1_bytes.len() - 1,
&det_id_info_suffix,
det_id_source_id0.as_deref(),
))
} else {
None
};
let xref_offsets = &pass1_output.xref_offsets;
let hint_stream_offset = pass1_output.hint_stream_offset;
let last_xref_offset = pass1_output.last_xref_offset;
let byte_lengths = compute_byte_lengths(
xref_offsets,
last_xref_offset,
hint_stream_new_num,
renumber.param_dict_ref().number,
);
let part3_plain_len: u64 = plan
.part3_objects
.iter()
.filter(|orig| !objstm_layout.member_to_container.contains_key(orig))
.map(|orig| {
renumber
.new_for_original(*orig)
.and_then(|new_ref| byte_lengths.get(&new_ref.number).copied())
.unwrap_or(0) as u64
})
.sum();
let part3_container_len: u64 = objstm_layout
.part3
.iter()
.map(|c| byte_lengths.get(&c.container_new_num).copied().unwrap_or(0) as u64)
.sum();
let part3_byte_len: u64 = part3_plain_len + part3_container_len;
validate_per_page_private_objects(plan)?;
let non_page_owned = crate::linearization::hint_page::non_page_owned_containers(
plan,
&objstm_layout.member_to_container,
);
let plain_byte_len = |orig: &ObjectRef| -> u64 {
renumber
.new_for_original(*orig)
.and_then(|new_ref| byte_lengths.get(&new_ref.number).copied())
.unwrap_or(0) as u64
};
let per_page_byte_lengths: Vec<u64> = plan
.per_page_private_objects
.iter()
.enumerate()
.map(|(page_idx, privates)| {
if page_idx == 0 {
let part2_len: u64 = privates.iter().map(plain_byte_len).sum();
let part6_plain_len: u64 = plan
.part6_outline_objects
.iter()
.filter(|orig| !objstm_layout.member_to_container.contains_key(*orig))
.map(plain_byte_len)
.sum();
part2_len + part3_byte_len + part6_plain_len
} else {
let mut len = 0u64;
let mut containers: std::collections::BTreeSet<u32> =
std::collections::BTreeSet::new();
for orig in privates {
match objstm_layout.member_to_container.get(orig) {
Some(&(container_num, _)) => {
if !non_page_owned.contains(&container_num) {
containers.insert(container_num);
}
}
None => len += plain_byte_len(orig),
}
}
len + containers
.iter()
.map(|c| byte_lengths.get(c).copied().unwrap_or(0) as u64)
.sum::<u64>()
}
})
.collect();
let mut po_table = PageOffsetHintTable::from_plan(
plan,
renumber,
&objstm_layout.member_to_container,
&container_shared_sort_key,
&second_half_container_nums,
&open_document_container_nums,
);
let mut so_table = SharedObjectHintTable::from_plan(
plan,
renumber,
&objstm_layout.member_to_container,
&second_half_container_nums,
&open_document_container_nums,
);
po_table.header.location_of_first_page = hint_stream_offset as u64;
let least_pl = per_page_byte_lengths.iter().copied().min().unwrap_or(0);
let max_pl = per_page_byte_lengths.iter().copied().max().unwrap_or(0);
let bits_delta_pl = bits_needed(max_pl.saturating_sub(least_pl));
po_table.header.least_page_length = least_pl;
po_table.header.bits_page_length_delta = bits_delta_pl;
po_table.header.least_content_length = least_pl;
po_table.header.bits_content_length_delta = bits_delta_pl;
for (entry, &bl) in po_table
.entries
.iter_mut()
.zip(per_page_byte_lengths.iter())
{
let delta = bl.saturating_sub(least_pl);
entry.page_length_minus_least = delta;
entry.content_stream_length = delta;
}
let folded_shared = plan.canonical_shared_hints(
&objstm_layout.member_to_container,
renumber,
&second_half_container_nums,
&open_document_container_nums,
);
let shared_section_lens: Vec<u64> = folded_shared
.iter()
.map(|h| -> Result<u64> {
if h.object_ref.generation == u16::MAX {
let len = byte_lengths
.get(&h.object_ref.number)
.copied()
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"shared hint container (new #{}) has no probed byte length",
h.object_ref.number
))
})?;
return Ok(len as u64);
}
let new_ref = renumber.new_for_original(h.object_ref).ok_or_else(|| {
crate::Error::Unsupported(format!(
"shared hint object {} has no renumber entry",
h.object_ref
))
})?;
let len = byte_lengths.get(&new_ref.number).copied().ok_or_else(|| {
crate::Error::Unsupported(format!(
"shared hint object {} (new #{}) has no probed byte length",
h.object_ref, new_ref.number
))
})?;
Ok(len as u64)
})
.collect::<Result<Vec<_>>>()?;
let least = shared_section_lens.iter().copied().min().unwrap_or(0);
let max = shared_section_lens.iter().copied().max().unwrap_or(0);
so_table.header.least_length = least;
so_table.header.bits_length_delta = bits_needed(max.saturating_sub(least));
let first_part8_lookup_num = so_table.header.first_object_number;
if first_part8_lookup_num != 0 {
let first_part8_off = xref_offsets
.get(&first_part8_lookup_num)
.copied()
.ok_or_else(|| {
crate::Error::Unsupported(format!(
"first Part-8 shared object (lookup #{first_part8_lookup_num}) \
has no probed offset"
))
})?;
so_table.header.location = first_part8_off as u64;
}
for (obj, &len) in so_table.objects.iter_mut().zip(&shared_section_lens) {
obj.length_minus_least = (len.saturating_sub(least)) as u32;
}
let outline_table = outline_info
.as_ref()
.map(|info| build_outline_hint_table(info, xref_offsets, &byte_lengths))
.transpose()?;
let new_hint_bytes = encode_hint_stream(&po_table, &so_table, outline_table.as_ref())?;
let new_hint_payload = if structural_streams_filtered {
new_hint_bytes.compressed
} else {
new_hint_bytes.uncompressed
};
let new_shared_s = new_hint_bytes.shared_section_offset_in_uncompressed;
let new_outline_o = new_hint_bytes.outline_section_offset_in_uncompressed;
let mut hint_stream_object = Vec::new();
append_hint_stream_object(
&mut hint_stream_object,
ObjectRef::new(hint_stream_new_num, 0),
&new_hint_payload,
new_shared_s,
new_outline_o,
structural_streams_filtered,
encrypt_ctx.as_ref(),
hint_stream_aes_iv,
)?;
let mut det_id_closure;
let id_writer: Option<crate::pdf_syntax::TrailerIdWriter> = match &classic_det_id {
Some((id0, id1)) => {
let id0 = id0.clone();
let id1 = *id1;
det_id_closure = move |out: &mut Vec<u8>| {
crate::writer::write_deterministic_id_array(out, &id0, &id1)
};
Some(&mut det_id_closure)
}
None => None,
};
let final_output = do_write_pass(
plan,
renumber,
pdf,
&part1,
catalog_new_ref,
hint_stream_new_num,
total_count,
info_new_ref,
first_page_object_new_num,
Some(&hint_stream_object),
structural_streams_filtered,
&source_trailer_handle,
&objstm_layout,
&relocation,
options,
false,
id_writer,
encrypt_ctx.as_ref(),
encrypted_string_emitter.as_mut(),
)?; let LinearizedPassOutput {
bytes: mut final_bytes,
xref_offsets: final_xref_offsets,
first_page_xref_offset: final_first_page_xref_offset,
hint_stream_offset: final_hint_stream_offset,
hint_stream_obj_total_len: final_hint_stream_obj_total_len,
end_of_first_page_offset: final_end_of_first_page_offset,
last_xref_offset: final_last_xref_keyword_offset,
last_xref_first_entry_offset: final_last_xref_first_entry_offset,
first_trailer_prev_range: final_first_trailer_prev_range,
id_ranges: final_id_ranges,
} = final_output;
if let (false, Some((id0, id1))) = (objstm_layout.is_empty(), &classic_det_id) {
patch_linearized_deterministic_id(&mut final_bytes, &final_id_ranges, id0, id1);
}
if let Some(pass1_path) = pass1_path {
let pass1_bytes = &pass1_output.bytes;
let pass1_hint_stream_offset = pass1_output.hint_stream_offset;
let pass1_main_xref_offset = pass1_output.last_xref_offset;
let second_xref_end = if objstm_layout.is_empty() {
0
} else {
let marker = b"startxref\n";
pass1_bytes
.windows(marker.len())
.rposition(|window| window == marker)
.ok_or_else(|| {
crate::Error::Unsupported(
"linearization writer: pass-1 xref-stream output has no trailing \
startxref marker (internal invariant violated)"
.to_string(),
)
})?
};
let debug_comments = format!(
"% hint_offset={pass1_hint_stream_offset}\n\
% hint_length={final_hint_stream_obj_total_len}\n\
% second_xref_offset={pass1_main_xref_offset}\n\
% second_xref_end={second_xref_end}\n"
);
let mut pass1_file = std::fs::File::create(pass1_path)
.map_err(|source| crate::Error::file_io("open", pass1_path, source))?;
write_pass1_stdio_body(&mut pass1_file, pass1_bytes, pass1_path)?;
write_pass1_debug_comments(&mut pass1_file, debug_comments.as_bytes());
}
let file_length = final_bytes.len();
let page_count = plan.page_hints.len() as u32;
let offsets = LinearizedOffsets {
file_length,
hint_stream_offset: final_hint_stream_offset,
hint_stream_length: final_hint_stream_obj_total_len,
first_page_object_new_num,
end_of_first_page_offset: final_end_of_first_page_offset,
last_xref_keyword_offset: final_last_xref_keyword_offset,
last_xref_offset: final_last_xref_first_entry_offset.saturating_sub(1),
page_count,
part1_placeholders,
xref_offsets: final_xref_offsets.clone(),
first_trailer_prev_range: final_first_trailer_prev_range,
dict_writable_region: part1_dict_region,
};
let mut old_to_new: BTreeMap<ObjectRef, ObjectRef> = renumber
.iter_in_layout_order()
.map(|(new_ref, old_ref)| (old_ref, new_ref))
.collect();
if options.object_streams == crate::writer::ObjectStreamMode::Preserve
&& !objstm_layout.is_empty()
{
let source_container_by_member: BTreeMap<ObjectRef, u32> = pdf
.source_xref_entries()
.into_iter()
.filter_map(|(object_ref, entry)| match entry {
crate::XrefEntry::Compressed { stream, .. } => Some((object_ref, stream)),
_ => None,
})
.collect();
for container in objstm_layout
.open_document
.iter()
.chain(&objstm_layout.part3)
.chain(&objstm_layout.part4)
{
let source_container_number =
preserved_source_container_number(container, &source_container_by_member)?;
old_to_new.insert(
ObjectRef::new(source_container_number, 0),
ObjectRef::new(container.container_new_num, 0),
);
}
}
let mut written_xref = final_xref_offsets
.iter()
.map(|(&number, &offset)| {
(
ObjectRef::new(number, 0),
crate::XrefEntry::Uncompressed {
offset: offset as u64,
},
)
})
.collect::<BTreeMap<_, _>>();
if !objstm_layout.is_empty() {
let first_xref_offset = final_first_page_xref_offset.ok_or_else(|| {
crate::Error::Unsupported(
"linearization result: missing first-page xref offset".to_string(),
)
})?;
written_xref.insert(
ObjectRef::new(relocation.first_xref_slot, 0),
crate::XrefEntry::Uncompressed {
offset: first_xref_offset as u64,
},
);
written_xref.insert(
ObjectRef::new(relocation.main_xref_slot, 0),
crate::XrefEntry::Uncompressed {
offset: final_last_xref_keyword_offset as u64,
},
);
for container in objstm_layout
.open_document
.iter()
.chain(&objstm_layout.part3)
.chain(&objstm_layout.part4)
{
for (index, &(_original, new_ref)) in container.members.iter().enumerate() {
written_xref.insert(
new_ref,
crate::XrefEntry::Compressed {
stream: container.container_new_num,
index: index as u32,
},
);
}
}
}
Ok((
LinearizedDocument {
bytes: final_bytes,
offsets,
},
WriterResult::new(old_to_new, written_xref),
))
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
use crate::writer::ProgressReporter;
fn one_page_pdf_with_direct_outlines() -> Vec<u8> {
let mut pdf = b"%PDF-1.4\n".to_vec();
let off1 = pdf.len();
pdf.extend_from_slice(
b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R /Outlines << /Count 0 >> >>\nendobj\n",
);
let off2 = pdf.len();
pdf.extend_from_slice(b"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n");
let off3 = pdf.len();
pdf.extend_from_slice(
b"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>\nendobj\n",
);
let xref = pdf.len();
pdf.extend_from_slice(
format!(
"xref\n0 4\n0000000000 65535 f \n{off1:010} 00000 n \n{off2:010} 00000 n \n{off3:010} 00000 n \ntrailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n"
)
.as_bytes(),
);
pdf
}
#[test]
fn linearized_writer_keeps_catalog_dirty_after_canonical_planning_mutation_and_progress_failure(
) {
let mut pdf =
Pdf::open(Cursor::new(one_page_pdf_with_direct_outlines())).expect("source parses");
let root_ref = pdf.root_ref().expect("Catalog present");
assert!(!pdf.is_dirty(root_ref), "fresh source must start clean");
let options = WriterOptions {
progress_reporter: Some(ProgressReporter::new(Box::new(|_| {
Err(crate::Error::System("test progress failure".to_owned()))
}))),
..WriterOptions::default()
};
let error = write_linearized_for_pdf_writer(&mut pdf, &options, None)
.expect_err("canonical progress reporter failure must abort writing");
assert!(
error.to_string().contains("test progress failure"),
"unexpected writer error: {error}"
);
let root = pdf.get_object_handle(root_ref);
pdf.resolve(&root).expect("Catalog resolves");
assert!(
root.try_get_key(b"/Outlines")
.expect("Outlines lookup")
.is_indirect(),
"planning must keep qpdf's direct /Outlines promotion"
);
assert!(
pdf.is_dirty(root_ref),
"a permanent planning mutation must keep the Catalog dirty after failure"
);
}
}