oxidize_pdf/writer/pdf_writer/

mod.rs

use crate::document::Document;
use crate::error::{PdfError, Result};
use crate::objects::{Dictionary, Object, ObjectId};
use crate::text::fonts::embedding::CjkFontType;
use crate::text::fonts::truetype::CmapSubtable;
use crate::writer::{ObjectStreamConfig, ObjectStreamWriter, XRefStreamWriter};
use chrono::{DateTime, Utc};
use std::collections::HashMap;
use std::io::{BufWriter, Write};
use std::path::Path;

/// Configuration for PDF writer
#[derive(Debug, Clone)]
pub struct WriterConfig {
    /// Use XRef streams instead of traditional XRef tables (PDF 1.5+)
    pub use_xref_streams: bool,
    /// Use Object Streams for compressing multiple objects together (PDF 1.5+)
    pub use_object_streams: bool,
    /// PDF version to write (default: 1.7)
    pub pdf_version: String,
    /// Enable compression for streams (default: true)
    pub compress_streams: bool,
    /// Enable incremental updates mode (ISO 32000-1 §7.5.6)
    pub incremental_update: bool,
}

impl Default for WriterConfig {
    fn default() -> Self {
        Self {
            use_xref_streams: false,
            use_object_streams: false,
            pdf_version: "1.7".to_string(),
            compress_streams: true,
            incremental_update: false,
        }
    }
}

impl WriterConfig {
    /// Create a modern PDF 1.5+ configuration with all compression features enabled
    pub fn modern() -> Self {
        Self {
            use_xref_streams: true,
            use_object_streams: true,
            pdf_version: "1.5".to_string(),
            compress_streams: true,
            incremental_update: false,
        }
    }

    /// Create a legacy PDF 1.4 configuration without modern compression
    pub fn legacy() -> Self {
        Self {
            use_xref_streams: false,
            use_object_streams: false,
            pdf_version: "1.4".to_string(),
            compress_streams: true,
            incremental_update: false,
        }
    }

    /// Create configuration for incremental updates (ISO 32000-1 §7.5.6)
    pub fn incremental() -> Self {
        Self {
            use_xref_streams: false,
            use_object_streams: false,
            pdf_version: "1.4".to_string(),
            compress_streams: true,
            incremental_update: true,
        }
    }
}

/// Escape the three characters that are meaningful inside a PDF literal
/// string (ISO 32000-1 §7.3.4.2): backslash introduces escape sequences
/// and MUST be doubled; parentheses delimit the string and MUST be
/// prefixed with a backslash when they appear in the payload.
///
/// Other control characters (CR, LF, HT, BS, FF) are legal inside a
/// literal string *unescaped*, so we leave them alone — the parser is
/// required to accept them verbatim per §7.3.4.2 Table 3. Octal
/// escapes are a valid alternative encoding but not required here.
///
/// Correct ordering is essential: `\` MUST be escaped first (otherwise
/// the `\` we insert to escape a `(` would itself get doubled). This
/// helper walks the input exactly once and emits the escaped form.
///
/// **Scope clarification (issue #240 follow-up):** this helper serves
/// only `Object::String` payloads (metadata, dict entries, array
/// elements). The show-text `(text) Tj` payloads inside content
/// streams take an independent path (`Op::ShowText` bytes are produced
/// by `text::encoding::escape_show_text_literal_bytes`, which DOES
/// escape the high byte range `0x80..=0xFF` as `\NNN` octal because
/// those payloads carry WinAnsi-encoded text whose bytes must survive
/// 7-bit-safe intermediaries). The two helpers solve different
/// problems and intentionally have different coverage; they are not
/// coordinated and one is not "downstream" of the other.
fn escape_pdf_string_bytes(input: &[u8]) -> Vec<u8> {
    let mut out = Vec::with_capacity(input.len());
    for &byte in input {
        match byte {
            b'\\' => out.extend_from_slice(b"\\\\"),
            b'(' => out.extend_from_slice(b"\\("),
            b')' => out.extend_from_slice(b"\\)"),
            other => out.push(other),
        }
    }
    out
}

pub struct PdfWriter<W: Write> {
    writer: W,
    xref_positions: HashMap<ObjectId, u64>,
    current_position: u64,
    next_object_id: u32,
    // Maps for tracking object IDs during writing
    catalog_id: Option<ObjectId>,
    pages_id: Option<ObjectId>,
    info_id: Option<ObjectId>,
    // Maps for tracking form fields and their widgets
    #[allow(dead_code)]
    field_widget_map: HashMap<String, Vec<ObjectId>>, // field name -> widget IDs
    #[allow(dead_code)]
    field_id_map: HashMap<String, ObjectId>, // field name -> field ID
    form_field_ids: Vec<ObjectId>, // form field IDs to add to page annotations
    page_ids: Vec<ObjectId>,       // page IDs for form field references
    // Configuration
    config: WriterConfig,
    // Characters used in document, bucketed by font name (issue #204).
    // The writer uses this to subset each custom font with only its
    // own characters — a single global set caused unused fonts to be
    // embedded with the active fonts' character coverage, doubling
    // emitted size when two fonts shared a family.
    document_used_chars_by_font: std::collections::HashMap<String, std::collections::HashSet<char>>,
    // Object stream buffering (when use_object_streams is enabled)
    buffered_objects: HashMap<ObjectId, Vec<u8>>,
    compressed_object_map: HashMap<ObjectId, (ObjectId, u32)>, // obj_id -> (stream_id, index)
    // Incremental update support (ISO 32000-1 §7.5.6)
    prev_xref_offset: Option<u64>,
    base_pdf_size: Option<u64>,
    // Encryption support
    encrypt_obj_id: Option<ObjectId>,
    file_id: Option<Vec<u8>>,
    encryption_state: Option<WriterEncryptionState>,
    pending_encrypt_dict: Option<Dictionary>,
    // FormManager field tracking:
    //  * `form_field_placeholder_map` translates the placeholder
    //    `ObjectReference` returned by `FormManager::add_text_field` et al.
    //    (those use a local counter unaware of writer-side allocation) into
    //    the real `ObjectId` chosen by `allocate_object_id`. Widgets created
    //    via `Page::add_form_widget_with_ref` store the placeholder in
    //    `Annotation::field_parent`; when the annotation dict is written we
    //    remap it through this table so `/Parent` points at the real field.
    //  * `form_manager_field_refs` is the ordered (alphabetical by field
    //    name) list of real refs; it's appended to `document.acro_form.fields`
    //    during `write_catalog` and is what ends up in
    //    `/AcroForm/Fields`.
    form_field_placeholder_map: HashMap<crate::objects::ObjectReference, ObjectId>,
    form_manager_field_refs: Vec<crate::objects::ObjectReference>,
}

/// Holds the encryption key and encryptor for encrypting objects during write
struct WriterEncryptionState {
    encryptor: crate::encryption::ObjectEncryptor,
}

impl<W: Write> PdfWriter<W> {
    pub fn new_with_writer(writer: W) -> Self {
        Self::with_config(writer, WriterConfig::default())
    }

    pub fn with_config(writer: W, config: WriterConfig) -> Self {
        Self {
            writer,
            xref_positions: HashMap::new(),
            current_position: 0,
            next_object_id: 1, // Start at 1 for sequential numbering
            catalog_id: None,
            pages_id: None,
            info_id: None,
            field_widget_map: HashMap::new(),
            field_id_map: HashMap::new(),
            form_field_ids: Vec::new(),
            page_ids: Vec::new(),
            config,
            document_used_chars_by_font: std::collections::HashMap::new(),
            buffered_objects: HashMap::new(),
            compressed_object_map: HashMap::new(),
            prev_xref_offset: None,
            base_pdf_size: None,
            encrypt_obj_id: None,
            file_id: None,
            encryption_state: None,
            pending_encrypt_dict: None,
            form_field_placeholder_map: HashMap::new(),
            form_manager_field_refs: Vec::new(),
        }
    }

    pub fn write_document(&mut self, document: &mut Document) -> Result<()> {
        // Store used characters for font subsetting
        if !document.used_characters_by_font.is_empty() {
            self.document_used_chars_by_font = document.used_characters_by_font.clone();
        }

        self.write_header()?;

        // Reserve object IDs for fixed objects (written in order)
        self.catalog_id = Some(self.allocate_object_id());
        self.pages_id = Some(self.allocate_object_id());
        self.info_id = Some(self.allocate_object_id());

        // Initialize encryption state BEFORE writing objects
        // (objects need to be encrypted as they are written)
        if let Some(ref encryption) = document.encryption {
            self.init_encryption(encryption)?;
        }

        // Write custom fonts first (so pages can reference them)
        let font_refs = self.write_fonts(document)?;

        // Pre-allocate object IDs for every field owned by the FormManager
        // BEFORE writing pages, so widget annotations on those pages can
        // emit `/Parent <real_id>` instead of pointing at the placeholder
        // refs returned by `FormManager::add_text_field`. This is the piece
        // that bridges the FormManager's local id counter and the writer's
        // global id allocator. See `form_field_placeholder_map` for details.
        self.preallocate_form_manager_fields(document)?;

        // Write pages (they contain widget annotations and font references)
        self.write_pages(document, &font_refs)?;

        // Write form fields (must be after pages so we can track widgets)
        self.write_form_fields(document)?;

        // Write catalog (must be after forms so AcroForm has correct field references)
        self.write_catalog(document)?;

        // Write document info
        self.write_info(document)?;

        // Write /Encrypt dict AFTER all objects (it must NOT be encrypted itself)
        self.write_encryption_dict()?;

        // Flush buffered objects as object streams (if enabled)
        if self.config.use_object_streams {
            self.flush_object_streams()?;
        }

        // Write xref table or stream
        let xref_position = self.current_position;
        if self.config.use_xref_streams {
            self.write_xref_stream()?;
        } else {
            self.write_xref()?;
        }

        // Write trailer (only for traditional xref)
        if !self.config.use_xref_streams {
            self.write_trailer(xref_position)?;
        }

        if let Ok(()) = self.writer.flush() {
            // Flush succeeded
        }
        Ok(())
    }

    /// Write an incremental update to an existing PDF (ISO 32000-1 §7.5.6)
    ///
    /// This appends new/modified objects to the end of an existing PDF file
    /// without modifying the original content. The base PDF is copied first,
    /// then new pages are ADDED to the end of the document.
    ///
    /// For REPLACING specific pages (e.g., form filling), use `write_incremental_with_page_replacement`.
    ///
    /// # Arguments
    ///
    /// * `base_pdf_path` - Path to the existing PDF file
    /// * `document` - Document containing NEW pages to add
    ///
    /// # Returns
    ///
    /// Returns Ok(()) if the incremental update was written successfully
    ///
    /// # Example - Adding Pages
    ///
    /// ```no_run
    /// use oxidize_pdf::{Document, Page, writer::{PdfWriter, WriterConfig}};
    /// use std::fs::File;
    /// use std::io::BufWriter;
    ///
    /// let mut doc = Document::new();
    /// doc.add_page(Page::a4()); // This will be added as a NEW page
    ///
    /// let file = File::create("output.pdf").unwrap();
    /// let writer = BufWriter::new(file);
    /// let config = WriterConfig::incremental();
    /// let mut pdf_writer = PdfWriter::with_config(writer, config);
    /// pdf_writer.write_incremental_update("base.pdf", &mut doc).unwrap();
    /// ```
    pub fn write_incremental_update(
        &mut self,
        base_pdf_path: impl AsRef<std::path::Path>,
        document: &mut Document,
    ) -> Result<()> {
        use std::io::{BufReader, Read, Seek, SeekFrom};

        // Step 1: Parse the base PDF to get catalog and page information
        let base_pdf_file = std::fs::File::open(base_pdf_path.as_ref())?;
        let mut pdf_reader = crate::parser::PdfReader::new(BufReader::new(base_pdf_file))?;

        // Get catalog from base PDF
        let base_catalog = pdf_reader.catalog()?;

        // Extract Pages reference from base catalog
        let (base_pages_id, base_pages_gen) = base_catalog
            .get("Pages")
            .and_then(|obj| {
                if let crate::parser::objects::PdfObject::Reference(id, gen) = obj {
                    Some((*id, *gen))
                } else {
                    None
                }
            })
            .ok_or_else(|| {
                crate::error::PdfError::InvalidStructure(
                    "Base PDF catalog missing /Pages reference".to_string(),
                )
            })?;

        // Get the pages dictionary from the base PDF using the reference
        let base_pages_obj = pdf_reader.get_object(base_pages_id, base_pages_gen)?;
        let base_pages_kids = if let crate::parser::objects::PdfObject::Dictionary(dict) =
            base_pages_obj
        {
            dict.get("Kids")
                .and_then(|obj| {
                    if let crate::parser::objects::PdfObject::Array(arr) = obj {
                        // Convert PdfObject::Reference to writer::Object::Reference
                        // PdfArray.0 gives access to the internal Vec<PdfObject>
                        Some(
                            arr.0
                                .iter()
                                .filter_map(|item| {
                                    if let crate::parser::objects::PdfObject::Reference(id, gen) =
                                        item
                                    {
                                        Some(crate::objects::Object::Reference(
                                            crate::objects::ObjectId::new(*id, *gen),
                                        ))
                                    } else {
                                        None
                                    }
                                })
                                .collect::<Vec<_>>(),
                        )
                    } else {
                        None
                    }
                })
                .unwrap_or_default()
        } else {
            Vec::new()
        };

        // Count existing pages
        let base_page_count = base_pages_kids.len();

        // Step 2: Copy the base PDF content
        let base_pdf = std::fs::File::open(base_pdf_path.as_ref())?;
        let mut base_reader = BufReader::new(base_pdf);

        // Find the startxref offset in the base PDF
        base_reader.seek(SeekFrom::End(-100))?;
        let mut end_buffer = vec![0u8; 100];
        let bytes_read = base_reader.read(&mut end_buffer)?;
        end_buffer.truncate(bytes_read);

        let end_str = String::from_utf8_lossy(&end_buffer);
        let prev_xref = if let Some(startxref_pos) = end_str.find("startxref") {
            let after_startxref = &end_str[startxref_pos + 9..];

            let number_str: String = after_startxref
                .chars()
                .skip_while(|c| c.is_whitespace())
                .take_while(|c| c.is_ascii_digit())
                .collect();

            number_str.parse::<u64>().map_err(|_| {
                crate::error::PdfError::InvalidStructure(
                    "Could not parse startxref offset".to_string(),
                )
            })?
        } else {
            return Err(crate::error::PdfError::InvalidStructure(
                "startxref not found in base PDF".to_string(),
            ));
        };

        // Copy entire base PDF
        base_reader.seek(SeekFrom::Start(0))?;
        let base_size = std::io::copy(&mut base_reader, &mut self.writer)? as u64;

        // Store base PDF info for trailer
        self.prev_xref_offset = Some(prev_xref);
        self.base_pdf_size = Some(base_size);
        self.current_position = base_size;

        // Step 3: Write new/modified objects only
        if !document.used_characters_by_font.is_empty() {
            self.document_used_chars_by_font = document.used_characters_by_font.clone();
        }

        // Allocate IDs for new objects
        self.catalog_id = Some(self.allocate_object_id());
        self.pages_id = Some(self.allocate_object_id());
        self.info_id = Some(self.allocate_object_id());

        // Write custom fonts first
        let font_refs = self.write_fonts(document)?;

        // Write NEW pages only (not rewriting all pages)
        self.write_pages(document, &font_refs)?;

        // Write form fields
        self.write_form_fields(document)?;

        // Step 4: Write modified catalog that references BOTH old and new pages
        let catalog_id = self.get_catalog_id()?;
        let new_pages_id = self.get_pages_id()?;

        let mut catalog = crate::objects::Dictionary::new();
        catalog.set("Type", crate::objects::Object::Name("Catalog".to_string()));
        catalog.set("Pages", crate::objects::Object::Reference(new_pages_id));

        // Note: For now, we only preserve the Pages reference.
        // Full catalog preservation (Outlines, AcroForm, etc.) would require
        // converting parser::PdfObject to writer::Object, which is a future enhancement.

        self.write_object(catalog_id, crate::objects::Object::Dictionary(catalog))?;

        // Step 5: Write new Pages tree that includes BOTH base pages and new pages
        let mut all_pages_kids = base_pages_kids;

        // Add references to new pages
        for page_id in &self.page_ids {
            all_pages_kids.push(crate::objects::Object::Reference(*page_id));
        }

        let mut pages_dict = crate::objects::Dictionary::new();
        pages_dict.set("Type", crate::objects::Object::Name("Pages".to_string()));
        pages_dict.set("Kids", crate::objects::Object::Array(all_pages_kids));
        pages_dict.set(
            "Count",
            crate::objects::Object::Integer((base_page_count + self.page_ids.len()) as i64),
        );

        self.write_object(new_pages_id, crate::objects::Object::Dictionary(pages_dict))?;

        // Write document info
        self.write_info(document)?;

        // Step 6: Write new XRef table with /Prev pointer
        let xref_position = self.current_position;
        self.write_xref()?;

        // Step 7: Write trailer with /Prev
        self.write_trailer(xref_position)?;

        self.writer.flush()?;
        Ok(())
    }

    /// Replaces pages in an existing PDF using incremental update structure (ISO 32000-1 §7.5.6).
    ///
    /// # Use Cases
    /// This API is ideal for:
    /// - **Dynamic page generation**: You have logic to generate complete pages from data
    /// - **Template variants**: Switching between multiple pre-generated page versions
    /// - **Page repair**: Regenerating corrupted or problematic pages from scratch
    ///
    /// # Manual Content Recreation Required
    /// **IMPORTANT**: This API requires you to **manually recreate** the entire page content.
    /// The replaced page will contain ONLY what you provide in `document.pages`.
    ///
    /// If you need to modify existing content (e.g., fill form fields on an existing page),
    /// you must recreate the base content AND add your modifications.
    ///
    /// # Example: Form Filling with Manual Recreation
    /// ```rust,no_run
    /// use oxidize_pdf::{Document, Page, text::Font, writer::{PdfWriter, WriterConfig}};
    /// use std::fs::File;
    /// use std::io::BufWriter;
    ///
    /// let mut filled_doc = Document::new();
    /// let mut page = Page::a4();
    ///
    /// // Step 1: Recreate the template content (REQUIRED - you must know this)
    /// page.text()
    ///     .set_font(Font::Helvetica, 12.0)
    ///     .at(50.0, 700.0)
    ///     .write("Name: _______________________________")?;
    ///
    /// // Step 2: Add your filled data at the appropriate position
    /// page.text()
    ///     .set_font(Font::Helvetica, 12.0)
    ///     .at(110.0, 700.0)
    ///     .write("John Smith")?;
    ///
    /// filled_doc.add_page(page);
    ///
    /// let file = File::create("filled.pdf")?;
    /// let writer = BufWriter::new(file);
    /// let mut pdf_writer = PdfWriter::with_config(writer, WriterConfig::incremental());
    ///
    /// pdf_writer.write_incremental_with_page_replacement("template.pdf", &mut filled_doc)?;
    /// # Ok::<(), Box<dyn std::error::Error>>(())
    /// ```
    ///
    /// # ISO Compliance
    /// This function implements ISO 32000-1 §7.5.6 incremental updates:
    /// - Preserves original PDF bytes (append-only)
    /// - Uses /Prev pointer in trailer
    /// - Maintains cross-reference chain
    /// - Compatible with digital signatures on base PDF
    ///
    /// # Future: Automatic Overlay API
    /// For automatic form filling (load + modify + save) without manual recreation,
    /// a future `write_incremental_with_overlay()` API is planned. This will require
    /// implementation of `Document::load()` and content overlay system.
    ///
    /// # Parameters
    /// - `base_pdf_path`: Path to the existing PDF to modify
    /// - `document`: Document containing replacement pages (first N pages will replace base pages 0..N-1)
    ///
    /// # Returns
    /// - `Ok(())` if incremental update was written successfully
    /// - `Err(PdfError)` if base PDF cannot be read, parsed, or structure is invalid
    pub fn write_incremental_with_page_replacement(
        &mut self,
        base_pdf_path: impl AsRef<std::path::Path>,
        document: &mut Document,
    ) -> Result<()> {
        use std::io::Cursor;

        // Step 1: Read the entire base PDF into memory (avoids double file open)
        let base_pdf_bytes = std::fs::read(base_pdf_path.as_ref())?;
        let base_size = base_pdf_bytes.len() as u64;

        // Step 2: Parse from memory to get page information
        let mut pdf_reader = crate::parser::PdfReader::new(Cursor::new(&base_pdf_bytes))?;

        let base_catalog = pdf_reader.catalog()?;

        let (base_pages_id, base_pages_gen) = base_catalog
            .get("Pages")
            .and_then(|obj| {
                if let crate::parser::objects::PdfObject::Reference(id, gen) = obj {
                    Some((*id, *gen))
                } else {
                    None
                }
            })
            .ok_or_else(|| {
                crate::error::PdfError::InvalidStructure(
                    "Base PDF catalog missing /Pages reference".to_string(),
                )
            })?;

        let base_pages_obj = pdf_reader.get_object(base_pages_id, base_pages_gen)?;
        let base_pages_kids = if let crate::parser::objects::PdfObject::Dictionary(dict) =
            base_pages_obj
        {
            dict.get("Kids")
                .and_then(|obj| {
                    if let crate::parser::objects::PdfObject::Array(arr) = obj {
                        Some(
                            arr.0
                                .iter()
                                .filter_map(|item| {
                                    if let crate::parser::objects::PdfObject::Reference(id, gen) =
                                        item
                                    {
                                        Some(crate::objects::Object::Reference(
                                            crate::objects::ObjectId::new(*id, *gen),
                                        ))
                                    } else {
                                        None
                                    }
                                })
                                .collect::<Vec<_>>(),
                        )
                    } else {
                        None
                    }
                })
                .unwrap_or_default()
        } else {
            Vec::new()
        };

        let base_page_count = base_pages_kids.len();

        // Step 3: Find startxref offset from the bytes
        let start_search = if base_size > 100 { base_size - 100 } else { 0 } as usize;
        let end_bytes = &base_pdf_bytes[start_search..];
        let end_str = String::from_utf8_lossy(end_bytes);

        let prev_xref = if let Some(startxref_pos) = end_str.find("startxref") {
            let after_startxref = &end_str[startxref_pos + 9..];
            let number_str: String = after_startxref
                .chars()
                .skip_while(|c| c.is_whitespace())
                .take_while(|c| c.is_ascii_digit())
                .collect();

            number_str.parse::<u64>().map_err(|_| {
                crate::error::PdfError::InvalidStructure(
                    "Could not parse startxref offset".to_string(),
                )
            })?
        } else {
            return Err(crate::error::PdfError::InvalidStructure(
                "startxref not found in base PDF".to_string(),
            ));
        };

        // Step 4: Copy base PDF bytes to output
        self.writer.write_all(&base_pdf_bytes)?;

        self.prev_xref_offset = Some(prev_xref);
        self.base_pdf_size = Some(base_size);
        self.current_position = base_size;

        // Step 3: Write replacement pages
        if !document.used_characters_by_font.is_empty() {
            self.document_used_chars_by_font = document.used_characters_by_font.clone();
        }

        self.catalog_id = Some(self.allocate_object_id());
        self.pages_id = Some(self.allocate_object_id());
        self.info_id = Some(self.allocate_object_id());

        let font_refs = self.write_fonts(document)?;
        self.write_pages(document, &font_refs)?;
        self.write_form_fields(document)?;

        // Step 4: Create Pages tree with REPLACEMENTS
        let catalog_id = self.get_catalog_id()?;
        let new_pages_id = self.get_pages_id()?;

        let mut catalog = crate::objects::Dictionary::new();
        catalog.set("Type", crate::objects::Object::Name("Catalog".to_string()));
        catalog.set("Pages", crate::objects::Object::Reference(new_pages_id));
        self.write_object(catalog_id, crate::objects::Object::Dictionary(catalog))?;

        // Build new Kids array: replace first N pages, keep rest from base
        let mut all_pages_kids = Vec::new();
        let replacement_count = document.pages.len();

        // Add replacement pages (these override base pages at same indices)
        for page_id in &self.page_ids {
            all_pages_kids.push(crate::objects::Object::Reference(*page_id));
        }

        // Add remaining base pages that weren't replaced
        if replacement_count < base_page_count {
            for i in replacement_count..base_page_count {
                if let Some(page_ref) = base_pages_kids.get(i) {
                    all_pages_kids.push(page_ref.clone());
                }
            }
        }

        let mut pages_dict = crate::objects::Dictionary::new();
        pages_dict.set("Type", crate::objects::Object::Name("Pages".to_string()));
        pages_dict.set(
            "Kids",
            crate::objects::Object::Array(all_pages_kids.clone()),
        );
        pages_dict.set(
            "Count",
            crate::objects::Object::Integer(all_pages_kids.len() as i64),
        );

        self.write_object(new_pages_id, crate::objects::Object::Dictionary(pages_dict))?;
        self.write_info(document)?;

        let xref_position = self.current_position;
        self.write_xref()?;
        self.write_trailer(xref_position)?;

        self.writer.flush()?;
        Ok(())
    }

    /// Overlays content onto existing PDF pages using incremental updates (PLANNED).
    ///
    /// **STATUS**: Not yet implemented. This API is planned for a future release.
    ///
    /// # What This Will Do
    /// When implemented, this function will allow you to:
    /// - Load an existing PDF
    /// - Modify specific elements (fill form fields, add annotations, watermarks)
    /// - Save incrementally without recreating entire pages
    ///
    /// # Difference from Page Replacement
    /// - **Page Replacement** (`write_incremental_with_page_replacement`): Replaces entire pages with manually recreated content
    /// - **Overlay** (this function): Modifies existing pages by adding/changing specific elements
    ///
    /// # Planned Usage (Future)
    /// ```rust,ignore
    /// // This code will work in a future release
    /// let mut pdf_writer = PdfWriter::with_config(writer, WriterConfig::incremental());
    ///
    /// let overlays = vec![
    ///     PageOverlay::new(0)
    ///         .add_text(110.0, 700.0, "John Smith")
    ///         .add_annotation(Annotation::text(200.0, 500.0, "Review this")),
    /// ];
    ///
    /// pdf_writer.write_incremental_with_overlay("form.pdf", overlays)?;
    /// ```
    ///
    /// # Implementation Requirements
    /// This function requires:
    /// 1. `Document::load()` - Load existing PDF into Document structure
    /// 2. `Page::from_parsed()` - Convert parsed pages to writable format
    /// 3. Content stream overlay system - Append to existing content streams
    /// 4. Resource merging - Combine new resources with existing ones
    ///
    /// Estimated implementation effort: 6-7 days
    ///
    /// # Current Workaround
    /// Until this is implemented, use `write_incremental_with_page_replacement()` with manual
    /// page recreation. See that function's documentation for examples.
    ///
    /// # Parameters
    /// - `base_pdf_path`: Path to the existing PDF to modify (future)
    /// - `overlays`: Content to overlay on existing pages (future)
    ///
    /// # Returns
    /// Currently always returns `PdfError::NotImplemented`
    pub fn write_incremental_with_overlay<P: AsRef<std::path::Path>>(
        &mut self,
        base_pdf_path: P,
        mut overlay_fn: impl FnMut(&mut crate::Page) -> Result<()>,
    ) -> Result<()> {
        use std::io::Cursor;

        // Step 1: Read the entire base PDF into memory
        let base_pdf_bytes = std::fs::read(base_pdf_path.as_ref())?;
        let base_size = base_pdf_bytes.len() as u64;

        // Step 2: Parse from memory to get page information
        let pdf_reader = crate::parser::PdfReader::new(Cursor::new(&base_pdf_bytes))?;
        let parsed_doc = crate::parser::PdfDocument::new(pdf_reader);

        // Get all pages from base PDF
        let page_count = parsed_doc.page_count()?;

        // Step 3: Find startxref offset from the bytes
        let start_search = if base_size > 100 { base_size - 100 } else { 0 } as usize;
        let end_bytes = &base_pdf_bytes[start_search..];
        let end_str = String::from_utf8_lossy(end_bytes);

        let prev_xref = if let Some(startxref_pos) = end_str.find("startxref") {
            let after_startxref = &end_str[startxref_pos + 9..];
            let number_str: String = after_startxref
                .chars()
                .skip_while(|c| c.is_whitespace())
                .take_while(|c| c.is_ascii_digit())
                .collect();

            number_str.parse::<u64>().map_err(|_| {
                crate::error::PdfError::InvalidStructure(
                    "Could not parse startxref offset".to_string(),
                )
            })?
        } else {
            return Err(crate::error::PdfError::InvalidStructure(
                "startxref not found in base PDF".to_string(),
            ));
        };

        // Step 5: Copy base PDF bytes to output
        self.writer.write_all(&base_pdf_bytes)?;

        self.prev_xref_offset = Some(prev_xref);
        self.base_pdf_size = Some(base_size);
        self.current_position = base_size;

        // Step 6: Build temporary document with overlaid pages
        let mut temp_doc = crate::Document::new();

        for page_idx in 0..page_count {
            // Convert parsed page to writable with content preservation
            let parsed_page = parsed_doc.get_page(page_idx)?;
            let mut writable_page =
                crate::Page::from_parsed_with_content(&parsed_page, &parsed_doc)?;

            // Apply overlay function
            overlay_fn(&mut writable_page)?;

            // Add to temporary document
            temp_doc.add_page(writable_page);
        }

        // Step 7: Write document with standard writer methods
        // This ensures consistent object numbering
        if !temp_doc.used_characters_by_font.is_empty() {
            self.document_used_chars_by_font = temp_doc.used_characters_by_font.clone();
        }

        self.catalog_id = Some(self.allocate_object_id());
        self.pages_id = Some(self.allocate_object_id());
        self.info_id = Some(self.allocate_object_id());

        let font_refs = self.write_fonts(&temp_doc)?;
        self.write_pages(&temp_doc, &font_refs)?;
        self.write_form_fields(&mut temp_doc)?;

        // Step 8: Create new catalog and pages tree
        let catalog_id = self.get_catalog_id()?;
        let new_pages_id = self.get_pages_id()?;

        let mut catalog = crate::objects::Dictionary::new();
        catalog.set("Type", crate::objects::Object::Name("Catalog".to_string()));
        catalog.set("Pages", crate::objects::Object::Reference(new_pages_id));
        self.write_object(catalog_id, crate::objects::Object::Dictionary(catalog))?;

        // Build new Kids array with ALL overlaid pages
        let mut all_pages_kids = Vec::new();
        for page_id in &self.page_ids {
            all_pages_kids.push(crate::objects::Object::Reference(*page_id));
        }

        let mut pages_dict = crate::objects::Dictionary::new();
        pages_dict.set("Type", crate::objects::Object::Name("Pages".to_string()));
        pages_dict.set(
            "Kids",
            crate::objects::Object::Array(all_pages_kids.clone()),
        );
        pages_dict.set(
            "Count",
            crate::objects::Object::Integer(all_pages_kids.len() as i64),
        );

        self.write_object(new_pages_id, crate::objects::Object::Dictionary(pages_dict))?;
        self.write_info(&temp_doc)?;

        let xref_position = self.current_position;
        self.write_xref()?;
        self.write_trailer(xref_position)?;

        self.writer.flush()?;
        Ok(())
    }

    fn write_header(&mut self) -> Result<()> {
        let header = format!("%PDF-{}\n", self.config.pdf_version);
        self.write_bytes(header.as_bytes())?;
        // Binary comment to ensure file is treated as binary
        self.write_bytes(&[b'%', 0xE2, 0xE3, 0xCF, 0xD3, b'\n'])?;
        Ok(())
    }

    /// Convert pdf_objects types to writer objects types
    /// This is a temporary bridge until type unification is complete
    fn convert_pdf_objects_dict_to_writer(
        &self,
        pdf_dict: &crate::pdf_objects::Dictionary,
    ) -> crate::objects::Dictionary {
        let mut writer_dict = crate::objects::Dictionary::new();

        for (key, value) in pdf_dict.iter() {
            let writer_obj = self.convert_pdf_object_to_writer(value);
            writer_dict.set(key.as_str(), writer_obj);
        }

        writer_dict
    }

    fn convert_pdf_object_to_writer(
        &self,
        obj: &crate::pdf_objects::Object,
    ) -> crate::objects::Object {
        use crate::objects::Object as WriterObj;
        use crate::pdf_objects::Object as PdfObj;

        match obj {
            PdfObj::Null => WriterObj::Null,
            PdfObj::Boolean(b) => WriterObj::Boolean(*b),
            PdfObj::Integer(i) => WriterObj::Integer(*i),
            PdfObj::Real(f) => WriterObj::Real(*f),
            PdfObj::String(s) => {
                WriterObj::String(String::from_utf8_lossy(s.as_bytes()).to_string())
            }
            PdfObj::Name(n) => WriterObj::Name(n.as_str().to_string()),
            PdfObj::Array(arr) => {
                let items: Vec<WriterObj> = arr
                    .iter()
                    .map(|item| self.convert_pdf_object_to_writer(item))
                    .collect();
                WriterObj::Array(items)
            }
            PdfObj::Dictionary(dict) => {
                WriterObj::Dictionary(self.convert_pdf_objects_dict_to_writer(dict))
            }
            PdfObj::Stream(stream) => {
                let dict = self.convert_pdf_objects_dict_to_writer(&stream.dict);
                WriterObj::Stream(dict, stream.data.clone())
            }
            PdfObj::Reference(id) => {
                WriterObj::Reference(crate::objects::ObjectId::new(id.number(), id.generation()))
            }
        }
    }

    fn write_catalog(&mut self, document: &mut Document) -> Result<()> {
        let catalog_id = self.get_catalog_id()?;
        let pages_id = self.get_pages_id()?;

        let mut catalog = Dictionary::new();
        catalog.set("Type", Object::Name("Catalog".to_string()));
        catalog.set("Pages", Object::Reference(pages_id));

        // Serialize fields owned by the FormManager (ISO 32000-1 §12.7.3).
        //
        // Before v2.5.6 this block did nothing: it bound `_form_manager`
        // but never read its `fields` map, so only fields appended manually
        // to `document.acro_form.fields` ever reached the output PDF. Any
        // field created via `FormManager::add_text_field` / `add_combo_box`
        // / etc. was silently dropped — exactly the gap the .NET wrapper
        // hit.
        //
        // Object IDs for these fields were pre-allocated in
        // `preallocate_form_manager_fields` (called before `write_pages`
        // so widget `/Parent` refs could resolve). Here we only have to:
        //   (a) write the field-body dict into each pre-allocated id, and
        //   (b) append those ids to `document.acro_form.fields` so the
        //       /AcroForm write block below emits
        //       `/AcroForm/Fields [N 0 R ...]`.
        //
        // Iteration follows the same deterministic order used at
        // pre-allocation time, so the order-vs-id pairing is stable.
        if let Some(form_manager) = &document.form_manager {
            if document.acro_form.is_none() {
                document.acro_form = Some(crate::forms::AcroForm::new());
            }

            // Write each field dict into its reserved id.
            // Surface a clean `PdfError` if the placeholder-ref → real-id
            // map is missing any entry — a "can't happen" breach of the
            // invariant established by `preallocate_form_manager_fields`,
            // which must run before this function.
            let mut sorted: Vec<(Dictionary, crate::objects::ObjectReference)> = Vec::new();
            for (name, form_field, placeholder) in form_manager.iter_fields_sorted() {
                let real_id = *self.form_field_placeholder_map.get(&placeholder).ok_or_else(
                    || {
                        PdfError::Internal(format!(
                            "AcroForm writer internal invariant broken: field '{name}' (placeholder {placeholder}) has no pre-allocated real object id — preallocate_form_manager_fields must run before write_catalog"
                        ))
                    },
                )?;
                sorted.push((form_field.field_dict.clone(), real_id));
            }
            for (field_dict, real_id) in sorted {
                self.write_object(real_id, Object::Dictionary(field_dict))?;
            }

            if let Some(acro) = document.acro_form.as_mut() {
                for r in &self.form_manager_field_refs {
                    if !acro.fields.contains(r) {
                        acro.fields.push(*r);
                    }
                }
            }
        }

        // Add AcroForm if present
        if let Some(acro_form) = &document.acro_form {
            // Reserve object ID for AcroForm
            let acro_form_id = self.allocate_object_id();

            // Write AcroForm object
            self.write_object(acro_form_id, Object::Dictionary(acro_form.to_dict()))?;

            // Reference it in catalog
            catalog.set("AcroForm", Object::Reference(acro_form_id));
        }

        // Add Outlines if present
        if let Some(outline_tree) = &document.outline {
            if !outline_tree.items.is_empty() {
                let outline_root_id = self.write_outline_tree(outline_tree)?;
                catalog.set("Outlines", Object::Reference(outline_root_id));
            }
        }

        // Add StructTreeRoot if present (Tagged PDF - ISO 32000-1 §14.8)
        if let Some(struct_tree) = &document.struct_tree {
            if !struct_tree.is_empty() {
                let struct_tree_root_id = self.write_struct_tree(struct_tree)?;
                catalog.set("StructTreeRoot", Object::Reference(struct_tree_root_id));
                // Mark as Tagged PDF
                catalog.set("MarkInfo", {
                    let mut mark_info = Dictionary::new();
                    mark_info.set("Marked", Object::Boolean(true));
                    Object::Dictionary(mark_info)
                });
            }
        }

        // Add XMP Metadata stream (ISO 32000-1 §14.3.2)
        // Generate XMP from document metadata and embed as stream
        let xmp_metadata = document.create_xmp_metadata();
        let xmp_packet = xmp_metadata.to_xmp_packet();
        let metadata_id = self.allocate_object_id();

        // Create metadata stream dictionary
        let mut metadata_dict = Dictionary::new();
        metadata_dict.set("Type", Object::Name("Metadata".to_string()));
        metadata_dict.set("Subtype", Object::Name("XML".to_string()));
        metadata_dict.set("Length", Object::Integer(xmp_packet.len() as i64));

        // Write XMP metadata stream
        self.write_object(
            metadata_id,
            Object::Stream(metadata_dict, xmp_packet.into_bytes()),
        )?;

        // Reference it in catalog
        catalog.set("Metadata", Object::Reference(metadata_id));

        // /OpenAction — ISO 32000-1 §7.7.2 Table 28
        if let Some(action) = &document.open_action {
            catalog.set("OpenAction", Object::Dictionary(action.to_dict()));
        }

        // /ViewerPreferences — ISO 32000-1 §7.7.2 Table 28, detailed in §12.2
        if let Some(prefs) = &document.viewer_preferences {
            catalog.set("ViewerPreferences", Object::Dictionary(prefs.to_dict()));
        }

        // /Names — ISO 32000-1 §7.7.4 Table 31 (Name Dictionary).
        // The /Dests sub-entry is the name tree for named destinations
        // (§12.3.2.3). Both the name tree and the Name Dictionary are
        // written as indirect objects.
        if let Some(named_dests) = &document.named_destinations {
            let dests_tree_id = self.allocate_object_id();
            self.write_object(dests_tree_id, Object::Dictionary(named_dests.to_dict()))?;

            let mut names_dict = Dictionary::new();
            names_dict.set("Dests", Object::Reference(dests_tree_id));
            let names_dict_id = self.allocate_object_id();
            self.write_object(names_dict_id, Object::Dictionary(names_dict))?;

            catalog.set("Names", Object::Reference(names_dict_id));
        }

        // /PageLabels — ISO 32000-1 §7.7.2 Table 28, §12.4.2.
        // The value is a number tree; we emit it as an indirect object so
        // large documents can grow without reshuffling the catalog.
        if let Some(page_labels) = &document.page_labels {
            let labels_id = self.allocate_object_id();
            self.write_object(labels_id, Object::Dictionary(page_labels.to_dict()))?;
            catalog.set("PageLabels", Object::Reference(labels_id));
        }

        self.write_object(catalog_id, Object::Dictionary(catalog))?;
        Ok(())
    }

    fn write_page_content(&mut self, content_id: ObjectId, page: &crate::page::Page) -> Result<()> {
        let mut page_copy = page.clone();
        let content = page_copy.generate_content()?;

        // Create stream with compression if enabled
        #[cfg(feature = "compression")]
        {
            use crate::objects::Stream;
            let mut stream = Stream::new(content);
            // Only compress if config allows it
            if self.config.compress_streams {
                stream.compress_flate()?;
            }

            self.write_object(
                content_id,
                Object::Stream(stream.dictionary().clone(), stream.data().to_vec()),
            )?;
        }

        #[cfg(not(feature = "compression"))]
        {
            let mut stream_dict = Dictionary::new();
            stream_dict.set("Length", Object::Integer(content.len() as i64));

            self.write_object(content_id, Object::Stream(stream_dict, content))?;
        }

        Ok(())
    }

    fn write_outline_tree(
        &mut self,
        outline_tree: &crate::structure::OutlineTree,
    ) -> Result<ObjectId> {
        // Create root outline dictionary
        let outline_root_id = self.allocate_object_id();

        let mut outline_root = Dictionary::new();
        outline_root.set("Type", Object::Name("Outlines".to_string()));

        if !outline_tree.items.is_empty() {
            // Reserve IDs for all outline items
            let mut item_ids = Vec::new();

            // Count all items and assign IDs
            fn count_items(items: &[crate::structure::OutlineItem]) -> usize {
                let mut count = items.len();
                for item in items {
                    count += count_items(&item.children);
                }
                count
            }

            let total_items = count_items(&outline_tree.items);

            // Reserve IDs for all items
            for _ in 0..total_items {
                item_ids.push(self.allocate_object_id());
            }

            let mut id_index = 0;

            // Write root items
            let first_id = item_ids[0];
            let last_id = item_ids[outline_tree.items.len() - 1];

            outline_root.set("First", Object::Reference(first_id));
            outline_root.set("Last", Object::Reference(last_id));

            // Visible count
            let visible_count = outline_tree.visible_count();
            outline_root.set("Count", Object::Integer(visible_count));

            // Write all items recursively
            let mut written_items = Vec::new();

            for (i, item) in outline_tree.items.iter().enumerate() {
                let item_id = item_ids[id_index];
                id_index += 1;

                let prev_id = if i > 0 { Some(item_ids[i - 1]) } else { None };
                let next_id = if i < outline_tree.items.len() - 1 {
                    Some(item_ids[i + 1])
                } else {
                    None
                };

                // Write this item and its children
                let children_ids = self.write_outline_item(
                    item,
                    item_id,
                    outline_root_id,
                    prev_id,
                    next_id,
                    &mut item_ids,
                    &mut id_index,
                )?;

                written_items.extend(children_ids);
            }
        }

        self.write_object(outline_root_id, Object::Dictionary(outline_root))?;
        Ok(outline_root_id)
    }

    #[allow(clippy::too_many_arguments)]
    fn write_outline_item(
        &mut self,
        item: &crate::structure::OutlineItem,
        item_id: ObjectId,
        parent_id: ObjectId,
        prev_id: Option<ObjectId>,
        next_id: Option<ObjectId>,
        all_ids: &mut Vec<ObjectId>,
        id_index: &mut usize,
    ) -> Result<Vec<ObjectId>> {
        let mut written_ids = vec![item_id];

        // Handle children if any
        let (first_child_id, last_child_id) = if !item.children.is_empty() {
            let first_idx = *id_index;
            let first_id = all_ids[first_idx];
            let last_idx = first_idx + item.children.len() - 1;
            let last_id = all_ids[last_idx];

            // Write children
            for (i, child) in item.children.iter().enumerate() {
                let child_id = all_ids[*id_index];
                *id_index += 1;

                let child_prev = if i > 0 {
                    Some(all_ids[first_idx + i - 1])
                } else {
                    None
                };
                let child_next = if i < item.children.len() - 1 {
                    Some(all_ids[first_idx + i + 1])
                } else {
                    None
                };

                let child_ids = self.write_outline_item(
                    child, child_id, item_id, // This item is the parent
                    child_prev, child_next, all_ids, id_index,
                )?;

                written_ids.extend(child_ids);
            }

            (Some(first_id), Some(last_id))
        } else {
            (None, None)
        };

        // Create item dictionary
        let item_dict = crate::structure::outline_item_to_dict(
            item,
            parent_id,
            first_child_id,
            last_child_id,
            prev_id,
            next_id,
        );

        self.write_object(item_id, Object::Dictionary(item_dict))?;

        Ok(written_ids)
    }

    /// Writes the structure tree for Tagged PDF (ISO 32000-1 §14.8)
    fn write_struct_tree(
        &mut self,
        struct_tree: &crate::structure::StructTree,
    ) -> Result<ObjectId> {
        // Allocate IDs for StructTreeRoot and all elements
        let struct_tree_root_id = self.allocate_object_id();
        let mut element_ids = Vec::new();
        for _ in 0..struct_tree.len() {
            element_ids.push(self.allocate_object_id());
        }

        // Build parent map: element_index -> parent_id
        let mut parent_map: std::collections::HashMap<usize, ObjectId> =
            std::collections::HashMap::new();

        // Root element's parent is StructTreeRoot
        if let Some(root_index) = struct_tree.root_index() {
            parent_map.insert(root_index, struct_tree_root_id);

            // Recursively map all children to their parents
            fn map_children_parents(
                tree: &crate::structure::StructTree,
                parent_index: usize,
                parent_id: ObjectId,
                element_ids: &[ObjectId],
                parent_map: &mut std::collections::HashMap<usize, ObjectId>,
            ) {
                if let Some(parent_elem) = tree.get(parent_index) {
                    for &child_index in &parent_elem.children {
                        parent_map.insert(child_index, parent_id);
                        map_children_parents(
                            tree,
                            child_index,
                            element_ids[child_index],
                            element_ids,
                            parent_map,
                        );
                    }
                }
            }

            map_children_parents(
                struct_tree,
                root_index,
                element_ids[root_index],
                &element_ids,
                &mut parent_map,
            );
        }

        // Write all structure elements with parent references
        for (index, element) in struct_tree.iter().enumerate() {
            let element_id = element_ids[index];
            let mut element_dict = Dictionary::new();

            element_dict.set("Type", Object::Name("StructElem".to_string()));
            element_dict.set("S", Object::Name(element.structure_type.as_pdf_name()));

            // Parent reference (ISO 32000-1 §14.7.2 - required)
            if let Some(&parent_id) = parent_map.get(&index) {
                element_dict.set("P", Object::Reference(parent_id));
            }

            // Element ID (optional)
            if let Some(ref id) = element.id {
                element_dict.set("ID", Object::String(id.clone()));
            }

            // Attributes
            if let Some(ref lang) = element.attributes.lang {
                element_dict.set("Lang", Object::String(lang.clone()));
            }
            if let Some(ref alt) = element.attributes.alt {
                element_dict.set("Alt", Object::String(alt.clone()));
            }
            if let Some(ref actual_text) = element.attributes.actual_text {
                element_dict.set("ActualText", Object::String(actual_text.clone()));
            }
            if let Some(ref title) = element.attributes.title {
                element_dict.set("T", Object::String(title.clone()));
            }
            if let Some(bbox) = element.attributes.bbox {
                element_dict.set(
                    "BBox",
                    Object::Array(vec![
                        Object::Real(bbox[0]),
                        Object::Real(bbox[1]),
                        Object::Real(bbox[2]),
                        Object::Real(bbox[3]),
                    ]),
                );
            }

            // Kids (children elements + marked content references)
            let mut kids = Vec::new();

            // Add child element references
            for &child_index in &element.children {
                kids.push(Object::Reference(element_ids[child_index]));
            }

            // Add marked content references (MCIDs)
            for mcid_ref in &element.mcids {
                let mut mcr = Dictionary::new();
                mcr.set("Type", Object::Name("MCR".to_string()));
                mcr.set("Pg", Object::Integer(mcid_ref.page_index as i64));
                mcr.set("MCID", Object::Integer(mcid_ref.mcid as i64));
                kids.push(Object::Dictionary(mcr));
            }

            if !kids.is_empty() {
                element_dict.set("K", Object::Array(kids));
            }

            self.write_object(element_id, Object::Dictionary(element_dict))?;
        }

        // Create StructTreeRoot dictionary
        let mut struct_tree_root = Dictionary::new();
        struct_tree_root.set("Type", Object::Name("StructTreeRoot".to_string()));

        // Add root element(s) as K entry
        if let Some(root_index) = struct_tree.root_index() {
            struct_tree_root.set("K", Object::Reference(element_ids[root_index]));
        }

        // Add RoleMap if not empty
        if !struct_tree.role_map.mappings().is_empty() {
            let mut role_map = Dictionary::new();
            for (custom_type, standard_type) in struct_tree.role_map.mappings() {
                role_map.set(
                    custom_type.as_str(),
                    Object::Name(standard_type.as_pdf_name().to_string()),
                );
            }
            struct_tree_root.set("RoleMap", Object::Dictionary(role_map));
        }

        self.write_object(struct_tree_root_id, Object::Dictionary(struct_tree_root))?;
        Ok(struct_tree_root_id)
    }

    /// Reserve an `ObjectId` for every field owned by `document.form_manager`
    /// and build the placeholder → real mapping used when widget annotations
    /// are serialised (see `Annotation::field_parent`).
    ///
    /// Called once from `write_document` before `write_pages`, so widget
    /// `/Parent` refs on pages resolve to real indirect objects. The field
    /// bodies themselves are written later, in `write_catalog`, reusing
    /// these pre-allocated IDs.
    ///
    /// Iteration order is deterministic (alphabetical by field name) via
    /// `FormManager::iter_fields_sorted` so object-ID allocation — and
    /// therefore the byte-for-byte output — is reproducible across builds.
    fn preallocate_form_manager_fields(&mut self, document: &Document) -> Result<()> {
        let Some(form_manager) = &document.form_manager else {
            return Ok(());
        };

        for (_name, _form_field, placeholder) in form_manager.iter_fields_sorted() {
            let real_id = self.allocate_object_id();
            self.form_field_placeholder_map.insert(placeholder, real_id);
            self.form_manager_field_refs.push(real_id);
        }
        Ok(())
    }

    fn write_form_fields(&mut self, document: &mut Document) -> Result<()> {
        // Add collected form field IDs to AcroForm
        if !self.form_field_ids.is_empty() {
            if let Some(acro_form) = &mut document.acro_form {
                // Clear any existing fields and add the ones we found
                acro_form.fields.clear();
                for field_id in &self.form_field_ids {
                    acro_form.add_field(*field_id);
                }

                // Ensure AcroForm has the right properties
                acro_form.need_appearances = true;
                if acro_form.da.is_none() {
                    acro_form.da = Some("/Helv 12 Tf 0 g".to_string());
                }
            }
        }
        Ok(())
    }

    fn write_info(&mut self, document: &Document) -> Result<()> {
        let info_id = self.get_info_id()?;
        let mut info_dict = Dictionary::new();

        if let Some(ref title) = document.metadata.title {
            info_dict.set("Title", Object::String(title.clone()));
        }
        if let Some(ref author) = document.metadata.author {
            info_dict.set("Author", Object::String(author.clone()));
        }
        if let Some(ref subject) = document.metadata.subject {
            info_dict.set("Subject", Object::String(subject.clone()));
        }
        if let Some(ref keywords) = document.metadata.keywords {
            info_dict.set("Keywords", Object::String(keywords.clone()));
        }
        if let Some(ref creator) = document.metadata.creator {
            info_dict.set("Creator", Object::String(creator.clone()));
        }
        if let Some(ref producer) = document.metadata.producer {
            info_dict.set("Producer", Object::String(producer.clone()));
        }

        // Add creation date
        if let Some(creation_date) = document.metadata.creation_date {
            let date_string = format_pdf_date(creation_date);
            info_dict.set("CreationDate", Object::String(date_string));
        }

        // Add modification date
        if let Some(mod_date) = document.metadata.modification_date {
            let date_string = format_pdf_date(mod_date);
            info_dict.set("ModDate", Object::String(date_string));
        }

        // Add PDF signature (anti-spoofing and licensing)
        // This is written AFTER user-configurable metadata so it cannot be overridden
        let edition = super::Edition::OpenSource;

        let signature = super::PdfSignature::new(document, edition);
        signature.write_to_info_dict(&mut info_dict);

        self.write_object(info_id, Object::Dictionary(info_dict))?;
        Ok(())
    }

    fn write_fonts(&mut self, document: &Document) -> Result<HashMap<String, ObjectId>> {
        let mut font_refs = HashMap::new();

        // Write custom fonts from the document. Fonts registered via
        // `add_font_from_bytes` but never referenced from any content
        // stream (i.e. never `set_font`'d on any page) are skipped —
        // embedding them waste space and was the direct cause of
        // issue #204 (two fonts in the same family both getting
        // subsetted with the active font's character set). The
        // per-font map is built during tracking by
        // `GraphicsContext::record_used_chars` / its `TextContext`
        // counterpart.
        for font_name in document.custom_font_names() {
            let has_usage = self
                .document_used_chars_by_font
                .get(&font_name)
                .map(|chars| !chars.is_empty())
                .unwrap_or(false);
            if !has_usage {
                continue;
            }
            if let Some(font) = document.get_custom_font(&font_name) {
                // For now, write all custom fonts as TrueType with Identity-H for Unicode support
                // The font from document is Arc<fonts::Font>, not text::font_manager::CustomFont
                let font_id = self.write_font_with_unicode_support(&font_name, &font)?;
                font_refs.insert(font_name.clone(), font_id);
            }
        }

        Ok(font_refs)
    }

    /// Write font with automatic Unicode support detection
    fn write_font_with_unicode_support(
        &mut self,
        font_name: &str,
        font: &crate::fonts::Font,
    ) -> Result<ObjectId> {
        // Check if any text in the document needs Unicode
        // For simplicity, always use Type0 for full Unicode support
        self.write_type0_font_from_font(font_name, font)
    }

    /// Write a Type0 font with CID support from fonts::Font
    fn write_type0_font_from_font(
        &mut self,
        font_name: &str,
        font: &crate::fonts::Font,
    ) -> Result<ObjectId> {
        // Per-font character set for subsetting (issue #204). Falls
        // back to a small ASCII/digit set only when the document
        // tracked no characters at all for this font — the ancient
        // code path pre-dating char tracking. Post-fix this fallback
        // shouldn't fire for any font reached through `write_fonts`
        // because that path already filters unused fonts out.
        let used_chars = self
            .document_used_chars_by_font
            .get(font_name)
            .cloned()
            .unwrap_or_else(|| {
                let mut chars = std::collections::HashSet::new();
                for ch in
                    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .,!?".chars()
                {
                    chars.insert(ch);
                }
                chars
            });

        // Diagnose characters the embedded font has no glyph for: they render
        // as .notdef (empty boxes). This is correct when the font genuinely
        // lacks the glyph, but warn so it is not a silent failure — the most
        // common cause of "my ✓/✗ show as boxes" reports (issue #287).
        // Fires once per font per save; a document saved repeatedly logs it
        // each time. `missing_glyphs` returns nothing when coverage is unknown
        // (e.g. an unparseable cmap), so this never produces false positives.
        let used_text: String = used_chars.iter().copied().collect();
        let mut missing = font.missing_glyphs(&used_text);
        if !missing.is_empty() {
            missing.sort_unstable();
            let list = missing
                .iter()
                .map(|c| format!("U+{:04X} {:?}", *c as u32, c))
                .collect::<Vec<_>>()
                .join(", ");
            tracing::warn!(
                "Custom font '{}' has no glyph for {} character(s): {}. \
                 They will render as .notdef (empty boxes); the embedded font \
                 does not contain these glyphs.",
                font_name,
                missing.len(),
                list
            );
        }

        // Allocate IDs for all font objects
        let font_id = self.allocate_object_id();
        let descendant_font_id = self.allocate_object_id();
        let descriptor_id = self.allocate_object_id();
        let font_file_id = self.allocate_object_id();
        let to_unicode_id = self.allocate_object_id();

        // Write font file. Large fonts are subsetted; the subsetter always
        // emits raw CFF for OpenType/CFF fonts, so OpenType font files are
        // embedded with /CIDFontType0C. TrueType fonts keep the SFNT wrapper.
        // IMPORTANT: We need the ORIGINAL font for width calculations, not the subset.
        let (font_data_to_embed, subset_glyph_mapping, original_font_for_widths) =
            if font.data.len() > 100_000 && !used_chars.is_empty() {
                match crate::text::fonts::truetype_subsetter::subset_font(
                    font.data.clone(),
                    &used_chars,
                ) {
                    Ok(subset_result) => (
                        subset_result.font_data,
                        Some(subset_result.glyph_mapping),
                        font.clone(),
                    ),
                    Err(_) => {
                        if font.data.len() < 25_000_000 {
                            (font.data.clone(), None, font.clone())
                        } else {
                            (Vec::new(), None, font.clone())
                        }
                    }
                }
            } else {
                (font.data.clone(), None, font.clone())
            };

        if !font_data_to_embed.is_empty() {
            // Build the initial font-file dictionary carrying the format-specific
            // metadata. `/Length1` (uncompressed byte count) is required for
            // TrueType FontFile2 streams per ISO 32000-1 §9.9. `/Subtype
            // /CIDFontType0C` marks raw CFF bytes for OpenType FontFile3 streams.
            let mut font_file_dict = Dictionary::new();
            match font.format {
                crate::fonts::FontFormat::OpenType => {
                    font_file_dict.set("Subtype", Object::Name("CIDFontType0C".to_string()));
                }
                crate::fonts::FontFormat::TrueType => {
                    font_file_dict.set("Length1", Object::Integer(font_data_to_embed.len() as i64));
                }
            }

            // Compress the font-file stream when the `compression` feature is
            // active and the writer config permits it. Uncompressed TTF glyf
            // data in particular compresses 60-70% with zlib — a 666 KB
            // subset PDF drops to under 200 KB after compression.
            #[cfg(feature = "compression")]
            {
                let font_stream_obj = if self.config.compress_streams {
                    let mut stream =
                        crate::objects::Stream::with_dictionary(font_file_dict, font_data_to_embed);
                    stream.compress_flate()?;
                    Object::Stream(stream.dictionary().clone(), stream.data().to_vec())
                } else {
                    Object::Stream(font_file_dict, font_data_to_embed)
                };
                self.write_object(font_file_id, font_stream_obj)?;
            }
            #[cfg(not(feature = "compression"))]
            {
                let font_stream_obj = Object::Stream(font_file_dict, font_data_to_embed);
                self.write_object(font_file_id, font_stream_obj)?;
            }
        } else {
            // No font data to embed
            let font_file_dict = Dictionary::new();
            let font_stream_obj = Object::Stream(font_file_dict, Vec::new());
            self.write_object(font_file_id, font_stream_obj)?;
        }

        // Write font descriptor
        let mut descriptor = Dictionary::new();
        descriptor.set("Type", Object::Name("FontDescriptor".to_string()));
        descriptor.set("FontName", Object::Name(font_name.to_string()));
        descriptor.set("Flags", Object::Integer(4)); // Symbolic font
        descriptor.set(
            "FontBBox",
            Object::Array(vec![
                Object::Integer(font.descriptor.font_bbox[0] as i64),
                Object::Integer(font.descriptor.font_bbox[1] as i64),
                Object::Integer(font.descriptor.font_bbox[2] as i64),
                Object::Integer(font.descriptor.font_bbox[3] as i64),
            ]),
        );
        descriptor.set(
            "ItalicAngle",
            Object::Real(font.descriptor.italic_angle as f64),
        );
        descriptor.set("Ascent", Object::Real(font.descriptor.ascent as f64));
        descriptor.set("Descent", Object::Real(font.descriptor.descent as f64));
        descriptor.set("CapHeight", Object::Real(font.descriptor.cap_height as f64));
        descriptor.set("StemV", Object::Real(font.descriptor.stem_v as f64));
        // Use appropriate FontFile type based on font format
        let font_file_key = match font.format {
            crate::fonts::FontFormat::OpenType => "FontFile3", // CFF/OpenType fonts
            crate::fonts::FontFormat::TrueType => "FontFile2", // TrueType fonts
        };
        descriptor.set(font_file_key, Object::Reference(font_file_id));
        self.write_object(descriptor_id, Object::Dictionary(descriptor))?;

        // Write CIDFont (descendant font)
        let mut cid_font = Dictionary::new();
        cid_font.set("Type", Object::Name("Font".to_string()));
        // ISO 32000-1 §9.7.4: CIDFontType0 for CFF/OpenType, CIDFontType2 for TrueType.
        let cid_font_subtype = match font.format {
            crate::fonts::FontFormat::OpenType => "CIDFontType0",
            crate::fonts::FontFormat::TrueType => "CIDFontType2",
        };
        cid_font.set("Subtype", Object::Name(cid_font_subtype.to_string()));
        cid_font.set("BaseFont", Object::Name(font_name.to_string()));

        // CIDSystemInfo - Use appropriate values for CJK fonts
        let mut cid_system_info = Dictionary::new();
        let (registry, ordering, supplement) =
            if let Some(cjk_type) = CjkFontType::detect_from_name(font_name) {
                cjk_type.cid_system_info()
            } else {
                ("Adobe", "Identity", 0)
            };

        cid_system_info.set("Registry", Object::String(registry.to_string()));
        cid_system_info.set("Ordering", Object::String(ordering.to_string()));
        cid_system_info.set("Supplement", Object::Integer(supplement as i64));
        cid_font.set("CIDSystemInfo", Object::Dictionary(cid_system_info));

        cid_font.set("FontDescriptor", Object::Reference(descriptor_id));

        // Calculate a better default width based on font metrics
        let default_width = self.calculate_default_width(font);
        cid_font.set("DW", Object::Integer(default_width));

        // Generate proper width array from font metrics
        // IMPORTANT: Use the ORIGINAL font for width calculations, not the subset
        // But pass the subset mapping to know which characters we're using
        let w_array = self.generate_width_array(
            &original_font_for_widths,
            default_width,
            subset_glyph_mapping.as_ref(),
        );
        cid_font.set("W", Object::Array(w_array));

        // CIDToGIDMap - Only required for CIDFontType2 (TrueType)
        // For CIDFontType0 (CFF/OpenType), CIDToGIDMap should NOT be present per ISO 32000-1:2008 §9.7.4.2
        // CFF fonts use CIDs directly as glyph identifiers, so no mapping is needed
        if cid_font_subtype == "CIDFontType2" {
            // TrueType fonts need CIDToGIDMap to map CIDs (Unicode code points) to Glyph IDs
            let cid_to_gid_map =
                self.generate_cid_to_gid_map(font_name, font, subset_glyph_mapping.as_ref())?;
            if !cid_to_gid_map.is_empty() {
                // Write the CIDToGIDMap as a stream, FlateDecode-compressed
                // when possible. The raw map is dimensioned to the highest
                // codepoint in use and is mostly zeros (only mapped code
                // points carry a 2-byte GID), so Flate compression typically
                // crushes it by 95-99%. For CJK-heavy documents this is the
                // difference between a 130 KB map (Issue #165) and a ~1 KB
                // stream.
                let cid_to_gid_map_id = self.allocate_object_id();
                let map_dict = Dictionary::new();
                #[cfg(feature = "compression")]
                let map_stream = if self.config.compress_streams {
                    let mut stream =
                        crate::objects::Stream::with_dictionary(map_dict, cid_to_gid_map);
                    stream.compress_flate()?;
                    Object::Stream(stream.dictionary().clone(), stream.data().to_vec())
                } else {
                    let mut d = map_dict;
                    d.set("Length", Object::Integer(cid_to_gid_map.len() as i64));
                    Object::Stream(d, cid_to_gid_map)
                };
                #[cfg(not(feature = "compression"))]
                let map_stream = {
                    let mut d = map_dict;
                    d.set("Length", Object::Integer(cid_to_gid_map.len() as i64));
                    Object::Stream(d, cid_to_gid_map)
                };
                self.write_object(cid_to_gid_map_id, map_stream)?;
                cid_font.set("CIDToGIDMap", Object::Reference(cid_to_gid_map_id));
            } else {
                cid_font.set("CIDToGIDMap", Object::Name("Identity".to_string()));
            }
        }
        // Note: For CIDFontType0 (CFF), we intentionally omit CIDToGIDMap

        self.write_object(descendant_font_id, Object::Dictionary(cid_font))?;

        // Write ToUnicode CMap. The CMap is filtered to the characters that
        // actually appear in the document (via `document_used_chars`) and the
        // stream is FlateDecode-compressed when the `compression` feature and
        // writer config allow it. The unfiltered, uncompressed version used to
        // dominate PDF output (~14 KB for a 2-char Latin document).
        let cmap_data = self.generate_tounicode_cmap_from_font(font_name, font);
        let cmap_dict = Dictionary::new();
        #[cfg(feature = "compression")]
        let cmap_stream = if self.config.compress_streams {
            let mut stream = crate::objects::Stream::with_dictionary(cmap_dict, cmap_data);
            stream.compress_flate()?;
            Object::Stream(stream.dictionary().clone(), stream.data().to_vec())
        } else {
            Object::Stream(cmap_dict, cmap_data)
        };
        #[cfg(not(feature = "compression"))]
        let cmap_stream = Object::Stream(cmap_dict, cmap_data);
        self.write_object(to_unicode_id, cmap_stream)?;

        // Write Type0 font (main font)
        let mut type0_font = Dictionary::new();
        type0_font.set("Type", Object::Name("Font".to_string()));
        type0_font.set("Subtype", Object::Name("Type0".to_string()));
        type0_font.set("BaseFont", Object::Name(font_name.to_string()));
        type0_font.set("Encoding", Object::Name("Identity-H".to_string()));
        type0_font.set(
            "DescendantFonts",
            Object::Array(vec![Object::Reference(descendant_font_id)]),
        );
        type0_font.set("ToUnicode", Object::Reference(to_unicode_id));

        self.write_object(font_id, Object::Dictionary(type0_font))?;

        Ok(font_id)
    }

    /// Calculate default width based on common characters
    fn calculate_default_width(&self, font: &crate::fonts::Font) -> i64 {
        use crate::text::fonts::truetype::TrueTypeFont;

        // Try to calculate from actual font metrics
        if let Ok(tt_font) = TrueTypeFont::parse(font.data.clone()) {
            if let Ok(cmap_tables) = tt_font.parse_cmap() {
                if let Some(cmap) = CmapSubtable::select_best_or_first(&cmap_tables) {
                    if let Ok(widths) = tt_font.get_glyph_widths(&cmap.mappings) {
                        // NOTE: get_glyph_widths already returns widths in PDF units (1000 per em)

                        // Calculate average width of common Latin characters
                        let common_chars =
                            "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ";
                        let mut total_width = 0;
                        let mut count = 0;

                        for ch in common_chars.chars() {
                            let unicode = ch as u32;
                            if let Some(&pdf_width) = widths.get(&unicode) {
                                total_width += pdf_width as i64;
                                count += 1;
                            }
                        }

                        if count > 0 {
                            return total_width / count;
                        }
                    }
                }
            }
        }

        // Fallback default if we can't calculate
        500
    }

    /// Generate width array for CID font
    fn generate_width_array(
        &self,
        font: &crate::fonts::Font,
        _default_width: i64,
        subset_mapping: Option<&HashMap<u32, u16>>,
    ) -> Vec<Object> {
        use crate::text::fonts::truetype::TrueTypeFont;

        let mut w_array = Vec::new();

        // Try to get actual glyph widths from the font
        if let Ok(tt_font) = TrueTypeFont::parse(font.data.clone()) {
            // IMPORTANT: Always use ORIGINAL mappings for width calculation
            // The subset_mapping has NEW GlyphIDs which don't correspond to the right glyphs
            // in the original font's width table
            let char_to_glyph = {
                // Parse cmap to get original mappings
                if let Ok(cmap_tables) = tt_font.parse_cmap() {
                    if let Some(cmap) = CmapSubtable::select_best_or_first(&cmap_tables) {
                        // If we have subset_mapping, filter to only include used characters
                        if let Some(subset_map) = subset_mapping {
                            let mut filtered = HashMap::new();
                            for unicode in subset_map.keys() {
                                // Get the ORIGINAL GlyphID for this Unicode
                                if let Some(&orig_glyph) = cmap.mappings.get(unicode) {
                                    filtered.insert(*unicode, orig_glyph);
                                }
                            }
                            filtered
                        } else {
                            cmap.mappings.clone()
                        }
                    } else {
                        HashMap::new()
                    }
                } else {
                    HashMap::new()
                }
            };

            if !char_to_glyph.is_empty() {
                // Get actual widths from the font
                if let Ok(widths) = tt_font.get_glyph_widths(&char_to_glyph) {
                    // NOTE: get_glyph_widths already returns widths scaled to PDF units (1000 per em)
                    // So we DON'T need to scale them again here

                    // Group consecutive characters with same width for efficiency
                    let mut sorted_chars: Vec<_> = widths.iter().collect();
                    sorted_chars.sort_by_key(|(unicode, _)| *unicode);

                    let mut i = 0;
                    while i < sorted_chars.len() {
                        let start_unicode = *sorted_chars[i].0;
                        // Width is already in PDF units from get_glyph_widths
                        let pdf_width = *sorted_chars[i].1 as i64;

                        // Find consecutive characters with same width
                        let mut end_unicode = start_unicode;
                        let mut j = i + 1;
                        while j < sorted_chars.len() && *sorted_chars[j].0 == end_unicode + 1 {
                            let next_pdf_width = *sorted_chars[j].1 as i64;
                            if next_pdf_width == pdf_width {
                                end_unicode = *sorted_chars[j].0;
                                j += 1;
                            } else {
                                break;
                            }
                        }

                        // Add to W array
                        if start_unicode == end_unicode {
                            // Single character
                            w_array.push(Object::Integer(start_unicode as i64));
                            w_array.push(Object::Array(vec![Object::Integer(pdf_width)]));
                        } else {
                            // Range of characters
                            w_array.push(Object::Integer(start_unicode as i64));
                            w_array.push(Object::Integer(end_unicode as i64));
                            w_array.push(Object::Integer(pdf_width));
                        }

                        i = j;
                    }

                    return w_array;
                }
            }
        }

        // Fallback to reasonable default widths if we can't parse the font
        let ranges = vec![
            // Space character should be narrower
            (0x20, 0x20, 250), // Space
            (0x21, 0x2F, 333), // Punctuation
            (0x30, 0x39, 500), // Numbers (0-9)
            (0x3A, 0x40, 333), // More punctuation
            (0x41, 0x5A, 667), // Uppercase letters (A-Z)
            (0x5B, 0x60, 333), // Brackets
            (0x61, 0x7A, 500), // Lowercase letters (a-z)
            (0x7B, 0x7E, 333), // More brackets
            // Extended Latin
            (0xA0, 0xA0, 250), // Non-breaking space
            (0xA1, 0xBF, 333), // Latin-1 punctuation
            (0xC0, 0xD6, 667), // Latin-1 uppercase
            (0xD7, 0xD7, 564), // Multiplication sign
            (0xD8, 0xDE, 667), // More Latin-1 uppercase
            (0xDF, 0xF6, 500), // Latin-1 lowercase
            (0xF7, 0xF7, 564), // Division sign
            (0xF8, 0xFF, 500), // More Latin-1 lowercase
            // Latin Extended-A
            (0x100, 0x17F, 500), // Latin Extended-A
            // Symbols and special characters
            (0x2000, 0x200F, 250), // Various spaces
            (0x2010, 0x2027, 333), // Hyphens and dashes
            (0x2028, 0x202F, 250), // More spaces
            (0x2030, 0x206F, 500), // General Punctuation
            (0x2070, 0x209F, 400), // Superscripts
            (0x20A0, 0x20CF, 600), // Currency symbols
            (0x2100, 0x214F, 700), // Letterlike symbols
            (0x2190, 0x21FF, 600), // Arrows
            (0x2200, 0x22FF, 600), // Mathematical operators
            (0x2300, 0x23FF, 600), // Miscellaneous technical
            (0x2500, 0x257F, 500), // Box drawing
            (0x2580, 0x259F, 500), // Block elements
            (0x25A0, 0x25FF, 600), // Geometric shapes
            (0x2600, 0x26FF, 600), // Miscellaneous symbols
            (0x2700, 0x27BF, 600), // Dingbats
        ];

        // Convert ranges to W array format
        for (start, end, width) in ranges {
            if start == end {
                // Single character
                w_array.push(Object::Integer(start));
                w_array.push(Object::Array(vec![Object::Integer(width)]));
            } else {
                // Range of characters
                w_array.push(Object::Integer(start));
                w_array.push(Object::Integer(end));
                w_array.push(Object::Integer(width));
            }
        }

        w_array
    }

    /// Generate CIDToGIDMap for Type0 font
    fn generate_cid_to_gid_map(
        &mut self,
        font_name: &str,
        font: &crate::fonts::Font,
        subset_mapping: Option<&HashMap<u32, u16>>,
    ) -> Result<Vec<u8>> {
        use crate::text::fonts::truetype::TrueTypeFont;

        // If we have a subset mapping, use it directly
        // Otherwise, parse the font to get the original cmap table
        let cmap_mappings = if let Some(subset_map) = subset_mapping {
            // Use the subset mapping directly
            subset_map.clone()
        } else {
            // Parse the font to get the original cmap table
            let tt_font = TrueTypeFont::parse(font.data.clone())?;
            let cmap_tables = tt_font.parse_cmap()?;

            // Find the best cmap table (prefer Format 12 for CJK)
            let cmap = CmapSubtable::select_best_or_first(&cmap_tables).ok_or_else(|| {
                crate::error::PdfError::FontError("No Unicode cmap table found".to_string())
            })?;

            cmap.mappings.clone()
        };

        // Build the CIDToGIDMap
        // Since we use Unicode code points as CIDs, we need to map Unicode → GlyphID
        // The map is a binary array where index = CID (Unicode) * 2, value = GlyphID (big-endian)

        // OPTIMIZATION: Only create map for characters actually used in the document
        // Get used characters from document tracking
        let used_chars = self
            .document_used_chars_by_font
            .get(font_name)
            .cloned()
            .unwrap_or_default();

        // Find the maximum Unicode value from used characters or full font
        let max_unicode = if !used_chars.is_empty() {
            // If we have used chars tracking, only map up to the highest used character
            used_chars
                .iter()
                .map(|ch| *ch as u32)
                .max()
                .unwrap_or(0x00FF) // At least Basic Latin
                .min(0xFFFF) as usize
        } else {
            // Fallback to original behavior if no tracking
            cmap_mappings
                .keys()
                .max()
                .copied()
                .unwrap_or(0xFFFF)
                .min(0xFFFF) as usize
        };

        // Create the map: 2 bytes per entry
        let mut map = vec![0u8; (max_unicode + 1) * 2];

        // Fill in the mappings
        let mut sample_mappings = Vec::new();
        for (&unicode, &glyph_id) in &cmap_mappings {
            if unicode <= max_unicode as u32 {
                let idx = (unicode as usize) * 2;
                // Write glyph_id in big-endian format
                map[idx] = (glyph_id >> 8) as u8;
                map[idx + 1] = (glyph_id & 0xFF) as u8;

                // Collect some sample mappings for debugging
                if unicode == 0x0041 || unicode == 0x0061 || unicode == 0x00E1 || unicode == 0x00F1
                {
                    sample_mappings.push((unicode, glyph_id));
                }
            }
        }

        Ok(map)
    }

    /// Generate ToUnicode CMap for Type0 font from fonts::Font
    fn generate_tounicode_cmap_from_font(
        &self,
        font_name: &str,
        font: &crate::fonts::Font,
    ) -> Vec<u8> {
        use crate::text::fonts::truetype::TrueTypeFont;

        let mut cmap = String::new();

        // CMap header
        cmap.push_str("/CIDInit /ProcSet findresource begin\n");
        cmap.push_str("12 dict begin\n");
        cmap.push_str("begincmap\n");
        cmap.push_str("/CIDSystemInfo\n");
        cmap.push_str("<< /Registry (Adobe)\n");
        cmap.push_str("   /Ordering (UCS)\n");
        cmap.push_str("   /Supplement 0\n");
        cmap.push_str(">> def\n");
        cmap.push_str("/CMapName /Adobe-Identity-UCS def\n");
        cmap.push_str("/CMapType 2 def\n");
        cmap.push_str("1 begincodespacerange\n");
        cmap.push_str("<0000> <FFFF>\n");
        cmap.push_str("endcodespacerange\n");

        // Build the set of code points that must appear in the ToUnicode CMap.
        // With Identity-H encoding, CID == Unicode, so each used character
        // produces a single `<CID> <unicode>` entry. If the document tracked
        // no used characters (legacy path), fall back to the font's full cmap
        // filtered to the BMP — but that path is a backstop, not the norm.
        let used_codepoints: Option<std::collections::HashSet<u32>> = self
            .document_used_chars_by_font
            .get(font_name)
            .map(|chars| {
                chars
                    .iter()
                    .map(|c| *c as u32)
                    .filter(|cp| *cp <= 0xFFFF)
                    .collect()
            });

        let mut mappings: Vec<(u32, u32)> = Vec::new();

        if let Some(used) = &used_codepoints {
            // Fast path: every used codepoint maps to itself under Identity-H.
            for cp in used {
                mappings.push((*cp, *cp));
            }
        } else if let Ok(tt_font) = TrueTypeFont::parse(font.data.clone()) {
            // Legacy backstop: no used-char tracking, emit every font mapping.
            if let Ok(cmap_tables) = tt_font.parse_cmap() {
                if let Some(cmap_table) = CmapSubtable::select_best_or_first(&cmap_tables) {
                    for (&unicode, &glyph_id) in &cmap_table.mappings {
                        if glyph_id > 0 && unicode <= 0xFFFF {
                            mappings.push((unicode, unicode));
                        }
                    }
                }
            }
        }

        // Sort mappings by CID for better organization
        mappings.sort_by_key(|&(cid, _)| cid);

        // Use more efficient bfrange where possible
        let mut i = 0;
        while i < mappings.len() {
            // Check if we can use a range
            let start_cid = mappings[i].0;
            let start_unicode = mappings[i].1;
            let mut end_idx = i;

            // Find consecutive mappings
            while end_idx + 1 < mappings.len()
                && mappings[end_idx + 1].0 == mappings[end_idx].0 + 1
                && mappings[end_idx + 1].1 == mappings[end_idx].1 + 1
                && end_idx - i < 99
            // Max 100 per block
            {
                end_idx += 1;
            }

            if end_idx > i {
                // Use bfrange for consecutive mappings
                cmap.push_str("1 beginbfrange\n");
                cmap.push_str(&format!(
                    "<{:04X}> <{:04X}> <{:04X}>\n",
                    start_cid, mappings[end_idx].0, start_unicode
                ));
                cmap.push_str("endbfrange\n");
                i = end_idx + 1;
            } else {
                // Use bfchar for individual mappings
                let mut chars = Vec::new();
                let chunk_end = (i + 100).min(mappings.len());

                for item in &mappings[i..chunk_end] {
                    chars.push(*item);
                }

                if !chars.is_empty() {
                    cmap.push_str(&format!("{} beginbfchar\n", chars.len()));
                    for (cid, unicode) in chars {
                        cmap.push_str(&format!("<{:04X}> <{:04X}>\n", cid, unicode));
                    }
                    cmap.push_str("endbfchar\n");
                }

                i = chunk_end;
            }
        }

        // CMap footer
        cmap.push_str("endcmap\n");
        cmap.push_str("CMapName currentdict /CMap defineresource pop\n");
        cmap.push_str("end\n");
        cmap.push_str("end\n");

        cmap.into_bytes()
    }

    /// Write a regular TrueType font
    #[allow(dead_code)]
    fn write_truetype_font(
        &mut self,
        font_name: &str,
        font: &crate::text::font_manager::CustomFont,
    ) -> Result<ObjectId> {
        // Allocate IDs for font objects
        let font_id = self.allocate_object_id();
        let descriptor_id = self.allocate_object_id();
        let font_file_id = self.allocate_object_id();

        // Write font file (embedded TTF data)
        if let Some(ref data) = font.font_data {
            let mut font_file_dict = Dictionary::new();
            font_file_dict.set("Length1", Object::Integer(data.len() as i64));
            let font_stream_obj = Object::Stream(font_file_dict, data.clone());
            self.write_object(font_file_id, font_stream_obj)?;
        }

        // Write font descriptor
        let mut descriptor = Dictionary::new();
        descriptor.set("Type", Object::Name("FontDescriptor".to_string()));
        descriptor.set("FontName", Object::Name(font_name.to_string()));
        descriptor.set("Flags", Object::Integer(32)); // Non-symbolic font
        descriptor.set(
            "FontBBox",
            Object::Array(vec![
                Object::Integer(-1000),
                Object::Integer(-1000),
                Object::Integer(2000),
                Object::Integer(2000),
            ]),
        );
        descriptor.set("ItalicAngle", Object::Integer(0));
        descriptor.set("Ascent", Object::Integer(font.descriptor.ascent as i64));
        descriptor.set("Descent", Object::Integer(font.descriptor.descent as i64));
        descriptor.set(
            "CapHeight",
            Object::Integer(font.descriptor.cap_height as i64),
        );
        descriptor.set("StemV", Object::Integer(font.descriptor.stem_v as i64));
        descriptor.set("FontFile2", Object::Reference(font_file_id));
        self.write_object(descriptor_id, Object::Dictionary(descriptor))?;

        // Write font dictionary
        let mut font_dict = Dictionary::new();
        font_dict.set("Type", Object::Name("Font".to_string()));
        font_dict.set("Subtype", Object::Name("TrueType".to_string()));
        font_dict.set("BaseFont", Object::Name(font_name.to_string()));
        font_dict.set("FirstChar", Object::Integer(0));
        font_dict.set("LastChar", Object::Integer(255));

        // Create widths array (simplified - all 600)
        let widths: Vec<Object> = (0..256).map(|_| Object::Integer(600)).collect();
        font_dict.set("Widths", Object::Array(widths));
        font_dict.set("FontDescriptor", Object::Reference(descriptor_id));

        // Use WinAnsiEncoding for regular TrueType
        font_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));

        self.write_object(font_id, Object::Dictionary(font_dict))?;

        Ok(font_id)
    }

    fn write_pages(
        &mut self,
        document: &Document,
        font_refs: &HashMap<String, ObjectId>,
    ) -> Result<()> {
        let pages_id = self.get_pages_id()?;
        let mut pages_dict = Dictionary::new();
        pages_dict.set("Type", Object::Name("Pages".to_string()));
        pages_dict.set("Count", Object::Integer(document.pages.len() as i64));

        let mut kids = Vec::new();

        // Allocate page object IDs sequentially
        let mut page_ids = Vec::new();
        let mut content_ids = Vec::new();
        for _ in 0..document.pages.len() {
            page_ids.push(self.allocate_object_id());
            content_ids.push(self.allocate_object_id());
        }

        for page_id in &page_ids {
            kids.push(Object::Reference(*page_id));
        }

        pages_dict.set("Kids", Object::Array(kids));

        self.write_object(pages_id, Object::Dictionary(pages_dict))?;

        // Store page IDs for form field references
        self.page_ids = page_ids.clone();

        // Write individual pages with font references
        for (i, page) in document.pages.iter().enumerate() {
            let page_id = page_ids[i];
            let content_id = content_ids[i];

            self.write_page_with_fonts(page_id, pages_id, content_id, page, document, font_refs)?;
            self.write_page_content(content_id, page)?;
        }

        Ok(())
    }

    /// Compatibility alias for `write_pages` to maintain backwards compatibility
    #[allow(dead_code)]
    fn write_pages_with_fonts(
        &mut self,
        document: &Document,
        font_refs: &HashMap<String, ObjectId>,
    ) -> Result<()> {
        self.write_pages(document, font_refs)
    }

    fn write_page_with_fonts(
        &mut self,
        page_id: ObjectId,
        parent_id: ObjectId,
        content_id: ObjectId,
        page: &crate::page::Page,
        _document: &Document,
        font_refs: &HashMap<String, ObjectId>,
    ) -> Result<()> {
        // Start with the page's dictionary which includes annotations
        let mut page_dict = page.to_dict();

        page_dict.set("Type", Object::Name("Page".to_string()));
        page_dict.set("Parent", Object::Reference(parent_id));
        page_dict.set("Contents", Object::Reference(content_id));

        // Get resources dictionary or create new one
        let mut resources = if let Some(Object::Dictionary(res)) = page_dict.get("Resources") {
            res.clone()
        } else {
            Dictionary::new()
        };

        // Add font resources
        let mut font_dict = Dictionary::new();

        // Add ALL standard PDF fonts (Type1) with WinAnsiEncoding
        // This fixes the text rendering issue in dashboards where HelveticaBold was missing

        // Helvetica family
        let mut helvetica_dict = Dictionary::new();
        helvetica_dict.set("Type", Object::Name("Font".to_string()));
        helvetica_dict.set("Subtype", Object::Name("Type1".to_string()));
        helvetica_dict.set("BaseFont", Object::Name("Helvetica".to_string()));
        helvetica_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set("Helvetica", Object::Dictionary(helvetica_dict));

        let mut helvetica_bold_dict = Dictionary::new();
        helvetica_bold_dict.set("Type", Object::Name("Font".to_string()));
        helvetica_bold_dict.set("Subtype", Object::Name("Type1".to_string()));
        helvetica_bold_dict.set("BaseFont", Object::Name("Helvetica-Bold".to_string()));
        helvetica_bold_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set("Helvetica-Bold", Object::Dictionary(helvetica_bold_dict));

        let mut helvetica_oblique_dict = Dictionary::new();
        helvetica_oblique_dict.set("Type", Object::Name("Font".to_string()));
        helvetica_oblique_dict.set("Subtype", Object::Name("Type1".to_string()));
        helvetica_oblique_dict.set("BaseFont", Object::Name("Helvetica-Oblique".to_string()));
        helvetica_oblique_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set(
            "Helvetica-Oblique",
            Object::Dictionary(helvetica_oblique_dict),
        );

        let mut helvetica_bold_oblique_dict = Dictionary::new();
        helvetica_bold_oblique_dict.set("Type", Object::Name("Font".to_string()));
        helvetica_bold_oblique_dict.set("Subtype", Object::Name("Type1".to_string()));
        helvetica_bold_oblique_dict.set(
            "BaseFont",
            Object::Name("Helvetica-BoldOblique".to_string()),
        );
        helvetica_bold_oblique_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set(
            "Helvetica-BoldOblique",
            Object::Dictionary(helvetica_bold_oblique_dict),
        );

        // Times family
        let mut times_dict = Dictionary::new();
        times_dict.set("Type", Object::Name("Font".to_string()));
        times_dict.set("Subtype", Object::Name("Type1".to_string()));
        times_dict.set("BaseFont", Object::Name("Times-Roman".to_string()));
        times_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set("Times-Roman", Object::Dictionary(times_dict));

        let mut times_bold_dict = Dictionary::new();
        times_bold_dict.set("Type", Object::Name("Font".to_string()));
        times_bold_dict.set("Subtype", Object::Name("Type1".to_string()));
        times_bold_dict.set("BaseFont", Object::Name("Times-Bold".to_string()));
        times_bold_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set("Times-Bold", Object::Dictionary(times_bold_dict));

        let mut times_italic_dict = Dictionary::new();
        times_italic_dict.set("Type", Object::Name("Font".to_string()));
        times_italic_dict.set("Subtype", Object::Name("Type1".to_string()));
        times_italic_dict.set("BaseFont", Object::Name("Times-Italic".to_string()));
        times_italic_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set("Times-Italic", Object::Dictionary(times_italic_dict));

        let mut times_bold_italic_dict = Dictionary::new();
        times_bold_italic_dict.set("Type", Object::Name("Font".to_string()));
        times_bold_italic_dict.set("Subtype", Object::Name("Type1".to_string()));
        times_bold_italic_dict.set("BaseFont", Object::Name("Times-BoldItalic".to_string()));
        times_bold_italic_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set(
            "Times-BoldItalic",
            Object::Dictionary(times_bold_italic_dict),
        );

        // Courier family
        let mut courier_dict = Dictionary::new();
        courier_dict.set("Type", Object::Name("Font".to_string()));
        courier_dict.set("Subtype", Object::Name("Type1".to_string()));
        courier_dict.set("BaseFont", Object::Name("Courier".to_string()));
        courier_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set("Courier", Object::Dictionary(courier_dict));

        let mut courier_bold_dict = Dictionary::new();
        courier_bold_dict.set("Type", Object::Name("Font".to_string()));
        courier_bold_dict.set("Subtype", Object::Name("Type1".to_string()));
        courier_bold_dict.set("BaseFont", Object::Name("Courier-Bold".to_string()));
        courier_bold_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set("Courier-Bold", Object::Dictionary(courier_bold_dict));

        let mut courier_oblique_dict = Dictionary::new();
        courier_oblique_dict.set("Type", Object::Name("Font".to_string()));
        courier_oblique_dict.set("Subtype", Object::Name("Type1".to_string()));
        courier_oblique_dict.set("BaseFont", Object::Name("Courier-Oblique".to_string()));
        courier_oblique_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set("Courier-Oblique", Object::Dictionary(courier_oblique_dict));

        let mut courier_bold_oblique_dict = Dictionary::new();
        courier_bold_oblique_dict.set("Type", Object::Name("Font".to_string()));
        courier_bold_oblique_dict.set("Subtype", Object::Name("Type1".to_string()));
        courier_bold_oblique_dict.set("BaseFont", Object::Name("Courier-BoldOblique".to_string()));
        courier_bold_oblique_dict.set("Encoding", Object::Name("WinAnsiEncoding".to_string()));
        font_dict.set(
            "Courier-BoldOblique",
            Object::Dictionary(courier_bold_oblique_dict),
        );

        // Add custom fonts (Type0 fonts for Unicode support)
        for (font_name, font_id) in font_refs {
            font_dict.set(font_name, Object::Reference(*font_id));
        }

        resources.set("Font", Object::Dictionary(font_dict));

        // Add images and Form XObjects as XObjects
        let has_images = !page.images().is_empty();
        let has_forms = !page.form_xobjects().is_empty();

        // Tracks name→ObjectId for every FormXObject written below.
        // Used downstream by the ExtGState SMask emission (ISO 32000-1
        // §11.6.4.3 Table 144 requires /G to be an INDIRECT reference
        // to a transparency-group Form XObject; the caller supplies the
        // group by name in `SoftMask::alpha(name)` and we resolve that
        // name to the ObjectId allocated here).
        let mut form_xobject_ids: HashMap<String, ObjectId> = HashMap::new();

        if has_images || has_forms {
            let mut xobject_dict = Dictionary::new();

            // Sort by name for reproducible output (images first, then
            // form xobjects — both sorted within their group). Sharing
            // the sort key produces the same layout across builds.
            let mut image_entries: Vec<(&String, &crate::graphics::Image)> =
                page.images().iter().collect();
            image_entries.sort_by_key(|(name, _)| name.as_str());
            for (name, image) in image_entries {
                // Use sequential ObjectId allocation to avoid conflicts
                let image_id = self.allocate_object_id();

                // Check if image has transparency (alpha channel)
                if image.has_transparency() {
                    // Handle transparent images with SMask
                    let (mut main_obj, smask_obj) = image.to_pdf_object_with_transparency()?;

                    // If we have a soft mask, write it as a separate object and reference it
                    if let Some(smask_stream) = smask_obj {
                        let smask_id = self.allocate_object_id();
                        self.write_object(smask_id, smask_stream)?;

                        // Add SMask reference to the main image dictionary
                        if let Object::Stream(ref mut dict, _) = main_obj {
                            dict.set("SMask", Object::Reference(smask_id));
                        }
                    }

                    // Write the main image XObject (now with SMask reference if applicable)
                    self.write_object(image_id, main_obj)?;
                } else {
                    // Write the image XObject without transparency
                    self.write_object(image_id, image.to_pdf_object())?;
                }

                // Add reference to XObject dictionary
                xobject_dict.set(name, Object::Reference(image_id));
            }

            // Write Form XObjects (used for overlay/watermark operations)
            let mut form_entries: Vec<(&String, &crate::graphics::FormXObject)> =
                page.form_xobjects().iter().collect();
            form_entries.sort_by_key(|(name, _)| name.as_str());
            for (name, form) in form_entries {
                let form_id = self.allocate_object_id();
                let stream = form.to_stream()?;
                let stream_obj =
                    Object::Stream(stream.dictionary().clone(), stream.data().to_vec());
                self.write_object(form_id, stream_obj)?;
                xobject_dict.set(name, Object::Reference(form_id));
                // Record the mapping so a downstream SoftMask with
                // `group_ref == name` can resolve to this indirect ref.
                form_xobject_ids.insert(name.clone(), form_id);
            }

            resources.set("XObject", Object::Dictionary(xobject_dict));
        }

        // Add ExtGState resources for transparency
        if let Some(extgstate_states) = page.get_extgstate_resources() {
            let mut extgstate_dict = Dictionary::new();
            // Sort ExtGState entries by name for reproducible output.
            let mut extgstate_entries: Vec<(&String, &crate::graphics::ExtGState)> =
                extgstate_states.iter().collect();
            extgstate_entries.sort_by_key(|(name, _)| name.as_str());
            for (name, state) in extgstate_entries {
                let mut state_dict = Dictionary::new();
                state_dict.set("Type", Object::Name("ExtGState".to_string()));

                // Add transparency parameters
                if let Some(alpha_stroke) = state.alpha_stroke {
                    state_dict.set("CA", Object::Real(alpha_stroke));
                }
                if let Some(alpha_fill) = state.alpha_fill {
                    state_dict.set("ca", Object::Real(alpha_fill));
                }

                // Add other parameters as needed
                if let Some(line_width) = state.line_width {
                    state_dict.set("LW", Object::Real(line_width));
                }
                if let Some(line_cap) = state.line_cap {
                    state_dict.set("LC", Object::Integer(line_cap as i64));
                }
                if let Some(line_join) = state.line_join {
                    state_dict.set("LJ", Object::Integer(line_join as i64));
                }
                if let Some(dash_pattern) = &state.dash_pattern {
                    let dash_objects: Vec<Object> = dash_pattern
                        .array
                        .iter()
                        .map(|&d| Object::Real(d))
                        .collect();
                    state_dict.set(
                        "D",
                        Object::Array(vec![
                            Object::Array(dash_objects),
                            Object::Real(dash_pattern.phase),
                        ]),
                    );
                }

                // Blend mode (ISO 32000-1 §11.3.5, Table 137). Emitted as
                // a single name; blend-mode *arrays* (multiple fallback
                // modes) are not currently exposed by ExtGState.
                if let Some(ref bm) = state.blend_mode {
                    state_dict.set("BM", Object::Name(bm.pdf_name().to_string()));
                }

                // Soft mask (ISO 32000-1 §11.6.4.3, Table 144).
                // `SoftMask::to_pdf_dictionary` returns a full mask dict
                // with /Type /Mask /S <Alpha|Luminosity|None> and,
                // when a transparency group is attached, the /G, /BC
                // and /TR entries. The `/SMask /None` Name shortcut is
                // *also* spec-legal per §11.6.4.3; we emit the dict
                // form unconditionally so callers see a consistent
                // shape (and because the builder already populated the
                // dict variant for them).
                //
                // /G MUST be an indirect reference (Table 144). The
                // `SoftMask` API models the group reference as a `String`
                // name matching a FormXObject registered on this page
                // via `Page::add_form_xobject(name, ...)`. Resolve the
                // name here to the indirect ObjectId allocated above.
                // If no matching FormXObject exists, surface a structured
                // error rather than emit a spec-invalid /G /<Name> token.
                if let Some(ref soft_mask) = state.soft_mask {
                    let mut mask_dict = soft_mask.to_pdf_dictionary()?;
                    if let Some(Object::Name(ref g_name)) = mask_dict.get("G").cloned() {
                        let form_id = form_xobject_ids.get(g_name).ok_or_else(|| {
                            crate::error::PdfError::InvalidStructure(format!(
                                "SoftMask references transparency group {:?} but no matching \
                                 FormXObject is registered on the page; call \
                                 Page::add_form_xobject({:?}, ...) before saving",
                                g_name, g_name
                            ))
                        })?;
                        mask_dict.set("G", Object::Reference(*form_id));
                    }
                    state_dict.set("SMask", Object::Dictionary(mask_dict));
                }

                extgstate_dict.set(name, Object::Dictionary(state_dict));
            }
            if !extgstate_dict.is_empty() {
                resources.set("ExtGState", Object::Dictionary(extgstate_dict));
            }
        }

        // ColorSpace resources (ISO 32000-1 §8.6, Table 62). Emitted as a
        // direct sub-dictionary — colour-space *parameters* (the dict
        // inside `[/CalRGB <<..>>]`) are generally small and inlining them
        // keeps the cross-reference table lean. Callers that need
        // larger / shared colour spaces can register them once and reuse
        // the same key across pages.
        // Deterministic emission of all three resource sub-dicts is
        // enforced at Dictionary write time (see QUAL-9 sort below in
        // `write_object_value`). We therefore iterate the source
        // HashMaps in any order here — the serializer reorders.
        // However we DO sort Pattern / Shading entries before
        // `allocate_object_id()` so object-id allocation is also
        // reproducible (two identical documents allocate ids in the
        // same sequence, producing byte-identical xref entries).
        if !page.color_spaces().is_empty() {
            let mut cs_dict = Dictionary::new();
            // Sort by name before allocating any stream object ids so id
            // allocation stays reproducible (mirrors the Pattern/Shading blocks).
            let mut entries: Vec<(&String, &crate::graphics::PageColorSpace)> =
                page.color_spaces().iter().collect();
            entries.sort_by_key(|(name, _)| name.as_str());
            for (name, cs) in entries {
                // ICCBased colour spaces MUST be an indirect stream carrying the
                // profile bytes (ISO 32000-1 §8.6.5.5) — a stream cannot be
                // inlined into the resource dict. Every other shape (device-name
                // alias, Cal*/Lab parameterised dict) is inline via `to_object`.
                if let Some((icc_dict, icc_data)) = cs.icc_stream_parts() {
                    let icc_id = self.allocate_object_id();
                    self.write_object(icc_id, Object::Stream(icc_dict, icc_data))?;
                    cs_dict.set(
                        name,
                        Object::Array(vec![
                            Object::Name("ICCBased".to_string()),
                            Object::Reference(icc_id),
                        ]),
                    );
                } else {
                    cs_dict.set(name, cs.to_object());
                }
            }
            resources.set("ColorSpace", Object::Dictionary(cs_dict));
        }

        if !page.patterns().is_empty() {
            let mut pat_dict = Dictionary::new();
            let mut entries: Vec<(&String, &crate::graphics::TilingPattern)> =
                page.patterns().iter().collect();
            entries.sort_by_key(|(name, _)| name.as_str());
            for (name, pattern) in entries {
                let pattern_id = self.allocate_object_id();
                let pattern_dict = pattern.to_pdf_dictionary()?;
                self.write_object(
                    pattern_id,
                    Object::Stream(pattern_dict, pattern.content_stream.clone()),
                )?;
                pat_dict.set(name, Object::Reference(pattern_id));
            }
            resources.set("Pattern", Object::Dictionary(pat_dict));
        }

        if !page.shadings().is_empty() {
            let mut sh_dict = Dictionary::new();
            let mut entries: Vec<(&String, &crate::graphics::ShadingDefinition)> =
                page.shadings().iter().collect();
            entries.sort_by_key(|(name, _)| name.as_str());
            for (name, shading) in entries {
                let mut shading_dict = shading.to_pdf_dictionary()?;
                // Hoist the inline /Function to an indirect object (issue #297 B).
                // ISO 32000-1 §8.7.4.5.2: functions are normally indirect. Only
                // a dictionary value is hoisted; FunctionBased shadings carry an
                // external function id (an Integer) which is left untouched.
                if let Some(Object::Dictionary(_)) = shading_dict.get("Function") {
                    if let Some(func_obj) = shading_dict.remove("Function") {
                        let func_id = self.allocate_object_id();
                        self.write_object(func_id, func_obj)?;
                        shading_dict.set("Function", Object::Reference(func_id));
                    }
                }
                let shading_id = self.allocate_object_id();
                self.write_object(shading_id, Object::Dictionary(shading_dict))?;
                sh_dict.set(name, Object::Reference(shading_id));
            }
            resources.set("Shading", Object::Dictionary(sh_dict));
        }

        // Merge preserved resources from original PDF (if any)
        // Phase 2.3: Rename preserved fonts to avoid conflicts with overlay fonts
        if let Some(preserved_res) = page.get_preserved_resources() {
            // Convert pdf_objects::Dictionary to writer Dictionary FIRST
            let mut preserved_writer_dict = self.convert_pdf_objects_dict_to_writer(preserved_res);

            // Step 1: Rename preserved fonts (F1 → OrigF1)
            if let Some(Object::Dictionary(fonts)) = preserved_writer_dict.get("Font") {
                // Rename font dictionary keys using our utility function
                let renamed_fonts = crate::writer::rename_preserved_fonts(fonts);

                // Replace Font dictionary with renamed version
                preserved_writer_dict.set("Font", Object::Dictionary(renamed_fonts));
            }

            // Phase 3.3: Write embedded font streams as indirect objects
            // Fonts that were resolved in Phase 3.2 have embedded Stream objects
            // We need to write these streams as separate PDF objects and replace with References
            if let Some(Object::Dictionary(fonts)) = preserved_writer_dict.get("Font") {
                let mut fonts_with_refs = crate::objects::Dictionary::new();

                for (font_name, font_obj) in fonts.iter() {
                    if let Object::Dictionary(font_dict) = font_obj {
                        // Try to extract and write embedded font streams
                        let updated_font = self.write_embedded_font_streams(font_dict)?;
                        fonts_with_refs.set(font_name, Object::Dictionary(updated_font));
                    } else {
                        // Not a dictionary, keep as-is
                        fonts_with_refs.set(font_name, font_obj.clone());
                    }
                }

                // Replace Font dictionary with version that has References instead of Streams
                preserved_writer_dict.set("Font", Object::Dictionary(fonts_with_refs));
            }

            // Write preserved XObject streams as indirect objects
            // XObjects resolved in from_parsed_with_content may contain inline Stream data.
            // Per ISO 32000-1 §7.3.8, streams MUST be indirect objects.
            if let Some(Object::Dictionary(xobjects)) = preserved_writer_dict.get("XObject") {
                let mut xobjects_with_refs = crate::objects::Dictionary::new();
                tracing::debug!(
                    "Externalizing {} preserved XObject entries as indirect objects",
                    xobjects.len()
                );

                for (xobj_name, xobj_obj) in xobjects.iter() {
                    match xobj_obj {
                        Object::Stream(dict, data) => {
                            let obj_id = self.allocate_object_id();
                            self.write_object(obj_id, Object::Stream(dict.clone(), data.clone()))?;
                            xobjects_with_refs.set(xobj_name, Object::Reference(obj_id));
                        }
                        Object::Dictionary(dict) => {
                            // Dictionary XObjects may contain nested streams (e.g., SMask)
                            let externalized = self.externalize_streams_in_dict(dict)?;
                            xobjects_with_refs.set(xobj_name, Object::Dictionary(externalized));
                        }
                        _ => {
                            xobjects_with_refs.set(xobj_name, xobj_obj.clone());
                        }
                    }
                }

                preserved_writer_dict.set("XObject", Object::Dictionary(xobjects_with_refs));
            }

            // Merge each resource category (Font, XObject, ColorSpace, etc.)
            for (key, value) in preserved_writer_dict.iter() {
                // If the resource category already exists, merge dictionaries
                if let Some(Object::Dictionary(existing)) = resources.get(key) {
                    if let Object::Dictionary(preserved_dict) = value {
                        let mut merged = existing.clone();
                        // Add all preserved resources, giving priority to existing (overlay wins)
                        for (res_name, res_obj) in preserved_dict.iter() {
                            if !merged.contains_key(res_name) {
                                merged.set(res_name, res_obj.clone());
                            }
                        }
                        resources.set(key, Object::Dictionary(merged));
                    }
                } else {
                    // Resource category doesn't exist yet, add it directly
                    resources.set(key, value.clone());
                }
            }
        }

        page_dict.set("Resources", Object::Dictionary(resources));

        // Collect all annotation references for the /Annots array
        let mut annot_refs: Vec<Object> = Vec::new();

        // 1. Process widget annotations already in page_dict (legacy form field path)
        if let Some(Object::Array(annots)) = page_dict.get("Annots") {
            for annot in annots {
                if let Object::Dictionary(ref annot_dict) = annot {
                    if let Some(Object::Name(subtype)) = annot_dict.get("Subtype") {
                        if subtype == "Widget" {
                            let widget_id = self.allocate_object_id();
                            self.write_object(widget_id, annot.clone())?;
                            annot_refs.push(Object::Reference(widget_id));

                            // Track widget for form fields
                            if let Some(Object::Name(_ft)) = annot_dict.get("FT") {
                                if let Some(Object::String(field_name)) = annot_dict.get("T") {
                                    self.field_widget_map
                                        .entry(field_name.clone())
                                        .or_default()
                                        .push(widget_id);
                                    self.field_id_map.insert(field_name.clone(), widget_id);
                                    self.form_field_ids.push(widget_id);
                                }
                            }
                            continue;
                        }
                    }
                }
                annot_refs.push(annot.clone());
            }
        }

        // 2. Write annotations from Page.annotations() (programmatic annotations)
        //    Handles highlights, text notes, stamps, links, etc. added via
        //    page.add_annotation(). Each is written as an indirect object.
        for annotation in page.annotations() {
            let annot_id = self.allocate_object_id();
            let mut annot_dict = annotation.to_dict();

            // Remap `/Parent` from FormManager placeholder → real ObjectId.
            // `Annotation::field_parent` stores the placeholder ref returned
            // by FormManager::add_*_field (which uses a counter disjoint
            // from the writer's allocator). At this point the writer has
            // already pre-allocated real ids for every FormManager field
            // via `preallocate_form_manager_fields`, so we translate.
            //
            // We read `field_parent` straight off the struct instead of
            // round-tripping through `annot_dict.get("Parent")`: the
            // dictionary representation is what we're producing, not a
            // source of truth. The struct field is authoritative and
            // avoids matching on a value we just computed.
            //
            // Widgets whose parent placeholder is NOT in the map (e.g.
            // the caller supplied a hand-built ref, or `field_parent` was
            // set from outside the FormManager) are left unchanged — not
            // every `/Parent` necessarily comes from the FormManager.
            if let Some(placeholder) = annotation.field_parent {
                if let Some(real_id) = self.form_field_placeholder_map.get(&placeholder) {
                    annot_dict.set("Parent", Object::Reference(*real_id));
                }
            }

            // Externalize inline streams inside /AP.
            //
            // `Widget::generate_appearance` (and any user-supplied appearance
            // dictionary) stores the /N, /R, /D entries as inline
            // `Object::Stream` values inside the /AP sub-dictionary. Per
            // ISO 32000-1 §7.3.8.1, "all streams shall be indirect objects" —
            // inline streams as dictionary values are not permitted. We
            // therefore externalize each inline stream to a freshly
            // allocated indirect object and replace it with a /Reference.
            //
            // /AP itself has two legal shapes (§12.5.5):
            //   * A single stream (direct or indirect) → the "default" state.
            //   * A sub-dictionary mapping state names (/N, /R, /D) to
            //     streams, where /D may further be a dict mapping values to
            //     streams (radio buttons, checkboxes).
            // We handle the sub-dict shape (which is what `fill_field`
            // emits); the legacy single-stream shape falls through to the
            // writer's default handling below.
            if let Some(Object::Dictionary(ap_dict)) = annot_dict.get("AP") {
                let mut updated_ap = crate::objects::Dictionary::new();
                for (state_key, state_val) in ap_dict.iter() {
                    match state_val {
                        Object::Stream(sd, data) => {
                            // Patch `/Resources/Font/<name>` placeholders to
                            // indirect references to the document-level fonts
                            // (issue #212 Fase 3). The placeholder is emitted
                            // by form-field appearance generators that don't
                            // know the Type0 font's ObjectId.
                            let patched_sd = Self::rewrite_ap_stream_font_resources(sd, font_refs);
                            let stream_id = self.allocate_object_id();
                            self.write_object(stream_id, Object::Stream(patched_sd, data.clone()))?;
                            updated_ap.set(state_key, Object::Reference(stream_id));
                        }
                        Object::Dictionary(down_dict) => {
                            // /D sub-dict case: map value → stream.
                            let externalized = self
                                .externalize_streams_in_dict_with_font_refs(down_dict, font_refs)?;
                            updated_ap.set(state_key, Object::Dictionary(externalized));
                        }
                        _ => {
                            updated_ap.set(state_key, state_val.clone());
                        }
                    }
                }
                annot_dict.set("AP", Object::Dictionary(updated_ap));
            }

            self.write_object(annot_id, Object::Dictionary(annot_dict))?;
            annot_refs.push(Object::Reference(annot_id));

            // Track widget annotations for AcroForm if they come through this path
            if annotation.annotation_type == crate::annotations::AnnotationType::Widget {
                if let Some(Object::String(field_name)) = annotation.properties.get("T") {
                    self.field_widget_map
                        .entry(field_name.clone())
                        .or_default()
                        .push(annot_id);
                    self.field_id_map.insert(field_name.clone(), annot_id);
                    self.form_field_ids.push(annot_id);
                }
            }
        }

        // Set or remove /Annots based on whether we have any
        if !annot_refs.is_empty() {
            page_dict.set("Annots", Object::Array(annot_refs));
        } else {
            page_dict.remove("Annots");
        }

        self.write_object(page_id, Object::Dictionary(page_dict))?;
        Ok(())
    }
}

impl PdfWriter<BufWriter<std::fs::File>> {
    pub fn new(path: impl AsRef<Path>) -> Result<Self> {
        let file = std::fs::File::create(path)?;
        let writer = BufWriter::new(file);

        Ok(Self {
            writer,
            xref_positions: HashMap::new(),
            current_position: 0,
            next_object_id: 1,
            catalog_id: None,
            pages_id: None,
            info_id: None,
            field_widget_map: HashMap::new(),
            field_id_map: HashMap::new(),
            form_field_ids: Vec::new(),
            page_ids: Vec::new(),
            config: WriterConfig::default(),
            document_used_chars_by_font: std::collections::HashMap::new(),
            buffered_objects: HashMap::new(),
            compressed_object_map: HashMap::new(),
            prev_xref_offset: None,
            base_pdf_size: None,
            encrypt_obj_id: None,
            file_id: None,
            encryption_state: None,
            pending_encrypt_dict: None,
            form_field_placeholder_map: HashMap::new(),
            form_manager_field_refs: Vec::new(),
        })
    }
}

impl<W: Write> PdfWriter<W> {
    /// Write embedded font streams as indirect objects (Phase 3.3 + Phase 3.4)
    ///
    /// Takes a font dictionary that may contain embedded Stream objects
    /// in its FontDescriptor, writes those streams as separate PDF objects,
    /// and returns an updated font dictionary with References instead of Streams.
    ///
    /// For Type0 (composite) fonts, also handles:
    /// - DescendantFonts array with embedded CIDFont dictionaries
    /// - ToUnicode stream embedded directly in Type0 font
    /// - CIDFont → FontDescriptor → FontFile2/FontFile3 chain
    ///
    /// # Example
    /// FontDescriptor:
    ///   FontFile2: Stream(dict, font_data)  → Write stream as obj 50
    ///   FontFile2: Reference(50, 0)          → Updated reference
    /// Walks a dictionary and writes any inline Stream values as indirect objects,
    /// replacing them with References. Required because PDF streams must be indirect
    /// objects (ISO 32000-1 §7.3.8).
    fn externalize_streams_in_dict(
        &mut self,
        dict: &crate::objects::Dictionary,
    ) -> Result<crate::objects::Dictionary> {
        self.externalize_streams_in_dict_with_font_refs(dict, &HashMap::new())
    }

    /// Same as [`externalize_streams_in_dict`] but also rewrites any
    /// `/Resources/Font/<name>` placeholders inside the externalised stream
    /// dictionaries to indirect references from `font_refs` (issue #212).
    fn externalize_streams_in_dict_with_font_refs(
        &mut self,
        dict: &crate::objects::Dictionary,
        font_refs: &HashMap<String, ObjectId>,
    ) -> Result<crate::objects::Dictionary> {
        let mut result = crate::objects::Dictionary::new();
        for (key, value) in dict.iter() {
            match value {
                Object::Stream(d, data) => {
                    let patched_d = Self::rewrite_ap_stream_font_resources(d, font_refs);
                    let obj_id = self.allocate_object_id();
                    self.write_object(obj_id, Object::Stream(patched_d, data.clone()))?;
                    result.set(key, Object::Reference(obj_id));
                }
                _ => {
                    result.set(key, value.clone());
                }
            }
        }
        Ok(result)
    }

    /// Rewrite `/Resources/Font/<name>` entries inside an appearance-stream
    /// dictionary: any entry whose name appears in `font_refs` is replaced
    /// by an `Object::Reference` to the document-level font object.
    ///
    /// Why: form-field appearance generators cannot know the ObjectId of
    /// the Type0 font at content-stream build time — they emit a
    /// placeholder dict (see `TextFieldAppearance::generate_appearance_with_font`).
    /// This pass wires that placeholder to the real indirect object produced
    /// by `write_fonts`. Built-in Type1 fonts (Helvetica etc.) stay as
    /// inline dictionaries, since they have no document-level object.
    ///
    /// Returns a copy of the input dictionary with the /Resources/Font
    /// rewrite applied. All non-/Resources keys are passed through intact.
    /// Called on the stream DICTIONARY (not the stream data) so the original
    /// content bytes remain untouched.
    fn rewrite_ap_stream_font_resources(
        stream_dict: &crate::objects::Dictionary,
        font_refs: &HashMap<String, ObjectId>,
    ) -> crate::objects::Dictionary {
        // Fast path: if the document has no custom fonts registered (i.e.
        // `font_refs` is empty), no placeholder entry can possibly match.
        // Skip the clone+walk entirely — this is the common case for
        // built-in-font forms, and `externalize_streams_in_dict` (the
        // legacy non-AP path) calls us with an empty map for every stream
        // it externalises.
        if font_refs.is_empty() {
            return stream_dict.clone();
        }

        let mut out = stream_dict.clone();

        // Drill /Resources → /Font. Both may be direct dicts; we rebuild
        // them rather than mutate in place so reference semantics are
        // explicit. Indirect /Resources isn't emitted by our generators, so
        // only the direct-dict shape is handled here (defensive: anything
        // else is left untouched).
        let Some(Object::Dictionary(resources)) = stream_dict.get("Resources") else {
            return out;
        };
        let Some(Object::Dictionary(fonts)) = resources.get("Font") else {
            return out;
        };

        let mut patched_fonts = crate::objects::Dictionary::new();
        let mut changed = false;
        for (font_name, entry) in fonts.iter() {
            // Rewrite when (a) this is the placeholder inline dict shape our
            // generator emits (Object::Dictionary with /Subtype /Type0), AND
            // (b) the name is registered as a document-level custom font.
            let should_rewrite = match entry {
                Object::Dictionary(d) => {
                    matches!(d.get("Subtype"), Some(Object::Name(s)) if s == "Type0")
                }
                _ => false,
            };
            if should_rewrite {
                if let Some(font_id) = font_refs.get(font_name.as_str()) {
                    patched_fonts.set(font_name, Object::Reference(*font_id));
                    changed = true;
                    continue;
                }
            }
            patched_fonts.set(font_name, entry.clone());
        }

        if changed {
            let mut patched_resources = resources.clone();
            patched_resources.set("Font", Object::Dictionary(patched_fonts));
            out.set("Resources", Object::Dictionary(patched_resources));
        }
        out
    }

    fn write_embedded_font_streams(
        &mut self,
        font_dict: &crate::objects::Dictionary,
    ) -> Result<crate::objects::Dictionary> {
        let mut updated_font = font_dict.clone();

        // Phase 3.4: Check for Type0 fonts with embedded DescendantFonts
        if let Some(Object::Name(subtype)) = font_dict.get("Subtype") {
            if subtype == "Type0" {
                // Process DescendantFonts array
                if let Some(Object::Array(descendants)) = font_dict.get("DescendantFonts") {
                    let mut updated_descendants = Vec::new();

                    for descendant in descendants {
                        match descendant {
                            Object::Dictionary(cidfont) => {
                                // CIDFont is embedded as Dictionary, process its FontDescriptor
                                let updated_cidfont =
                                    self.write_cidfont_embedded_streams(cidfont)?;
                                // Write CIDFont as a separate object
                                let cidfont_id = self.allocate_object_id();
                                self.write_object(cidfont_id, Object::Dictionary(updated_cidfont))?;
                                // Replace with reference
                                updated_descendants.push(Object::Reference(cidfont_id));
                            }
                            Object::Reference(_) => {
                                // Already a reference, keep as-is
                                updated_descendants.push(descendant.clone());
                            }
                            _ => {
                                updated_descendants.push(descendant.clone());
                            }
                        }
                    }

                    updated_font.set("DescendantFonts", Object::Array(updated_descendants));
                }

                // Process ToUnicode stream if embedded
                if let Some(Object::Stream(stream_dict, stream_data)) = font_dict.get("ToUnicode") {
                    let tounicode_id = self.allocate_object_id();
                    self.write_object(
                        tounicode_id,
                        Object::Stream(stream_dict.clone(), stream_data.clone()),
                    )?;
                    updated_font.set("ToUnicode", Object::Reference(tounicode_id));
                }

                return Ok(updated_font);
            }
        }

        // Original Phase 3.3 logic for simple fonts (Type1, TrueType, etc.)
        // Check if font has a FontDescriptor
        if let Some(Object::Dictionary(descriptor)) = font_dict.get("FontDescriptor") {
            let mut updated_descriptor = descriptor.clone();
            let font_file_keys = ["FontFile", "FontFile2", "FontFile3"];

            // Check each font file key for embedded streams
            for key in &font_file_keys {
                if let Some(Object::Stream(stream_dict, stream_data)) = descriptor.get(*key) {
                    // Found embedded stream! Write it as a separate object
                    let stream_id = self.allocate_object_id();
                    let stream_obj = Object::Stream(stream_dict.clone(), stream_data.clone());
                    self.write_object(stream_id, stream_obj)?;

                    // Replace Stream with Reference to the newly written object
                    updated_descriptor.set(*key, Object::Reference(stream_id));
                }
                // If it's already a Reference, leave it as-is
            }

            // Update FontDescriptor in font dictionary
            updated_font.set("FontDescriptor", Object::Dictionary(updated_descriptor));
        }

        Ok(updated_font)
    }

    /// Helper function to process CIDFont embedded streams (Phase 3.4)
    fn write_cidfont_embedded_streams(
        &mut self,
        cidfont: &crate::objects::Dictionary,
    ) -> Result<crate::objects::Dictionary> {
        let mut updated_cidfont = cidfont.clone();

        // Process FontDescriptor
        if let Some(Object::Dictionary(descriptor)) = cidfont.get("FontDescriptor") {
            let mut updated_descriptor = descriptor.clone();
            let font_file_keys = ["FontFile", "FontFile2", "FontFile3"];

            // Write embedded font streams
            for key in &font_file_keys {
                if let Some(Object::Stream(stream_dict, stream_data)) = descriptor.get(*key) {
                    let stream_id = self.allocate_object_id();
                    self.write_object(
                        stream_id,
                        Object::Stream(stream_dict.clone(), stream_data.clone()),
                    )?;
                    updated_descriptor.set(*key, Object::Reference(stream_id));
                }
            }

            // Write FontDescriptor as a separate object
            let descriptor_id = self.allocate_object_id();
            self.write_object(descriptor_id, Object::Dictionary(updated_descriptor))?;

            // Update CIDFont to reference the FontDescriptor
            updated_cidfont.set("FontDescriptor", Object::Reference(descriptor_id));
        }

        // Process CIDToGIDMap if present and embedded as stream
        if let Some(Object::Stream(map_dict, map_data)) = cidfont.get("CIDToGIDMap") {
            let map_id = self.allocate_object_id();
            self.write_object(map_id, Object::Stream(map_dict.clone(), map_data.clone()))?;
            updated_cidfont.set("CIDToGIDMap", Object::Reference(map_id));
        }

        Ok(updated_cidfont)
    }

    fn allocate_object_id(&mut self) -> ObjectId {
        let id = ObjectId::new(self.next_object_id, 0);
        self.next_object_id += 1;
        id
    }

    /// Get catalog_id, returning error if not initialized
    fn get_catalog_id(&self) -> Result<ObjectId> {
        self.catalog_id.ok_or_else(|| {
            PdfError::InvalidOperation(
                "catalog_id not initialized - write_document() must be called first".to_string(),
            )
        })
    }

    /// Get pages_id, returning error if not initialized
    fn get_pages_id(&self) -> Result<ObjectId> {
        self.pages_id.ok_or_else(|| {
            PdfError::InvalidOperation(
                "pages_id not initialized - write_document() must be called first".to_string(),
            )
        })
    }

    /// Get info_id, returning error if not initialized
    fn get_info_id(&self) -> Result<ObjectId> {
        self.info_id.ok_or_else(|| {
            PdfError::InvalidOperation(
                "info_id not initialized - write_document() must be called first".to_string(),
            )
        })
    }

    fn write_object(&mut self, id: ObjectId, object: Object) -> Result<()> {
        use crate::writer::ObjectStreamWriter;

        // Encrypt the object if encryption is active
        let object = if let Some(ref enc_state) = self.encryption_state {
            let mut obj = object;
            enc_state.encryptor.encrypt_object(&mut obj, &id)?;
            obj
        } else {
            object
        };

        // If object streams enabled and object is compressible, buffer it
        if self.config.use_object_streams && ObjectStreamWriter::can_compress(&object) {
            let mut buffer = Vec::new();
            self.write_object_value_to_buffer(&object, &mut buffer)?;
            self.buffered_objects.insert(id, buffer);
            return Ok(());
        }

        // Otherwise write immediately (streams, encryption dicts, etc.)
        self.xref_positions.insert(id, self.current_position);

        // Pre-format header to count exact bytes once
        let header = format!("{} {} obj\n", id.number(), id.generation());
        self.write_bytes(header.as_bytes())?;

        self.write_object_value(&object)?;

        self.write_bytes(b"\nendobj\n")?;
        Ok(())
    }

    fn write_object_value(&mut self, object: &Object) -> Result<()> {
        match object {
            Object::Null => self.write_bytes(b"null")?,
            Object::Boolean(b) => self.write_bytes(if *b { b"true" } else { b"false" })?,
            Object::Integer(i) => self.write_bytes(i.to_string().as_bytes())?,
            Object::Real(f) => self.write_bytes(
                format!("{f:.6}")
                    .trim_end_matches('0')
                    .trim_end_matches('.')
                    .as_bytes(),
            )?,
            Object::String(s) => {
                // ISO 32000-1 §7.3.4.2: inside a literal string, the
                // characters `\`, `(` and `)` MUST be escaped (as `\\`,
                // `\(`, `\)` respectively) so the parser does not
                // terminate the string early or treat `\` as an escape
                // introducer for the following byte. Without this, a
                // caller-supplied value containing `)` (e.g. through
                // `Document::fill_field`) would close the literal and
                // allow dict-level injection into the enclosing object.
                self.write_bytes(b"(")?;
                self.write_bytes(&escape_pdf_string_bytes(s.as_bytes()))?;
                self.write_bytes(b")")?;
            }
            Object::ByteString(bytes) => {
                // Write as PDF hex string <AABB...> for byte-perfect binary data
                self.write_bytes(b"<")?;
                for byte in bytes {
                    self.write_bytes(format!("{byte:02X}").as_bytes())?;
                }
                self.write_bytes(b">")?;
            }
            Object::Name(n) => {
                self.write_bytes(b"/")?;
                self.write_bytes(n.as_bytes())?;
            }
            Object::Array(arr) => {
                self.write_bytes(b"[")?;
                for (i, obj) in arr.iter().enumerate() {
                    if i > 0 {
                        self.write_bytes(b" ")?;
                    }
                    self.write_object_value(obj)?;
                }
                self.write_bytes(b"]")?;
            }
            Object::Dictionary(dict) => {
                // Sort entries lexicographically by key for reproducible
                // output. `Dictionary` is backed by `HashMap` (with
                // per-instance randomised iteration order), so two
                // identical logical documents would otherwise emit
                // byte-different PDFs. PDF dict entries are unordered
                // by spec (ISO 32000-1 §7.3.7 Table 5: "the order of
                // entries ... is not significant"), so sorting is safe.
                self.write_bytes(b"<<")?;
                let mut entries: Vec<(&String, &Object)> = dict.entries().collect();
                entries.sort_by_key(|(k, _)| k.as_str());
                for (key, value) in entries {
                    self.write_bytes(b"\n/")?;
                    self.write_bytes(key.as_bytes())?;
                    self.write_bytes(b" ")?;
                    self.write_object_value(value)?;
                }
                self.write_bytes(b"\n>>")?;
            }
            Object::Stream(dict, data) => {
                // CRITICAL: Ensure Length in dictionary matches actual data length
                // This prevents "Bad Length" PDF syntax errors
                let mut corrected_dict = dict.clone();
                corrected_dict.set("Length", Object::Integer(data.len() as i64));

                self.write_object_value(&Object::Dictionary(corrected_dict))?;
                self.write_bytes(b"\nstream\n")?;
                self.write_bytes(data)?;
                self.write_bytes(b"\nendstream")?;
            }
            Object::Reference(id) => {
                let ref_str = format!("{} {} R", id.number(), id.generation());
                self.write_bytes(ref_str.as_bytes())?;
            }
        }
        Ok(())
    }

    /// Write object value to a buffer (for object streams)
    fn write_object_value_to_buffer(&self, object: &Object, buffer: &mut Vec<u8>) -> Result<()> {
        match object {
            Object::Null => buffer.extend_from_slice(b"null"),
            Object::Boolean(b) => buffer.extend_from_slice(if *b { b"true" } else { b"false" }),
            Object::Integer(i) => buffer.extend_from_slice(i.to_string().as_bytes()),
            Object::Real(f) => buffer.extend_from_slice(
                format!("{f:.6}")
                    .trim_end_matches('0')
                    .trim_end_matches('.')
                    .as_bytes(),
            ),
            Object::String(s) => {
                // Same escape rules as the streaming `write_object_value`
                // path — see ISO 32000-1 §7.3.4.2.
                buffer.push(b'(');
                buffer.extend_from_slice(&escape_pdf_string_bytes(s.as_bytes()));
                buffer.push(b')');
            }
            Object::ByteString(bytes) => {
                buffer.push(b'<');
                for byte in bytes {
                    buffer.extend_from_slice(format!("{byte:02X}").as_bytes());
                }
                buffer.push(b'>');
            }
            Object::Name(n) => {
                buffer.push(b'/');
                buffer.extend_from_slice(n.as_bytes());
            }
            Object::Array(arr) => {
                buffer.push(b'[');
                for (i, obj) in arr.iter().enumerate() {
                    if i > 0 {
                        buffer.push(b' ');
                    }
                    self.write_object_value_to_buffer(obj, buffer)?;
                }
                buffer.push(b']');
            }
            Object::Dictionary(dict) => {
                // Same deterministic-order rule as the streaming writer
                // (see `write_object_value`): sort entries by key for
                // reproducible output across builds.
                buffer.extend_from_slice(b"<<");
                let mut entries: Vec<(&String, &Object)> = dict.entries().collect();
                entries.sort_by_key(|(k, _)| k.as_str());
                for (key, value) in entries {
                    buffer.extend_from_slice(b"\n/");
                    buffer.extend_from_slice(key.as_bytes());
                    buffer.push(b' ');
                    self.write_object_value_to_buffer(value, buffer)?;
                }
                buffer.extend_from_slice(b"\n>>");
            }
            Object::Stream(_, _) => {
                // Streams should never be compressed in object streams
                return Err(crate::error::PdfError::ObjectStreamError(
                    "Cannot compress stream objects in object streams".to_string(),
                ));
            }
            Object::Reference(id) => {
                let ref_str = format!("{} {} R", id.number(), id.generation());
                buffer.extend_from_slice(ref_str.as_bytes());
            }
        }
        Ok(())
    }

    /// Flush buffered objects as compressed object streams
    fn flush_object_streams(&mut self) -> Result<()> {
        if self.buffered_objects.is_empty() {
            return Ok(());
        }

        // Create object stream writer
        let config = ObjectStreamConfig {
            max_objects_per_stream: 100,
            compression_level: 6,
            enabled: true,
        };
        let mut os_writer = ObjectStreamWriter::new(config);

        // Sort buffered objects by ID for deterministic output
        let mut buffered: Vec<_> = self.buffered_objects.iter().collect();
        buffered.sort_by_key(|(id, _)| id.number());

        // Add all buffered objects to the stream writer
        for (id, data) in buffered {
            os_writer.add_object(*id, data.clone())?;
        }

        // Finalize and get completed streams
        let streams = os_writer.finalize()?;

        // Write each object stream to the PDF
        for mut stream in streams {
            let stream_id = stream.stream_id;

            // Generate compressed stream data
            let compressed_data = stream.generate_stream_data(6)?;

            // Generate stream dictionary
            let dict = stream.generate_dictionary(&compressed_data);

            // Track compressed object mapping for xref
            for (index, (obj_id, _)) in stream.objects.iter().enumerate() {
                self.compressed_object_map
                    .insert(*obj_id, (stream_id, index as u32));
            }

            // Write the object stream itself
            self.xref_positions.insert(stream_id, self.current_position);

            let header = format!("{} {} obj\n", stream_id.number(), stream_id.generation());
            self.write_bytes(header.as_bytes())?;

            self.write_object_value(&Object::Dictionary(dict))?;

            self.write_bytes(b"\nstream\n")?;
            self.write_bytes(&compressed_data)?;
            self.write_bytes(b"\nendstream\nendobj\n")?;
        }

        Ok(())
    }

    fn write_xref(&mut self) -> Result<()> {
        self.write_bytes(b"xref\n")?;

        // Sort by object number and write entries
        let mut entries: Vec<_> = self
            .xref_positions
            .iter()
            .map(|(id, pos)| (*id, *pos))
            .collect();
        entries.sort_by_key(|(id, _)| id.number());

        // Find the highest object number to determine size
        let max_obj_num = entries.iter().map(|(id, _)| id.number()).max().unwrap_or(0);

        // Write subsection header - PDF 1.7 spec allows multiple subsections
        // For simplicity, write one subsection from 0 to max
        self.write_bytes(b"0 ")?;
        self.write_bytes((max_obj_num + 1).to_string().as_bytes())?;
        self.write_bytes(b"\n")?;

        // Write free object entry
        self.write_bytes(b"0000000000 65535 f \n")?;

        // Write entries for all object numbers from 1 to max
        // Fill in gaps with free entries
        for obj_num in 1..=max_obj_num {
            let _obj_id = ObjectId::new(obj_num, 0);
            if let Some((_, position)) = entries.iter().find(|(id, _)| id.number() == obj_num) {
                let entry = format!("{:010} {:05} n \n", position, 0);
                self.write_bytes(entry.as_bytes())?;
            } else {
                // Free entry for gap
                self.write_bytes(b"0000000000 00000 f \n")?;
            }
        }

        Ok(())
    }

    fn write_xref_stream(&mut self) -> Result<()> {
        let catalog_id = self.get_catalog_id()?;
        let info_id = self.get_info_id()?;

        // Allocate object ID for the xref stream
        let xref_stream_id = self.allocate_object_id();
        let xref_position = self.current_position;

        // Create XRef stream writer with trailer information
        let mut xref_writer = XRefStreamWriter::new(xref_stream_id);
        xref_writer.set_trailer_info(catalog_id, info_id);

        // Add free entry for object 0
        xref_writer.add_free_entry(0, 65535);

        // Sort entries by object number
        let mut entries: Vec<_> = self
            .xref_positions
            .iter()
            .map(|(id, pos)| (*id, *pos))
            .collect();
        entries.sort_by_key(|(id, _)| id.number());

        // Find the highest object number (including the xref stream itself)
        let max_obj_num = entries
            .iter()
            .map(|(id, _)| id.number())
            .max()
            .unwrap_or(0)
            .max(xref_stream_id.number());

        // Add entries for all objects (including compressed objects)
        for obj_num in 1..=max_obj_num {
            let obj_id = ObjectId::new(obj_num, 0);

            if obj_num == xref_stream_id.number() {
                // The xref stream entry will be added with the correct position
                xref_writer.add_in_use_entry(xref_position, 0);
            } else if let Some((stream_id, index)) = self.compressed_object_map.get(&obj_id) {
                // Type 2: Object is compressed in an object stream
                xref_writer.add_compressed_entry(stream_id.number(), *index);
            } else if let Some((id, position)) =
                entries.iter().find(|(id, _)| id.number() == obj_num)
            {
                // Type 1: Regular in-use entry
                xref_writer.add_in_use_entry(*position, id.generation());
            } else {
                // Type 0: Free entry for gap
                xref_writer.add_free_entry(0, 0);
            }
        }

        // Mark position for xref stream object
        self.xref_positions.insert(xref_stream_id, xref_position);

        // Write object header
        self.write_bytes(
            format!(
                "{} {} obj\n",
                xref_stream_id.number(),
                xref_stream_id.generation()
            )
            .as_bytes(),
        )?;

        // Get the encoded data
        let uncompressed_data = xref_writer.encode_entries();
        let final_data = if self.config.compress_streams {
            crate::compression::compress(&uncompressed_data)?
        } else {
            uncompressed_data
        };

        // Create and write dictionary
        let mut dict = xref_writer.create_dictionary(None);
        dict.set("Length", Object::Integer(final_data.len() as i64));

        // Add filter if compression is enabled
        if self.config.compress_streams {
            dict.set("Filter", Object::Name("FlateDecode".to_string()));
        }
        self.write_bytes(b"<<")?;
        for (key, value) in dict.iter() {
            self.write_bytes(b"\n/")?;
            self.write_bytes(key.as_bytes())?;
            self.write_bytes(b" ")?;
            self.write_object_value(value)?;
        }
        self.write_bytes(b"\n>>\n")?;

        // Write stream
        self.write_bytes(b"stream\n")?;
        self.write_bytes(&final_data)?;
        self.write_bytes(b"\nendstream\n")?;
        self.write_bytes(b"endobj\n")?;

        // Write startxref and EOF
        self.write_bytes(b"\nstartxref\n")?;
        self.write_bytes(xref_position.to_string().as_bytes())?;
        self.write_bytes(b"\n%%EOF\n")?;

        Ok(())
    }

    /// Write the encryption dictionary as an indirect object and store
    /// the object ID and file ID for the trailer.
    /// Initialize encryption state: generates file ID, creates encryption dict,
    /// computes encryption key, and builds the ObjectEncryptor.
    /// The /Encrypt dict object is written later (after all other objects) since it
    /// must NOT be encrypted itself (ISO 32000-1 §7.6.1).
    fn init_encryption(&mut self, encryption: &crate::document::DocumentEncryption) -> Result<()> {
        use crate::encryption::{
            CryptFilterManager, CryptFilterMethod, FunctionalCryptFilter, ObjectEncryptor,
        };
        use std::sync::Arc;

        // Generate file ID (16 random bytes, required by ISO 32000-1 §7.5.5)
        let mut fid = vec![0u8; 16];
        use rand::Rng;
        rand::rng().fill_bytes(&mut fid);

        let enc_dict = encryption
            .create_encryption_dict(Some(&fid))
            .map_err(|e| PdfError::EncryptionError(format!("encryption dict: {}", e)))?;

        // Compute encryption key
        let enc_key = encryption
            .get_encryption_key(&enc_dict, Some(&fid))
            .map_err(|e| PdfError::EncryptionError(format!("encryption key: {}", e)))?;

        // Build CryptFilterManager based on encryption strength
        let handler = encryption.handler();
        let (method, key_len) = match encryption.strength {
            crate::document::EncryptionStrength::Rc4_40bit => (CryptFilterMethod::V2, Some(5)),
            crate::document::EncryptionStrength::Rc4_128bit => (CryptFilterMethod::V2, Some(16)),
            crate::document::EncryptionStrength::Aes128 => (CryptFilterMethod::AESV2, Some(16)),
            crate::document::EncryptionStrength::Aes256 => (CryptFilterMethod::AESV3, Some(32)),
        };

        let std_filter = FunctionalCryptFilter {
            name: "StdCF".to_string(),
            method,
            length: key_len,
            auth_event: crate::encryption::AuthEvent::DocOpen,
            recipients: None,
        };

        let mut filter_manager =
            CryptFilterManager::new(Box::new(handler), "StdCF".to_string(), "StdCF".to_string());
        filter_manager.add_filter(std_filter);

        let encryptor =
            ObjectEncryptor::new(Arc::new(filter_manager), enc_key, enc_dict.encrypt_metadata);

        // Reserve ID for /Encrypt dict (will be written at the end)
        let encrypt_id = self.allocate_object_id();
        self.encrypt_obj_id = Some(encrypt_id);
        self.file_id = Some(fid);
        self.encryption_state = Some(WriterEncryptionState { encryptor });

        // Store the dict to write later
        self.pending_encrypt_dict = Some(enc_dict.to_dict());

        Ok(())
    }

    /// Write the /Encrypt dictionary object (must NOT be encrypted per ISO 32000-1 §7.6.1)
    fn write_encryption_dict(&mut self) -> Result<()> {
        if let (Some(encrypt_id), Some(dict)) =
            (self.encrypt_obj_id, self.pending_encrypt_dict.take())
        {
            // Temporarily disable encryption so the /Encrypt dict is not encrypted
            let enc_state = self.encryption_state.take();
            self.write_object(encrypt_id, Object::Dictionary(dict))?;
            self.encryption_state = enc_state;
        }
        Ok(())
    }

    fn write_trailer(&mut self, xref_position: u64) -> Result<()> {
        let catalog_id = self.get_catalog_id()?;
        let info_id = self.get_info_id()?;
        // Find the highest object number to determine size
        let max_obj_num = self
            .xref_positions
            .keys()
            .map(|id| id.number())
            .max()
            .unwrap_or(0);

        let mut trailer = Dictionary::new();
        trailer.set("Size", Object::Integer((max_obj_num + 1) as i64));
        trailer.set("Root", Object::Reference(catalog_id));
        trailer.set("Info", Object::Reference(info_id));

        // Add /Prev pointer for incremental updates (ISO 32000-1 §7.5.6)
        if let Some(prev_xref) = self.prev_xref_offset {
            trailer.set("Prev", Object::Integer(prev_xref as i64));
        }

        // Add /Encrypt reference and /ID array for encrypted documents
        if let Some(encrypt_id) = self.encrypt_obj_id {
            trailer.set("Encrypt", Object::Reference(encrypt_id));
        }
        if let Some(ref fid) = self.file_id {
            trailer.set(
                "ID",
                Object::Array(vec![
                    Object::ByteString(fid.clone()),
                    Object::ByteString(fid.clone()),
                ]),
            );
        }

        self.write_bytes(b"trailer\n")?;
        self.write_object_value(&Object::Dictionary(trailer))?;
        self.write_bytes(b"\nstartxref\n")?;
        self.write_bytes(xref_position.to_string().as_bytes())?;
        self.write_bytes(b"\n%%EOF\n")?;

        Ok(())
    }

    fn write_bytes(&mut self, data: &[u8]) -> Result<()> {
        self.writer.write_all(data)?;
        self.current_position += data.len() as u64;
        Ok(())
    }

    #[allow(dead_code)]
    fn create_widget_appearance_stream(&mut self, widget_dict: &Dictionary) -> Result<ObjectId> {
        // Get widget rectangle
        let rect = if let Some(Object::Array(rect_array)) = widget_dict.get("Rect") {
            if rect_array.len() >= 4 {
                if let (
                    Some(Object::Real(x1)),
                    Some(Object::Real(y1)),
                    Some(Object::Real(x2)),
                    Some(Object::Real(y2)),
                ) = (
                    rect_array.first(),
                    rect_array.get(1),
                    rect_array.get(2),
                    rect_array.get(3),
                ) {
                    (*x1, *y1, *x2, *y2)
                } else {
                    (0.0, 0.0, 100.0, 20.0) // Default
                }
            } else {
                (0.0, 0.0, 100.0, 20.0) // Default
            }
        } else {
            (0.0, 0.0, 100.0, 20.0) // Default
        };

        let width = rect.2 - rect.0;
        let height = rect.3 - rect.1;

        // Create appearance stream content
        let mut content = String::new();

        // Set graphics state
        content.push_str("q\n");

        // Draw border (black) — single source of truth for color emission.
        crate::graphics::color::write_stroke_color(&mut content, crate::graphics::Color::black());
        content.push_str("1 w\n"); // 1pt line width

        // Draw rectangle border
        content.push_str(&format!("0 0 {width} {height} re\n"));
        content.push_str("S\n"); // Stroke

        // Fill with white background
        crate::graphics::color::write_fill_color(&mut content, crate::graphics::Color::white());
        content.push_str(&format!("0.5 0.5 {} {} re\n", width - 1.0, height - 1.0));
        content.push_str("f\n"); // Fill

        // Restore graphics state
        content.push_str("Q\n");

        // Create stream dictionary
        let mut stream_dict = Dictionary::new();
        stream_dict.set("Type", Object::Name("XObject".to_string()));
        stream_dict.set("Subtype", Object::Name("Form".to_string()));
        stream_dict.set(
            "BBox",
            Object::Array(vec![
                Object::Real(0.0),
                Object::Real(0.0),
                Object::Real(width),
                Object::Real(height),
            ]),
        );
        stream_dict.set("Resources", Object::Dictionary(Dictionary::new()));
        stream_dict.set("Length", Object::Integer(content.len() as i64));

        // Write the appearance stream
        let stream_id = self.allocate_object_id();
        self.write_object(stream_id, Object::Stream(stream_dict, content.into_bytes()))?;

        Ok(stream_id)
    }

    #[allow(dead_code)]
    fn create_field_appearance_stream(
        &mut self,
        field_dict: &Dictionary,
        widget: &crate::forms::Widget,
    ) -> Result<ObjectId> {
        let width = widget.rect.upper_right.x - widget.rect.lower_left.x;
        let height = widget.rect.upper_right.y - widget.rect.lower_left.y;

        // Create appearance stream content
        let mut content = String::new();

        // Set graphics state
        content.push_str("q\n");

        // Draw background if specified — routed through the shared
        // NaN-sanitising helpers (issues #220, #221).
        if let Some(bg_color) = &widget.appearance.background_color {
            crate::graphics::color::write_fill_color(&mut content, *bg_color);
            content.push_str(&format!("0 0 {width} {height} re\n"));
            content.push_str("f\n");
        }

        // Draw border
        if let Some(border_color) = &widget.appearance.border_color {
            crate::graphics::color::write_stroke_color(&mut content, *border_color);
            content.push_str(&format!("{} w\n", widget.appearance.border_width));
            content.push_str(&format!("0 0 {width} {height} re\n"));
            content.push_str("S\n");
        }

        // For checkboxes, add a checkmark if checked
        if let Some(Object::Name(ft)) = field_dict.get("FT") {
            if ft == "Btn" {
                if let Some(Object::Name(v)) = field_dict.get("V") {
                    if v == "Yes" {
                        // Draw checkmark
                        crate::graphics::color::write_stroke_color(
                            &mut content,
                            crate::graphics::Color::black(),
                        );
                        content.push_str("2 w\n");
                        let margin = width * 0.2;
                        content.push_str(&format!("{} {} m\n", margin, height / 2.0));
                        content.push_str(&format!("{} {} l\n", width / 2.0, margin));
                        content.push_str(&format!("{} {} l\n", width - margin, height - margin));
                        content.push_str("S\n");
                    }
                }
            }
        }

        // Restore graphics state
        content.push_str("Q\n");

        // Create stream dictionary
        let mut stream_dict = Dictionary::new();
        stream_dict.set("Type", Object::Name("XObject".to_string()));
        stream_dict.set("Subtype", Object::Name("Form".to_string()));
        stream_dict.set(
            "BBox",
            Object::Array(vec![
                Object::Real(0.0),
                Object::Real(0.0),
                Object::Real(width),
                Object::Real(height),
            ]),
        );
        stream_dict.set("Resources", Object::Dictionary(Dictionary::new()));
        stream_dict.set("Length", Object::Integer(content.len() as i64));

        // Write the appearance stream
        let stream_id = self.allocate_object_id();
        self.write_object(stream_id, Object::Stream(stream_dict, content.into_bytes()))?;

        Ok(stream_id)
    }
}

/// Format a DateTime as a PDF date string (D:YYYYMMDDHHmmSSOHH'mm)
fn format_pdf_date(date: DateTime<Utc>) -> String {
    // Format the UTC date according to PDF specification
    // D:YYYYMMDDHHmmSSOHH'mm where O is the relationship of local time to UTC (+ or -)
    let formatted = date.format("D:%Y%m%d%H%M%S");

    // For UTC, the offset is always +00'00
    format!("{formatted}+00'00")
}

#[cfg(test)]
mod tests;

#[cfg(test)]
mod rigorous_tests;