oxidize_pdf/parser/

content.rs

//! PDF Content Stream Parser - Complete support for PDF graphics operators
//!
//! This module implements comprehensive parsing of PDF content streams according to the PDF specification.
//! Content streams contain the actual drawing instructions (operators) that render text, graphics, and images
//! on PDF pages.
//!
//! # Overview
//!
//! Content streams are sequences of PDF operators that describe:
//! - Text positioning and rendering
//! - Path construction and painting
//! - Color and graphics state management
//! - Image and XObject placement
//! - Coordinate transformations
//!
//! # Architecture
//!
//! The parser is divided into two main components:
//! - `ContentTokenizer`: Low-level tokenization of content stream bytes
//! - `ContentParser`: High-level parsing of tokens into structured operations
//!
//! # Example
//!
//! ```rust,no_run
//! use oxidize_pdf::parser::content::{ContentParser, ContentOperation};
//!
//! # fn example() -> Result<(), Box<dyn std::error::Error>> {
//! // Parse a content stream
//! let content_stream = b"BT /F1 12 Tf 100 200 Td (Hello World) Tj ET";
//! let operations = ContentParser::parse_content(content_stream)?;
//!
//! // Process operations
//! for op in operations {
//!     match op {
//!         ContentOperation::BeginText => println!("Start text object"),
//!         ContentOperation::SetFont(name, size) => println!("Font: {} at {}", name, size),
//!         ContentOperation::ShowText(text) => println!("Text: {:?}", text),
//!         _ => {}
//!     }
//! }
//! # Ok(())
//! # }
//! ```
//!
//! # Supported Operators
//!
//! This parser supports all standard PDF operators including:
//! - Text operators (BT, ET, Tj, TJ, Tf, Td, etc.)
//! - Graphics state operators (q, Q, cm, w, J, etc.)
//! - Path construction operators (m, l, c, re, h)
//! - Path painting operators (S, f, B, n, etc.)
//! - Color operators (g, rg, k, cs, scn, etc.)
//! - XObject operators (Do)
//! - Marked content operators (BMC, BDC, EMC, etc.)

use super::{ParseError, ParseResult};
use crate::objects::Object;
use std::collections::HashMap;

/// A single value inside a marked-content properties dictionary or array.
///
/// PDF marked-content properties (BDC, DP) carry typed values: strings,
/// integers, real numbers, names, arrays, and nested dictionaries. The
/// previous `HashMap<String, String>` carrier was lossy for `/ActualText`
/// (UTF-16BE bytes mangled by `String::from_utf8_lossy`) and for `/MCID`
/// (integer values stored as their decimal string representation). This
/// enum preserves the original token type and bytes; decoding happens
/// lazily at the extractor level (e.g. UTF-16BE detection via BOM).
///
/// Hex strings (`<FEFF00660069>`) and literal strings (`(text)`) both
/// land here as `MarkedContentValue::String(Vec<u8>)` because both are
/// raw byte sequences at the PDF tokenizer level.
#[derive(Debug, Clone, PartialEq)]
pub enum MarkedContentValue {
    /// Raw PDF string bytes (from either `Token::String` or `Token::HexString`).
    /// Decoded lazily by consumers — UTF-16BE detection via BOM happens in the
    /// extractor's `decode_pdf_string` helper.
    String(Vec<u8>),
    /// PDF integer (e.g. `/MCID 0`).
    Integer(i64),
    /// PDF real number.
    Real(f64),
    /// PDF name token (e.g. `/Pagination`).
    Name(String),
    /// PDF array; nested values are themselves `MarkedContentValue`.
    Array(Vec<MarkedContentValue>),
    /// Nested dictionary; keys are PDF name strings (the leading `/` is stripped).
    Dict(HashMap<String, MarkedContentValue>),
}

/// Properties operand of a BDC/DP operator. Two shapes per ISO 32000-1
/// §14.6.2:
///
/// - **Inline**: the second BDC operand is an inline dictionary literal
///   (`<< /MCID 0 /ActualText (fi) >>`). Keys map to `MarkedContentValue`.
/// - **ResourceRef**: the second BDC operand is a name (`/PropsName`) that
///   references the page's `/Resources /Properties /<name>` dictionary.
///   Resolution against the page's resource tree happens in the extractor
///   (parser does not have access to the page object).
#[derive(Debug, Clone, PartialEq)]
pub enum MarkedContentProps {
    Inline(HashMap<String, MarkedContentValue>),
    ResourceRef(String),
}

/// Represents a single operator in a PDF content stream.
///
/// Each variant corresponds to a specific PDF operator and carries the associated
/// operands. These operations form a complete instruction set for rendering PDF content.
///
/// # Categories
///
/// Operations are grouped into several categories:
/// - **Text Object**: BeginText, EndText
/// - **Text State**: Font, spacing, scaling, rendering mode
/// - **Text Positioning**: Matrix transforms, moves, line advances
/// - **Text Showing**: Display text with various formatting
/// - **Graphics State**: Save/restore, transforms, line properties
/// - **Path Construction**: Move, line, curve, rectangle operations
/// - **Path Painting**: Stroke, fill, clipping operations
/// - **Color**: RGB, CMYK, grayscale, and color space operations
/// - **XObject**: External graphics and form placement
/// - **Marked Content**: Semantic tagging for accessibility
///
/// # Example
///
/// ```rust
/// use oxidize_pdf::parser::content::{ContentOperation};
///
/// // Text operation
/// let op1 = ContentOperation::ShowText(b"Hello".to_vec());
///
/// // Graphics operation
/// let op2 = ContentOperation::SetLineWidth(2.0);
///
/// // Path operation
/// let op3 = ContentOperation::Rectangle(10.0, 10.0, 100.0, 50.0);
/// ```
#[derive(Debug, Clone, PartialEq)]
pub enum ContentOperation {
    // Text object operators
    /// Begin a text object (BT operator).
    /// All text showing operations must occur within a text object.
    BeginText,

    /// End a text object (ET operator).
    /// Closes the current text object started with BeginText.
    EndText,

    // Text state operators
    /// Set character spacing (Tc operator).
    /// Additional space between characters in unscaled text units.
    SetCharSpacing(f32),

    /// Set word spacing (Tw operator).
    /// Additional space for ASCII space character (0x20) in unscaled text units.
    SetWordSpacing(f32),

    /// Set horizontal text scaling (Tz operator).
    /// Percentage of normal width (100 = normal).
    SetHorizontalScaling(f32),

    /// Set text leading (TL operator).
    /// Vertical distance between baselines for T* operator.
    SetLeading(f32),

    /// Set font and size (Tf operator).
    /// Font name must match a key in the Resources/Font dictionary.
    SetFont(String, f32),

    /// Set text rendering mode (Tr operator).
    /// 0=fill, 1=stroke, 2=fill+stroke, 3=invisible, 4=fill+clip, 5=stroke+clip, 6=fill+stroke+clip, 7=clip
    SetTextRenderMode(i32),

    /// Set text rise (Ts operator).
    /// Vertical displacement for superscripts/subscripts in text units.
    SetTextRise(f32),

    // Text positioning operators
    /// Move text position (Td operator).
    /// Translates the text matrix by (tx, ty).
    MoveText(f32, f32),

    /// Move text position and set leading (TD operator).
    /// Equivalent to: -ty TL tx ty Td
    MoveTextSetLeading(f32, f32),

    /// Set text matrix directly (Tm operator).
    /// Parameters: [a, b, c, d, e, f] for transformation matrix.
    SetTextMatrix(f32, f32, f32, f32, f32, f32),

    /// Move to start of next line (T* operator).
    /// Uses the current leading value set with TL.
    NextLine,

    // Text showing operators
    /// Show text string (Tj operator).
    /// The bytes are encoded according to the current font's encoding.
    ShowText(Vec<u8>),

    /// Show text with individual positioning (TJ operator).
    /// Array elements can be strings or position adjustments.
    ShowTextArray(Vec<TextElement>),

    /// Move to next line and show text (' operator).
    /// Equivalent to: T* string Tj
    NextLineShowText(Vec<u8>),

    /// Set spacing, move to next line, and show text (" operator).
    /// Equivalent to: word_spacing Tw char_spacing Tc string '
    SetSpacingNextLineShowText(f32, f32, Vec<u8>),

    // Graphics state operators
    /// Save current graphics state (q operator).
    /// Pushes the entire graphics state onto a stack.
    SaveGraphicsState,

    /// Restore graphics state (Q operator).
    /// Pops the graphics state from the stack.
    RestoreGraphicsState,

    /// Concatenate matrix to current transformation matrix (cm operator).
    /// Modifies the CTM: CTM' = CTM × [a b c d e f]
    SetTransformMatrix(f32, f32, f32, f32, f32, f32),

    /// Set line width (w operator) in user space units.
    SetLineWidth(f32),

    /// Set line cap style (J operator).
    /// 0=butt cap, 1=round cap, 2=projecting square cap
    SetLineCap(i32),

    /// Set line join style (j operator).
    /// 0=miter join, 1=round join, 2=bevel join
    SetLineJoin(i32),

    /// Set miter limit (M operator).
    /// Maximum ratio of miter length to line width.
    SetMiterLimit(f32),

    /// Set dash pattern (d operator).
    /// Array of dash/gap lengths and starting phase.
    SetDashPattern(Vec<f32>, f32),

    /// Set rendering intent (ri operator).
    /// Color rendering intent: /AbsoluteColorimetric, /RelativeColorimetric, /Saturation, /Perceptual
    SetIntent(String),

    /// Set flatness tolerance (i operator).
    /// Maximum error when rendering curves as line segments.
    SetFlatness(f32),

    /// Set graphics state from parameter dictionary (gs operator).
    /// References ExtGState resource dictionary.
    SetGraphicsStateParams(String),

    // Path construction operators
    /// Begin new subpath at point (m operator).
    MoveTo(f32, f32),

    /// Append straight line segment (l operator).
    LineTo(f32, f32),

    /// Append cubic Bézier curve (c operator).
    /// Control points: (x1,y1), (x2,y2), endpoint: (x3,y3)
    CurveTo(f32, f32, f32, f32, f32, f32),

    /// Append cubic Bézier curve with first control point = current point (v operator).
    CurveToV(f32, f32, f32, f32),

    /// Append cubic Bézier curve with second control point = endpoint (y operator).
    CurveToY(f32, f32, f32, f32),

    /// Close current subpath (h operator).
    /// Appends straight line to starting point.
    ClosePath,

    /// Append rectangle as complete subpath (re operator).
    /// Parameters: x, y, width, height
    Rectangle(f32, f32, f32, f32),

    // Path painting operators
    /// Stroke the path (S operator).
    Stroke,

    /// Close and stroke the path (s operator).
    /// Equivalent to: h S
    CloseStroke,

    /// Fill the path using nonzero winding rule (f or F operator).
    Fill,

    /// Fill the path using even-odd rule (f* operator).
    FillEvenOdd,

    /// Fill then stroke the path (B operator).
    /// Uses nonzero winding rule.
    FillStroke,

    /// Fill then stroke using even-odd rule (B* operator).
    FillStrokeEvenOdd,

    /// Close, fill, and stroke the path (b operator).
    /// Equivalent to: h B
    CloseFillStroke,

    /// Close, fill, and stroke using even-odd rule (b* operator).
    CloseFillStrokeEvenOdd,

    /// End path without filling or stroking (n operator).
    /// Used primarily before clipping.
    EndPath,

    // Clipping path operators
    Clip,        // W
    ClipEvenOdd, // W*

    // Color operators
    /// Set stroking color space (CS operator).
    /// References ColorSpace resource dictionary.
    SetStrokingColorSpace(String),

    /// Set non-stroking color space (cs operator).
    /// References ColorSpace resource dictionary.
    SetNonStrokingColorSpace(String),

    /// Set stroking color (SC, SCN operators).
    /// Number of components depends on current color space.
    SetStrokingColor(Vec<f32>),

    /// Set non-stroking color (sc, scn operators).
    /// Number of components depends on current color space.
    SetNonStrokingColor(Vec<f32>),

    /// Set stroking color to DeviceGray (G operator).
    /// 0.0 = black, 1.0 = white
    SetStrokingGray(f32),

    /// Set non-stroking color to DeviceGray (g operator).
    SetNonStrokingGray(f32),

    /// Set stroking color to DeviceRGB (RG operator).
    /// Components range from 0.0 to 1.0.
    SetStrokingRGB(f32, f32, f32),

    /// Set non-stroking color to DeviceRGB (rg operator).
    SetNonStrokingRGB(f32, f32, f32),

    /// Set stroking color to DeviceCMYK (K operator).
    SetStrokingCMYK(f32, f32, f32, f32),

    /// Set non-stroking color to DeviceCMYK (k operator).
    SetNonStrokingCMYK(f32, f32, f32, f32),

    // Shading operators
    ShadingFill(String), // sh

    // Inline image operators
    /// Begin inline image (BI operator)
    BeginInlineImage,
    /// Inline image with parsed dictionary and data
    InlineImage {
        /// Image parameters (width, height, colorspace, etc.)
        params: HashMap<String, Object>,
        /// Raw image data
        data: Vec<u8>,
    },

    // XObject operators
    /// Paint external object (Do operator).
    /// References XObject resource dictionary (images, forms).
    PaintXObject(String),

    // Marked content operators
    BeginMarkedContent(String),                                    // BMC
    BeginMarkedContentWithProps(String, MarkedContentProps),       // BDC
    EndMarkedContent,                                              // EMC
    DefineMarkedContentPoint(String),                              // MP
    DefineMarkedContentPointWithProps(String, MarkedContentProps), // DP

    // Compatibility operators
    BeginCompatibility, // BX
    EndCompatibility,   // EX
}

/// Represents a text element in a TJ array for ShowTextArray operations.
///
/// The TJ operator takes an array of strings and position adjustments,
/// allowing fine control over character and word spacing.
///
/// # Example
///
/// ```rust
/// use oxidize_pdf::parser::content::{TextElement, ContentOperation};
///
/// // TJ array: [(Hello) -50 (World)]
/// let tj_array = vec![
///     TextElement::Text(b"Hello".to_vec()),
///     TextElement::Spacing(-50.0), // Move left 50 units
///     TextElement::Text(b"World".to_vec()),
/// ];
/// let op = ContentOperation::ShowTextArray(tj_array);
/// ```
#[derive(Debug, Clone, PartialEq)]
pub enum TextElement {
    /// Text string to show
    Text(Vec<u8>),
    /// Position adjustment in thousandths of text space units
    /// Negative values move to the right (decrease spacing)
    Spacing(f32),
}

/// Token types in content streams
#[derive(Debug, Clone, PartialEq)]
pub(super) enum Token {
    Number(f32),
    Integer(i32),
    String(Vec<u8>),
    HexString(Vec<u8>),
    Name(String),
    Operator(String),
    ArrayStart,
    ArrayEnd,
    DictStart,
    DictEnd,
    /// Raw binary data between ID and EI in an inline image.
    /// The tokenizer captures this as opaque bytes to prevent
    /// binary image data from being mis-parsed as operators.
    InlineImageData(Vec<u8>),
}

/// Content stream tokenizer
pub struct ContentTokenizer<'a> {
    input: &'a [u8],
    position: usize,
    /// Set after returning an "ID" operator token.
    /// The next call to next_token() will read raw inline image bytes.
    in_inline_image: bool,
}

impl<'a> ContentTokenizer<'a> {
    /// Create a new tokenizer for the given input
    pub fn new(input: &'a [u8]) -> Self {
        Self {
            input,
            position: 0,
            in_inline_image: false,
        }
    }

    /// Get the next token from the stream
    pub(super) fn next_token(&mut self) -> ParseResult<Option<Token>> {
        // If we just returned an "ID" token, read raw inline image binary data
        if self.in_inline_image {
            self.in_inline_image = false;
            return self.read_inline_image_data();
        }

        self.skip_whitespace();

        if self.position >= self.input.len() {
            return Ok(None);
        }

        let ch = self.input[self.position];

        match ch {
            // Numbers
            b'+' | b'-' | b'.' | b'0'..=b'9' => self.read_number(),

            // Strings
            b'(' => self.read_literal_string(),
            b'<' => {
                if self.peek_next() == Some(b'<') {
                    self.position += 2;
                    Ok(Some(Token::DictStart))
                } else {
                    self.read_hex_string()
                }
            }
            b'>' => {
                if self.peek_next() == Some(b'>') {
                    self.position += 2;
                    Ok(Some(Token::DictEnd))
                } else {
                    Err(ParseError::SyntaxError {
                        position: self.position,
                        message: "Unexpected '>'".to_string(),
                    })
                }
            }

            // Arrays
            b'[' => {
                self.position += 1;
                Ok(Some(Token::ArrayStart))
            }
            b']' => {
                self.position += 1;
                Ok(Some(Token::ArrayEnd))
            }

            // Names
            b'/' => self.read_name(),

            // Skip unhandled delimiters (corrupted content / binary data recovery)
            // These bytes are delimiters in read_operator() but have no valid meaning
            // at the top level of a content stream. Skipping them prevents infinite loops
            // where read_operator() would return an empty operator without advancing.
            b';' | b')' | b'{' | b'}' => {
                self.position += 1;
                self.next_token() // Recursively get next valid token
            }

            // Operators or other tokens
            _ => {
                let token = self.read_operator()?;
                // After "ID" operator, switch to raw binary mode for inline image data
                if let Some(Token::Operator(ref op)) = token {
                    if op == "ID" {
                        self.in_inline_image = true;
                    }
                }
                Ok(token)
            }
        }
    }

    fn skip_whitespace(&mut self) {
        while self.position < self.input.len() {
            match self.input[self.position] {
                b' ' | b'\t' | b'\r' | b'\n' | b'\x0C' => self.position += 1,
                b'%' => self.skip_comment(),
                _ => break,
            }
        }
    }

    fn skip_comment(&mut self) {
        while self.position < self.input.len() && self.input[self.position] != b'\n' {
            self.position += 1;
        }
    }

    fn peek_next(&self) -> Option<u8> {
        if self.position + 1 < self.input.len() {
            Some(self.input[self.position + 1])
        } else {
            None
        }
    }

    fn read_number(&mut self) -> ParseResult<Option<Token>> {
        let start = self.position;
        let mut has_dot = false;

        // Handle optional sign
        if self.position < self.input.len()
            && (self.input[self.position] == b'+' || self.input[self.position] == b'-')
        {
            self.position += 1;
        }

        // Read digits and optional decimal point
        while self.position < self.input.len() {
            match self.input[self.position] {
                b'0'..=b'9' => self.position += 1,
                b'.' if !has_dot => {
                    has_dot = true;
                    self.position += 1;
                }
                _ => break,
            }
        }

        let num_str = std::str::from_utf8(&self.input[start..self.position]).map_err(|_| {
            ParseError::SyntaxError {
                position: start,
                message: "Invalid number format".to_string(),
            }
        })?;

        if has_dot {
            let value = num_str
                .parse::<f32>()
                .map_err(|_| ParseError::SyntaxError {
                    position: start,
                    message: "Invalid float number".to_string(),
                })?;
            Ok(Some(Token::Number(value)))
        } else {
            let value = num_str
                .parse::<i32>()
                .map_err(|_| ParseError::SyntaxError {
                    position: start,
                    message: "Invalid integer number".to_string(),
                })?;
            Ok(Some(Token::Integer(value)))
        }
    }

    fn read_literal_string(&mut self) -> ParseResult<Option<Token>> {
        self.position += 1; // Skip opening '('
        let mut result = Vec::new();
        let mut paren_depth = 1;
        let mut escape = false;

        while self.position < self.input.len() && paren_depth > 0 {
            let ch = self.input[self.position];
            self.position += 1;

            if escape {
                match ch {
                    b'n' => result.push(b'\n'),
                    b'r' => result.push(b'\r'),
                    b't' => result.push(b'\t'),
                    b'b' => result.push(b'\x08'),
                    b'f' => result.push(b'\x0C'),
                    b'(' => result.push(b'('),
                    b')' => result.push(b')'),
                    b'\\' => result.push(b'\\'),
                    b'0'..=b'7' => {
                        // Octal escape sequence
                        self.position -= 1;
                        let octal_value = self.read_octal_escape()?;
                        result.push(octal_value);
                    }
                    _ => result.push(ch), // Unknown escape, treat as literal
                }
                escape = false;
            } else {
                match ch {
                    b'\\' => escape = true,
                    b'(' => {
                        paren_depth += 1;
                        result.push(ch);
                    }
                    b')' => {
                        paren_depth -= 1;
                        if paren_depth > 0 {
                            result.push(ch);
                        }
                    }
                    _ => result.push(ch),
                }
            }
        }

        Ok(Some(Token::String(result)))
    }

    fn read_octal_escape(&mut self) -> ParseResult<u8> {
        // Use u16 to avoid overflow panic on malformed octal sequences (e.g. \777).
        // Per ISO 32000-1:2008 §7.3.4.2: "high-order overflow shall be ignored".
        let mut value = 0u16;
        let mut count = 0;

        while count < 3 && self.position < self.input.len() {
            match self.input[self.position] {
                b'0'..=b'7' => {
                    value = value * 8 + u16::from(self.input[self.position] - b'0');
                    self.position += 1;
                    count += 1;
                }
                _ => break,
            }
        }

        Ok(value as u8)
    }

    fn read_hex_string(&mut self) -> ParseResult<Option<Token>> {
        self.position += 1; // Skip opening '<'
        let mut result = Vec::new();
        let mut nibble = None;

        while self.position < self.input.len() {
            let ch = self.input[self.position];

            match ch {
                b'>' => {
                    self.position += 1;
                    // Handle odd number of hex digits
                    if let Some(n) = nibble {
                        result.push(n << 4);
                    }
                    return Ok(Some(Token::HexString(result)));
                }
                b'0'..=b'9' | b'A'..=b'F' | b'a'..=b'f' => {
                    let digit = if ch <= b'9' {
                        ch - b'0'
                    } else if ch <= b'F' {
                        ch - b'A' + 10
                    } else {
                        ch - b'a' + 10
                    };

                    if let Some(n) = nibble {
                        result.push((n << 4) | digit);
                        nibble = None;
                    } else {
                        nibble = Some(digit);
                    }
                    self.position += 1;
                }
                b' ' | b'\t' | b'\r' | b'\n' | b'\x0C' => {
                    // Skip whitespace in hex strings
                    self.position += 1;
                }
                _ => {
                    return Err(ParseError::SyntaxError {
                        position: self.position,
                        message: format!("Invalid character in hex string: {:?}", ch as char),
                    });
                }
            }
        }

        Err(ParseError::SyntaxError {
            position: self.position,
            message: "Unterminated hex string".to_string(),
        })
    }

    fn read_name(&mut self) -> ParseResult<Option<Token>> {
        self.position += 1; // Skip '/'
        let start = self.position;

        while self.position < self.input.len() {
            let ch = self.input[self.position];
            match ch {
                b' ' | b'\t' | b'\r' | b'\n' | b'\x0C' | b'(' | b')' | b'<' | b'>' | b'['
                | b']' | b'{' | b'}' | b'/' | b'%' => break,
                b'#' => {
                    // Handle hex escape in name
                    self.position += 1;
                    if self.position + 1 < self.input.len() {
                        self.position += 2;
                    }
                }
                _ => self.position += 1,
            }
        }

        let name_bytes = &self.input[start..self.position];
        let name = self.decode_name(name_bytes)?;
        Ok(Some(Token::Name(name)))
    }

    fn decode_name(&self, bytes: &[u8]) -> ParseResult<String> {
        let mut result = Vec::new();
        let mut i = 0;

        while i < bytes.len() {
            if bytes[i] == b'#' && i + 2 < bytes.len() {
                // Hex escape
                let hex_str = std::str::from_utf8(&bytes[i + 1..i + 3]).map_err(|_| {
                    ParseError::SyntaxError {
                        position: self.position,
                        message: "Invalid hex escape in name".to_string(),
                    }
                })?;
                let value =
                    u8::from_str_radix(hex_str, 16).map_err(|_| ParseError::SyntaxError {
                        position: self.position,
                        message: "Invalid hex escape in name".to_string(),
                    })?;
                result.push(value);
                i += 3;
            } else {
                result.push(bytes[i]);
                i += 1;
            }
        }

        String::from_utf8(result).map_err(|_| ParseError::SyntaxError {
            position: self.position,
            message: "Invalid UTF-8 in name".to_string(),
        })
    }

    fn read_operator(&mut self) -> ParseResult<Option<Token>> {
        let start = self.position;

        while self.position < self.input.len() {
            let ch = self.input[self.position];
            match ch {
                b' ' | b'\t' | b'\r' | b'\n' | b'\x0C' | b'(' | b')' | b'<' | b'>' | b'['
                | b']' | b'{' | b'}' | b'/' | b'%' | b';' => break,
                _ => self.position += 1,
            }
        }

        let op_bytes = &self.input[start..self.position];
        let op = std::str::from_utf8(op_bytes).map_err(|_| ParseError::SyntaxError {
            position: start,
            message: "Invalid operator".to_string(),
        })?;

        Ok(Some(Token::Operator(op.to_string())))
    }

    /// Read raw binary data for an inline image (between ID and EI).
    ///
    /// Per PDF spec §4.8.6, after the ID operator and a single whitespace byte,
    /// all subsequent bytes are raw image data until the EI marker is found.
    /// The EI marker is: whitespace + 'E' + 'I' + (whitespace, delimiter, or EOF).
    fn read_inline_image_data(&mut self) -> ParseResult<Option<Token>> {
        // Skip single whitespace byte after ID (per PDF spec §4.8.6)
        if self.position < self.input.len() {
            let ch = self.input[self.position];
            if ch == b' ' || ch == b'\n' || ch == b'\r' || ch == b'\t' {
                self.position += 1;
                // Handle \r\n as single whitespace
                if ch == b'\r'
                    && self.position < self.input.len()
                    && self.input[self.position] == b'\n'
                {
                    self.position += 1;
                }
            }
        }

        let start = self.position;

        // Scan for EI marker: preceded by whitespace + 'E' + 'I' + (whitespace/delimiter/EOF)
        while self.position + 1 < self.input.len() {
            let preceded_by_whitespace = self.position == start
                || matches!(
                    self.input[self.position - 1],
                    b' ' | b'\t' | b'\r' | b'\n' | b'\x0C'
                );

            if preceded_by_whitespace
                && self.input[self.position] == b'E'
                && self.input[self.position + 1] == b'I'
            {
                let after_ei = self.position + 2;
                let followed_by_boundary = after_ei >= self.input.len()
                    || matches!(
                        self.input[after_ei],
                        b' ' | b'\t' | b'\r' | b'\n' | b'\x0C' | b'/' | b'<' | b'(' | b'[' | b'%'
                    );

                if followed_by_boundary {
                    // Trim trailing whitespace that preceded EI from the data
                    let mut end = self.position;
                    if end > start
                        && matches!(self.input[end - 1], b' ' | b'\t' | b'\r' | b'\n' | b'\x0C')
                    {
                        end -= 1;
                    }
                    let data = self.input[start..end].to_vec();
                    self.position = after_ei; // Skip past "EI"
                    return Ok(Some(Token::InlineImageData(data)));
                }
            }
            self.position += 1;
        }

        // No EI found — return remaining bytes as best-effort recovery
        let data = self.input[start..].to_vec();
        self.position = self.input.len();
        Ok(Some(Token::InlineImageData(data)))
    }
}

/// High-level content stream parser.
///
/// Converts tokenized content streams into structured `ContentOperation` values.
/// This parser handles the operand stack and operator parsing according to PDF specifications.
///
/// # Usage
///
/// The parser is typically used through its static methods:
///
/// ```rust
/// use oxidize_pdf::parser::content::ContentParser;
///
/// # fn example() -> Result<(), Box<dyn std::error::Error>> {
/// let content = b"q 1 0 0 1 50 50 cm 100 100 200 150 re S Q";
/// let operations = ContentParser::parse(content)?;
/// # Ok(())
/// # }
/// ```
pub struct ContentParser {
    tokens: Vec<Token>,
    position: usize,
}

pub(crate) struct ParsedType3CharProc {
    pub(crate) width: (f64, f64),
    pub(crate) bbox: Option<[f64; 4]>,
    pub(crate) operations: Vec<ContentOperation>,
}

impl ContentParser {
    /// Create a new content parser
    pub fn new(_content: &[u8]) -> Self {
        Self {
            tokens: Vec::new(),
            position: 0,
        }
    }

    /// Parse a content stream into a vector of operators.
    ///
    /// This is a convenience method that creates a parser and processes the entire stream.
    ///
    /// # Arguments
    ///
    /// * `content` - Raw content stream bytes (may be compressed)
    ///
    /// # Returns
    ///
    /// A vector of parsed `ContentOperation` values in the order they appear.
    ///
    /// # Errors
    ///
    /// Returns an error if:
    /// - Invalid operator syntax is encountered
    /// - Operators have incorrect number/type of operands
    /// - Unknown operators are found
    ///
    /// # Example
    ///
    /// ```rust
    /// use oxidize_pdf::parser::content::{ContentParser, ContentOperation};
    ///
    /// # fn example() -> Result<(), Box<dyn std::error::Error>> {
    /// let content = b"BT /F1 12 Tf 100 200 Td (Hello) Tj ET";
    /// let operations = ContentParser::parse(content)?;
    ///
    /// assert_eq!(operations.len(), 5);
    /// assert!(matches!(operations[0], ContentOperation::BeginText));
    /// # Ok(())
    /// # }
    /// ```
    pub fn parse(content: &[u8]) -> ParseResult<Vec<ContentOperation>> {
        Self::parse_content(content)
    }

    /// Parse a content stream without best-effort recovery.
    ///
    /// Unlike [`Self::parse`], malformed or unknown operators are returned to
    /// the caller. This is intended for self-contained programs such as Type 3
    /// character procedures, where silently dropping an operation could alter
    /// the rendered glyph.
    pub fn parse_strict(content: &[u8]) -> ParseResult<Vec<ContentOperation>> {
        let mut tokenizer = ContentTokenizer::new(content);
        let mut tokens = Vec::new();
        while let Some(token) = tokenizer.next_token()? {
            tokens.push(token);
        }
        Self {
            tokens,
            position: 0,
        }
        .parse_operators_strict()
    }

    pub(crate) fn parse_type3_charproc(content: &[u8]) -> ParseResult<ParsedType3CharProc> {
        let mut tokenizer = ContentTokenizer::new(content);
        let mut tokens = Vec::new();
        while let Some(token) = tokenizer.next_token()? {
            tokens.push(token);
        }
        let mut parser = Self {
            tokens,
            position: 0,
        };
        let mut operations = Vec::new();
        let mut operands = Vec::new();
        let mut metrics = None;
        while parser.position < parser.tokens.len() {
            let token = parser.tokens[parser.position].clone();
            parser.position += 1;
            match token {
                Token::Operator(op) if op == "d0" => {
                    if metrics.is_some() || !operations.is_empty() {
                        return Err(ParseError::SyntaxError {
                            position: parser.position,
                            message: "d0/d1 must be the first Type 3 CharProc operator".into(),
                        });
                    }
                    let wy = parser.pop_number(&mut operands)?;
                    let wx = parser.pop_number(&mut operands)?;
                    if !operands.is_empty() {
                        return Err(ParseError::SyntaxError {
                            position: parser.position,
                            message: "d0 has extra operands".into(),
                        });
                    }
                    metrics = Some(((f64::from(wx), f64::from(wy)), None));
                }
                Token::Operator(op) if op == "d1" => {
                    if metrics.is_some() || !operations.is_empty() {
                        return Err(ParseError::SyntaxError {
                            position: parser.position,
                            message: "d0/d1 must be the first Type 3 CharProc operator".into(),
                        });
                    }
                    let ury = parser.pop_number(&mut operands)?;
                    let urx = parser.pop_number(&mut operands)?;
                    let lly = parser.pop_number(&mut operands)?;
                    let llx = parser.pop_number(&mut operands)?;
                    let wy = parser.pop_number(&mut operands)?;
                    let wx = parser.pop_number(&mut operands)?;
                    if !operands.is_empty() {
                        return Err(ParseError::SyntaxError {
                            position: parser.position,
                            message: "d1 has extra operands".into(),
                        });
                    }
                    metrics = Some((
                        (f64::from(wx), f64::from(wy)),
                        Some([
                            f64::from(llx),
                            f64::from(lly),
                            f64::from(urx),
                            f64::from(ury),
                        ]),
                    ));
                }
                Token::Operator(op) => {
                    operations.push(parser.parse_operator(&op, &mut operands)?);
                }
                operand => operands.push(operand),
            }
        }
        if !operands.is_empty() {
            return Err(ParseError::SyntaxError {
                position: parser.position,
                message: "trailing operands without an operator".into(),
            });
        }
        let (width, bbox) = metrics.ok_or_else(|| ParseError::SyntaxError {
            position: 0,
            message: "Type 3 CharProc must begin with d0 or d1".into(),
        })?;
        Ok(ParsedType3CharProc {
            width,
            bbox,
            operations,
        })
    }

    /// Parse a content stream into a vector of operators.
    ///
    /// This method tokenizes the input and converts it to operations.
    /// It handles the PDF postfix notation where operands precede operators.
    pub fn parse_content(content: &[u8]) -> ParseResult<Vec<ContentOperation>> {
        let mut tokenizer = ContentTokenizer::new(content);
        let mut tokens = Vec::new();

        // Tokenize the entire stream. Best-effort recovery (issue #319):
        // if the tokenizer hits an unrecoverable construct (e.g. an
        // unterminated hex string), stop there but KEEP every token parsed
        // so far instead of discarding the whole page. `next_token` already
        // recovers internally from skippable garbage; a hard error here is
        // the tail of the stream, and losing the tail beats losing the page.
        loop {
            match tokenizer.next_token() {
                Ok(Some(token)) => tokens.push(token),
                Ok(None) => break,
                Err(_e) => {
                    tracing::debug!("content tokenizer stopped early: {_e}");
                    break;
                }
            }
        }

        let mut parser = Self {
            tokens,
            position: 0,
        };

        parser.parse_operators()
    }

    fn parse_operators(&mut self) -> ParseResult<Vec<ContentOperation>> {
        let mut operators = Vec::new();
        let mut operand_stack: Vec<Token> = Vec::new();

        while self.position < self.tokens.len() {
            let token = self.tokens[self.position].clone();
            self.position += 1;

            match &token {
                Token::Operator(op) => {
                    // Best-effort recovery (issue #319): a single malformed
                    // operator must NOT discard the entire content stream.
                    // Content streams are not safety-critical; an operand
                    // mismatch on one operator (e.g. a `Td` missing its
                    // numbers, common in some producers) previously aborted
                    // the whole page via `?`, so the extractor dropped every
                    // valid operator before it. Instead, skip the bad
                    // operator, resync by clearing its pending operands, and
                    // continue parsing the rest of the stream.
                    match self.parse_operator(op, &mut operand_stack) {
                        Ok(operator) => operators.push(operator),
                        Err(_e) => {
                            tracing::debug!("skipping malformed content operator '{op}': {_e}");
                            operand_stack.clear();
                        }
                    }
                }
                _ => {
                    // Not an operator, push to operand stack
                    operand_stack.push(token);
                }
            }
        }

        Ok(operators)
    }

    fn parse_operators_strict(&mut self) -> ParseResult<Vec<ContentOperation>> {
        let mut operators = Vec::new();
        let mut operand_stack = Vec::new();
        while self.position < self.tokens.len() {
            let token = self.tokens[self.position].clone();
            self.position += 1;
            match token {
                Token::Operator(op) => {
                    operators.push(self.parse_operator(&op, &mut operand_stack)?);
                }
                operand => operand_stack.push(operand),
            }
        }
        if operand_stack.is_empty() {
            Ok(operators)
        } else {
            Err(ParseError::SyntaxError {
                position: self.position,
                message: "trailing operands without an operator".to_string(),
            })
        }
    }

    fn parse_operator(
        &mut self,
        op: &str,
        operands: &mut Vec<Token>,
    ) -> ParseResult<ContentOperation> {
        let operator = match op {
            // Text object operators
            "BT" => ContentOperation::BeginText,
            "ET" => ContentOperation::EndText,

            // Text state operators
            "Tc" => {
                let spacing = self.pop_number(operands)?;
                ContentOperation::SetCharSpacing(spacing)
            }
            "Tw" => {
                let spacing = self.pop_number(operands)?;
                ContentOperation::SetWordSpacing(spacing)
            }
            "Tz" => {
                let scale = self.pop_number(operands)?;
                ContentOperation::SetHorizontalScaling(scale)
            }
            "TL" => {
                let leading = self.pop_number(operands)?;
                ContentOperation::SetLeading(leading)
            }
            "Tf" => {
                let size = self.pop_number(operands)?;
                let font = self.pop_name(operands)?;
                ContentOperation::SetFont(font, size)
            }
            "Tr" => {
                let mode = self.pop_integer(operands)?;
                ContentOperation::SetTextRenderMode(mode)
            }
            "Ts" => {
                let rise = self.pop_number(operands)?;
                ContentOperation::SetTextRise(rise)
            }

            // Text positioning operators
            "Td" => {
                let ty = self.pop_number(operands)?;
                let tx = self.pop_number(operands)?;
                ContentOperation::MoveText(tx, ty)
            }
            "TD" => {
                let ty = self.pop_number(operands)?;
                let tx = self.pop_number(operands)?;
                ContentOperation::MoveTextSetLeading(tx, ty)
            }
            "Tm" => {
                let f = self.pop_number(operands)?;
                let e = self.pop_number(operands)?;
                let d = self.pop_number(operands)?;
                let c = self.pop_number(operands)?;
                let b = self.pop_number(operands)?;
                let a = self.pop_number(operands)?;
                ContentOperation::SetTextMatrix(a, b, c, d, e, f)
            }
            "T*" => ContentOperation::NextLine,

            // Text showing operators
            "Tj" => {
                let text = self.pop_string(operands)?;
                ContentOperation::ShowText(text)
            }
            "TJ" => {
                let array = self.pop_array(operands)?;
                let elements = self.parse_text_array(array)?;
                ContentOperation::ShowTextArray(elements)
            }
            "'" => {
                let text = self.pop_string(operands)?;
                ContentOperation::NextLineShowText(text)
            }
            "\"" => {
                // ISO 32000-1 §9.4.3: operand order is `aw ac string "`
                // (aw at the bottom of the operand stack). `pop_*` is LIFO,
                // so we pop string first, then `ac`, then `aw`. The enum
                // variant is `(word_spacing, char_spacing, text)` to match
                // the spec field names — pass aw first, then ac.
                let text = self.pop_string(operands)?;
                let ac = self.pop_number(operands)?;
                let aw = self.pop_number(operands)?;
                ContentOperation::SetSpacingNextLineShowText(aw, ac, text)
            }

            // Graphics state operators
            "q" => ContentOperation::SaveGraphicsState,
            "Q" => ContentOperation::RestoreGraphicsState,
            "cm" => {
                let f = self.pop_number(operands)?;
                let e = self.pop_number(operands)?;
                let d = self.pop_number(operands)?;
                let c = self.pop_number(operands)?;
                let b = self.pop_number(operands)?;
                let a = self.pop_number(operands)?;
                ContentOperation::SetTransformMatrix(a, b, c, d, e, f)
            }
            "w" => {
                let width = self.pop_number(operands)?;
                ContentOperation::SetLineWidth(width)
            }
            "J" => {
                let cap = self.pop_integer(operands)?;
                ContentOperation::SetLineCap(cap)
            }
            "j" => {
                let join = self.pop_integer(operands)?;
                ContentOperation::SetLineJoin(join)
            }
            "M" => {
                let limit = self.pop_number(operands)?;
                ContentOperation::SetMiterLimit(limit)
            }
            "d" => {
                let phase = self.pop_number(operands)?;
                let array = self.pop_array(operands)?;
                let pattern = self.parse_dash_array(array)?;
                ContentOperation::SetDashPattern(pattern, phase)
            }
            "ri" => {
                let intent = self.pop_name(operands)?;
                ContentOperation::SetIntent(intent)
            }
            "i" => {
                let flatness = self.pop_number(operands)?;
                ContentOperation::SetFlatness(flatness)
            }
            "gs" => {
                let name = self.pop_name(operands)?;
                ContentOperation::SetGraphicsStateParams(name)
            }

            // Path construction operators
            "m" => {
                let y = self.pop_number(operands)?;
                let x = self.pop_number(operands)?;
                ContentOperation::MoveTo(x, y)
            }
            "l" => {
                let y = self.pop_number(operands)?;
                let x = self.pop_number(operands)?;
                ContentOperation::LineTo(x, y)
            }
            "c" => {
                let y3 = self.pop_number(operands)?;
                let x3 = self.pop_number(operands)?;
                let y2 = self.pop_number(operands)?;
                let x2 = self.pop_number(operands)?;
                let y1 = self.pop_number(operands)?;
                let x1 = self.pop_number(operands)?;
                ContentOperation::CurveTo(x1, y1, x2, y2, x3, y3)
            }
            "v" => {
                let y3 = self.pop_number(operands)?;
                let x3 = self.pop_number(operands)?;
                let y2 = self.pop_number(operands)?;
                let x2 = self.pop_number(operands)?;
                ContentOperation::CurveToV(x2, y2, x3, y3)
            }
            "y" => {
                let y3 = self.pop_number(operands)?;
                let x3 = self.pop_number(operands)?;
                let y1 = self.pop_number(operands)?;
                let x1 = self.pop_number(operands)?;
                ContentOperation::CurveToY(x1, y1, x3, y3)
            }
            "h" => ContentOperation::ClosePath,
            "re" => {
                let height = self.pop_number(operands)?;
                let width = self.pop_number(operands)?;
                let y = self.pop_number(operands)?;
                let x = self.pop_number(operands)?;
                ContentOperation::Rectangle(x, y, width, height)
            }

            // Path painting operators
            "S" => ContentOperation::Stroke,
            "s" => ContentOperation::CloseStroke,
            "f" | "F" => ContentOperation::Fill,
            "f*" => ContentOperation::FillEvenOdd,
            "B" => ContentOperation::FillStroke,
            "B*" => ContentOperation::FillStrokeEvenOdd,
            "b" => ContentOperation::CloseFillStroke,
            "b*" => ContentOperation::CloseFillStrokeEvenOdd,
            "n" => ContentOperation::EndPath,

            // Clipping path operators
            "W" => ContentOperation::Clip,
            "W*" => ContentOperation::ClipEvenOdd,

            // Color operators
            "CS" => {
                let name = self.pop_name(operands)?;
                ContentOperation::SetStrokingColorSpace(name)
            }
            "cs" => {
                let name = self.pop_name(operands)?;
                ContentOperation::SetNonStrokingColorSpace(name)
            }
            "SC" | "SCN" => {
                let components = self.pop_color_components(operands)?;
                ContentOperation::SetStrokingColor(components)
            }
            "sc" | "scn" => {
                let components = self.pop_color_components(operands)?;
                ContentOperation::SetNonStrokingColor(components)
            }
            "G" => {
                let gray = self.pop_number(operands)?;
                ContentOperation::SetStrokingGray(gray)
            }
            "g" => {
                let gray = self.pop_number(operands)?;
                ContentOperation::SetNonStrokingGray(gray)
            }
            "RG" => {
                let b = self.pop_number(operands)?;
                let g = self.pop_number(operands)?;
                let r = self.pop_number(operands)?;
                ContentOperation::SetStrokingRGB(r, g, b)
            }
            "rg" => {
                let b = self.pop_number(operands)?;
                let g = self.pop_number(operands)?;
                let r = self.pop_number(operands)?;
                ContentOperation::SetNonStrokingRGB(r, g, b)
            }
            "K" => {
                let k = self.pop_number(operands)?;
                let y = self.pop_number(operands)?;
                let m = self.pop_number(operands)?;
                let c = self.pop_number(operands)?;
                ContentOperation::SetStrokingCMYK(c, m, y, k)
            }
            "k" => {
                let k = self.pop_number(operands)?;
                let y = self.pop_number(operands)?;
                let m = self.pop_number(operands)?;
                let c = self.pop_number(operands)?;
                ContentOperation::SetNonStrokingCMYK(c, m, y, k)
            }

            // Shading operators
            "sh" => {
                let name = self.pop_name(operands)?;
                ContentOperation::ShadingFill(name)
            }

            // XObject operators
            "Do" => {
                let name = self.pop_name(operands)?;
                ContentOperation::PaintXObject(name)
            }

            // Marked content operators
            "BMC" => {
                let tag = self.pop_name(operands)?;
                ContentOperation::BeginMarkedContent(tag)
            }
            "BDC" => {
                let props = self.pop_dict_or_name(operands)?;
                let tag = self.pop_name(operands)?;
                ContentOperation::BeginMarkedContentWithProps(tag, props)
            }
            "EMC" => ContentOperation::EndMarkedContent,
            "MP" => {
                let tag = self.pop_name(operands)?;
                ContentOperation::DefineMarkedContentPoint(tag)
            }
            "DP" => {
                let props = self.pop_dict_or_name(operands)?;
                let tag = self.pop_name(operands)?;
                ContentOperation::DefineMarkedContentPointWithProps(tag, props)
            }

            // Compatibility operators
            "BX" => ContentOperation::BeginCompatibility,
            "EX" => ContentOperation::EndCompatibility,

            // Inline images are handled specially
            "BI" => {
                operands.clear(); // Clear any remaining operands
                self.parse_inline_image()?
            }

            _ => {
                return Err(ParseError::SyntaxError {
                    position: self.position,
                    message: format!("Unknown operator: {op}"),
                });
            }
        };

        operands.clear(); // Clear operands after processing
        Ok(operator)
    }

    // Helper methods for popping operands
    fn pop_number(&self, operands: &mut Vec<Token>) -> ParseResult<f32> {
        match operands.pop() {
            Some(Token::Number(n)) => Ok(n),
            Some(Token::Integer(i)) => Ok(i as f32),
            _ => Err(ParseError::SyntaxError {
                position: self.position,
                message: "Expected number operand".to_string(),
            }),
        }
    }

    fn pop_integer(&self, operands: &mut Vec<Token>) -> ParseResult<i32> {
        match operands.pop() {
            Some(Token::Integer(i)) => Ok(i),
            _ => Err(ParseError::SyntaxError {
                position: self.position,
                message: "Expected integer operand".to_string(),
            }),
        }
    }

    fn pop_name(&self, operands: &mut Vec<Token>) -> ParseResult<String> {
        match operands.pop() {
            Some(Token::Name(n)) => Ok(n),
            _ => Err(ParseError::SyntaxError {
                position: self.position,
                message: "Expected name operand".to_string(),
            }),
        }
    }

    fn pop_string(&self, operands: &mut Vec<Token>) -> ParseResult<Vec<u8>> {
        match operands.pop() {
            Some(Token::String(s)) => Ok(s),
            Some(Token::HexString(s)) => Ok(s),
            _ => Err(ParseError::SyntaxError {
                position: self.position,
                message: "Expected string operand".to_string(),
            }),
        }
    }

    fn pop_array(&self, operands: &mut Vec<Token>) -> ParseResult<Vec<Token>> {
        // First check if we have an ArrayEnd at the top (which we should for a complete array)
        let has_array_end = matches!(operands.last(), Some(Token::ArrayEnd));
        if has_array_end {
            operands.pop(); // Remove the ArrayEnd
        }

        let mut array = Vec::new();
        let mut found_start = false;

        // Pop tokens until we find ArrayStart
        while let Some(token) = operands.pop() {
            match token {
                Token::ArrayStart => {
                    found_start = true;
                    break;
                }
                Token::ArrayEnd => {
                    // Skip any additional ArrayEnd tokens (shouldn't happen in well-formed PDFs)
                    continue;
                }
                _ => array.push(token),
            }
        }

        if !found_start {
            return Err(ParseError::SyntaxError {
                position: self.position,
                message: "Expected array".to_string(),
            });
        }

        array.reverse(); // We collected in reverse order
        Ok(array)
    }

    fn pop_dict_or_name(&self, operands: &mut Vec<Token>) -> ParseResult<MarkedContentProps> {
        let token = operands.pop().ok_or_else(|| ParseError::SyntaxError {
            position: self.position,
            message: "Expected dict or name operand for BDC/DP".to_string(),
        })?;

        match token {
            Token::Name(name) => Ok(MarkedContentProps::ResourceRef(name)),
            Token::DictEnd => {
                // Inline dictionary. Stack layout (newest on top):
                //   ... DictStart Name(k1) Value(v1) Name(k2) Value(v2) DictEnd
                // We pop value-then-key pairs in reverse until we hit DictStart.
                let mut map: HashMap<String, MarkedContentValue> = HashMap::new();
                loop {
                    let next = operands.pop().ok_or_else(|| ParseError::SyntaxError {
                        position: self.position,
                        message: "Unterminated inline dict in BDC/DP".to_string(),
                    })?;
                    if matches!(next, Token::DictStart) {
                        break;
                    }
                    let value = Self::token_to_mc_value(next, operands)?;
                    let key = match operands.pop() {
                        Some(Token::Name(k)) => k,
                        Some(other) => {
                            return Err(ParseError::SyntaxError {
                                position: self.position,
                                message: format!(
                                    "Expected Name as inline dict key, got {:?}",
                                    other
                                ),
                            });
                        }
                        None => {
                            return Err(ParseError::SyntaxError {
                                position: self.position,
                                message: "Unterminated inline dict (missing key)".to_string(),
                            });
                        }
                    };
                    map.insert(key, value);
                }
                Ok(MarkedContentProps::Inline(map))
            }
            other => Err(ParseError::SyntaxError {
                position: self.position,
                message: format!("Expected name or inline dict for BDC/DP, got {:?}", other),
            }),
        }
    }

    /// Convert a popped token to a `MarkedContentValue`. For `ArrayEnd` and
    /// `DictEnd` tokens we recursively collect the matching container; all
    /// other tokens map to leaf variants.
    fn token_to_mc_value(
        token: Token,
        operands: &mut Vec<Token>,
    ) -> ParseResult<MarkedContentValue> {
        match token {
            Token::String(b) | Token::HexString(b) => Ok(MarkedContentValue::String(b)),
            Token::Integer(i) => Ok(MarkedContentValue::Integer(i as i64)),
            Token::Number(f) => Ok(MarkedContentValue::Real(f as f64)),
            Token::Name(n) => Ok(MarkedContentValue::Name(n)),
            Token::ArrayEnd => {
                let mut items: Vec<MarkedContentValue> = Vec::new();
                loop {
                    let next = operands.pop().ok_or_else(|| ParseError::SyntaxError {
                        position: 0,
                        message: "Unterminated array in marked-content props".to_string(),
                    })?;
                    if matches!(next, Token::ArrayStart) {
                        break;
                    }
                    items.push(Self::token_to_mc_value(next, operands)?);
                }
                items.reverse();
                Ok(MarkedContentValue::Array(items))
            }
            Token::DictEnd => {
                let mut nested: HashMap<String, MarkedContentValue> = HashMap::new();
                loop {
                    let next = operands.pop().ok_or_else(|| ParseError::SyntaxError {
                        position: 0,
                        message: "Unterminated nested dict in marked-content props".to_string(),
                    })?;
                    if matches!(next, Token::DictStart) {
                        break;
                    }
                    let value = Self::token_to_mc_value(next, operands)?;
                    let key = match operands.pop() {
                        Some(Token::Name(k)) => k,
                        _ => {
                            return Err(ParseError::SyntaxError {
                                position: 0,
                                message: "Expected name key in nested dict".to_string(),
                            });
                        }
                    };
                    nested.insert(key, value);
                }
                Ok(MarkedContentValue::Dict(nested))
            }
            other => Err(ParseError::SyntaxError {
                position: 0,
                message: format!("Unexpected token type in marked-content value: {:?}", other),
            }),
        }
    }

    fn pop_color_components(&self, operands: &mut Vec<Token>) -> ParseResult<Vec<f32>> {
        let mut components = Vec::new();

        // Pop all numeric values from the stack
        while let Some(token) = operands.last() {
            match token {
                Token::Number(n) => {
                    components.push(*n);
                    operands.pop();
                }
                Token::Integer(i) => {
                    components.push(*i as f32);
                    operands.pop();
                }
                _ => break,
            }
        }

        components.reverse();
        Ok(components)
    }

    fn parse_text_array(&self, tokens: Vec<Token>) -> ParseResult<Vec<TextElement>> {
        let mut elements = Vec::new();

        for token in tokens {
            match token {
                Token::String(s) | Token::HexString(s) => {
                    elements.push(TextElement::Text(s));
                }
                Token::Number(n) => {
                    elements.push(TextElement::Spacing(n));
                }
                Token::Integer(i) => {
                    elements.push(TextElement::Spacing(i as f32));
                }
                _ => {
                    return Err(ParseError::SyntaxError {
                        position: self.position,
                        message: "Invalid element in text array".to_string(),
                    });
                }
            }
        }

        Ok(elements)
    }

    fn parse_dash_array(&self, tokens: Vec<Token>) -> ParseResult<Vec<f32>> {
        let mut pattern = Vec::new();

        for token in tokens {
            match token {
                Token::Number(n) => pattern.push(n),
                Token::Integer(i) => pattern.push(i as f32),
                _ => {
                    return Err(ParseError::SyntaxError {
                        position: self.position,
                        message: "Invalid element in dash array".to_string(),
                    });
                }
            }
        }

        Ok(pattern)
    }

    fn parse_inline_image(&mut self) -> ParseResult<ContentOperation> {
        // Parse inline image dictionary until we find ID
        let mut params = HashMap::new();

        while self.position < self.tokens.len() {
            // Check if we've reached the ID operator
            if let Token::Operator(op) = &self.tokens[self.position] {
                if op == "ID" {
                    self.position += 1;
                    break;
                }
            }

            // Parse key-value pairs for image parameters
            // Keys are abbreviated in inline images:
            // /W -> Width, /H -> Height, /CS -> ColorSpace, /BPC -> BitsPerComponent
            // /F -> Filter, /DP -> DecodeParms, /IM -> ImageMask, /I -> Interpolate
            if let Token::Name(key) = &self.tokens[self.position] {
                self.position += 1;
                if self.position >= self.tokens.len() {
                    break;
                }

                // Parse the value
                let value = match &self.tokens[self.position] {
                    Token::Integer(n) => Object::Integer(*n as i64),
                    Token::Number(n) => Object::Real(*n as f64),
                    Token::Name(s) => Object::Name(expand_inline_name(s)),
                    Token::String(s) => Object::String(String::from_utf8_lossy(s).to_string()),
                    Token::HexString(s) => Object::String(String::from_utf8_lossy(s).to_string()),
                    _ => Object::Null,
                };

                // Expand abbreviated keys to full names
                let full_key = expand_inline_key(key);
                params.insert(full_key, value);
                self.position += 1;
            } else {
                self.position += 1;
            }
        }

        // Get inline image data from dedicated InlineImageData token
        // (the tokenizer reads raw bytes between ID whitespace and EI)
        let data = if self.position < self.tokens.len() {
            if let Token::InlineImageData(bytes) = &self.tokens[self.position] {
                let d = bytes.clone();
                self.position += 1;
                d
            } else {
                // Fallback: collect tokens until EI (for backwards compat with edge cases)
                self.collect_inline_image_data_from_tokens()?
            }
        } else {
            Vec::new()
        };

        Ok(ContentOperation::InlineImage { params, data })
    }

    /// Fallback data collection when InlineImageData token is not present.
    /// This handles edge cases where the tokenizer couldn't detect the ID/EI boundary.
    fn collect_inline_image_data_from_tokens(&mut self) -> ParseResult<Vec<u8>> {
        let mut data = Vec::new();
        while self.position < self.tokens.len() {
            if let Token::Operator(op) = &self.tokens[self.position] {
                if op == "EI" {
                    self.position += 1;
                    break;
                }
            }
            match &self.tokens[self.position] {
                Token::String(bytes) | Token::HexString(bytes) => {
                    data.extend_from_slice(bytes);
                }
                Token::Integer(n) => data.extend_from_slice(n.to_string().as_bytes()),
                Token::Number(n) => data.extend_from_slice(n.to_string().as_bytes()),
                Token::Name(s) | Token::Operator(s) => data.extend_from_slice(s.as_bytes()),
                _ => {}
            }
            self.position += 1;
        }
        Ok(data)
    }
}

/// Expand abbreviated inline image key names to full names
fn expand_inline_key(key: &str) -> String {
    match key {
        "W" => "Width".to_string(),
        "H" => "Height".to_string(),
        "CS" | "ColorSpace" => "ColorSpace".to_string(),
        "BPC" | "BitsPerComponent" => "BitsPerComponent".to_string(),
        "F" => "Filter".to_string(),
        "DP" | "DecodeParms" => "DecodeParms".to_string(),
        "IM" => "ImageMask".to_string(),
        "I" => "Interpolate".to_string(),
        "Intent" => "Intent".to_string(),
        "D" => "Decode".to_string(),
        _ => key.to_string(),
    }
}

/// Expand abbreviated inline image color space names
fn expand_inline_name(name: &str) -> String {
    match name {
        "G" => "DeviceGray".to_string(),
        "RGB" => "DeviceRGB".to_string(),
        "CMYK" => "DeviceCMYK".to_string(),
        "I" => "Indexed".to_string(),
        "AHx" => "ASCIIHexDecode".to_string(),
        "A85" => "ASCII85Decode".to_string(),
        "LZW" => "LZWDecode".to_string(),
        "Fl" => "FlateDecode".to_string(),
        "RL" => "RunLengthDecode".to_string(),
        "DCT" => "DCTDecode".to_string(),
        "CCF" => "CCITTFaxDecode".to_string(),
        _ => name.to_string(),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_tokenize_numbers() {
        let input = b"123 -45 3.14159 -0.5 .5";
        let mut tokenizer = ContentTokenizer::new(input);

        assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Integer(123)));
        assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Integer(-45)));
        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::Number(3.14159))
        );
        assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Number(-0.5)));
        assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Number(0.5)));
        assert_eq!(tokenizer.next_token().unwrap(), None);
    }

    #[test]
    fn test_tokenize_strings() {
        let input = b"(Hello World) (Hello\\nWorld) (Nested (paren))";
        let mut tokenizer = ContentTokenizer::new(input);

        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::String(b"Hello World".to_vec()))
        );
        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::String(b"Hello\nWorld".to_vec()))
        );
        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::String(b"Nested (paren)".to_vec()))
        );
    }

    #[test]
    fn test_tokenize_hex_strings() {
        let input = b"<48656C6C6F> <48 65 6C 6C 6F>";
        let mut tokenizer = ContentTokenizer::new(input);

        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::HexString(b"Hello".to_vec()))
        );
        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::HexString(b"Hello".to_vec()))
        );
    }

    #[test]
    fn test_tokenize_names() {
        let input = b"/Name /Name#20with#20spaces /A#42C";
        let mut tokenizer = ContentTokenizer::new(input);

        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::Name("Name".to_string()))
        );
        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::Name("Name with spaces".to_string()))
        );
        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::Name("ABC".to_string()))
        );
    }

    #[test]
    fn test_tokenize_operators() {
        let input = b"BT Tj ET q Q";
        let mut tokenizer = ContentTokenizer::new(input);

        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::Operator("BT".to_string()))
        );
        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::Operator("Tj".to_string()))
        );
        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::Operator("ET".to_string()))
        );
        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::Operator("q".to_string()))
        );
        assert_eq!(
            tokenizer.next_token().unwrap(),
            Some(Token::Operator("Q".to_string()))
        );
    }

    #[test]
    fn test_parse_text_operators() {
        let content = b"BT /F1 12 Tf 100 200 Td (Hello World) Tj ET";
        let operators = ContentParser::parse(content).unwrap();

        assert_eq!(operators.len(), 5);
        assert_eq!(operators[0], ContentOperation::BeginText);
        assert_eq!(
            operators[1],
            ContentOperation::SetFont("F1".to_string(), 12.0)
        );
        assert_eq!(operators[2], ContentOperation::MoveText(100.0, 200.0));
        assert_eq!(
            operators[3],
            ContentOperation::ShowText(b"Hello World".to_vec())
        );
        assert_eq!(operators[4], ContentOperation::EndText);
    }

    #[test]
    fn test_parse_graphics_operators() {
        let content = b"q 1 0 0 1 50 50 cm 2 w 0 0 100 100 re S Q";
        let operators = ContentParser::parse(content).unwrap();

        assert_eq!(operators.len(), 6);
        assert_eq!(operators[0], ContentOperation::SaveGraphicsState);
        assert_eq!(
            operators[1],
            ContentOperation::SetTransformMatrix(1.0, 0.0, 0.0, 1.0, 50.0, 50.0)
        );
        assert_eq!(operators[2], ContentOperation::SetLineWidth(2.0));
        assert_eq!(
            operators[3],
            ContentOperation::Rectangle(0.0, 0.0, 100.0, 100.0)
        );
        assert_eq!(operators[4], ContentOperation::Stroke);
        assert_eq!(operators[5], ContentOperation::RestoreGraphicsState);
    }

    #[test]
    fn test_parse_color_operators() {
        let content = b"0.5 g 1 0 0 rg 0 0 0 1 k";
        let operators = ContentParser::parse(content).unwrap();

        assert_eq!(operators.len(), 3);
        assert_eq!(operators[0], ContentOperation::SetNonStrokingGray(0.5));
        assert_eq!(
            operators[1],
            ContentOperation::SetNonStrokingRGB(1.0, 0.0, 0.0)
        );
        assert_eq!(
            operators[2],
            ContentOperation::SetNonStrokingCMYK(0.0, 0.0, 0.0, 1.0)
        );
    }

    // Comprehensive tests for all ContentOperation variants
    mod comprehensive_tests {
        use super::*;

        #[test]
        fn test_all_text_operators() {
            // Test basic text operators that work with current parser
            let content = b"BT 5 Tc 10 Tw 120 Tz 15 TL /F1 12 Tf 1 Tr 5 Ts 100 200 Td 50 150 TD T* (Hello) Tj ET";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators[0], ContentOperation::BeginText);
            assert_eq!(operators[1], ContentOperation::SetCharSpacing(5.0));
            assert_eq!(operators[2], ContentOperation::SetWordSpacing(10.0));
            assert_eq!(operators[3], ContentOperation::SetHorizontalScaling(120.0));
            assert_eq!(operators[4], ContentOperation::SetLeading(15.0));
            assert_eq!(
                operators[5],
                ContentOperation::SetFont("F1".to_string(), 12.0)
            );
            assert_eq!(operators[6], ContentOperation::SetTextRenderMode(1));
            assert_eq!(operators[7], ContentOperation::SetTextRise(5.0));
            assert_eq!(operators[8], ContentOperation::MoveText(100.0, 200.0));
            assert_eq!(
                operators[9],
                ContentOperation::MoveTextSetLeading(50.0, 150.0)
            );
            assert_eq!(operators[10], ContentOperation::NextLine);
            assert_eq!(operators[11], ContentOperation::ShowText(b"Hello".to_vec()));
            assert_eq!(operators[12], ContentOperation::EndText);
        }

        #[test]
        fn test_all_graphics_state_operators() {
            // Test basic graphics state operators without arrays
            let content = b"q Q 1 0 0 1 50 50 cm 2 w 1 J 2 j 10 M /GS1 gs 0.5 i /Perceptual ri";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators[0], ContentOperation::SaveGraphicsState);
            assert_eq!(operators[1], ContentOperation::RestoreGraphicsState);
            assert_eq!(
                operators[2],
                ContentOperation::SetTransformMatrix(1.0, 0.0, 0.0, 1.0, 50.0, 50.0)
            );
            assert_eq!(operators[3], ContentOperation::SetLineWidth(2.0));
            assert_eq!(operators[4], ContentOperation::SetLineCap(1));
            assert_eq!(operators[5], ContentOperation::SetLineJoin(2));
            assert_eq!(operators[6], ContentOperation::SetMiterLimit(10.0));
            assert_eq!(
                operators[7],
                ContentOperation::SetGraphicsStateParams("GS1".to_string())
            );
            assert_eq!(operators[8], ContentOperation::SetFlatness(0.5));
            assert_eq!(
                operators[9],
                ContentOperation::SetIntent("Perceptual".to_string())
            );
        }

        #[test]
        fn test_all_path_construction_operators() {
            let content = b"100 200 m 150 200 l 200 200 250 250 300 200 c 250 180 300 200 v 200 180 300 200 y h 50 50 100 100 re";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators[0], ContentOperation::MoveTo(100.0, 200.0));
            assert_eq!(operators[1], ContentOperation::LineTo(150.0, 200.0));
            assert_eq!(
                operators[2],
                ContentOperation::CurveTo(200.0, 200.0, 250.0, 250.0, 300.0, 200.0)
            );
            assert_eq!(
                operators[3],
                ContentOperation::CurveToV(250.0, 180.0, 300.0, 200.0)
            );
            assert_eq!(
                operators[4],
                ContentOperation::CurveToY(200.0, 180.0, 300.0, 200.0)
            );
            assert_eq!(operators[5], ContentOperation::ClosePath);
            assert_eq!(
                operators[6],
                ContentOperation::Rectangle(50.0, 50.0, 100.0, 100.0)
            );
        }

        #[test]
        fn test_all_path_painting_operators() {
            let content = b"S s f F f* B B* b b* n W W*";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators[0], ContentOperation::Stroke);
            assert_eq!(operators[1], ContentOperation::CloseStroke);
            assert_eq!(operators[2], ContentOperation::Fill);
            assert_eq!(operators[3], ContentOperation::Fill); // F is alias for f
            assert_eq!(operators[4], ContentOperation::FillEvenOdd);
            assert_eq!(operators[5], ContentOperation::FillStroke);
            assert_eq!(operators[6], ContentOperation::FillStrokeEvenOdd);
            assert_eq!(operators[7], ContentOperation::CloseFillStroke);
            assert_eq!(operators[8], ContentOperation::CloseFillStrokeEvenOdd);
            assert_eq!(operators[9], ContentOperation::EndPath);
            assert_eq!(operators[10], ContentOperation::Clip);
            assert_eq!(operators[11], ContentOperation::ClipEvenOdd);
        }

        #[test]
        fn test_all_color_operators() {
            // Test basic color operators that work with current parser
            let content = b"/DeviceRGB CS /DeviceGray cs 0.7 G 0.4 g 1 0 0 RG 0 1 0 rg 0 0 0 1 K 0.2 0.3 0.4 0.5 k /Shade1 sh";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(
                operators[0],
                ContentOperation::SetStrokingColorSpace("DeviceRGB".to_string())
            );
            assert_eq!(
                operators[1],
                ContentOperation::SetNonStrokingColorSpace("DeviceGray".to_string())
            );
            assert_eq!(operators[2], ContentOperation::SetStrokingGray(0.7));
            assert_eq!(operators[3], ContentOperation::SetNonStrokingGray(0.4));
            assert_eq!(
                operators[4],
                ContentOperation::SetStrokingRGB(1.0, 0.0, 0.0)
            );
            assert_eq!(
                operators[5],
                ContentOperation::SetNonStrokingRGB(0.0, 1.0, 0.0)
            );
            assert_eq!(
                operators[6],
                ContentOperation::SetStrokingCMYK(0.0, 0.0, 0.0, 1.0)
            );
            assert_eq!(
                operators[7],
                ContentOperation::SetNonStrokingCMYK(0.2, 0.3, 0.4, 0.5)
            );
            assert_eq!(
                operators[8],
                ContentOperation::ShadingFill("Shade1".to_string())
            );
        }

        #[test]
        fn test_xobject_and_marked_content_operators() {
            // Test basic XObject and marked content operators
            let content = b"/Image1 Do /MC1 BMC EMC /MP1 MP BX EX";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(
                operators[0],
                ContentOperation::PaintXObject("Image1".to_string())
            );
            assert_eq!(
                operators[1],
                ContentOperation::BeginMarkedContent("MC1".to_string())
            );
            assert_eq!(operators[2], ContentOperation::EndMarkedContent);
            assert_eq!(
                operators[3],
                ContentOperation::DefineMarkedContentPoint("MP1".to_string())
            );
            assert_eq!(operators[4], ContentOperation::BeginCompatibility);
            assert_eq!(operators[5], ContentOperation::EndCompatibility);
        }

        #[test]
        fn test_complex_content_stream() {
            let content = b"q 0.5 0 0 0.5 100 100 cm BT /F1 12 Tf 0 0 Td (Complex) Tj ET Q";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 8);
            assert_eq!(operators[0], ContentOperation::SaveGraphicsState);
            assert_eq!(
                operators[1],
                ContentOperation::SetTransformMatrix(0.5, 0.0, 0.0, 0.5, 100.0, 100.0)
            );
            assert_eq!(operators[2], ContentOperation::BeginText);
            assert_eq!(
                operators[3],
                ContentOperation::SetFont("F1".to_string(), 12.0)
            );
            assert_eq!(operators[4], ContentOperation::MoveText(0.0, 0.0));
            assert_eq!(
                operators[5],
                ContentOperation::ShowText(b"Complex".to_vec())
            );
            assert_eq!(operators[6], ContentOperation::EndText);
            assert_eq!(operators[7], ContentOperation::RestoreGraphicsState);
        }

        #[test]
        fn test_tokenizer_whitespace_handling() {
            let input = b"  \t\n\r  BT  \t\n  /F1   12.5  \t Tf  \n\r  ET  ";
            let mut tokenizer = ContentTokenizer::new(input);

            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::Operator("BT".to_string()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::Name("F1".to_string()))
            );
            assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Number(12.5)));
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::Operator("Tf".to_string()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::Operator("ET".to_string()))
            );
            assert_eq!(tokenizer.next_token().unwrap(), None);
        }

        #[test]
        fn test_tokenizer_edge_cases() {
            // Test basic number formats that are actually supported
            let input = b"0 .5 -.5 +.5 123. .123 1.23 -1.23";
            let mut tokenizer = ContentTokenizer::new(input);

            assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Integer(0)));
            assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Number(0.5)));
            assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Number(-0.5)));
            assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Number(0.5)));
            assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Number(123.0)));
            assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Number(0.123)));
            assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Number(1.23)));
            assert_eq!(tokenizer.next_token().unwrap(), Some(Token::Number(-1.23)));
        }

        #[test]
        fn test_string_parsing_edge_cases() {
            let input = b"(Simple) (With\\\\backslash) (With\\)paren) (With\\newline) (With\\ttab) (With\\rcarriage) (With\\bbackspace) (With\\fformfeed) (With\\(leftparen) (With\\)rightparen) (With\\377octal) (With\\dddoctal)";
            let mut tokenizer = ContentTokenizer::new(input);

            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::String(b"Simple".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::String(b"With\\backslash".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::String(b"With)paren".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::String(b"With\newline".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::String(b"With\ttab".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::String(b"With\rcarriage".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::String(b"With\x08backspace".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::String(b"With\x0Cformfeed".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::String(b"With(leftparen".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::String(b"With)rightparen".to_vec()))
            );
        }

        #[test]
        fn test_hex_string_parsing() {
            let input = b"<48656C6C6F> <48 65 6C 6C 6F> <48656C6C6F57> <48656C6C6F5>";
            let mut tokenizer = ContentTokenizer::new(input);

            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::HexString(b"Hello".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::HexString(b"Hello".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::HexString(b"HelloW".to_vec()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::HexString(b"Hello\x50".to_vec()))
            );
        }

        #[test]
        fn test_name_parsing_edge_cases() {
            let input = b"/Name /Name#20with#20spaces /Name#23with#23hash /Name#2Fwith#2Fslash /#45mptyName";
            let mut tokenizer = ContentTokenizer::new(input);

            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::Name("Name".to_string()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::Name("Name with spaces".to_string()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::Name("Name#with#hash".to_string()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::Name("Name/with/slash".to_string()))
            );
            assert_eq!(
                tokenizer.next_token().unwrap(),
                Some(Token::Name("EmptyName".to_string()))
            );
        }

        #[test]
        fn test_operator_parsing_edge_cases() {
            let content = b"q q q Q Q Q BT BT ET ET";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 10);
            assert_eq!(operators[0], ContentOperation::SaveGraphicsState);
            assert_eq!(operators[1], ContentOperation::SaveGraphicsState);
            assert_eq!(operators[2], ContentOperation::SaveGraphicsState);
            assert_eq!(operators[3], ContentOperation::RestoreGraphicsState);
            assert_eq!(operators[4], ContentOperation::RestoreGraphicsState);
            assert_eq!(operators[5], ContentOperation::RestoreGraphicsState);
            assert_eq!(operators[6], ContentOperation::BeginText);
            assert_eq!(operators[7], ContentOperation::BeginText);
            assert_eq!(operators[8], ContentOperation::EndText);
            assert_eq!(operators[9], ContentOperation::EndText);
        }

        #[test]
        fn test_error_handling_insufficient_operands() {
            // Best-effort recovery (issue #319): `Td` is missing its y
            // coordinate. The malformed operator is skipped, but a following
            // valid text operator must still be recovered — the page is NOT
            // discarded wholesale.
            let content = b"100 Td (kept) Tj";
            let ops = ContentParser::parse(content).expect("recovers from bad Td");
            assert!(
                ops.iter()
                    .any(|op| matches!(op, ContentOperation::ShowText(t) if t == b"kept")),
                "valid Tj after the malformed Td must survive: {ops:?}"
            );
        }

        #[test]
        fn test_error_handling_invalid_operator() {
            // Unknown operator `INVALID` is skipped; the following valid
            // MoveTo survives (issue #319 recovery contract).
            let content = b"100 200 INVALID 10 20 m";
            let ops = ContentParser::parse(content).expect("recovers from unknown operator");
            assert!(
                ops.iter()
                    .any(|op| matches!(op, ContentOperation::MoveTo(_, _))),
                "valid MoveTo after the unknown operator must survive: {ops:?}"
            );
        }

        #[test]
        fn test_error_handling_malformed_string() {
            // Test that the tokenizer handles malformed strings appropriately
            let input = b"(Unclosed string";
            let mut tokenizer = ContentTokenizer::new(input);
            let result = tokenizer.next_token();
            // The current implementation may not detect this as an error
            // so we'll just test that we get some result
            assert!(result.is_ok() || result.is_err());
        }

        #[test]
        fn test_error_handling_malformed_hex_string() {
            let input = b"<48656C6C6G>";
            let mut tokenizer = ContentTokenizer::new(input);
            let result = tokenizer.next_token();
            assert!(result.is_err());
        }

        #[test]
        fn test_error_handling_malformed_name() {
            let input = b"/Name#GG";
            let mut tokenizer = ContentTokenizer::new(input);
            let result = tokenizer.next_token();
            assert!(result.is_err());
        }

        #[test]
        fn test_empty_content_stream() {
            let content = b"";
            let operators = ContentParser::parse(content).unwrap();
            assert_eq!(operators.len(), 0);
        }

        #[test]
        fn test_whitespace_only_content_stream() {
            let content = b"   \t\n\r   ";
            let operators = ContentParser::parse(content).unwrap();
            assert_eq!(operators.len(), 0);
        }

        #[test]
        fn test_mixed_integer_and_real_operands() {
            // Test with simple operands that work with current parser
            let content = b"100 200 m 150 200 l";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 2);
            assert_eq!(operators[0], ContentOperation::MoveTo(100.0, 200.0));
            assert_eq!(operators[1], ContentOperation::LineTo(150.0, 200.0));
        }

        #[test]
        fn test_negative_operands() {
            let content = b"-100 -200 Td -50.5 -75.2 TD";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 2);
            assert_eq!(operators[0], ContentOperation::MoveText(-100.0, -200.0));
            assert_eq!(
                operators[1],
                ContentOperation::MoveTextSetLeading(-50.5, -75.2)
            );
        }

        #[test]
        fn test_large_numbers() {
            let content = b"999999.999999 -999999.999999 m";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 1);
            assert_eq!(
                operators[0],
                ContentOperation::MoveTo(999999.999999, -999999.999999)
            );
        }

        #[test]
        fn test_scientific_notation() {
            // Test with simple decimal numbers since scientific notation isn't implemented
            let content = b"123.45 -456.78 m";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 1);
            assert_eq!(operators[0], ContentOperation::MoveTo(123.45, -456.78));
        }

        #[test]
        fn test_show_text_array_complex() {
            // `TJ` expects an array operand, not a plain string. The malformed
            // operator is skipped; a following valid Tj is recovered
            // (issue #319 recovery contract).
            let content = b"(Hello) TJ (kept) Tj";
            let ops = ContentParser::parse(content).expect("recovers from malformed TJ");
            assert!(
                ops.iter()
                    .any(|op| matches!(op, ContentOperation::ShowText(t) if t == b"kept")),
                "valid Tj after the malformed TJ must survive: {ops:?}"
            );
        }

        #[test]
        fn test_dash_pattern_empty() {
            // `d` needs an array operand; the malformed operator is skipped and
            // a following valid MoveTo survives (issue #319 recovery contract).
            let content = b"0 d 10 20 m";
            let ops = ContentParser::parse(content).expect("recovers from malformed d");
            assert!(
                ops.iter()
                    .any(|op| matches!(op, ContentOperation::MoveTo(_, _))),
                "valid MoveTo after the malformed dash op must survive: {ops:?}"
            );
        }

        #[test]
        fn test_dash_pattern_complex() {
            // Same recovery contract with a real-number operand before `d`.
            let content = b"2.5 d 10 20 m";
            let ops = ContentParser::parse(content).expect("recovers from malformed d");
            assert!(
                ops.iter()
                    .any(|op| matches!(op, ContentOperation::MoveTo(_, _))),
                "valid MoveTo after the malformed dash op must survive: {ops:?}"
            );
        }

        #[test]
        fn test_pop_array_removes_array_end() {
            // Test that pop_array correctly handles ArrayEnd tokens
            let parser = ContentParser::new(b"");

            // Test normal array: [1 2 3]
            let mut operands = vec![
                Token::ArrayStart,
                Token::Integer(1),
                Token::Integer(2),
                Token::Integer(3),
                Token::ArrayEnd,
            ];
            let result = parser.pop_array(&mut operands).unwrap();
            assert_eq!(result.len(), 3);
            assert!(operands.is_empty());

            // Test array without ArrayEnd (backwards compatibility)
            let mut operands = vec![Token::ArrayStart, Token::Number(1.5), Token::Number(2.5)];
            let result = parser.pop_array(&mut operands).unwrap();
            assert_eq!(result.len(), 2);
            assert!(operands.is_empty());
        }

        #[test]
        fn test_dash_array_parsing_valid() {
            // Test that parser correctly parses valid dash arrays
            let parser = ContentParser::new(b"");

            // Test with valid numbers only
            let valid_tokens = vec![Token::Number(3.0), Token::Integer(2)];
            let result = parser.parse_dash_array(valid_tokens).unwrap();
            assert_eq!(result, vec![3.0, 2.0]);

            // Test empty dash array
            let empty_tokens = vec![];
            let result = parser.parse_dash_array(empty_tokens).unwrap();
            let expected: Vec<f32> = vec![];
            assert_eq!(result, expected);
        }

        #[test]
        fn test_text_array_parsing_valid() {
            // Test that parser correctly parses valid text arrays
            let parser = ContentParser::new(b"");

            // Test with valid elements only
            let valid_tokens = vec![
                Token::String(b"Hello".to_vec()),
                Token::Number(-100.0),
                Token::String(b"World".to_vec()),
            ];
            let result = parser.parse_text_array(valid_tokens).unwrap();
            assert_eq!(result.len(), 3);
        }

        #[test]
        fn test_inline_image_handling() {
            let content = b"BI /W 100 /H 100 /BPC 8 /CS /RGB ID some_image_data EI";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 1);
            match &operators[0] {
                ContentOperation::InlineImage { params, data: _ } => {
                    // Check parsed parameters
                    assert_eq!(params.get("Width"), Some(&Object::Integer(100)));
                    assert_eq!(params.get("Height"), Some(&Object::Integer(100)));
                    assert_eq!(params.get("BitsPerComponent"), Some(&Object::Integer(8)));
                    assert_eq!(
                        params.get("ColorSpace"),
                        Some(&Object::Name("DeviceRGB".to_string()))
                    );
                    // Data field is not captured, just verify params
                }
                _ => panic!("Expected InlineImage operation"),
            }
        }

        #[test]
        fn test_inline_image_with_filter() {
            let content = b"BI /W 50 /H 50 /CS /G /BPC 1 /F /AHx ID 00FF00FF EI";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 1);
            match &operators[0] {
                ContentOperation::InlineImage { params, data: _ } => {
                    assert_eq!(params.get("Width"), Some(&Object::Integer(50)));
                    assert_eq!(params.get("Height"), Some(&Object::Integer(50)));
                    assert_eq!(
                        params.get("ColorSpace"),
                        Some(&Object::Name("DeviceGray".to_string()))
                    );
                    assert_eq!(params.get("BitsPerComponent"), Some(&Object::Integer(1)));
                    assert_eq!(
                        params.get("Filter"),
                        Some(&Object::Name("ASCIIHexDecode".to_string()))
                    );
                }
                _ => panic!("Expected InlineImage operation"),
            }
        }

        #[test]
        fn test_content_parser_performance() {
            let mut content = Vec::new();
            for i in 0..1000 {
                content.extend_from_slice(format!("{} {} m ", i, i + 1).as_bytes());
            }

            let start = std::time::Instant::now();
            let operators = ContentParser::parse(&content).unwrap();
            let duration = start.elapsed();

            assert_eq!(operators.len(), 1000);
            assert!(duration.as_millis() < 100); // Should parse 1000 operators in under 100ms
        }

        #[test]
        fn test_tokenizer_performance() {
            let mut input = Vec::new();
            for i in 0..1000 {
                input.extend_from_slice(format!("{} {} ", i, i + 1).as_bytes());
            }

            let start = std::time::Instant::now();
            let mut tokenizer = ContentTokenizer::new(&input);
            let mut count = 0;
            while tokenizer.next_token().unwrap().is_some() {
                count += 1;
            }
            let duration = start.elapsed();

            assert_eq!(count, 2000); // 1000 pairs of numbers
            assert!(duration.as_millis() < 50); // Should tokenize 2000 tokens in under 50ms
        }

        #[test]
        fn test_memory_usage_large_content() {
            let mut content = Vec::new();
            for i in 0..10000 {
                content.extend_from_slice(
                    format!("{} {} {} {} {} {} c ", i, i + 1, i + 2, i + 3, i + 4, i + 5)
                        .as_bytes(),
                );
            }

            let operators = ContentParser::parse(&content).unwrap();
            assert_eq!(operators.len(), 10000);

            // Verify all operations are CurveTo
            for op in operators {
                matches!(op, ContentOperation::CurveTo(_, _, _, _, _, _));
            }
        }

        #[test]
        fn test_concurrent_parsing() {
            use std::sync::Arc;
            use std::thread;

            let content = Arc::new(b"BT /F1 12 Tf 100 200 Td (Hello) Tj ET".to_vec());
            let handles: Vec<_> = (0..10)
                .map(|_| {
                    let content_clone = content.clone();
                    thread::spawn(move || ContentParser::parse(&content_clone).unwrap())
                })
                .collect();

            for handle in handles {
                let operators = handle.join().unwrap();
                assert_eq!(operators.len(), 5);
                assert_eq!(operators[0], ContentOperation::BeginText);
                assert_eq!(operators[4], ContentOperation::EndText);
            }
        }

        // ========== NEW COMPREHENSIVE TESTS ==========

        #[test]
        fn test_tokenizer_hex_string_edge_cases() {
            let mut tokenizer = ContentTokenizer::new(b"<>");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::HexString(data) => assert!(data.is_empty()),
                _ => panic!("Expected empty hex string"),
            }

            // Odd number of hex digits
            let mut tokenizer = ContentTokenizer::new(b"<123>");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::HexString(data) => assert_eq!(data, vec![0x12, 0x30]),
                _ => panic!("Expected hex string with odd digits"),
            }

            // Hex string with whitespace
            let mut tokenizer = ContentTokenizer::new(b"<12 34\t56\n78>");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::HexString(data) => assert_eq!(data, vec![0x12, 0x34, 0x56, 0x78]),
                _ => panic!("Expected hex string with whitespace"),
            }
        }

        #[test]
        fn test_tokenizer_literal_string_escape_sequences() {
            // Test all standard escape sequences
            let mut tokenizer = ContentTokenizer::new(b"(\\n\\r\\t\\b\\f\\(\\)\\\\)");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::String(data) => {
                    assert_eq!(
                        data,
                        vec![b'\n', b'\r', b'\t', 0x08, 0x0C, b'(', b')', b'\\']
                    );
                }
                _ => panic!("Expected string with escapes"),
            }

            // Test octal escape sequences
            let mut tokenizer = ContentTokenizer::new(b"(\\101\\040\\377)");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::String(data) => assert_eq!(data, vec![b'A', b' ', 255]),
                _ => panic!("Expected string with octal escapes"),
            }
        }

        #[test]
        fn test_tokenizer_nested_parentheses() {
            let mut tokenizer = ContentTokenizer::new(b"(outer (inner) text)");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::String(data) => {
                    assert_eq!(data, b"outer (inner) text");
                }
                _ => panic!("Expected string with nested parentheses"),
            }

            // Multiple levels of nesting
            let mut tokenizer = ContentTokenizer::new(b"(level1 (level2 (level3) back2) back1)");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::String(data) => {
                    assert_eq!(data, b"level1 (level2 (level3) back2) back1");
                }
                _ => panic!("Expected string with deep nesting"),
            }
        }

        #[test]
        fn test_tokenizer_name_hex_escapes() {
            let mut tokenizer = ContentTokenizer::new(b"/Name#20With#20Spaces");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::Name(name) => assert_eq!(name, "Name With Spaces"),
                _ => panic!("Expected name with hex escapes"),
            }

            // Test various special characters
            let mut tokenizer = ContentTokenizer::new(b"/Special#2F#28#29#3C#3E");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::Name(name) => assert_eq!(name, "Special/()<>"),
                _ => panic!("Expected name with special character escapes"),
            }
        }

        #[test]
        fn test_tokenizer_number_edge_cases() {
            // Very large integers
            let mut tokenizer = ContentTokenizer::new(b"2147483647");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::Integer(n) => assert_eq!(n, 2147483647),
                _ => panic!("Expected large integer"),
            }

            // Very small numbers
            let mut tokenizer = ContentTokenizer::new(b"0.00001");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::Number(n) => assert!((n - 0.00001).abs() < f32::EPSILON),
                _ => panic!("Expected small float"),
            }

            // Numbers starting with dot
            let mut tokenizer = ContentTokenizer::new(b".5");
            let token = tokenizer.next_token().unwrap().unwrap();
            match token {
                Token::Number(n) => assert!((n - 0.5).abs() < f32::EPSILON),
                _ => panic!("Expected float starting with dot"),
            }
        }

        #[test]
        fn test_parser_complex_path_operations() {
            let content = b"100 200 m 150 200 l 150 250 l 100 250 l h f";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 6);
            assert_eq!(operators[0], ContentOperation::MoveTo(100.0, 200.0));
            assert_eq!(operators[1], ContentOperation::LineTo(150.0, 200.0));
            assert_eq!(operators[2], ContentOperation::LineTo(150.0, 250.0));
            assert_eq!(operators[3], ContentOperation::LineTo(100.0, 250.0));
            assert_eq!(operators[4], ContentOperation::ClosePath);
            assert_eq!(operators[5], ContentOperation::Fill);
        }

        #[test]
        fn test_parser_bezier_curves() {
            let content = b"100 100 150 50 200 150 c";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 1);
            match &operators[0] {
                ContentOperation::CurveTo(x1, y1, x2, y2, x3, y3) => {
                    // Values are parsed in reverse order: last 6 values for c operator
                    // Stack order: 100 100 150 50 200 150
                    // Pop order: x1=100, y1=100, x2=150, y2=50, x3=200, y3=150
                    assert!(x1.is_finite() && y1.is_finite());
                    assert!(x2.is_finite() && y2.is_finite());
                    assert!(x3.is_finite() && y3.is_finite());
                    // Verify we have 6 coordinate values
                    assert!(*x1 >= 50.0 && *x1 <= 200.0);
                    assert!(*y1 >= 50.0 && *y1 <= 200.0);
                }
                _ => panic!("Expected CurveTo operation"),
            }
        }

        #[test]
        fn test_parser_color_operations() {
            let content = b"0.5 g 1 0 0 rg 0 1 0 1 k /DeviceRGB cs 0.2 0.4 0.6 sc";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 5);
            match &operators[0] {
                ContentOperation::SetNonStrokingGray(gray) => assert_eq!(*gray, 0.5),
                _ => panic!("Expected SetNonStrokingGray"),
            }
            match &operators[1] {
                ContentOperation::SetNonStrokingRGB(r, g, b) => {
                    assert_eq!((*r, *g, *b), (1.0, 0.0, 0.0));
                }
                _ => panic!("Expected SetNonStrokingRGB"),
            }
        }

        #[test]
        fn test_parser_text_positioning_advanced() {
            let content = b"BT 1 0 0 1 100 200 Tm 0 TL 10 TL (Line 1) ' (Line 2) ' ET";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 7);
            assert_eq!(operators[0], ContentOperation::BeginText);
            match &operators[1] {
                ContentOperation::SetTextMatrix(a, b, c, d, e, f) => {
                    assert_eq!((*a, *b, *c, *d, *e, *f), (1.0, 0.0, 0.0, 1.0, 100.0, 200.0));
                }
                _ => panic!("Expected SetTextMatrix"),
            }
            assert_eq!(operators[6], ContentOperation::EndText);
        }

        #[test]
        fn test_parser_graphics_state_operations() {
            let content = b"q 2 0 0 2 100 100 cm 5 w 1 J 2 j 10 M Q";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 7);
            assert_eq!(operators[0], ContentOperation::SaveGraphicsState);
            match &operators[1] {
                ContentOperation::SetTransformMatrix(a, b, c, d, e, f) => {
                    assert_eq!((*a, *b, *c, *d, *e, *f), (2.0, 0.0, 0.0, 2.0, 100.0, 100.0));
                }
                _ => panic!("Expected SetTransformMatrix"),
            }
            assert_eq!(operators[6], ContentOperation::RestoreGraphicsState);
        }

        #[test]
        fn test_parser_xobject_operations() {
            let content = b"/Image1 Do /Form2 Do /Pattern3 Do";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 3);
            for (i, expected_name) in ["Image1", "Form2", "Pattern3"].iter().enumerate() {
                match &operators[i] {
                    ContentOperation::PaintXObject(name) => assert_eq!(name, expected_name),
                    _ => panic!("Expected PaintXObject"),
                }
            }
        }

        #[test]
        fn test_parser_marked_content_operations() {
            let content = b"/P BMC (Tagged content) Tj EMC";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 3);
            match &operators[0] {
                ContentOperation::BeginMarkedContent(tag) => assert_eq!(tag, "P"),
                _ => panic!("Expected BeginMarkedContent"),
            }
            assert_eq!(operators[2], ContentOperation::EndMarkedContent);
        }

        #[test]
        fn test_parser_error_handling_invalid_operators() {
            // Best-effort recovery contract (issue #319).

            // Missing operands for `m`: the malformed operator is skipped but
            // a following valid `l` is recovered.
            let content = b"m 10 20 l";
            let ops = ContentParser::parse(content).expect("recovers from operand-less m");
            assert!(
                ops.iter()
                    .any(|op| matches!(op, ContentOperation::LineTo(_, _))),
                "valid LineTo after the operand-less m must survive: {ops:?}"
            );

            // Unterminated hex string: the tokenizer stops at the malformed
            // tail but keeps every token before it, so valid text ahead of the
            // bad hex is still extracted.
            let content = b"(kept) Tj <ABC DEF";
            let ops = ContentParser::parse(content).expect("recovers, keeping pre-error tokens");
            assert!(
                ops.iter()
                    .any(|op| matches!(op, ContentOperation::ShowText(t) if t == b"kept")),
                "text before the unterminated hex must survive: {ops:?}"
            );

            // Numbers without an operator parse OK (no operator attempted).
            let content = b"100 200 300";
            assert!(ContentParser::parse(content).is_ok());
        }

        #[test]
        fn test_parser_whitespace_tolerance() {
            let content = b"  \n\t  100   \r\n  200  \t m  \n";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 1);
            assert_eq!(operators[0], ContentOperation::MoveTo(100.0, 200.0));
        }

        #[test]
        fn test_tokenizer_comment_handling() {
            let content = b"100 % This is a comment\n200 m % Another comment";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 1);
            assert_eq!(operators[0], ContentOperation::MoveTo(100.0, 200.0));
        }

        #[test]
        fn test_parser_stream_with_binary_data() {
            // Test content stream with comment containing binary-like data
            let content = b"100 200 m % Comment with \xFF binary\n150 250 l";

            let operators = ContentParser::parse(content).unwrap();
            assert_eq!(operators.len(), 2);
            assert_eq!(operators[0], ContentOperation::MoveTo(100.0, 200.0));
            assert_eq!(operators[1], ContentOperation::LineTo(150.0, 250.0));
        }

        #[test]
        fn test_tokenizer_array_parsing() {
            // Test simple operations that don't require complex array parsing
            let content = b"100 200 m 150 250 l";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 2);
            assert_eq!(operators[0], ContentOperation::MoveTo(100.0, 200.0));
            assert_eq!(operators[1], ContentOperation::LineTo(150.0, 250.0));
        }

        #[test]
        fn test_parser_rectangle_operations() {
            let content = b"10 20 100 50 re 0 0 200 300 re";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 2);
            match &operators[0] {
                ContentOperation::Rectangle(x, y, width, height) => {
                    assert_eq!((*x, *y, *width, *height), (10.0, 20.0, 100.0, 50.0));
                }
                _ => panic!("Expected Rectangle operation"),
            }
            match &operators[1] {
                ContentOperation::Rectangle(x, y, width, height) => {
                    assert_eq!((*x, *y, *width, *height), (0.0, 0.0, 200.0, 300.0));
                }
                _ => panic!("Expected Rectangle operation"),
            }
        }

        #[test]
        fn test_parser_clipping_operations() {
            let content = b"100 100 50 50 re W n 200 200 75 75 re W* n";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 6);
            assert_eq!(operators[1], ContentOperation::Clip);
            assert_eq!(operators[2], ContentOperation::EndPath);
            assert_eq!(operators[4], ContentOperation::ClipEvenOdd);
            assert_eq!(operators[5], ContentOperation::EndPath);
        }

        #[test]
        fn test_parser_painting_operations() {
            let content = b"S s f f* B B* b b*";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 8);
            assert_eq!(operators[0], ContentOperation::Stroke);
            assert_eq!(operators[1], ContentOperation::CloseStroke);
            assert_eq!(operators[2], ContentOperation::Fill);
            assert_eq!(operators[3], ContentOperation::FillEvenOdd);
            assert_eq!(operators[4], ContentOperation::FillStroke);
            assert_eq!(operators[5], ContentOperation::FillStrokeEvenOdd);
            assert_eq!(operators[6], ContentOperation::CloseFillStroke);
            assert_eq!(operators[7], ContentOperation::CloseFillStrokeEvenOdd);
        }

        #[test]
        fn test_parser_line_style_operations() {
            let content = b"5 w 1 J 2 j 10 M [ 3 2 ] 0 d";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 5);
            assert_eq!(operators[0], ContentOperation::SetLineWidth(5.0));
            assert_eq!(operators[1], ContentOperation::SetLineCap(1));
            assert_eq!(operators[2], ContentOperation::SetLineJoin(2));
            assert_eq!(operators[3], ContentOperation::SetMiterLimit(10.0));
            // Dash pattern test would need array support
        }

        #[test]
        fn test_parser_text_state_operations() {
            let content = b"12 Tc 3 Tw 100 Tz 1 Tr 2 Ts";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 5);
            assert_eq!(operators[0], ContentOperation::SetCharSpacing(12.0));
            assert_eq!(operators[1], ContentOperation::SetWordSpacing(3.0));
            assert_eq!(operators[2], ContentOperation::SetHorizontalScaling(100.0));
            assert_eq!(operators[3], ContentOperation::SetTextRenderMode(1));
            assert_eq!(operators[4], ContentOperation::SetTextRise(2.0));
        }

        #[test]
        fn test_parser_unicode_text() {
            let content = b"BT (Hello \xC2\xA9 World \xE2\x9C\x93) Tj ET";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 3);
            assert_eq!(operators[0], ContentOperation::BeginText);
            match &operators[1] {
                ContentOperation::ShowText(text) => {
                    assert!(text.len() > 5); // Should contain Unicode bytes
                }
                _ => panic!("Expected ShowText operation"),
            }
            assert_eq!(operators[2], ContentOperation::EndText);
        }

        #[test]
        fn test_parser_stress_test_large_coordinates() {
            let content = b"999999.999 -999999.999 999999.999 -999999.999 999999.999 -999999.999 c";
            let operators = ContentParser::parse(content).unwrap();

            assert_eq!(operators.len(), 1);
            match &operators[0] {
                ContentOperation::CurveTo(_x1, _y1, _x2, _y2, _x3, _y3) => {
                    assert!((*_x1 - 999999.999).abs() < 0.1);
                    assert!((*_y1 - (-999999.999)).abs() < 0.1);
                    assert!((*_x3 - 999999.999).abs() < 0.1);
                }
                _ => panic!("Expected CurveTo operation"),
            }
        }

        #[test]
        fn test_parser_empty_content_stream() {
            let content = b"";
            let operators = ContentParser::parse(content).unwrap();
            assert!(operators.is_empty());

            let content = b"   \n\t\r   ";
            let operators = ContentParser::parse(content).unwrap();
            assert!(operators.is_empty());
        }

        #[test]
        fn test_tokenizer_error_recovery() {
            // A comment carrying a stray binary byte sits between two valid
            // path operators. The comment (and its binary) is skipped and
            // BOTH operators are recovered (issue #319 recovery contract).
            let content = b"100 200 m % Comment with\xFFbinary\n150 250 l";
            let ops = ContentParser::parse(content).expect("recovers around binary comment");
            assert!(
                ops.iter()
                    .any(|op| matches!(op, ContentOperation::MoveTo(_, _))),
                "MoveTo before the comment must survive: {ops:?}"
            );
            assert!(
                ops.iter()
                    .any(|op| matches!(op, ContentOperation::LineTo(_, _))),
                "LineTo after the comment must survive: {ops:?}"
            );
        }

        #[test]
        fn malformed_operator_does_not_discard_surrounding_text() {
            // Issue #319: a single malformed operator must NOT drop the whole
            // page's content. Here a bare `Td` (missing its two operands)
            // sits between two valid text-show operators. Before the fix,
            // `parse_operators` propagated the operand error with `?`, so the
            // entire stream returned Err and BOTH show-text ops were lost
            // (the extractor then dropped the page). The parser must recover:
            // skip the bad operator, keep the valid ones.
            let content = b"BT /F1 12 Tf 72 700 Td (First line) Tj Td (Second line) Tj ET";
            let ops = ContentParser::parse_content(content)
                .expect("malformed operator must not fail the whole stream");
            let shown: Vec<&Vec<u8>> = ops
                .iter()
                .filter_map(|op| match op {
                    ContentOperation::ShowText(t) => Some(t),
                    _ => None,
                })
                .collect();
            assert_eq!(
                shown.len(),
                2,
                "both valid Tj operators must survive the malformed Td"
            );
            assert_eq!(shown[0], b"First line");
            assert_eq!(shown[1], b"Second line");
        }

        #[test]
        fn test_parser_optimization_repeated_operations() {
            // Test performance with many repeated operations
            let mut content = Vec::new();
            for i in 0..1000 {
                content.extend_from_slice(format!("{} {} m ", i, i * 2).as_bytes());
            }

            let start = std::time::Instant::now();
            let operators = ContentParser::parse(&content).unwrap();
            let duration = start.elapsed();

            assert_eq!(operators.len(), 1000);
            assert!(duration.as_millis() < 200); // Should be fast
        }

        #[test]
        fn test_parser_memory_efficiency_large_strings() {
            // Test with large text content
            let large_text = "A".repeat(10000);
            let content = format!("BT ({}) Tj ET", large_text);
            let operators = ContentParser::parse(content.as_bytes()).unwrap();

            assert_eq!(operators.len(), 3);
            match &operators[1] {
                ContentOperation::ShowText(text) => {
                    assert_eq!(text.len(), 10000);
                }
                _ => panic!("Expected ShowText operation"),
            }
        }
    }

    #[test]
    fn test_content_stream_too_large() {
        // Test handling of very large content streams (covering potential size limits)
        let mut large_content = Vec::new();

        // Create a content stream with many operations
        for i in 0..10000 {
            large_content.extend_from_slice(format!("{} {} m ", i, i).as_bytes());
        }
        large_content.extend_from_slice(b"S");

        // Should handle large content without panic
        let result = ContentParser::parse_content(&large_content);
        assert!(result.is_ok());

        let operations = result.unwrap();
        // Should have many MoveTo operations plus one Stroke
        assert!(operations.len() > 10000);
    }

    #[test]
    fn test_invalid_operator_handling() {
        // Test parsing with invalid operators
        let content = b"100 200 INVALID_OP 300 400 m";
        let result = ContentParser::parse_content(content);

        // Should either handle gracefully or return error
        if let Ok(operations) = result {
            // If it succeeds, should have at least the valid MoveTo
            assert!(operations
                .iter()
                .any(|op| matches!(op, ContentOperation::MoveTo(_, _))));
        }
    }

    #[test]
    fn test_nested_arrays_malformed() {
        // Test malformed nested arrays in TJ operator
        let content = b"[[(Hello] [World)]] TJ";
        let result = ContentParser::parse_content(content);

        // Should handle malformed arrays gracefully
        assert!(result.is_ok() || result.is_err());
    }

    #[test]
    fn test_escape_sequences_in_strings() {
        // Test various escape sequences in strings
        let test_cases = vec![
            (b"(\\n\\r\\t)".as_slice(), b"\n\r\t".as_slice()),
            (b"(\\\\)".as_slice(), b"\\".as_slice()),
            (b"(\\(\\))".as_slice(), b"()".as_slice()),
            (b"(\\123)".as_slice(), b"S".as_slice()), // Octal 123 = 83 = 'S'
            (b"(\\0)".as_slice(), b"\0".as_slice()),
        ];

        for (input, expected) in test_cases {
            let mut content = Vec::new();
            content.extend_from_slice(input);
            content.extend_from_slice(b" Tj");

            let result = ContentParser::parse_content(&content);
            assert!(result.is_ok());

            let operations = result.unwrap();
            if let ContentOperation::ShowText(text) = &operations[0] {
                assert_eq!(text, expected, "Failed for input: {:?}", input);
            } else {
                panic!("Expected ShowText operation");
            }
        }
    }

    #[test]
    fn test_content_with_inline_images() {
        // Test handling of inline images in content stream
        let content = b"BI /W 10 /H 10 /CS /RGB ID \x00\x01\x02\x03 EI";
        let result = ContentParser::parse_content(content);

        // Should handle inline images (even if not fully implemented)
        assert!(result.is_ok() || result.is_err());
    }

    #[test]
    fn test_operator_with_missing_operands() {
        // Test operators with insufficient operands
        let test_cases = vec![
            b"Tj" as &[u8], // ShowText without string
            b"m",           // MoveTo without coordinates
            b"rg",          // SetRGBColor without values
            b"Tf",          // SetFont without name and size
        ];

        for content in test_cases {
            let result = ContentParser::parse_content(content);
            // Should handle gracefully (error or skip)
            assert!(result.is_ok() || result.is_err());
        }
    }

    // --- Tests for infinite loop fix (curly braces, stray parens, inline images) ---

    #[test]
    fn test_tokenizer_handles_curly_braces() {
        // Curly braces { } are not valid PDF content operators but appear in
        // binary inline image data. The tokenizer must skip them without hanging.
        let input = b"q { } Q";
        let mut tokenizer = ContentTokenizer::new(input);

        let mut tokens = Vec::new();
        while let Some(token) = tokenizer.next_token().unwrap() {
            tokens.push(token);
        }

        // Should produce tokens for q and Q, skipping { and }
        assert!(tokens.contains(&Token::Operator("q".to_string())));
        assert!(tokens.contains(&Token::Operator("Q".to_string())));
    }

    #[test]
    fn test_tokenizer_handles_closing_paren() {
        // A stray ) outside a string literal should be skipped, not cause a hang
        let input = b"q ) Q";
        let mut tokenizer = ContentTokenizer::new(input);

        let mut tokens = Vec::new();
        while let Some(token) = tokenizer.next_token().unwrap() {
            tokens.push(token);
        }

        assert!(tokens.contains(&Token::Operator("q".to_string())));
        assert!(tokens.contains(&Token::Operator("Q".to_string())));
    }

    #[test]
    fn test_inline_image_binary_with_curly_braces() {
        // Inline image binary data containing { and } bytes must be handled
        // correctly — the tokenizer should capture them as raw image data
        let content = b"BI /W 2 /H 2 /BPC 8 /CS /G ID \x7B\x7D\x00\xFF EI Q";
        let result = ContentParser::parse_content(content);
        assert!(
            result.is_ok(),
            "Parsing inline image with curly braces failed: {:?}",
            result.err()
        );

        let ops = result.unwrap();
        // Should have InlineImage + RestoreGraphicsState
        let has_inline = ops
            .iter()
            .any(|op| matches!(op, ContentOperation::InlineImage { .. }));
        let has_q = ops
            .iter()
            .any(|op| matches!(op, ContentOperation::RestoreGraphicsState));
        assert!(has_inline, "Expected InlineImage operation");
        assert!(has_q, "Expected RestoreGraphicsState after EI");
    }

    #[test]
    fn test_inline_image_binary_with_all_byte_values() {
        // Inline image with bytes 0x00-0xFF to ensure no byte causes a hang
        let mut content = Vec::new();
        content.extend_from_slice(b"BI /W 16 /H 16 /BPC 8 /CS /G ID ");
        // Add all 256 byte values as image data
        for b in 0u8..=255 {
            content.push(b);
        }
        content.extend_from_slice(b" EI Q");

        let result = ContentParser::parse_content(&content);
        assert!(
            result.is_ok(),
            "Parsing inline image with all byte values failed: {:?}",
            result.err()
        );
    }

    #[test]
    fn test_inline_image_ei_detection() {
        // EI must be preceded by whitespace to be recognized as end marker
        // "EI" within binary data (not preceded by whitespace) should NOT end the image
        let content = b"BI /W 2 /H 1 /BPC 8 /CS /G ID \x45\x49\x00\n EI Q";
        //                                               ^E  ^I  (within data)  ^real EI
        let result = ContentParser::parse_content(content);
        assert!(result.is_ok(), "EI detection failed: {:?}", result.err());

        let ops = result.unwrap();
        let has_inline = ops
            .iter()
            .any(|op| matches!(op, ContentOperation::InlineImage { .. }));
        assert!(has_inline, "Expected InlineImage operation");
    }

    #[test]
    fn test_tokenizer_no_infinite_loop_on_consecutive_delimiters() {
        // Multiple consecutive unhandled delimiters must not cause a hang
        let input = b"q {{{}}})))) Q";
        let mut tokenizer = ContentTokenizer::new(input);

        let mut tokens = Vec::new();
        while let Some(token) = tokenizer.next_token().unwrap() {
            tokens.push(token);
            if tokens.len() > 100 {
                panic!("Tokenizer produced too many tokens — possible infinite loop");
            }
        }

        assert!(tokens.contains(&Token::Operator("q".to_string())));
        assert!(tokens.contains(&Token::Operator("Q".to_string())));
    }

    #[test]
    fn test_content_parser_inline_image_produces_correct_operation() {
        // Full parse of a simple inline image should produce correct params
        let content = b"BI /W 4 /H 4 /BPC 8 /CS /G ID \x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F EI";
        let result = ContentParser::parse_content(content);
        assert!(result.is_ok(), "Parse failed: {:?}", result.err());

        let ops = result.unwrap();
        assert_eq!(
            ops.len(),
            1,
            "Expected exactly 1 operation, got {}",
            ops.len()
        );

        if let ContentOperation::InlineImage { params, data } = &ops[0] {
            assert_eq!(params.get("Width"), Some(&Object::Integer(4)));
            assert_eq!(params.get("Height"), Some(&Object::Integer(4)));
            assert_eq!(params.get("BitsPerComponent"), Some(&Object::Integer(8)));
            assert!(!data.is_empty(), "Image data should not be empty");
        } else {
            panic!("Expected InlineImage operation, got {:?}", ops[0]);
        }
    }

    #[test]
    fn test_octal_escape_overflow_777() {
        // \777 = octal 777 = 511 decimal, overflows u8.
        // Per ISO 32000-1:2008 §7.3.4.2: "high-order overflow shall be ignored"
        // 511 as u8 = 255 (0x1FF truncated to 0xFF)
        let mut tokenizer = ContentTokenizer::new(b"(\\777)");
        let token = tokenizer.next_token().unwrap().unwrap();
        match token {
            Token::String(data) => assert_eq!(data, vec![0xFF]),
            _ => panic!("Expected string token"),
        }
    }

    #[test]
    fn test_octal_escape_overflow_400() {
        // \400 = octal 400 = 256 decimal, just overflows u8.
        // 256 as u8 = 0
        let mut tokenizer = ContentTokenizer::new(b"(\\400)");
        let token = tokenizer.next_token().unwrap().unwrap();
        match token {
            Token::String(data) => assert_eq!(data, vec![0x00]),
            _ => panic!("Expected string token"),
        }
    }

    #[test]
    fn test_octal_escape_overflow_577() {
        // \577 = octal 577 = 383 decimal.
        // 383 as u8 = 127 (0x17F truncated to 0x7F)
        let mut tokenizer = ContentTokenizer::new(b"(\\577)");
        let token = tokenizer.next_token().unwrap().unwrap();
        match token {
            Token::String(data) => assert_eq!(data, vec![0x7F]),
            _ => panic!("Expected string token"),
        }
    }

    #[test]
    fn test_octal_escape_max_valid_377() {
        // \377 = 255, max valid octal for u8 - should still work correctly
        let mut tokenizer = ContentTokenizer::new(b"(\\377)");
        let token = tokenizer.next_token().unwrap().unwrap();
        match token {
            Token::String(data) => assert_eq!(data, vec![0xFF]),
            _ => panic!("Expected string token"),
        }
    }

    #[test]
    fn test_octal_escape_overflow_mixed_with_valid() {
        // Mix of overflow octal and normal text
        let mut tokenizer = ContentTokenizer::new(b"(A\\777B\\101C)");
        let token = tokenizer.next_token().unwrap().unwrap();
        match token {
            Token::String(data) => {
                assert_eq!(data, vec![b'A', 0xFF, b'B', b'A', b'C']);
            }
            _ => panic!("Expected string token"),
        }
    }
}