rpdfium_parser/

content_stream.rs

// Derived from PDFium's cpdf_streamparser.cpp / cpdf_contentparser.cpp
// Original: Copyright 2014 The PDFium Authors
// Licensed under BSD-3-Clause / Apache-2.0
// See pdfium-upstream/LICENSE for the original license.

//! Content stream operator tokenization (Stage 1).
//!
//! Tokenizes content stream bytes into a sequence of PDF operators.
//! This module understands the PostScript-like operand stack + operator
//! keyword pattern but not the semantics of operators.
//!
//! ```text
//! rpdfium-parser:   &[u8] → Vec<Operator>          (tokenization — syntactic)
//! rpdfium-page:     Vec<Operator> → DisplayTree     (interpretation — semantic)
//! ```

use std::collections::HashMap;

use rpdfium_core::Name;
use rpdfium_core::error::PdfError;

use crate::tokenizer::{Token, Tokenizer, is_delimiter, is_whitespace};

/// A content stream operand value.
#[derive(Debug, Clone, PartialEq)]
pub enum Operand {
    Integer(i64),
    Real(f64),
    Name(Name),
    String(Vec<u8>),
    Array(Vec<Operand>),
    Boolean(bool),
    Null,
}

impl Operand {
    /// Get as f32, converting from integer if needed.
    pub fn as_f32(&self) -> Option<f32> {
        match self {
            Operand::Integer(n) => Some(*n as f32),
            Operand::Real(f) => Some(*f as f32),
            _ => None,
        }
    }

    /// Get as i64.
    pub fn as_i64(&self) -> Option<i64> {
        match self {
            Operand::Integer(n) => Some(*n),
            Operand::Real(f) => Some(*f as i64),
            _ => None,
        }
    }
}

/// A parsed content stream operator with its operands.
#[derive(Debug, Clone, PartialEq)]
pub enum Operator {
    // --- Text state operators ---
    /// BT — Begin text object.
    BeginText,
    /// ET — End text object.
    EndText,
    /// Tf name size — Set font and size.
    SetFont { name: Name, size: f32 },
    /// Td tx ty — Move text position.
    MoveText { tx: f32, ty: f32 },
    /// TD tx ty — Move text position and set leading.
    MoveTextSetLeading { tx: f32, ty: f32 },
    /// Tm a b c d e f — Set text matrix.
    SetTextMatrix {
        a: f32,
        b: f32,
        c: f32,
        d: f32,
        e: f32,
        f: f32,
    },
    /// T* — Move to next line.
    NextLine,
    /// Tj string — Show text.
    ShowText { bytes: Vec<u8> },
    /// TJ array — Show text with positioning.
    ShowTextArray { elements: Vec<TextArrayElement> },
    /// ' string — Move to next line and show text.
    NextLineShowText { bytes: Vec<u8> },
    /// " aw ac string — Set word/char spacing, move to next line, show text.
    SetSpacingShowText {
        word_space: f32,
        char_space: f32,
        bytes: Vec<u8>,
    },
    /// Tc charSpace — Set character spacing.
    SetCharSpacing { spacing: f32 },
    /// Tw wordSpace — Set word spacing.
    SetWordSpacing { spacing: f32 },
    /// Tz scale — Set horizontal scaling.
    SetHorizontalScaling { scale: f32 },
    /// TL leading — Set text leading.
    SetTextLeading { leading: f32 },
    /// Ts rise — Set text rise.
    SetTextRise { rise: f32 },
    /// Tr render — Set text rendering mode.
    SetTextRenderingMode { mode: i64 },

    // --- Graphics state operators ---
    /// q — Save graphics state.
    SaveState,
    /// Q — Restore graphics state.
    RestoreState,
    /// cm a b c d e f — Concatenate matrix.
    ConcatMatrix {
        a: f32,
        b: f32,
        c: f32,
        d: f32,
        e: f32,
        f: f32,
    },
    /// w lineWidth — Set line width.
    SetLineWidth { width: f32 },
    /// J lineCap — Set line cap style.
    SetLineCap { cap: i64 },
    /// j lineJoin — Set line join style.
    SetLineJoin { join: i64 },
    /// M miterLimit — Set miter limit.
    SetMiterLimit { limit: f32 },
    /// d dashArray dashPhase — Set line dash pattern.
    SetDashPattern { array: Vec<f32>, phase: f32 },
    /// ri intent — Set rendering intent.
    SetRenderingIntent { intent: Name },
    /// i flatness — Set flatness tolerance.
    SetFlatness { flatness: f32 },
    /// gs name — Set graphics state from external dictionary.
    SetGraphicsState { name: Name },

    // --- Path construction operators ---
    /// m x y — Begin new subpath.
    MoveTo { x: f32, y: f32 },
    /// l x y — Append straight line.
    LineTo { x: f32, y: f32 },
    /// c x1 y1 x2 y2 x3 y3 — Append cubic Bezier curve.
    CurveTo {
        x1: f32,
        y1: f32,
        x2: f32,
        y2: f32,
        x3: f32,
        y3: f32,
    },
    /// v x2 y2 x3 y3 — Append cubic Bezier (initial point replicated).
    CurveToInitial { x2: f32, y2: f32, x3: f32, y3: f32 },
    /// y x1 y1 x3 y3 — Append cubic Bezier (final point replicated).
    CurveToFinal { x1: f32, y1: f32, x3: f32, y3: f32 },
    /// h — Close subpath.
    ClosePath,
    /// re x y w h — Append rectangle.
    Rectangle { x: f32, y: f32, w: f32, h: f32 },

    // --- Path painting operators ---
    /// S — Stroke path.
    Stroke,
    /// s — Close and stroke path.
    CloseAndStroke,
    /// f — Fill path (non-zero winding rule).
    Fill,
    /// F — Fill path (non-zero winding rule, obsolete).
    FillObsolete,
    /// f* — Fill path (even-odd rule).
    FillEvenOdd,
    /// B — Fill and stroke (non-zero winding).
    FillStroke,
    /// B* — Fill and stroke (even-odd).
    FillStrokeEvenOdd,
    /// b — Close, fill, and stroke (non-zero winding).
    CloseFillStroke,
    /// b* — Close, fill, and stroke (even-odd).
    CloseFillStrokeEvenOdd,
    /// n — End path (no-op painting).
    EndPath,

    // --- Clipping operators ---
    /// W — Set clipping path (non-zero winding).
    Clip,
    /// W* — Set clipping path (even-odd).
    ClipEvenOdd,

    // --- Color operators ---
    /// CS name — Set color space (stroking).
    SetColorSpaceStroke { name: Name },
    /// cs name — Set color space (non-stroking).
    SetColorSpaceFill { name: Name },
    /// SC c1...cn — Set color (stroking).
    SetColorStroke { components: Vec<f32> },
    /// sc c1...cn — Set color (non-stroking).
    SetColorFill { components: Vec<f32> },
    /// SCN c1...cn \[name\] — Set color (stroking, with pattern).
    SetColorStrokeN {
        components: Vec<f32>,
        name: Option<Name>,
    },
    /// scn c1...cn \[name\] — Set color (non-stroking, with pattern).
    SetColorFillN {
        components: Vec<f32>,
        name: Option<Name>,
    },
    /// G gray — Set gray (stroking).
    SetGrayStroke { gray: f32 },
    /// g gray — Set gray (non-stroking).
    SetGrayFill { gray: f32 },
    /// RG r g b — Set RGB (stroking).
    SetRgbStroke { r: f32, g: f32, b: f32 },
    /// rg r g b — Set RGB (non-stroking).
    SetRgbFill { r: f32, g: f32, b: f32 },
    /// K c m y k — Set CMYK (stroking).
    SetCmykStroke { c: f32, m: f32, y: f32, k: f32 },
    /// k c m y k — Set CMYK (non-stroking).
    SetCmykFill { c: f32, m: f32, y: f32, k: f32 },

    // --- XObject, inline image, and shading operators ---
    /// Do name — Paint XObject.
    PaintXObject { name: Name },
    /// sh name — Paint shading.
    PaintShading { name: Name },
    /// BI ... ID ... EI — Inline image with parsed properties and raw data.
    InlineImage {
        properties: HashMap<Name, Operand>,
        data: Vec<u8>,
    },

    // --- Marked content operators ---
    /// BMC tag — Begin marked content.
    BeginMarkedContent { tag: Name },
    /// BDC tag properties — Begin marked content with properties.
    BeginMarkedContentDict { tag: Name, properties: Operand },
    /// EMC — End marked content.
    EndMarkedContent,
    /// MP tag — Marked content point.
    MarkedContentPoint { tag: Name },
    /// DP tag properties — Marked content point with properties.
    MarkedContentPointDict { tag: Name, properties: Operand },

    // --- Type 3 character width operators ---
    /// d0 wx wy — Set character width (Type 3).
    SetCharWidth { wx: f32, wy: f32 },
    /// d1 wx wy llx lly urx ury — Set cache device (Type 3).
    SetCacheDevice {
        wx: f32,
        wy: f32,
        llx: f32,
        lly: f32,
        urx: f32,
        ury: f32,
    },

    // --- Compatibility operators ---
    /// BX — Begin compatibility section.
    BeginCompat,
    /// EX — End compatibility section.
    EndCompat,

    /// An unknown/unrecognized operator keyword with its raw operands.
    Unknown {
        keyword: Vec<u8>,
        operands: Vec<Operand>,
    },
}

/// An element in a TJ (show text array) operator.
#[derive(Debug, Clone, PartialEq)]
pub enum TextArrayElement {
    /// A text string to show.
    Text(Vec<u8>),
    /// A position adjustment (negative = move right in text direction).
    Adjustment(f32),
}

/// Tokenize a content stream into a sequence of operators.
///
/// Content streams use a PostScript-like syntax: operands are pushed onto
/// a stack, then an operator keyword consumes them.
pub fn tokenize_content_stream(data: &[u8]) -> Result<Vec<Operator>, PdfError> {
    let mut tok = Tokenizer::new(data);
    let mut operands: Vec<Operand> = Vec::new();
    let mut operators = Vec::new();

    loop {
        let token = match tok.next_token() {
            Some(Ok(t)) => t,
            Some(Err(_)) => {
                // Skip invalid tokens in content streams (lenient)
                continue;
            }
            None => break,
        };

        match token {
            Token::Integer(n) => operands.push(Operand::Integer(n)),
            Token::Real(f) => operands.push(Operand::Real(f)),
            Token::Name(n) => operands.push(Operand::Name(n)),
            Token::String(s) => operands.push(Operand::String(s.as_bytes().to_vec())),
            Token::Boolean(b) => operands.push(Operand::Boolean(b)),
            Token::Null => operands.push(Operand::Null),
            Token::ArrayStart => {
                // Read array operand
                let arr = read_operand_array(&mut tok)?;
                operands.push(Operand::Array(arr));
            }
            Token::Ref(_) => {
                // References shouldn't appear in content streams; treat as unknown
                tracing::warn!("indirect reference in content stream, ignoring");
            }
            Token::Keyword(kw) => {
                let op = build_operator(&kw, &mut operands, &mut tok);
                operators.push(op);
            }
            Token::DictStart | Token::DictEnd | Token::ArrayEnd | Token::Comment(_) => {
                // These shouldn't normally appear at the top level, skip
            }
        }
    }

    Ok(operators)
}

/// Read an array of operands from the tokenizer (for TJ arrays, dash patterns, etc.).
fn read_operand_array(tok: &mut Tokenizer<'_>) -> Result<Vec<Operand>, PdfError> {
    let mut arr = Vec::new();
    loop {
        match tok.next_token() {
            Some(Ok(Token::ArrayEnd)) => return Ok(arr),
            Some(Ok(Token::Integer(n))) => arr.push(Operand::Integer(n)),
            Some(Ok(Token::Real(f))) => arr.push(Operand::Real(f)),
            Some(Ok(Token::String(s))) => {
                arr.push(Operand::String(s.as_bytes().to_vec()));
            }
            Some(Ok(Token::Name(n))) => arr.push(Operand::Name(n)),
            Some(Ok(Token::Boolean(b))) => arr.push(Operand::Boolean(b)),
            Some(Ok(Token::Null)) => arr.push(Operand::Null),
            None => return Ok(arr),
            _ => continue,
        }
    }
}

/// Build an operator from the keyword and operand stack.
fn build_operator(
    keyword: &[u8],
    operands: &mut Vec<Operand>,
    tok: &mut Tokenizer<'_>,
) -> Operator {
    let op = match keyword {
        // Text state
        b"BT" => Operator::BeginText,
        b"ET" => Operator::EndText,
        b"Tf" => {
            let size = pop_f32(operands);
            let name = pop_name(operands);
            Operator::SetFont { name, size }
        }
        b"Td" => {
            let ty = pop_f32(operands);
            let tx = pop_f32(operands);
            Operator::MoveText { tx, ty }
        }
        b"TD" => {
            let ty = pop_f32(operands);
            let tx = pop_f32(operands);
            Operator::MoveTextSetLeading { tx, ty }
        }
        b"Tm" => {
            let f = pop_f32(operands);
            let e = pop_f32(operands);
            let d = pop_f32(operands);
            let c = pop_f32(operands);
            let b = pop_f32(operands);
            let a = pop_f32(operands);
            Operator::SetTextMatrix { a, b, c, d, e, f }
        }
        b"T*" => Operator::NextLine,
        b"Tj" => {
            let bytes = pop_bytes(operands);
            Operator::ShowText { bytes }
        }
        b"TJ" => {
            let elements = pop_text_array(operands);
            Operator::ShowTextArray { elements }
        }
        b"'" => {
            let bytes = pop_bytes(operands);
            Operator::NextLineShowText { bytes }
        }
        b"\"" => {
            let bytes = pop_bytes(operands);
            let char_space = pop_f32(operands);
            let word_space = pop_f32(operands);
            Operator::SetSpacingShowText {
                word_space,
                char_space,
                bytes,
            }
        }
        b"Tc" => Operator::SetCharSpacing {
            spacing: pop_f32(operands),
        },
        b"Tw" => Operator::SetWordSpacing {
            spacing: pop_f32(operands),
        },
        b"Tz" => Operator::SetHorizontalScaling {
            scale: pop_f32(operands),
        },
        b"TL" => Operator::SetTextLeading {
            leading: pop_f32(operands),
        },
        b"Ts" => Operator::SetTextRise {
            rise: pop_f32(operands),
        },
        b"Tr" => Operator::SetTextRenderingMode {
            mode: pop_i64(operands),
        },

        // Graphics state
        b"q" => Operator::SaveState,
        b"Q" => Operator::RestoreState,
        b"cm" => {
            let f = pop_f32(operands);
            let e = pop_f32(operands);
            let d = pop_f32(operands);
            let c = pop_f32(operands);
            let b = pop_f32(operands);
            let a = pop_f32(operands);
            Operator::ConcatMatrix { a, b, c, d, e, f }
        }
        b"w" => Operator::SetLineWidth {
            width: pop_f32(operands),
        },
        b"J" => Operator::SetLineCap {
            cap: pop_i64(operands),
        },
        b"j" => Operator::SetLineJoin {
            join: pop_i64(operands),
        },
        b"M" => Operator::SetMiterLimit {
            limit: pop_f32(operands),
        },
        b"d" => {
            let phase = pop_f32(operands);
            let array = pop_f32_array(operands);
            Operator::SetDashPattern { array, phase }
        }
        b"ri" => Operator::SetRenderingIntent {
            intent: pop_name(operands),
        },
        b"i" => Operator::SetFlatness {
            flatness: pop_f32(operands),
        },
        b"gs" => Operator::SetGraphicsState {
            name: pop_name(operands),
        },

        // Path construction
        b"m" => {
            let y = pop_f32(operands);
            let x = pop_f32(operands);
            Operator::MoveTo { x, y }
        }
        b"l" => {
            let y = pop_f32(operands);
            let x = pop_f32(operands);
            Operator::LineTo { x, y }
        }
        b"c" => {
            let y3 = pop_f32(operands);
            let x3 = pop_f32(operands);
            let y2 = pop_f32(operands);
            let x2 = pop_f32(operands);
            let y1 = pop_f32(operands);
            let x1 = pop_f32(operands);
            Operator::CurveTo {
                x1,
                y1,
                x2,
                y2,
                x3,
                y3,
            }
        }
        b"v" => {
            let y3 = pop_f32(operands);
            let x3 = pop_f32(operands);
            let y2 = pop_f32(operands);
            let x2 = pop_f32(operands);
            Operator::CurveToInitial { x2, y2, x3, y3 }
        }
        b"y" => {
            let y3 = pop_f32(operands);
            let x3 = pop_f32(operands);
            let y1 = pop_f32(operands);
            let x1 = pop_f32(operands);
            Operator::CurveToFinal { x1, y1, x3, y3 }
        }
        b"h" => Operator::ClosePath,
        b"re" => {
            let h = pop_f32(operands);
            let w = pop_f32(operands);
            let y = pop_f32(operands);
            let x = pop_f32(operands);
            Operator::Rectangle { x, y, w, h }
        }

        // Path painting
        b"S" => Operator::Stroke,
        b"s" => Operator::CloseAndStroke,
        b"f" => Operator::Fill,
        b"F" => Operator::FillObsolete,
        b"f*" => Operator::FillEvenOdd,
        b"B" => Operator::FillStroke,
        b"B*" => Operator::FillStrokeEvenOdd,
        b"b" => Operator::CloseFillStroke,
        b"b*" => Operator::CloseFillStrokeEvenOdd,
        b"n" => Operator::EndPath,

        // Clipping
        b"W" => Operator::Clip,
        b"W*" => Operator::ClipEvenOdd,

        // Color
        b"CS" => Operator::SetColorSpaceStroke {
            name: pop_name(operands),
        },
        b"cs" => Operator::SetColorSpaceFill {
            name: pop_name(operands),
        },
        b"SC" => Operator::SetColorStroke {
            components: drain_f32(operands),
        },
        b"sc" => Operator::SetColorFill {
            components: drain_f32(operands),
        },
        b"SCN" => {
            let (components, name) = drain_f32_with_optional_name(operands);
            Operator::SetColorStrokeN { components, name }
        }
        b"scn" => {
            let (components, name) = drain_f32_with_optional_name(operands);
            Operator::SetColorFillN { components, name }
        }
        b"G" => Operator::SetGrayStroke {
            gray: pop_f32(operands),
        },
        b"g" => Operator::SetGrayFill {
            gray: pop_f32(operands),
        },
        b"RG" => {
            let b = pop_f32(operands);
            let g = pop_f32(operands);
            let r = pop_f32(operands);
            Operator::SetRgbStroke { r, g, b }
        }
        b"rg" => {
            let b = pop_f32(operands);
            let g = pop_f32(operands);
            let r = pop_f32(operands);
            Operator::SetRgbFill { r, g, b }
        }
        b"K" => {
            let k = pop_f32(operands);
            let y = pop_f32(operands);
            let m = pop_f32(operands);
            let c = pop_f32(operands);
            Operator::SetCmykStroke { c, m, y, k }
        }
        b"k" => {
            let k = pop_f32(operands);
            let y = pop_f32(operands);
            let m = pop_f32(operands);
            let c = pop_f32(operands);
            Operator::SetCmykFill { c, m, y, k }
        }

        // XObject
        b"Do" => Operator::PaintXObject {
            name: pop_name(operands),
        },

        // Shading
        b"sh" => Operator::PaintShading {
            name: pop_name(operands),
        },

        // Inline image: BI ... ID <data> EI
        b"BI" => {
            let (properties, data) = read_inline_image(tok);
            Operator::InlineImage { properties, data }
        }

        // Marked content
        b"BMC" => Operator::BeginMarkedContent {
            tag: pop_name(operands),
        },
        b"BDC" => {
            let properties = operands.pop().unwrap_or(Operand::Null);
            let tag = pop_name(operands);
            Operator::BeginMarkedContentDict { tag, properties }
        }
        b"EMC" => Operator::EndMarkedContent,
        b"MP" => Operator::MarkedContentPoint {
            tag: pop_name(operands),
        },
        b"DP" => {
            let properties = operands.pop().unwrap_or(Operand::Null);
            let tag = pop_name(operands);
            Operator::MarkedContentPointDict { tag, properties }
        }

        // Type 3 character width
        b"d0" => {
            let wy = pop_f32(operands);
            let wx = pop_f32(operands);
            Operator::SetCharWidth { wx, wy }
        }
        b"d1" => {
            let ury = pop_f32(operands);
            let urx = pop_f32(operands);
            let lly = pop_f32(operands);
            let llx = pop_f32(operands);
            let wy = pop_f32(operands);
            let wx = pop_f32(operands);
            Operator::SetCacheDevice {
                wx,
                wy,
                llx,
                lly,
                urx,
                ury,
            }
        }

        // Compatibility
        b"BX" => Operator::BeginCompat,
        b"EX" => Operator::EndCompat,

        _ => {
            let all_operands = std::mem::take(operands);
            return Operator::Unknown {
                keyword: keyword.to_vec(),
                operands: all_operands,
            };
        }
    };

    operands.clear();
    op
}

/// Expand an abbreviated inline-image key to its full name.
fn expand_inline_key(abbr: &[u8]) -> Name {
    match abbr {
        b"BPC" => Name::from("BitsPerComponent"),
        b"CS" => Name::from("ColorSpace"),
        b"D" => Name::from("Decode"),
        b"DP" => Name::from("DecodeParms"),
        b"F" => Name::from("Filter"),
        b"H" => Name::from("Height"),
        b"IM" => Name::from("ImageMask"),
        b"I" => Name::from("Interpolate"),
        b"W" => Name::from("Width"),
        b"L" => Name::from("Length"),
        _ => Name::from_bytes(abbr.to_vec()),
    }
}

/// Expand abbreviated inline-image color space or filter name values.
fn expand_inline_name_value(name: &Name) -> Name {
    match name.as_bytes() {
        // Color spaces
        b"G" => Name::from("DeviceGray"),
        b"RGB" => Name::from("DeviceRGB"),
        b"CMYK" => Name::from("DeviceCMYK"),
        b"I" => Name::from("Indexed"),
        // Filters
        b"AHx" => Name::from("ASCIIHexDecode"),
        b"A85" => Name::from("ASCII85Decode"),
        b"LZW" => Name::from("LZWDecode"),
        b"Fl" => Name::from("FlateDecode"),
        b"RL" => Name::from("RunLengthDecode"),
        b"CCF" => Name::from("CCITTFaxDecode"),
        b"DCT" => Name::from("DCTDecode"),
        _ => name.clone(),
    }
}

/// Expand abbreviated names within an operand value (for CS and F values).
fn expand_inline_value(operand: Operand) -> Operand {
    match operand {
        Operand::Name(n) => Operand::Name(expand_inline_name_value(&n)),
        Operand::Array(arr) => Operand::Array(arr.into_iter().map(expand_inline_value).collect()),
        other => other,
    }
}

/// Read inline image properties and data between BI...ID...EI.
fn read_inline_image(tok: &mut Tokenizer<'_>) -> (HashMap<Name, Operand>, Vec<u8>) {
    let empty = (HashMap::new(), Vec::new());

    // Parse key-value pairs until we hit "ID"
    let mut properties = HashMap::new();
    loop {
        match tok.next_token() {
            Some(Ok(Token::Keyword(ref kw))) if kw == b"ID" => break,
            Some(Ok(Token::Name(key))) => {
                let expanded_key = expand_inline_key(key.as_bytes());
                // Read the value
                let value = match tok.next_token() {
                    Some(Ok(Token::Integer(n))) => Operand::Integer(n),
                    Some(Ok(Token::Real(f))) => Operand::Real(f),
                    Some(Ok(Token::Name(n))) => Operand::Name(n),
                    Some(Ok(Token::String(s))) => Operand::String(s.as_bytes().to_vec()),
                    Some(Ok(Token::Boolean(b))) => Operand::Boolean(b),
                    Some(Ok(Token::Null)) => Operand::Null,
                    Some(Ok(Token::ArrayStart)) => match read_operand_array(tok) {
                        Ok(arr) => Operand::Array(arr),
                        Err(_) => return empty,
                    },
                    Some(Ok(Token::Keyword(ref kw))) if kw == b"ID" => {
                        // Value-less key right before ID; store Null and break
                        properties.insert(expanded_key, Operand::Null);
                        break;
                    }
                    _ => return empty,
                };
                let value = expand_inline_value(value);
                properties.insert(expanded_key, value);
            }
            None => return empty,
            _ => continue,
        }
    }

    // After "ID", skip exactly 1 whitespace byte, then binary data begins
    let source = tok.source();
    let pos = tok.position();
    if pos >= source.len() {
        return (properties, Vec::new());
    }
    // Skip the single whitespace byte after ID
    let data_start = pos + 1;

    // Binary scan: find whitespace + "EI" + (whitespace|EOF|delimiter)
    let mut i = data_start;
    let data;
    loop {
        if i + 2 >= source.len() {
            // Reached end without finding EI — take everything remaining
            data = source[data_start..source.len()].to_vec();
            tok.set_position(source.len());
            return (properties, data);
        }
        // Look for whitespace byte followed by 'E' 'I'
        if is_whitespace(source[i]) && source[i + 1] == b'E' && source[i + 2] == b'I' {
            // Check that EI is followed by whitespace, delimiter, or EOF
            let after_ei = i + 3;
            if after_ei >= source.len()
                || is_whitespace(source[after_ei])
                || is_delimiter(source[after_ei])
            {
                data = source[data_start..i].to_vec();
                tok.set_position(after_ei);
                return (properties, data);
            }
        }
        i += 1;
    }
}

// --- Operand stack helper functions ---

fn pop_f32(operands: &mut Vec<Operand>) -> f32 {
    operands.pop().and_then(|op| op.as_f32()).unwrap_or(0.0)
}

fn pop_i64(operands: &mut Vec<Operand>) -> i64 {
    operands.pop().and_then(|op| op.as_i64()).unwrap_or(0)
}

fn pop_name(operands: &mut Vec<Operand>) -> Name {
    match operands.pop() {
        Some(Operand::Name(n)) => n,
        _ => Name::from_bytes(Vec::new()),
    }
}

fn pop_bytes(operands: &mut Vec<Operand>) -> Vec<u8> {
    match operands.pop() {
        Some(Operand::String(b)) => b,
        _ => Vec::new(),
    }
}

fn pop_text_array(operands: &mut Vec<Operand>) -> Vec<TextArrayElement> {
    match operands.pop() {
        Some(Operand::Array(arr)) => arr
            .into_iter()
            .map(|op| match op {
                Operand::String(b) => TextArrayElement::Text(b),
                Operand::Integer(n) => TextArrayElement::Adjustment(n as f32),
                Operand::Real(f) => TextArrayElement::Adjustment(f as f32),
                _ => TextArrayElement::Adjustment(0.0),
            })
            .collect(),
        _ => Vec::new(),
    }
}

fn pop_f32_array(operands: &mut Vec<Operand>) -> Vec<f32> {
    match operands.pop() {
        Some(Operand::Array(arr)) => arr.iter().filter_map(|op| op.as_f32()).collect(),
        _ => Vec::new(),
    }
}

fn drain_f32(operands: &mut Vec<Operand>) -> Vec<f32> {
    let result: Vec<f32> = operands.iter().filter_map(|op| op.as_f32()).collect();
    operands.clear();
    result
}

fn drain_f32_with_optional_name(operands: &mut Vec<Operand>) -> (Vec<f32>, Option<Name>) {
    let mut name = None;
    let mut components = Vec::new();

    for op in operands.drain(..) {
        match op {
            Operand::Name(n) => name = Some(n),
            Operand::Integer(n) => components.push(n as f32),
            Operand::Real(f) => components.push(f as f32),
            _ => {}
        }
    }

    (components, name)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_tokenize_simple_text() {
        let data = b"BT /F1 12 Tf 100 700 Td (Hello World) Tj ET";
        let ops = tokenize_content_stream(data).unwrap();

        assert!(matches!(ops[0], Operator::BeginText));
        assert!(matches!(ops[1], Operator::SetFont { .. }));
        assert!(matches!(ops[2], Operator::MoveText { .. }));
        assert!(matches!(ops[3], Operator::ShowText { .. }));
        assert!(matches!(ops[4], Operator::EndText));
    }

    #[test]
    fn test_tokenize_path_operators() {
        let data = b"100 200 300 400 re f S";
        let ops = tokenize_content_stream(data).unwrap();

        match &ops[0] {
            Operator::Rectangle { x, y, w, h } => {
                assert_eq!(*x, 100.0);
                assert_eq!(*y, 200.0);
                assert_eq!(*w, 300.0);
                assert_eq!(*h, 400.0);
            }
            _ => panic!("expected Rectangle"),
        }
        assert!(matches!(ops[1], Operator::Fill));
        assert!(matches!(ops[2], Operator::Stroke));
    }

    #[test]
    fn test_tokenize_graphics_state() {
        let data = b"q 1 0 0 1 100 200 cm Q";
        let ops = tokenize_content_stream(data).unwrap();

        assert!(matches!(ops[0], Operator::SaveState));
        assert!(matches!(ops[1], Operator::ConcatMatrix { .. }));
        assert!(matches!(ops[2], Operator::RestoreState));
    }

    #[test]
    fn test_tokenize_color_operators() {
        let data = b"1 0 0 rg 0.5 G";
        let ops = tokenize_content_stream(data).unwrap();

        match &ops[0] {
            Operator::SetRgbFill { r, g, b } => {
                assert_eq!(*r, 1.0);
                assert_eq!(*g, 0.0);
                assert_eq!(*b, 0.0);
            }
            _ => panic!("expected SetRgbFill"),
        }
        match &ops[1] {
            Operator::SetGrayStroke { gray } => {
                assert_eq!(*gray, 0.5);
            }
            _ => panic!("expected SetGrayStroke"),
        }
    }

    #[test]
    fn test_tokenize_text_array() {
        let data = b"[(Hello) -50 (World)] TJ";
        let ops = tokenize_content_stream(data).unwrap();

        match &ops[0] {
            Operator::ShowTextArray { elements } => {
                assert_eq!(elements.len(), 3);
                assert!(matches!(&elements[0], TextArrayElement::Text(b) if b == b"Hello"));
                assert!(matches!(
                    &elements[1],
                    TextArrayElement::Adjustment(a) if *a == -50.0
                ));
                assert!(matches!(&elements[2], TextArrayElement::Text(b) if b == b"World"));
            }
            _ => panic!("expected ShowTextArray"),
        }
    }

    #[test]
    fn test_tokenize_marked_content() {
        let data = b"/OC BMC (Hello) Tj EMC";
        let ops = tokenize_content_stream(data).unwrap();

        assert!(matches!(ops[0], Operator::BeginMarkedContent { .. }));
        assert!(matches!(ops[1], Operator::ShowText { .. }));
        assert!(matches!(ops[2], Operator::EndMarkedContent));
    }

    #[test]
    fn test_tokenize_xobject() {
        let data = b"/Im0 Do";
        let ops = tokenize_content_stream(data).unwrap();

        match &ops[0] {
            Operator::PaintXObject { name } => {
                assert_eq!(name.as_bytes(), b"Im0");
            }
            _ => panic!("expected PaintXObject"),
        }
    }

    #[test]
    fn test_tokenize_unknown_operator() {
        let data = b"42 ZZ";
        let ops = tokenize_content_stream(data).unwrap();

        match &ops[0] {
            Operator::Unknown { keyword, operands } => {
                assert_eq!(keyword, b"ZZ");
                assert_eq!(operands.len(), 1);
            }
            _ => panic!("expected Unknown operator"),
        }
    }

    #[test]
    fn test_tokenize_empty_stream() {
        let ops = tokenize_content_stream(b"").unwrap();
        assert!(ops.is_empty());
    }

    #[test]
    #[allow(clippy::approx_constant)]
    fn test_operand_as_f32() {
        assert_eq!(Operand::Integer(42).as_f32(), Some(42.0));
        assert_eq!(Operand::Real(3.14).as_f32(), Some(3.14));
        assert_eq!(Operand::Null.as_f32(), None);
    }

    #[test]
    fn test_operand_as_i64() {
        assert_eq!(Operand::Integer(42).as_i64(), Some(42));
        assert_eq!(Operand::Null.as_i64(), None);
    }

    #[test]
    fn test_set_font_operator() {
        let data = b"BT /F1 12 Tf ET";
        let ops = tokenize_content_stream(data).unwrap();
        match &ops[1] {
            Operator::SetFont { name, size } => {
                assert_eq!(name.as_bytes(), b"F1");
                assert_eq!(*size, 12.0);
            }
            _ => panic!("expected SetFont"),
        }
    }

    #[test]
    fn test_set_dash_pattern() {
        let data = b"[3 5] 0 d";
        let ops = tokenize_content_stream(data).unwrap();
        match &ops[0] {
            Operator::SetDashPattern { array, phase } => {
                assert_eq!(array, &[3.0, 5.0]);
                assert_eq!(*phase, 0.0);
            }
            _ => panic!("expected SetDashPattern"),
        }
    }

    #[test]
    fn test_inline_image_basic() {
        // BI /W 10 /H 10 /BPC 8 /CS /G ID <10 bytes of data> EI
        let mut data = Vec::new();
        data.extend_from_slice(b"BI /W 10 /H 10 /BPC 8 /CS /G ID ");
        // 10 bytes of image data
        data.extend_from_slice(&[0xFF; 10]);
        data.extend_from_slice(b" EI");
        let ops = tokenize_content_stream(&data).unwrap();
        assert_eq!(ops.len(), 1);
        match &ops[0] {
            Operator::InlineImage { properties, data } => {
                assert_eq!(
                    properties.get(&Name::from("Width")),
                    Some(&Operand::Integer(10))
                );
                assert_eq!(
                    properties.get(&Name::from("Height")),
                    Some(&Operand::Integer(10))
                );
                assert_eq!(
                    properties.get(&Name::from("BitsPerComponent")),
                    Some(&Operand::Integer(8))
                );
                assert_eq!(
                    properties.get(&Name::from("ColorSpace")),
                    Some(&Operand::Name(Name::from("DeviceGray")))
                );
                assert_eq!(data.len(), 10);
                assert!(data.iter().all(|&b| b == 0xFF));
            }
            _ => panic!("expected InlineImage"),
        }
    }

    #[test]
    fn test_inline_image_abbreviation_expansion() {
        let mut data = Vec::new();
        data.extend_from_slice(b"BI /W 4 /H 4 /CS /RGB /F /Fl ID ");
        data.extend_from_slice(&[0xAA; 4]);
        data.extend_from_slice(b" EI");
        let ops = tokenize_content_stream(&data).unwrap();
        match &ops[0] {
            Operator::InlineImage { properties, .. } => {
                assert_eq!(
                    properties.get(&Name::from("ColorSpace")),
                    Some(&Operand::Name(Name::from("DeviceRGB")))
                );
                assert_eq!(
                    properties.get(&Name::from("Filter")),
                    Some(&Operand::Name(Name::from("FlateDecode")))
                );
            }
            _ => panic!("expected InlineImage"),
        }
    }

    #[test]
    fn test_inline_image_binary_data_extraction() {
        // Data containing bytes that look like EI but aren't terminated properly
        let mut data = Vec::new();
        data.extend_from_slice(b"BI /W 2 /H 2 /BPC 8 /CS /G ID ");
        // Binary data includes "EI" in middle but not at a valid boundary
        data.extend_from_slice(&[0x45, 0x49, 0x45, 0x49]); // "EIEI"
        data.extend_from_slice(b" EI");
        let ops = tokenize_content_stream(&data).unwrap();
        match &ops[0] {
            Operator::InlineImage { data, .. } => {
                assert_eq!(data, &[0x45, 0x49, 0x45, 0x49]);
            }
            _ => panic!("expected InlineImage"),
        }
    }

    #[test]
    fn test_inline_image_tokenizer_resumes_after_ei() {
        let mut data = Vec::new();
        data.extend_from_slice(b"BI /W 1 /H 1 /BPC 8 /CS /G ID ");
        data.extend_from_slice(&[0xAB]);
        data.extend_from_slice(b"\nEI\n");
        data.extend_from_slice(b"100 200 m");
        let ops = tokenize_content_stream(&data).unwrap();
        assert_eq!(ops.len(), 2);
        assert!(matches!(&ops[0], Operator::InlineImage { .. }));
        assert!(matches!(&ops[1], Operator::MoveTo { x, y } if *x == 100.0 && *y == 200.0));
    }

    #[test]
    fn test_tokenize_paint_shading() {
        let data = b"/Sh0 sh";
        let ops = tokenize_content_stream(data).unwrap();
        assert_eq!(ops.len(), 1);
        match &ops[0] {
            Operator::PaintShading { name } => {
                assert_eq!(name.as_bytes(), b"Sh0");
            }
            _ => panic!("expected PaintShading"),
        }
    }

    #[test]
    fn test_tokenize_d0_operator() {
        let data = b"500 0 d0";
        let ops = tokenize_content_stream(data).unwrap();
        assert_eq!(ops.len(), 1);
        match &ops[0] {
            Operator::SetCharWidth { wx, wy } => {
                assert_eq!(*wx, 500.0);
                assert_eq!(*wy, 0.0);
            }
            _ => panic!("expected SetCharWidth"),
        }
    }

    #[test]
    fn test_tokenize_d1_operator() {
        let data = b"500 0 10 -20 400 700 d1";
        let ops = tokenize_content_stream(data).unwrap();
        assert_eq!(ops.len(), 1);
        match &ops[0] {
            Operator::SetCacheDevice {
                wx,
                wy,
                llx,
                lly,
                urx,
                ury,
            } => {
                assert_eq!(*wx, 500.0);
                assert_eq!(*wy, 0.0);
                assert_eq!(*llx, 10.0);
                assert_eq!(*lly, -20.0);
                assert_eq!(*urx, 400.0);
                assert_eq!(*ury, 700.0);
            }
            _ => panic!("expected SetCacheDevice"),
        }
    }

    #[test]
    fn test_d0_in_char_proc_stream() {
        let data = b"500 0 d0 100 200 m 300 400 l S";
        let ops = tokenize_content_stream(data).unwrap();
        assert_eq!(ops.len(), 4);
        assert!(matches!(ops[0], Operator::SetCharWidth { .. }));
        assert!(matches!(ops[1], Operator::MoveTo { .. }));
        assert!(matches!(ops[2], Operator::LineTo { .. }));
        assert!(matches!(ops[3], Operator::Stroke));
    }

    // ===================================================================
    // Upstream-ported tests
    // ===================================================================

    /// Upstream: TEST(CPDFStreamContentParserTest, PDFFindKeyAbbreviation)
    ///
    /// Inline image key abbreviation lookup: BPC->BitsPerComponent, W->Width, etc.
    #[test]
    fn test_find_key_abbreviation() {
        assert_eq!(expand_inline_key(b"BPC"), Name::from("BitsPerComponent"));
        assert_eq!(expand_inline_key(b"W"), Name::from("Width"));
        assert_eq!(expand_inline_key(b"H"), Name::from("Height"));
        assert_eq!(expand_inline_key(b"CS"), Name::from("ColorSpace"));
        assert_eq!(expand_inline_key(b"F"), Name::from("Filter"));
        assert_eq!(expand_inline_key(b"D"), Name::from("Decode"));
        assert_eq!(expand_inline_key(b"DP"), Name::from("DecodeParms"));
        assert_eq!(expand_inline_key(b"IM"), Name::from("ImageMask"));
        assert_eq!(expand_inline_key(b"I"), Name::from("Interpolate"));
        assert_eq!(expand_inline_key(b"L"), Name::from("Length"));

        // Empty and not-in-list should return the input unchanged
        assert_eq!(expand_inline_key(b""), Name::from(""));
        assert_eq!(expand_inline_key(b"NoInList"), Name::from("NoInList"));

        // Prefix should not match
        assert_eq!(expand_inline_key(b"WW"), Name::from("WW"));
    }

    /// Upstream: TEST(CPDFStreamContentParserTest, PDFFindValueAbbreviation)
    ///
    /// Inline image value abbreviation lookup: G->DeviceGray, DCT->DCTDecode, etc.
    #[test]
    fn test_find_value_abbreviation() {
        assert_eq!(
            expand_inline_name_value(&Name::from("G")),
            Name::from("DeviceGray")
        );
        assert_eq!(
            expand_inline_name_value(&Name::from("RGB")),
            Name::from("DeviceRGB")
        );
        assert_eq!(
            expand_inline_name_value(&Name::from("CMYK")),
            Name::from("DeviceCMYK")
        );
        assert_eq!(
            expand_inline_name_value(&Name::from("DCT")),
            Name::from("DCTDecode")
        );
        assert_eq!(
            expand_inline_name_value(&Name::from("Fl")),
            Name::from("FlateDecode")
        );
        assert_eq!(
            expand_inline_name_value(&Name::from("AHx")),
            Name::from("ASCIIHexDecode")
        );
        assert_eq!(
            expand_inline_name_value(&Name::from("A85")),
            Name::from("ASCII85Decode")
        );
        assert_eq!(
            expand_inline_name_value(&Name::from("LZW")),
            Name::from("LZWDecode")
        );
        assert_eq!(
            expand_inline_name_value(&Name::from("RL")),
            Name::from("RunLengthDecode")
        );
        assert_eq!(
            expand_inline_name_value(&Name::from("CCF")),
            Name::from("CCITTFaxDecode")
        );
        assert_eq!(
            expand_inline_name_value(&Name::from("I")),
            Name::from("Indexed")
        );

        // Empty and not-in-list should return input unchanged
        assert_eq!(expand_inline_name_value(&Name::from("")), Name::from(""));
        assert_eq!(
            expand_inline_name_value(&Name::from("NoInList")),
            Name::from("NoInList")
        );

        // Prefix should not match
        assert_eq!(
            expand_inline_name_value(&Name::from("II")),
            Name::from("II")
        );
    }
}