rpdfium_parser/

object_parser.rs

// Derived from PDFium's cpdf_syntax_parser.cpp / cpdf_parser.cpp
// Original: Copyright 2014 The PDFium Authors
// Licensed under BSD-3-Clause / Apache-2.0
// See pdfium-upstream/LICENSE for the original license.

//! PDF object parsing — builds `Object` values from token streams.
//!
//! Handles indirect object definitions (`N G obj ... endobj`), direct objects,
//! and stream objects with length resolution via direct `/Length` + endstream scan fallback.

use std::collections::HashMap;

use rpdfium_core::error::{ParseError, PdfError};
use rpdfium_core::fx_system::DEFAULT_MAX_ENDSTREAM_SCAN_DISTANCE;
use rpdfium_core::{Name, ParsingMode};

use crate::object::{Object, ObjectId, StreamData};
use crate::tokenizer::{Token, Tokenizer, is_whitespace};

/// Parse an indirect object at the given offset: `N G obj <value> endobj`.
///
/// Returns the parsed `Object` and the object's `ObjectId`.
pub fn parse_indirect_object(
    source: &[u8],
    offset: u64,
    mode: ParsingMode,
) -> Result<(ObjectId, Object), PdfError> {
    let mut tok = Tokenizer::new_at(source, offset as usize);

    // Read object number
    let number = match tok.next_token() {
        Some(Ok(Token::Integer(n))) if n >= 0 => n as u32,
        _ => {
            return Err(PdfError::Parse(ParseError::InvalidObjectHeader { offset }));
        }
    };

    // Read generation number
    let generation = match tok.next_token() {
        Some(Ok(Token::Integer(g))) if g >= 0 && g <= u16::MAX as i64 => g as u16,
        _ => {
            return Err(PdfError::Parse(ParseError::InvalidObjectHeader { offset }));
        }
    };

    // Read "obj" keyword
    match tok.next_token() {
        Some(Ok(Token::Keyword(ref kw))) if kw == b"obj" => {}
        _ => {
            return Err(PdfError::Parse(ParseError::InvalidObjectHeader { offset }));
        }
    }

    let id = ObjectId::new(number, generation);
    let obj = parse_direct_object_from_tokenizer(&mut tok, source, mode)?;

    // Check for stream keyword
    let obj = try_parse_stream(obj, &mut tok, source, mode)?;

    // Expect "endobj" keyword (lenient mode tolerates missing endobj)
    tok.skip_whitespace_and_comments();
    match tok.next_token() {
        Some(Ok(Token::Keyword(ref kw))) if kw == b"endobj" => {}
        _ => {
            if mode == ParsingMode::Strict {
                return Err(PdfError::Parse(ParseError::MissingEndobj {
                    offset: tok.position() as u64,
                }));
            }
            tracing::warn!(
                object_id = %id,
                offset = tok.position(),
                "missing endobj keyword"
            );
        }
    }

    Ok((id, obj))
}

/// Parse a direct (non-indirect) object starting at the given offset.
pub fn parse_object(source: &[u8], offset: u64, mode: ParsingMode) -> Result<Object, PdfError> {
    let mut tok = Tokenizer::new_at(source, offset as usize);
    parse_direct_object_from_tokenizer(&mut tok, source, mode)
}

/// Parse a single direct object from the tokenizer's current position.
fn parse_direct_object_from_tokenizer(
    tok: &mut Tokenizer<'_>,
    source: &[u8],
    mode: ParsingMode,
) -> Result<Object, PdfError> {
    let token = tok
        .next_token()
        .ok_or(PdfError::Parse(ParseError::UnexpectedEof {
            offset: tok.position() as u64,
        }))??;

    match token {
        Token::Null => Ok(Object::Null),
        Token::Boolean(b) => Ok(Object::Boolean(b)),
        Token::Integer(n) => Ok(Object::Integer(n)),
        Token::Real(f) => Ok(Object::Real(f)),
        Token::String(s) => Ok(Object::String(s)),
        Token::Name(n) => Ok(Object::Name(n)),
        Token::Ref(id) => Ok(Object::Reference(id)),
        Token::ArrayStart => parse_array(tok, source, mode),
        Token::DictStart => parse_dictionary(tok, source, mode),
        Token::Keyword(ref kw) => {
            // Some PDFs have bare keywords where we expect objects.
            // In lenient mode, treat unexpected keywords as null.
            if mode == ParsingMode::Lenient {
                tracing::warn!(
                    keyword = %String::from_utf8_lossy(kw),
                    offset = tok.position(),
                    "unexpected keyword where object expected, treating as null"
                );
                Ok(Object::Null)
            } else {
                Err(PdfError::Parse(ParseError::UnexpectedToken {
                    offset: tok.position() as u64,
                    expected: "object".into(),
                    found: format!("keyword '{}'", String::from_utf8_lossy(kw)),
                }))
            }
        }
        other => Err(PdfError::Parse(ParseError::UnexpectedToken {
            offset: tok.position() as u64,
            expected: "object value".into(),
            found: format!("{:?}", other),
        })),
    }
}

/// Parse an array `[ ... ]` from the token stream.
fn parse_array(
    tok: &mut Tokenizer<'_>,
    source: &[u8],
    mode: ParsingMode,
) -> Result<Object, PdfError> {
    let mut elements = Vec::new();

    loop {
        tok.skip_whitespace_and_comments();

        // Peek to check for ArrayEnd
        let saved = tok.position();
        match tok.next_token() {
            Some(Ok(Token::ArrayEnd)) => {
                return Ok(Object::Array(elements));
            }
            None => {
                return Err(PdfError::Parse(ParseError::UnexpectedEof {
                    offset: tok.position() as u64,
                }));
            }
            _ => {
                // Put back the token by resetting position
                tok.set_position(saved);
            }
        }

        let element = parse_direct_object_from_tokenizer(tok, source, mode)?;
        elements.push(element);
    }
}

/// Parse a dictionary `<< ... >>` from the token stream.
fn parse_dictionary(
    tok: &mut Tokenizer<'_>,
    source: &[u8],
    mode: ParsingMode,
) -> Result<Object, PdfError> {
    let mut map = HashMap::new();

    loop {
        tok.skip_whitespace_and_comments();

        // Peek to check for DictEnd
        let saved = tok.position();
        match tok.next_token() {
            Some(Ok(Token::DictEnd)) => {
                return Ok(Object::Dictionary(map));
            }
            None => {
                return Err(PdfError::Parse(ParseError::UnexpectedEof {
                    offset: tok.position() as u64,
                }));
            }
            Some(Ok(Token::Name(key))) => {
                // Parse the value
                let value = parse_direct_object_from_tokenizer(tok, source, mode)?;
                if map.contains_key(&key) {
                    if mode == ParsingMode::Strict {
                        return Err(PdfError::Parse(ParseError::DuplicateKey {
                            key: String::from_utf8_lossy(key.as_bytes()).into_owned(),
                        }));
                    }
                    tracing::warn!(
                        key = %String::from_utf8_lossy(key.as_bytes()),
                        "duplicate dictionary key, using last value"
                    );
                }
                map.insert(key, value);
            }
            Some(Ok(other)) => {
                if mode == ParsingMode::Lenient {
                    tracing::warn!(
                        token = ?other,
                        "expected name key in dictionary, skipping"
                    );
                    // Skip this token and try to continue
                    continue;
                }
                return Err(PdfError::Parse(ParseError::UnexpectedToken {
                    offset: saved as u64,
                    expected: "name key".into(),
                    found: format!("{:?}", other),
                }));
            }
            Some(Err(e)) => return Err(e),
        }
    }
}

/// After parsing a dictionary, check if it's followed by a `stream` keyword.
/// If so, convert it to a Stream object.
fn try_parse_stream(
    obj: Object,
    tok: &mut Tokenizer<'_>,
    source: &[u8],
    mode: ParsingMode,
) -> Result<Object, PdfError> {
    let dict = match obj {
        Object::Dictionary(d) => d,
        other => return Ok(other),
    };

    // Save position to revert if no stream keyword
    let saved = tok.position();
    tok.skip_whitespace_and_comments();

    match tok.next_token() {
        Some(Ok(Token::Keyword(ref kw))) if kw == b"stream" => {
            // After "stream", skip exactly one EOL marker (CR, LF, or CR+LF)
            let mut pos = tok.position();
            if pos < source.len() && source[pos] == b'\r' {
                pos += 1;
            }
            if pos < source.len() && source[pos] == b'\n' {
                pos += 1;
            }

            let stream_start = pos as u64;
            let length = resolve_stream_length(source, &dict, stream_start, mode)?;

            let data = StreamData::Raw {
                offset: stream_start,
                length,
            };

            // Skip past the stream data and endstream keyword
            let end_pos = (stream_start + length) as usize;
            tok.set_position(end_pos);
            tok.skip_whitespace_and_comments();

            // Consume "endstream" keyword
            match tok.next_token() {
                Some(Ok(Token::Keyword(ref kw))) if kw == b"endstream" => {}
                _ => {
                    if mode == ParsingMode::Strict {
                        return Err(PdfError::Parse(ParseError::MissingEndstream {
                            offset: tok.position() as u64,
                        }));
                    }
                    tracing::warn!(offset = tok.position(), "missing endstream keyword");
                }
            }

            Ok(Object::Stream { dict, data })
        }
        _ => {
            // Not a stream, revert position
            tok.set_position(saved);
            Ok(Object::Dictionary(dict))
        }
    }
}

/// Resolve the byte length of a stream.
///
/// 1. Try direct `/Length` integer value.
/// 2. Verify `endstream` at the expected position.
/// 3. Scan for `endstream` as fallback (bounded by `MAX_ENDSTREAM_SCAN_DISTANCE`).
fn resolve_stream_length(
    source: &[u8],
    dict: &HashMap<Name, Object>,
    stream_start: u64,
    mode: ParsingMode,
) -> Result<u64, PdfError> {
    // Step 1: Try direct /Length value (only direct integers, not indirect refs)
    if let Some(Object::Integer(len)) = dict.get(&Name::length()) {
        if *len >= 0 {
            let len = *len as u64;
            // Step 2: Verify endstream at expected position
            if verify_endstream(source, stream_start + len) {
                return Ok(len);
            }
            tracing::warn!(
                expected_length = len,
                stream_start = stream_start,
                "endstream not found at expected offset"
            );
        }
    }

    // Step 3: Scan for endstream (bounded)
    scan_for_endstream(
        source,
        stream_start,
        DEFAULT_MAX_ENDSTREAM_SCAN_DISTANCE,
        mode,
    )
}

/// Check if the `endstream` keyword appears at the expected position.
fn verify_endstream(source: &[u8], expected_end: u64) -> bool {
    let pos = expected_end as usize;
    let marker = b"endstream";

    // Allow optional whitespace before endstream
    let mut check_pos = pos;
    while check_pos < source.len() && is_whitespace(source[check_pos]) {
        check_pos += 1;
    }

    if check_pos + marker.len() <= source.len() {
        &source[check_pos..check_pos + marker.len()] == marker
    } else {
        false
    }
}

/// Scan forward from `stream_start` looking for `endstream`, bounded by `max_distance`.
fn scan_for_endstream(
    source: &[u8],
    stream_start: u64,
    max_distance: u64,
    mode: ParsingMode,
) -> Result<u64, PdfError> {
    let start = stream_start as usize;
    let limit = source
        .len()
        .min(start.saturating_add(max_distance as usize));
    let marker = b"endstream";

    let search_slice = &source[start..limit];

    // Search for "endstream" in the range
    for i in 0..search_slice.len().saturating_sub(marker.len()) {
        if &search_slice[i..i + marker.len()] == marker {
            // Found endstream. The stream length is the offset before it,
            // trimming any trailing whitespace.
            let mut end = i;
            while end > 0 && is_whitespace(search_slice[end - 1]) {
                end -= 1;
            }
            return Ok(end as u64);
        }
    }

    if mode == ParsingMode::Lenient {
        tracing::warn!(
            stream_start = stream_start,
            "endstream not found within scan distance, using remainder"
        );
        Ok((limit - start) as u64)
    } else {
        Err(PdfError::EndstreamScanFailed)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_parse_null_object() {
        let obj = parse_object(b"null", 0, ParsingMode::Strict).unwrap();
        assert!(obj.is_null());
    }

    #[test]
    fn test_parse_boolean_object() {
        let obj = parse_object(b"true", 0, ParsingMode::Strict).unwrap();
        assert_eq!(obj.as_bool(), Some(true));
    }

    #[test]
    fn test_parse_integer_object() {
        let obj = parse_object(b"42", 0, ParsingMode::Strict).unwrap();
        assert_eq!(obj.as_i64(), Some(42));
    }

    #[test]
    #[allow(clippy::approx_constant)]
    fn test_parse_real_object() {
        let obj = parse_object(b"3.14", 0, ParsingMode::Strict).unwrap();
        assert_eq!(obj.as_f64(), Some(3.14));
    }

    #[test]
    fn test_parse_string_object() {
        let obj = parse_object(b"(Hello)", 0, ParsingMode::Strict).unwrap();
        assert_eq!(obj.as_string().unwrap().as_bytes(), b"Hello");
    }

    #[test]
    fn test_parse_name_object() {
        let obj = parse_object(b"/Type", 0, ParsingMode::Strict).unwrap();
        assert!(obj.as_name().is_some());
    }

    #[test]
    fn test_parse_array_object() {
        let obj = parse_object(b"[1 2 3]", 0, ParsingMode::Strict).unwrap();
        let arr = obj.as_array().unwrap();
        assert_eq!(arr.len(), 3);
        assert_eq!(arr[0].as_i64(), Some(1));
        assert_eq!(arr[2].as_i64(), Some(3));
    }

    #[test]
    fn test_parse_nested_array() {
        let obj = parse_object(b"[[1 2] [3 4]]", 0, ParsingMode::Strict).unwrap();
        let arr = obj.as_array().unwrap();
        assert_eq!(arr.len(), 2);
        assert_eq!(arr[0].as_array().unwrap().len(), 2);
    }

    #[test]
    fn test_parse_dictionary_object() {
        let obj =
            parse_object(b"<< /Type /Catalog /Pages 3 0 R >>", 0, ParsingMode::Strict).unwrap();
        let dict = obj.as_dict().unwrap();
        assert!(dict.contains_key(&Name::r#type()));
    }

    #[test]
    fn test_parse_reference_in_dict() {
        let obj = parse_object(b"<< /Pages 3 0 R >>", 0, ParsingMode::Strict).unwrap();
        let dict = obj.as_dict().unwrap();
        let pages = dict.get(&Name::pages()).unwrap();
        assert_eq!(pages.as_reference(), Some(ObjectId::new(3, 0)));
    }

    #[test]
    fn test_parse_indirect_object_simple() {
        let source = b"1 0 obj\n42\nendobj";
        let (id, obj) = parse_indirect_object(source, 0, ParsingMode::Strict).unwrap();
        assert_eq!(id, ObjectId::new(1, 0));
        assert_eq!(obj.as_i64(), Some(42));
    }

    #[test]
    fn test_parse_indirect_object_dict() {
        let source = b"5 0 obj\n<< /Type /Page >>\nendobj";
        let (id, obj) = parse_indirect_object(source, 0, ParsingMode::Strict).unwrap();
        assert_eq!(id, ObjectId::new(5, 0));
        assert!(obj.as_dict().is_some());
    }

    #[test]
    fn test_parse_stream_object() {
        let source = b"1 0 obj\n<< /Length 5 >>\nstream\nHelloendstream\nendobj";
        let (id, obj) = parse_indirect_object(source, 0, ParsingMode::Strict).unwrap();
        assert_eq!(id, ObjectId::new(1, 0));
        match &obj {
            Object::Stream { dict, data } => {
                assert!(dict.contains_key(&Name::length()));
                match data {
                    StreamData::Raw { length, .. } => assert_eq!(*length, 5),
                    StreamData::Decoded { .. } => panic!("expected raw stream"),
                }
            }
            _ => panic!("expected stream object"),
        }
    }

    #[test]
    fn test_parse_stream_with_scan_fallback() {
        // No /Length key — should scan for endstream
        let source = b"1 0 obj\n<< >>\nstream\nSome data here\nendstream\nendobj";
        let (_, obj) = parse_indirect_object(source, 0, ParsingMode::Lenient).unwrap();
        match &obj {
            Object::Stream { data, .. } => match data {
                StreamData::Raw { length, .. } => {
                    assert!(*length > 0);
                }
                StreamData::Decoded { .. } => panic!("expected raw stream"),
            },
            _ => panic!("expected stream object"),
        }
    }

    #[test]
    fn test_verify_endstream_function() {
        let source = b"Hello\nendstream";
        assert!(verify_endstream(source, 5));
    }

    #[test]
    fn test_verify_endstream_with_whitespace() {
        let source = b"Hello \n endstream";
        assert!(verify_endstream(source, 5));
    }

    #[test]
    fn test_missing_endobj_lenient() {
        let source = b"1 0 obj\n42\n";
        let result = parse_indirect_object(source, 0, ParsingMode::Lenient);
        assert!(result.is_ok());
    }

    #[test]
    fn test_missing_endobj_strict() {
        let source = b"1 0 obj\n42\n";
        let result = parse_indirect_object(source, 0, ParsingMode::Strict);
        assert!(result.is_err());
    }

    #[test]
    fn test_parse_empty_array() {
        let obj = parse_object(b"[]", 0, ParsingMode::Strict).unwrap();
        assert_eq!(obj.as_array().unwrap().len(), 0);
    }

    #[test]
    fn test_parse_empty_dict() {
        let obj = parse_object(b"<< >>", 0, ParsingMode::Strict).unwrap();
        assert_eq!(obj.as_dict().unwrap().len(), 0);
    }

    #[test]
    fn test_duplicate_key_strict() {
        let result = parse_object(b"<< /Type /A /Type /B >>", 0, ParsingMode::Strict);
        assert!(result.is_err());
    }

    #[test]
    fn test_duplicate_key_lenient() {
        let obj = parse_object(b"<< /Type /A /Type /B >>", 0, ParsingMode::Lenient).unwrap();
        let dict = obj.as_dict().unwrap();
        // Should have the last value
        assert!(dict.contains_key(&Name::r#type()));
    }

    // -----------------------------------------------------------------------
    // Malformed input tests (upstream-inspired)
    // -----------------------------------------------------------------------

    /// Completely empty input should fail.
    #[test]
    fn test_parse_empty_input() {
        let result = parse_object(b"", 0, ParsingMode::Strict);
        assert!(result.is_err());
    }

    /// Whitespace-only input should fail.
    #[test]
    fn test_parse_whitespace_only() {
        let result = parse_object(b"   \n\t  ", 0, ParsingMode::Strict);
        assert!(result.is_err());
    }

    /// Unterminated array.
    #[test]
    fn test_parse_unterminated_array() {
        let result = parse_object(b"[1 2 3", 0, ParsingMode::Strict);
        assert!(result.is_err());
    }

    /// Unterminated dictionary.
    #[test]
    fn test_parse_unterminated_dict() {
        let result = parse_object(b"<< /Type /Catalog", 0, ParsingMode::Strict);
        assert!(result.is_err());
    }

    /// Array with mixed types.
    #[test]
    #[allow(clippy::approx_constant)]
    fn test_parse_array_mixed_types() {
        let obj = parse_object(
            b"[1 3.14 true false null (hello) /Name 5 0 R]",
            0,
            ParsingMode::Strict,
        )
        .unwrap();
        let arr = obj.as_array().unwrap();
        assert_eq!(arr.len(), 8);
        assert_eq!(arr[0].as_i64(), Some(1));
        assert_eq!(arr[1].as_f64(), Some(3.14));
        assert_eq!(arr[2].as_bool(), Some(true));
        assert_eq!(arr[3].as_bool(), Some(false));
        assert!(arr[4].is_null());
        assert!(arr[5].as_string().is_some());
        assert!(arr[6].as_name().is_some());
        assert_eq!(arr[7].as_reference(), Some(ObjectId::new(5, 0)));
    }

    /// Deeply nested arrays (not recursive, just deep nesting).
    #[test]
    fn test_parse_deeply_nested_array() {
        let source = b"[[[[[[42]]]]]]";
        let obj = parse_object(source, 0, ParsingMode::Strict).unwrap();
        // Navigate to the innermost value
        let mut current = &obj;
        for _ in 0..5 {
            current = &current.as_array().unwrap()[0];
        }
        let inner = current.as_array().unwrap();
        assert_eq!(inner[0].as_i64(), Some(42));
    }

    /// Dictionary with non-name key in strict mode should fail.
    #[test]
    fn test_parse_dict_non_name_key_strict() {
        let result = parse_object(b"<< 42 /Value >>", 0, ParsingMode::Strict);
        assert!(result.is_err());
    }

    /// Dictionary with non-name key in lenient mode: `42` is skipped, then
    /// `/Value` becomes a key needing a value, but `>>` closes the dict.
    /// The parser may treat `>>` as Dict end after reading `/Value` as key
    /// with the next token (DictEnd) causing a parse attempt. This is an
    /// edge case — test that lenient mode at least doesn't panic.
    #[test]
    fn test_parse_dict_non_name_key_lenient() {
        // Use a form where the skipped token doesn't break key-value pairing:
        // `<< 42 /Key /Value >>` — skip `42`, then parse /Key → /Value.
        let result = parse_object(b"<< 42 /Key /Value >>", 0, ParsingMode::Lenient);
        assert!(result.is_ok());
        let dict = result.unwrap().as_dict().unwrap().clone();
        let key = Name::from_bytes(b"Key".to_vec());
        assert!(dict.contains_key(&key));
    }

    /// Parsing at a non-zero offset.
    #[test]
    fn test_parse_at_offset() {
        let source = b"garbage42";
        let obj = parse_object(source, 7, ParsingMode::Strict).unwrap();
        assert_eq!(obj.as_i64(), Some(42));
    }

    /// Indirect object with stream: verify stream length is correct.
    #[test]
    fn test_parse_stream_with_exact_length() {
        let content = b"Hello World!"; // 12 bytes
        let source = format!(
            "1 0 obj\n<< /Length {} >>\nstream\n{}endstream\nendobj",
            content.len(),
            String::from_utf8_lossy(content)
        );
        let (id, obj) = parse_indirect_object(source.as_bytes(), 0, ParsingMode::Strict).unwrap();
        assert_eq!(id, ObjectId::new(1, 0));
        match &obj {
            Object::Stream { data, .. } => match data {
                StreamData::Raw { length, .. } => assert_eq!(*length, 12),
                StreamData::Decoded { .. } => panic!("expected raw stream"),
            },
            _ => panic!("expected stream object"),
        }
    }

    /// Indirect object with generation > 0.
    #[test]
    fn test_parse_indirect_object_with_generation() {
        let source = b"5 3 obj\nnull\nendobj";
        let (id, obj) = parse_indirect_object(source, 0, ParsingMode::Strict).unwrap();
        assert_eq!(id, ObjectId::new(5, 3));
        assert!(obj.is_null());
    }

    /// Invalid object header (negative number).
    #[test]
    fn test_parse_indirect_object_negative_number() {
        let source = b"-1 0 obj\n42\nendobj";
        let result = parse_indirect_object(source, 0, ParsingMode::Strict);
        assert!(result.is_err());
    }

    /// Bare keyword in strict mode should fail.
    #[test]
    fn test_parse_bare_keyword_strict() {
        let result = parse_object(b"endobj", 0, ParsingMode::Strict);
        assert!(result.is_err());
    }

    /// Bare keyword in lenient mode returns Null.
    #[test]
    fn test_parse_bare_keyword_lenient() {
        let obj = parse_object(b"endobj", 0, ParsingMode::Lenient).unwrap();
        assert!(obj.is_null());
    }
}