rpdfium_parser/

tokenizer.rs

// Derived from PDFium's cpdf_syntax_parser.cpp
// Original: Copyright 2014 The PDFium Authors
// Licensed under BSD-3-Clause / Apache-2.0
// See pdfium-upstream/LICENSE for the original license.

//! Low-level PDF tokenizer.
//!
//! Tokenizes raw PDF bytes into a sequence of tokens: numbers, strings,
//! names, booleans, null, delimiters, keywords, and indirect references.

use rpdfium_core::error::{ParseError, PdfError};
use rpdfium_core::{Name, PdfString};

use crate::object::ObjectId;

/// A PDF token produced by the tokenizer.
#[derive(Debug, Clone, PartialEq)]
pub enum Token {
    /// An integer literal.
    Integer(i64),
    /// A real (floating-point) literal.
    Real(f64),
    /// A string literal (parenthesis-delimited or hex).
    String(PdfString),
    /// A name object (`/SomeName`).
    Name(Name),
    /// A boolean `true` or `false`.
    Boolean(bool),
    /// The `null` keyword.
    Null,
    /// Start of array `[`.
    ArrayStart,
    /// End of array `]`.
    ArrayEnd,
    /// Start of dictionary `<<`.
    DictStart,
    /// End of dictionary `>>`.
    DictEnd,
    /// An indirect reference `N G R`.
    Ref(ObjectId),
    /// A keyword (e.g., `obj`, `endobj`, `stream`, `endstream`, `xref`, `trailer`).
    Keyword(Vec<u8>),
    /// A comment `% ...`.
    Comment(Vec<u8>),
}

/// PDF tokenizer: produces tokens from a byte slice.
pub struct Tokenizer<'a> {
    source: &'a [u8],
    pos: usize,
}

impl<'a> Tokenizer<'a> {
    /// Create a new tokenizer starting at the given byte slice.
    pub fn new(source: &'a [u8]) -> Self {
        Self { source, pos: 0 }
    }

    /// Create a new tokenizer starting at the given position.
    pub fn new_at(source: &'a [u8], pos: usize) -> Self {
        Self { source, pos }
    }

    /// Current byte position in the source.
    pub fn position(&self) -> usize {
        self.pos
    }

    /// Set the current position.
    pub fn set_position(&mut self, pos: usize) {
        self.pos = pos;
    }

    /// Access the underlying source bytes.
    pub fn source(&self) -> &'a [u8] {
        self.source
    }

    /// Skip whitespace and comments, returning `true` if any were skipped.
    pub fn skip_whitespace_and_comments(&mut self) {
        loop {
            // Skip whitespace
            while self.pos < self.source.len() && is_whitespace(self.source[self.pos]) {
                self.pos += 1;
            }
            // Skip comments
            if self.pos < self.source.len() && self.source[self.pos] == b'%' {
                while self.pos < self.source.len()
                    && self.source[self.pos] != b'\r'
                    && self.source[self.pos] != b'\n'
                {
                    self.pos += 1;
                }
            } else {
                break;
            }
        }
    }

    /// Read the next token, or `None` at EOF.
    pub fn next_token(&mut self) -> Option<Result<Token, PdfError>> {
        self.skip_whitespace_and_comments();

        if self.pos >= self.source.len() {
            return None;
        }

        let offset = self.pos as u64;
        let b = self.source[self.pos];

        let result = match b {
            b'[' => {
                self.pos += 1;
                Ok(Token::ArrayStart)
            }
            b']' => {
                self.pos += 1;
                Ok(Token::ArrayEnd)
            }
            b'<' => {
                if self.pos + 1 < self.source.len() && self.source[self.pos + 1] == b'<' {
                    self.pos += 2;
                    Ok(Token::DictStart)
                } else {
                    self.read_hex_string(offset)
                }
            }
            b'>' => {
                if self.pos + 1 < self.source.len() && self.source[self.pos + 1] == b'>' {
                    self.pos += 2;
                    Ok(Token::DictEnd)
                } else {
                    Err(PdfError::Parse(ParseError::UnexpectedToken {
                        offset,
                        expected: ">>".into(),
                        found: ">".into(),
                    }))
                }
            }
            b'(' => self.read_literal_string(offset),
            b'/' => self.read_name(offset),
            b'+' | b'-' | b'.' | b'0'..=b'9' => self.read_number(offset),
            _ if b.is_ascii_alphabetic() => self.read_keyword_or_bool(offset),
            _ => {
                self.pos += 1;
                Err(PdfError::Parse(ParseError::UnexpectedToken {
                    offset,
                    expected: "token".into(),
                    found: format!("byte 0x{:02x}", b),
                }))
            }
        };

        Some(result)
    }

    /// Read a literal string delimited by parentheses, handling nesting and escapes.
    fn read_literal_string(&mut self, offset: u64) -> Result<Token, PdfError> {
        debug_assert_eq!(self.source[self.pos], b'(');
        self.pos += 1; // skip opening '('

        let mut bytes = Vec::new();
        let mut depth: u32 = 1;

        while self.pos < self.source.len() {
            let b = self.source[self.pos];
            self.pos += 1;

            match b {
                b'(' => {
                    depth += 1;
                    bytes.push(b'(');
                }
                b')' => {
                    depth -= 1;
                    if depth == 0 {
                        return Ok(Token::String(PdfString::from_bytes(bytes)));
                    }
                    bytes.push(b')');
                }
                b'\\' => {
                    if self.pos >= self.source.len() {
                        return Err(PdfError::Parse(ParseError::InvalidString {
                            offset,
                            detail: "unexpected end of input in escape sequence".into(),
                        }));
                    }
                    let escaped = self.source[self.pos];
                    self.pos += 1;
                    match escaped {
                        b'n' => bytes.push(b'\n'),
                        b'r' => bytes.push(b'\r'),
                        b't' => bytes.push(b'\t'),
                        b'b' => bytes.push(0x08),
                        b'f' => bytes.push(0x0C),
                        b'(' => bytes.push(b'('),
                        b')' => bytes.push(b')'),
                        b'\\' => bytes.push(b'\\'),
                        b'\r' => {
                            // Backslash + CR (or CR+LF) = line continuation
                            if self.pos < self.source.len() && self.source[self.pos] == b'\n' {
                                self.pos += 1;
                            }
                        }
                        b'\n' => {
                            // Backslash + LF = line continuation
                        }
                        b'0'..=b'7' => {
                            // Octal escape: 1-3 octal digits
                            let mut val: u8 = escaped - b'0';
                            for _ in 0..2 {
                                if self.pos < self.source.len() {
                                    let next = self.source[self.pos];
                                    if (b'0'..=b'7').contains(&next) {
                                        val = val * 8 + (next - b'0');
                                        self.pos += 1;
                                    } else {
                                        break;
                                    }
                                }
                            }
                            bytes.push(val);
                        }
                        _ => {
                            // Per spec, if the character following the backslash
                            // is not recognized, the backslash is ignored.
                            bytes.push(escaped);
                        }
                    }
                }
                _ => bytes.push(b),
            }
        }

        Err(PdfError::Parse(ParseError::InvalidString {
            offset,
            detail: "unterminated literal string".into(),
        }))
    }

    /// Read a hex string `<...>`.
    fn read_hex_string(&mut self, offset: u64) -> Result<Token, PdfError> {
        debug_assert_eq!(self.source[self.pos], b'<');
        self.pos += 1; // skip opening '<'

        let mut hex_chars = Vec::new();

        while self.pos < self.source.len() {
            let b = self.source[self.pos];
            self.pos += 1;

            match b {
                b'>' => {
                    // If odd number of hex digits, append 0
                    if hex_chars.len() % 2 != 0 {
                        hex_chars.push(0);
                    }
                    let bytes: Vec<u8> = hex_chars
                        .chunks(2)
                        .map(|pair| pair[0] * 16 + pair[1])
                        .collect();
                    return Ok(Token::String(PdfString::from_bytes(bytes)));
                }
                _ if is_whitespace(b) => continue,
                _ => {
                    let nibble =
                        hex_digit(b).ok_or(PdfError::Parse(ParseError::InvalidString {
                            offset,
                            detail: format!("invalid hex digit 0x{:02x}", b),
                        }))?;
                    hex_chars.push(nibble);
                }
            }
        }

        Err(PdfError::Parse(ParseError::InvalidString {
            offset,
            detail: "unterminated hex string".into(),
        }))
    }

    /// Read a PDF name `/SomeName` with `#XX` hex decoding.
    fn read_name(&mut self, _offset: u64) -> Result<Token, PdfError> {
        debug_assert_eq!(self.source[self.pos], b'/');
        self.pos += 1; // skip '/'

        let mut bytes = Vec::new();

        while self.pos < self.source.len() {
            let b = self.source[self.pos];

            if is_whitespace(b) || is_delimiter(b) {
                break;
            }

            self.pos += 1;

            if b == b'#' {
                // Hex-encoded byte: #XX
                if self.pos + 1 < self.source.len() {
                    let hi = hex_digit(self.source[self.pos]);
                    let lo = hex_digit(self.source[self.pos + 1]);
                    if let (Some(h), Some(l)) = (hi, lo) {
                        bytes.push(h * 16 + l);
                        self.pos += 2;
                        continue;
                    }
                }
                // If hex decode fails, treat '#' as literal (lenient behavior)
                bytes.push(b'#');
            } else {
                bytes.push(b);
            }
        }

        Ok(Token::Name(Name::from_bytes(bytes)))
    }

    /// Read a number (integer or real).
    fn read_number(&mut self, offset: u64) -> Result<Token, PdfError> {
        let start = self.pos;
        let mut has_dot = false;

        // Optional sign
        if self.pos < self.source.len()
            && (self.source[self.pos] == b'+' || self.source[self.pos] == b'-')
        {
            self.pos += 1;
        }

        // Check for leading dot
        if self.pos < self.source.len() && self.source[self.pos] == b'.' {
            has_dot = true;
            self.pos += 1;
        }

        let digit_start = self.pos;

        while self.pos < self.source.len() {
            let b = self.source[self.pos];
            if b == b'.' && !has_dot {
                has_dot = true;
                self.pos += 1;
            } else if b.is_ascii_digit() {
                self.pos += 1;
            } else {
                break;
            }
        }

        // Must have consumed at least one digit
        if self.pos == digit_start && !has_dot {
            // Just a sign with no digits — this happens with `+` or `-` alone
            // which shouldn't be tokenized as a number. Treat as keyword.
            self.pos = start;
            return self.read_keyword_or_bool(offset);
        }

        let num_str = std::str::from_utf8(&self.source[start..self.pos])
            .map_err(|_| PdfError::Parse(ParseError::InvalidNumber { offset }))?;

        if has_dot {
            let val: f64 = num_str
                .parse()
                .map_err(|_| PdfError::Parse(ParseError::InvalidNumber { offset }))?;
            Ok(Token::Real(val))
        } else {
            // Try to parse as integer. If it doesn't fit, fall back to real.
            match num_str.parse::<i64>() {
                Ok(val) => {
                    // Check if followed by spaces + another integer + spaces + 'R' → indirect ref
                    let saved_pos = self.pos;
                    if let Some(obj_ref) = self.try_read_reference(val, offset) {
                        return Ok(Token::Ref(obj_ref));
                    }
                    self.pos = saved_pos;
                    Ok(Token::Integer(val))
                }
                Err(_) => {
                    let val: f64 = num_str
                        .parse()
                        .map_err(|_| PdfError::Parse(ParseError::InvalidNumber { offset }))?;
                    Ok(Token::Real(val))
                }
            }
        }
    }

    /// Try to read `G R` after an integer to form an indirect reference.
    fn try_read_reference(&mut self, number: i64, _offset: u64) -> Option<ObjectId> {
        if number < 0 {
            return None;
        }

        let saved = self.pos;

        self.skip_whitespace_and_comments();

        // Read generation number
        let gen_start = self.pos;
        while self.pos < self.source.len()
            && self.source[self.pos] >= b'0'
            && self.source[self.pos] <= b'9'
        {
            self.pos += 1;
        }

        if self.pos == gen_start {
            self.pos = saved;
            return None;
        }

        let gen_str = std::str::from_utf8(&self.source[gen_start..self.pos]).ok()?;
        let generation: u16 = gen_str.parse().ok()?;

        self.skip_whitespace_and_comments();

        // Expect 'R' followed by a delimiter or whitespace or EOF
        if self.pos < self.source.len() && self.source[self.pos] == b'R' {
            let after_r = self.pos + 1;
            if after_r >= self.source.len()
                || is_whitespace(self.source[after_r])
                || is_delimiter(self.source[after_r])
            {
                self.pos = after_r;
                return Some(ObjectId::new(number as u32, generation));
            }
        }

        self.pos = saved;
        None
    }

    /// Read a keyword (`true`, `false`, `null`, `obj`, `endobj`, etc.) or boolean.
    fn read_keyword_or_bool(&mut self, _offset: u64) -> Result<Token, PdfError> {
        let start = self.pos;

        while self.pos < self.source.len() {
            let b = self.source[self.pos];
            if is_whitespace(b) || is_delimiter(b) {
                break;
            }
            self.pos += 1;
        }

        let word = &self.source[start..self.pos];

        match word {
            b"true" => Ok(Token::Boolean(true)),
            b"false" => Ok(Token::Boolean(false)),
            b"null" => Ok(Token::Null),
            _ => Ok(Token::Keyword(word.to_vec())),
        }
    }
}

/// Returns `true` if `b` is a PDF whitespace character (ISO 32000 Table 1).
#[inline]
pub fn is_whitespace(b: u8) -> bool {
    matches!(b, b'\0' | b'\t' | b'\n' | 0x0C | b'\r' | b' ')
}

/// Returns `true` if `b` is a PDF delimiter character (ISO 32000 Table 2).
#[inline]
pub fn is_delimiter(b: u8) -> bool {
    matches!(
        b,
        b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%'
    )
}

/// Convert a hex digit character to its value (0-15).
#[inline]
fn hex_digit(b: u8) -> Option<u8> {
    match b {
        b'0'..=b'9' => Some(b - b'0'),
        b'a'..=b'f' => Some(b - b'a' + 10),
        b'A'..=b'F' => Some(b - b'A' + 10),
        _ => None,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn tokenize_all(source: &[u8]) -> Vec<Token> {
        let mut tok = Tokenizer::new(source);
        let mut tokens = Vec::new();
        while let Some(result) = tok.next_token() {
            tokens.push(result.unwrap());
        }
        tokens
    }

    #[test]
    fn integer_token() {
        let tokens = tokenize_all(b"42");
        assert_eq!(tokens, vec![Token::Integer(42)]);
    }

    #[test]
    fn negative_integer() {
        let tokens = tokenize_all(b"-7");
        assert_eq!(tokens, vec![Token::Integer(-7)]);
    }

    #[test]
    fn positive_integer() {
        let tokens = tokenize_all(b"+3");
        assert_eq!(tokens, vec![Token::Integer(3)]);
    }

    #[test]
    #[allow(clippy::approx_constant)]
    fn real_number() {
        let tokens = tokenize_all(b"3.14");
        assert_eq!(tokens, vec![Token::Real(3.14)]);
    }

    #[test]
    fn real_leading_dot() {
        let tokens = tokenize_all(b".5");
        assert_eq!(tokens, vec![Token::Real(0.5)]);
    }

    #[test]
    fn real_negative() {
        let tokens = tokenize_all(b"-1.5");
        assert_eq!(tokens, vec![Token::Real(-1.5)]);
    }

    #[test]
    fn boolean_true() {
        let tokens = tokenize_all(b"true");
        assert_eq!(tokens, vec![Token::Boolean(true)]);
    }

    #[test]
    fn boolean_false() {
        let tokens = tokenize_all(b"false");
        assert_eq!(tokens, vec![Token::Boolean(false)]);
    }

    #[test]
    fn null_token() {
        let tokens = tokenize_all(b"null");
        assert_eq!(tokens, vec![Token::Null]);
    }

    #[test]
    fn name_simple() {
        let tokens = tokenize_all(b"/Type");
        assert_eq!(
            tokens,
            vec![Token::Name(Name::from_bytes(b"Type".to_vec()))]
        );
    }

    #[test]
    fn name_with_hex_escape() {
        let tokens = tokenize_all(b"/Name#20With#20Spaces");
        assert_eq!(
            tokens,
            vec![Token::Name(Name::from_bytes(b"Name With Spaces".to_vec()))]
        );
    }

    #[test]
    fn name_empty() {
        let tokens = tokenize_all(b"/ ");
        assert_eq!(tokens, vec![Token::Name(Name::from_bytes(Vec::new()))]);
    }

    #[test]
    fn literal_string_simple() {
        let tokens = tokenize_all(b"(Hello World)");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(
                b"Hello World".to_vec()
            ))]
        );
    }

    #[test]
    fn literal_string_nested_parens() {
        let tokens = tokenize_all(b"(A (B) C)");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"A (B) C".to_vec()))]
        );
    }

    #[test]
    fn literal_string_escape_sequences() {
        let tokens = tokenize_all(b"(\\n\\r\\t\\\\)");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"\n\r\t\\".to_vec()))]
        );
    }

    #[test]
    fn literal_string_octal_escape() {
        let tokens = tokenize_all(b"(\\101)"); // 0o101 = 65 = 'A'
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"A".to_vec()))]
        );
    }

    #[test]
    fn hex_string() {
        let tokens = tokenize_all(b"<48656C6C6F>");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"Hello".to_vec()))]
        );
    }

    #[test]
    fn hex_string_with_whitespace() {
        let tokens = tokenize_all(b"<48 65 6C 6C 6F>");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"Hello".to_vec()))]
        );
    }

    #[test]
    fn hex_string_odd_digits() {
        // Odd number of hex digits: last nibble padded with 0
        let tokens = tokenize_all(b"<ABC>");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(vec![0xAB, 0xC0]))]
        );
    }

    #[test]
    fn array_delimiters() {
        let tokens = tokenize_all(b"[1 2]");
        assert_eq!(
            tokens,
            vec![
                Token::ArrayStart,
                Token::Integer(1),
                Token::Integer(2),
                Token::ArrayEnd,
            ]
        );
    }

    #[test]
    fn dict_delimiters() {
        let tokens = tokenize_all(b"<< /Type /Catalog >>");
        assert_eq!(
            tokens,
            vec![
                Token::DictStart,
                Token::Name(Name::from_bytes(b"Type".to_vec())),
                Token::Name(Name::from_bytes(b"Catalog".to_vec())),
                Token::DictEnd,
            ]
        );
    }

    #[test]
    fn indirect_reference() {
        let tokens = tokenize_all(b"5 0 R");
        assert_eq!(tokens, vec![Token::Ref(ObjectId::new(5, 0))]);
    }

    #[test]
    fn indirect_reference_in_array() {
        let tokens = tokenize_all(b"[5 0 R]");
        assert_eq!(
            tokens,
            vec![
                Token::ArrayStart,
                Token::Ref(ObjectId::new(5, 0)),
                Token::ArrayEnd,
            ]
        );
    }

    #[test]
    fn keyword_obj() {
        let tokens = tokenize_all(b"obj");
        assert_eq!(tokens, vec![Token::Keyword(b"obj".to_vec())]);
    }

    #[test]
    fn keyword_stream() {
        let tokens = tokenize_all(b"stream");
        assert_eq!(tokens, vec![Token::Keyword(b"stream".to_vec())]);
    }

    #[test]
    fn skip_comments() {
        let tokens = tokenize_all(b"% this is a comment\n42");
        assert_eq!(tokens, vec![Token::Integer(42)]);
    }

    #[test]
    fn mixed_tokens() {
        let source = b"1 0 obj\n<< /Type /Page /MediaBox [0 0 612 792] >>\nendobj";
        let tokens = tokenize_all(source);
        assert_eq!(tokens[0], Token::Integer(1));
        assert_eq!(tokens[1], Token::Integer(0));
        assert_eq!(tokens[2], Token::Keyword(b"obj".to_vec()));
        assert_eq!(tokens[3], Token::DictStart);
    }

    #[test]
    fn position_tracking() {
        let mut tok = Tokenizer::new(b"  42");
        assert_eq!(tok.position(), 0);
        let _ = tok.next_token();
        assert_eq!(tok.position(), 4);
    }

    #[test]
    fn empty_input() {
        let tokens = tokenize_all(b"");
        assert!(tokens.is_empty());
    }

    #[test]
    fn whitespace_only() {
        let tokens = tokenize_all(b"   \n\r\t  ");
        assert!(tokens.is_empty());
    }

    #[test]
    fn literal_string_line_continuation() {
        let tokens = tokenize_all(b"(line1\\\nline2)");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"line1line2".to_vec()))]
        );
    }

    // -----------------------------------------------------------------------
    // Upstream-derived hex string tests (cpdf_syntax_parser_unittest.cpp)
    // -----------------------------------------------------------------------

    /// Upstream: ReadHexString — empty hex string `<>`.
    #[test]
    fn hex_string_empty() {
        let tokens = tokenize_all(b"<>");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(Vec::new()))]
        );
    }

    /// Upstream: ReadHexString — regular conversion stops at `>`.
    #[test]
    fn hex_string_stops_at_closing_angle() {
        let tokens = tokenize_all(b"<1A2b>abcd");
        assert_eq!(
            tokens,
            vec![
                Token::String(PdfString::from_bytes(vec![0x1A, 0x2B])),
                Token::Keyword(b"abcd".to_vec()),
            ]
        );
    }

    /// Upstream: ReadHexString — uneven number of hex digits pads last nibble with 0.
    #[test]
    fn hex_string_uneven_single_digit() {
        // Single hex digit "A" → 0xA0
        let tokens = tokenize_all(b"<A>");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(vec![0xA0]))]
        );
    }

    /// Upstream: ReadHexString — unterminated hex string (missing `>`).
    #[test]
    fn hex_string_unterminated_is_error() {
        let mut tok = Tokenizer::new(b"<1A2b");
        let result = tok.next_token();
        assert!(result.is_some());
        assert!(result.unwrap().is_err());
    }

    /// Upstream: ReadHexString — invalid hex digit is error in our strict implementation.
    #[test]
    fn hex_string_invalid_hex_digit() {
        let mut tok = Tokenizer::new(b"<zz>");
        let result = tok.next_token();
        assert!(result.is_some());
        assert!(result.unwrap().is_err());
    }

    /// Upstream: ReadHexString — just closing angle `>` yields empty string.
    #[test]
    fn hex_string_just_closing_angle() {
        // `<>` should return empty string
        let tokens = tokenize_all(b"<>");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(Vec::new()))]
        );
    }

    /// Upstream: ReadHexString — whitespace inside hex string is ignored.
    #[test]
    fn hex_string_whitespace_varieties() {
        let tokens = tokenize_all(b"<4 8\t6\n5\r6C>");
        // "48656C" → "Hel"
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"Hel".to_vec()))]
        );
    }

    /// Upstream: ReadHexString — lowercase hex digits.
    #[test]
    fn hex_string_lowercase() {
        let tokens = tokenize_all(b"<48656c6c6f>");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"Hello".to_vec()))]
        );
    }

    // -----------------------------------------------------------------------
    // Upstream-derived invalid reference test (cpdf_syntax_parser_unittest.cpp)
    // -----------------------------------------------------------------------

    /// Upstream: GetInvalidReference — object number `4294967295` (u32::MAX) should not
    /// produce a valid reference. Our tokenizer reads it as an integer.
    #[test]
    fn invalid_reference_max_u32() {
        let tokens = tokenize_all(b"4294967295 0 R");
        // This is too large for u32 in ObjectId::new, but the tokenizer
        // may parse it differently. The key check is that it doesn't panic.
        assert!(!tokens.is_empty());
    }

    // -----------------------------------------------------------------------
    // Upstream-derived word/name/string tests (cpdf_simple_parser_unittest.cpp)
    // -----------------------------------------------------------------------

    /// Upstream: GetWord — words separated by whitespace and delimiters.
    #[test]
    fn multiple_words_whitespace_separated() {
        let tokens = tokenize_all(b"  foo  bar  ");
        assert_eq!(
            tokens,
            vec![
                Token::Keyword(b"foo".to_vec()),
                Token::Keyword(b"bar".to_vec()),
            ]
        );
    }

    /// Upstream: GetWord — words around delimiters.
    #[test]
    fn words_around_delimiters() {
        let tokens = tokenize_all(b"/Name[42]");
        assert_eq!(
            tokens,
            vec![
                Token::Name(Name::from_bytes(b"Name".to_vec())),
                Token::ArrayStart,
                Token::Integer(42),
                Token::ArrayEnd,
            ]
        );
    }

    /// Upstream: GetWord — comment skipping.
    #[test]
    fn comment_then_token() {
        let tokens = tokenize_all(b"% comment line\r\n/Next");
        assert_eq!(
            tokens,
            vec![Token::Name(Name::from_bytes(b"Next".to_vec()))]
        );
    }

    /// Upstream: GetWord — multiple consecutive comments.
    #[test]
    fn multiple_comments() {
        let tokens = tokenize_all(b"% first\n% second\n42");
        assert_eq!(tokens, vec![Token::Integer(42)]);
    }

    /// Upstream: literal string — nested parens with escapes.
    #[test]
    fn literal_string_deeply_nested_parens() {
        let tokens = tokenize_all(b"(a(b(c)d)e)");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"a(b(c)d)e".to_vec()))]
        );
    }

    /// Upstream: literal string — escape sequences \\b and \\f.
    #[test]
    fn literal_string_escape_backspace_formfeed() {
        let tokens = tokenize_all(b"(\\b\\f)");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(vec![0x08, 0x0C]))]
        );
    }

    /// Upstream: literal string — escaped parens.
    #[test]
    fn literal_string_escaped_parens() {
        let tokens = tokenize_all(b"(\\(\\))");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"()".to_vec()))]
        );
    }

    /// Upstream: literal string — unterminated (missing closing paren).
    #[test]
    fn literal_string_unterminated() {
        let mut tok = Tokenizer::new(b"(no closing paren");
        let result = tok.next_token();
        assert!(result.is_some());
        assert!(result.unwrap().is_err());
    }

    /// Upstream: literal string — octal escapes at range boundaries.
    #[test]
    fn literal_string_octal_range() {
        // \000 = NUL, \377 = 0xFF
        let tokens = tokenize_all(b"(\\000\\377)");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(vec![0x00, 0xFF]))]
        );
    }

    /// Upstream: literal string — CR+LF continuation.
    #[test]
    fn literal_string_cr_lf_continuation() {
        let tokens = tokenize_all(b"(line1\\\r\nline2)");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"line1line2".to_vec()))]
        );
    }

    /// Upstream: literal string — unrecognized escape is ignored per spec.
    #[test]
    fn literal_string_unknown_escape() {
        // \z is not a recognized escape → backslash is ignored, 'z' is kept.
        let tokens = tokenize_all(b"(\\z)");
        assert_eq!(
            tokens,
            vec![Token::String(PdfString::from_bytes(b"z".to_vec()))]
        );
    }

    // -----------------------------------------------------------------------
    // Name edge cases
    // -----------------------------------------------------------------------

    /// Name with all-hex encoding.
    #[test]
    fn name_all_hex_encoded() {
        // /Type encoded as /#54#79#70#65
        let tokens = tokenize_all(b"/#54#79#70#65");
        assert_eq!(
            tokens,
            vec![Token::Name(Name::from_bytes(b"Type".to_vec()))]
        );
    }

    /// Name at end of input (no trailing whitespace).
    #[test]
    fn name_at_eof() {
        let tokens = tokenize_all(b"/EOF");
        assert_eq!(tokens, vec![Token::Name(Name::from_bytes(b"EOF".to_vec()))]);
    }

    // -----------------------------------------------------------------------
    // Number edge cases
    // -----------------------------------------------------------------------

    /// Large integer that overflows i64 falls back to real.
    #[test]
    fn large_integer_becomes_real() {
        let tokens = tokenize_all(b"99999999999999999999");
        assert_eq!(tokens.len(), 1);
        // Should have been parsed as a real (f64) since it overflows i64
        match &tokens[0] {
            Token::Real(_) => {}
            other => panic!("expected Real, got {:?}", other),
        }
    }

    /// Zero integer.
    #[test]
    fn zero_integer() {
        let tokens = tokenize_all(b"0");
        assert_eq!(tokens, vec![Token::Integer(0)]);
    }

    /// Negative zero.
    #[test]
    fn negative_zero() {
        let tokens = tokenize_all(b"-0");
        assert_eq!(tokens, vec![Token::Integer(0)]);
    }

    /// Tokenizer position after multiple tokens.
    #[test]
    fn position_tracks_through_tokens() {
        let mut tok = Tokenizer::new(b"true false null");
        assert_eq!(tok.position(), 0);
        let _ = tok.next_token(); // "true"
        assert_eq!(tok.position(), 4);
        let _ = tok.next_token(); // "false"
        assert_eq!(tok.position(), 10);
        let _ = tok.next_token(); // "null"
        assert_eq!(tok.position(), 15);
        assert!(tok.next_token().is_none()); // EOF
    }

    /// Set position mid-stream and re-tokenize.
    #[test]
    fn set_position_reparse() {
        let source = b"42 true";
        let mut tok = Tokenizer::new(source);
        let _ = tok.next_token(); // "42"
        tok.set_position(3); // back to "true"
        let t = tok.next_token().unwrap().unwrap();
        assert_eq!(t, Token::Boolean(true));
    }

    /// Lone `>` produces an error (not `DictEnd`).
    #[test]
    fn lone_greater_than_is_error() {
        let mut tok = Tokenizer::new(b">");
        let result = tok.next_token();
        assert!(result.is_some());
        assert!(result.unwrap().is_err());
    }

    /// NUL byte is treated as whitespace.
    #[test]
    fn nul_byte_is_whitespace() {
        let tokens = tokenize_all(&[0x00, b'4', b'2']);
        assert_eq!(tokens, vec![Token::Integer(42)]);
    }
}