stet_pdf_reader/
lexer.rs

1// stet-pdf-reader
2// Copyright (c) 2026 Scott Bowman
3// SPDX-License-Identifier: Apache-2.0 OR MIT
4
5//! PDF tokenizer.
6
7use crate::error::PdfError;
8use crate::objects::{PdfDict, PdfObj};
9
10/// PDF token types.
11#[derive(Debug, Clone, PartialEq)]
12pub enum Token {
13    Bool(bool),
14    Int(i64),
15    Real(f64),
16    /// Name without leading `/`.
17    Name(Vec<u8>),
18    /// Literal string `(...)`, decoded.
19    LitString(Vec<u8>),
20    /// Hex string `<...>`, decoded.
21    HexString(Vec<u8>),
22    /// `[`
23    ArrayBegin,
24    /// `]`
25    ArrayEnd,
26    /// `<<`
27    DictBegin,
28    /// `>>`
29    DictEnd,
30    /// Keywords: `obj`, `endobj`, `stream`, `endstream`, `R`, `null`, `xref`, `trailer`, etc.
31    Keyword(Vec<u8>),
32    Eof,
33}
34
35/// PDF lexer operating on a byte slice with a cursor.
36pub struct Lexer<'a> {
37    data: &'a [u8],
38    pos: usize,
39}
40
41impl<'a> Lexer<'a> {
42    pub fn new(data: &'a [u8]) -> Self {
43        Self { data, pos: 0 }
44    }
45
46    /// Create a lexer starting at a given offset.
47    pub fn at(data: &'a [u8], pos: usize) -> Self {
48        Self { data, pos }
49    }
50
51    /// Current byte offset.
52    pub fn pos(&self) -> usize {
53        self.pos
54    }
55
56    /// Set the cursor position.
57    pub fn set_pos(&mut self, pos: usize) {
58        self.pos = pos;
59    }
60
61    /// Underlying data slice.
62    pub fn data(&self) -> &'a [u8] {
63        self.data
64    }
65
66    /// Read the next token, advancing the cursor.
67    pub fn next_token(&mut self) -> Result<Token, PdfError> {
68        self.skip_whitespace_and_comments();
69
70        if self.pos >= self.data.len() {
71            return Ok(Token::Eof);
72        }
73
74        let b = self.data[self.pos];
75        match b {
76            b'/' => self.read_name(),
77            b'(' => self.read_literal_string(),
78            b'<' => {
79                if self.pos + 1 < self.data.len() && self.data[self.pos + 1] == b'<' {
80                    self.pos += 2;
81                    Ok(Token::DictBegin)
82                } else {
83                    self.read_hex_string()
84                }
85            }
86            b'>' => {
87                if self.pos + 1 < self.data.len() && self.data[self.pos + 1] == b'>' {
88                    self.pos += 2;
89                    Ok(Token::DictEnd)
90                } else {
91                    self.pos += 1;
92                    Err(PdfError::UnexpectedToken {
93                        expected: ">>".into(),
94                        got: ">".into(),
95                    })
96                }
97            }
98            b'[' => {
99                self.pos += 1;
100                Ok(Token::ArrayBegin)
101            }
102            b']' => {
103                self.pos += 1;
104                Ok(Token::ArrayEnd)
105            }
106            b'+' | b'-' | b'.' | b'0'..=b'9' => self.read_number(),
107            b'\'' | b'"' => {
108                // PDF text operators: ' (move to next line and show) and " (set spacing and show)
109                self.pos += 1;
110                Ok(Token::Keyword(vec![b]))
111            }
112            _ if b.is_ascii_alphabetic() => self.read_keyword(),
113            _ => {
114                let ch = b as char;
115                self.pos += 1;
116                Err(PdfError::UnexpectedToken {
117                    expected: "token".into(),
118                    got: format!("byte 0x{b:02x} '{ch}'"),
119                })
120            }
121        }
122    }
123
124    /// Peek at the next token without advancing.
125    pub fn peek_token(&mut self) -> Result<Token, PdfError> {
126        let saved = self.pos;
127        let tok = self.next_token();
128        self.pos = saved;
129        tok
130    }
131
132    /// Skip whitespace (space, tab, CR, LF, FF, NUL) and comments (% to EOL).
133    fn skip_whitespace_and_comments(&mut self) {
134        loop {
135            // Skip whitespace
136            while self.pos < self.data.len() && is_whitespace(self.data[self.pos]) {
137                self.pos += 1;
138            }
139            // Skip comments
140            if self.pos < self.data.len() && self.data[self.pos] == b'%' {
141                while self.pos < self.data.len()
142                    && self.data[self.pos] != b'\n'
143                    && self.data[self.pos] != b'\r'
144                {
145                    self.pos += 1;
146                }
147            } else {
148                break;
149            }
150        }
151    }
152
153    /// Read a number token (integer or real).
154    fn read_number(&mut self) -> Result<Token, PdfError> {
155        let start = self.pos;
156        let mut has_dot = false;
157
158        // Optional sign
159        if self.pos < self.data.len()
160            && (self.data[self.pos] == b'+' || self.data[self.pos] == b'-')
161        {
162            self.pos += 1;
163        }
164
165        // Digits and optional decimal point
166        while self.pos < self.data.len() {
167            let b = self.data[self.pos];
168            if b == b'.' && !has_dot {
169                has_dot = true;
170                self.pos += 1;
171            } else if b.is_ascii_digit() {
172                self.pos += 1;
173            } else {
174                break;
175            }
176        }
177
178        // Handle implicit exponent: "0.00-50" means "0.00e-50".
179        // Some PDF writers omit the 'e', producing a sign+digits suffix
180        // immediately after a real number.
181        let mut implicit_exp = false;
182        if has_dot
183            && self.pos < self.data.len()
184            && (self.data[self.pos] == b'+' || self.data[self.pos] == b'-')
185        {
186            // Peek ahead to check for digits after the sign
187            let sign_pos = self.pos;
188            let mut peek = sign_pos + 1;
189            while peek < self.data.len() && self.data[peek].is_ascii_digit() {
190                peek += 1;
191            }
192            if peek > sign_pos + 1 {
193                // Consume sign + exponent digits
194                self.pos = peek;
195                implicit_exp = true;
196            }
197        }
198
199        let s = &self.data[start..self.pos];
200        if s == b"+" || s == b"-" || s == b"." || s == b"+." || s == b"-." {
201            // Bare sign or dot without digits.  Treat as zero to match pdf.js
202            // behavior — some malformed PDFs use `--2.5` meaning `0 -2.5`, and
203            // returning a keyword would desynchronize the operand stack.
204            return Ok(Token::Int(0));
205        }
206
207        if has_dot {
208            let f: f64 = if implicit_exp {
209                // Insert 'e' before the exponent sign: "0.00-50" → "0.00e-50"
210                let s_str =
211                    std::str::from_utf8(s).map_err(|_| PdfError::Other("invalid number".into()))?;
212                let sign_idx = s_str.rfind(['+', '-']).unwrap();
213                let mut with_e = String::from(&s_str[..sign_idx]);
214                with_e.push('e');
215                with_e.push_str(&s_str[sign_idx..]);
216                with_e
217                    .parse()
218                    .map_err(|_| PdfError::Other(format!("invalid real: {s_str}")))?
219            } else {
220                let s_str =
221                    std::str::from_utf8(s).map_err(|_| PdfError::Other("invalid number".into()))?;
222                s_str
223                    .parse()
224                    .map_err(|_| PdfError::Other(format!("invalid real: {s_str}")))?
225            };
226            Ok(Token::Real(f))
227        } else {
228            let s_str =
229                std::str::from_utf8(s).map_err(|_| PdfError::Other("invalid number".into()))?;
230            let n: i64 = s_str
231                .parse()
232                .map_err(|_| PdfError::Other(format!("invalid integer: {s_str}")))?;
233            Ok(Token::Int(n))
234        }
235    }
236
237    /// Read a name token (after consuming `/`).
238    fn read_name(&mut self) -> Result<Token, PdfError> {
239        self.pos += 1; // skip '/'
240        let mut name = Vec::new();
241
242        while self.pos < self.data.len() {
243            let b = self.data[self.pos];
244            if is_whitespace(b) || is_delimiter(b) {
245                break;
246            }
247            if b == b'#' && self.pos + 2 < self.data.len() {
248                // Hex escape
249                let hi = hex_digit(self.data[self.pos + 1]);
250                let lo = hex_digit(self.data[self.pos + 2]);
251                if let (Some(h), Some(l)) = (hi, lo) {
252                    name.push(h << 4 | l);
253                    self.pos += 3;
254                    continue;
255                }
256            }
257            name.push(b);
258            self.pos += 1;
259        }
260
261        Ok(Token::Name(name))
262    }
263
264    /// Read a literal string `(...)` with escapes and nested parens.
265    fn read_literal_string(&mut self) -> Result<Token, PdfError> {
266        self.pos += 1; // skip '('
267        let mut result = Vec::new();
268        let mut depth = 1u32;
269
270        while self.pos < self.data.len() {
271            let b = self.data[self.pos];
272            match b {
273                b'(' => {
274                    depth += 1;
275                    result.push(b);
276                    self.pos += 1;
277                }
278                b')' => {
279                    depth -= 1;
280                    if depth == 0 {
281                        self.pos += 1;
282                        return Ok(Token::LitString(result));
283                    }
284                    result.push(b);
285                    self.pos += 1;
286                }
287                b'\\' => {
288                    self.pos += 1;
289                    if self.pos >= self.data.len() {
290                        break;
291                    }
292                    let esc = self.data[self.pos];
293                    match esc {
294                        b'n' => {
295                            result.push(b'\n');
296                            self.pos += 1;
297                        }
298                        b'r' => {
299                            result.push(b'\r');
300                            self.pos += 1;
301                        }
302                        b't' => {
303                            result.push(b'\t');
304                            self.pos += 1;
305                        }
306                        b'b' => {
307                            result.push(0x08);
308                            self.pos += 1;
309                        }
310                        b'f' => {
311                            result.push(0x0C);
312                            self.pos += 1;
313                        }
314                        b'(' | b')' | b'\\' => {
315                            result.push(esc);
316                            self.pos += 1;
317                        }
318                        b'\r' => {
319                            // Line continuation
320                            self.pos += 1;
321                            if self.pos < self.data.len() && self.data[self.pos] == b'\n' {
322                                self.pos += 1;
323                            }
324                        }
325                        b'\n' => {
326                            // Line continuation
327                            self.pos += 1;
328                        }
329                        b'0'..=b'7' => {
330                            // Octal escape (1-3 digits)
331                            let mut val = esc - b'0';
332                            self.pos += 1;
333                            if self.pos < self.data.len()
334                                && self.data[self.pos] >= b'0'
335                                && self.data[self.pos] <= b'7'
336                            {
337                                val = val * 8 + (self.data[self.pos] - b'0');
338                                self.pos += 1;
339                                if self.pos < self.data.len()
340                                    && self.data[self.pos] >= b'0'
341                                    && self.data[self.pos] <= b'7'
342                                {
343                                    val = val * 8 + (self.data[self.pos] - b'0');
344                                    self.pos += 1;
345                                }
346                            }
347                            result.push(val);
348                        }
349                        _ => {
350                            // Unknown escape — just include the character
351                            result.push(esc);
352                            self.pos += 1;
353                        }
354                    }
355                }
356                _ => {
357                    result.push(b);
358                    self.pos += 1;
359                }
360            }
361        }
362
363        Err(PdfError::Unterminated("string"))
364    }
365
366    /// Read a hex string `<...>`.
367    fn read_hex_string(&mut self) -> Result<Token, PdfError> {
368        self.pos += 1; // skip '<'
369        let mut result = Vec::new();
370        let mut high_nibble: Option<u8> = None;
371
372        while self.pos < self.data.len() {
373            let b = self.data[self.pos];
374            if b == b'>' {
375                self.pos += 1;
376                // Odd number of hex digits: implicit trailing 0
377                if let Some(h) = high_nibble {
378                    result.push(h << 4);
379                }
380                return Ok(Token::HexString(result));
381            }
382            if is_whitespace(b) {
383                self.pos += 1;
384                continue;
385            }
386            if let Some(nibble) = hex_digit(b) {
387                match high_nibble {
388                    None => high_nibble = Some(nibble),
389                    Some(h) => {
390                        result.push(h << 4 | nibble);
391                        high_nibble = None;
392                    }
393                }
394                self.pos += 1;
395            } else {
396                self.pos += 1;
397                return Err(PdfError::UnexpectedToken {
398                    expected: "hex digit".into(),
399                    got: format!("byte 0x{b:02x}"),
400                });
401            }
402        }
403
404        Err(PdfError::Unterminated("hex string"))
405    }
406
407    /// Read a keyword (alphabetic sequence).
408    fn read_keyword(&mut self) -> Result<Token, PdfError> {
409        let start = self.pos;
410        while self.pos < self.data.len() && self.data[self.pos].is_ascii_alphabetic() {
411            self.pos += 1;
412        }
413        let word = &self.data[start..self.pos];
414        match word {
415            b"true" => Ok(Token::Bool(true)),
416            b"false" => Ok(Token::Bool(false)),
417            _ => Ok(Token::Keyword(word.to_vec())),
418        }
419    }
420}
421
422/// Parse a PDF object from the lexer (recursive descent).
423///
424/// This handles arrays, dicts, and indirect references (`N G R`).
425pub fn parse_object(lexer: &mut Lexer) -> Result<PdfObj, PdfError> {
426    let tok = lexer.next_token()?;
427    parse_object_from_token(lexer, tok)
428}
429
430/// Parse a PDF object given an already-consumed first token.
431pub fn parse_object_from_token(lexer: &mut Lexer, tok: Token) -> Result<PdfObj, PdfError> {
432    match tok {
433        Token::Bool(b) => Ok(PdfObj::Bool(b)),
434        Token::Real(f) => Ok(PdfObj::Real(f)),
435        Token::Int(n) => {
436            // Could be start of indirect reference: N G R
437            let saved = lexer.pos();
438            match lexer.next_token() {
439                Ok(Token::Int(g)) => match lexer.next_token() {
440                    Ok(Token::Keyword(ref kw)) if kw == b"R" => Ok(PdfObj::Ref(n as u32, g as u16)),
441                    _ => {
442                        lexer.set_pos(saved);
443                        Ok(PdfObj::Int(n))
444                    }
445                },
446                _ => {
447                    lexer.set_pos(saved);
448                    Ok(PdfObj::Int(n))
449                }
450            }
451        }
452        Token::Name(n) => Ok(PdfObj::Name(n)),
453        Token::LitString(s) => Ok(PdfObj::Str(s)),
454        Token::HexString(s) => Ok(PdfObj::Str(s)),
455        Token::Keyword(ref kw) if kw == b"null" => Ok(PdfObj::Null),
456        Token::ArrayBegin => {
457            let mut elems = Vec::new();
458            loop {
459                let t = lexer.next_token()?;
460                if t == Token::ArrayEnd || t == Token::Eof {
461                    break;
462                }
463                match parse_object_from_token(lexer, t) {
464                    Ok(obj) => elems.push(obj),
465                    Err(_) => {} // skip unparseable tokens in arrays (corrupt PDF)
466                }
467            }
468            Ok(PdfObj::Array(elems))
469        }
470        Token::DictBegin => {
471            let dict = parse_dict_body(lexer)?;
472            Ok(PdfObj::Dict(dict))
473        }
474        _ => Err(PdfError::UnexpectedToken {
475            expected: "object".into(),
476            got: format!("{tok:?}"),
477        }),
478    }
479}
480
481/// Parse dictionary entries until `>>`, returning a PdfDict.
482pub fn parse_dict_body(lexer: &mut Lexer) -> Result<PdfDict, PdfError> {
483    let mut dict = PdfDict::new();
484    loop {
485        // Tolerate garbage bytes between entries: a lexer error here just means
486        // next_token hit a byte that isn't a valid PDF token start (e.g. a
487        // stray backtick in a malformed dict like `/Encoding 30 0`R`). The
488        // lexer has already advanced past the bad byte, so we can retry.
489        let t = match lexer.next_token() {
490            Ok(t) => t,
491            Err(_) => continue,
492        };
493        match t {
494            Token::DictEnd | Token::Eof => break,
495            Token::Name(key) => {
496                // Parse the value. On a value-level parse error (e.g. a garbage
497                // byte inside the value slot), insert /Null and resync on the
498                // next token rather than discarding the whole dict. Keeping
499                // already-parsed entries is what lets the Times-Roman /BaseFont
500                // survive a later /Encoding parse failure.
501                match parse_object(lexer) {
502                    Ok(val) => {
503                        dict.insert(key, val);
504                    }
505                    Err(_) => {
506                        dict.insert(key, PdfObj::Null);
507                    }
508                }
509            }
510            _ => {
511                // Tolerate unexpected tokens in dict (skip and continue)
512                continue;
513            }
514        }
515    }
516    Ok(dict)
517}
518
519/// PDF whitespace characters (PDF spec 7.2.2).
520fn is_whitespace(b: u8) -> bool {
521    matches!(b, b' ' | b'\t' | b'\r' | b'\n' | 0x0C | 0x00)
522}
523
524/// PDF delimiter characters.
525fn is_delimiter(b: u8) -> bool {
526    matches!(
527        b,
528        b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%'
529    )
530}
531
532/// Convert a hex digit to its value (0-15).
533fn hex_digit(b: u8) -> Option<u8> {
534    match b {
535        b'0'..=b'9' => Some(b - b'0'),
536        b'a'..=b'f' => Some(b - b'a' + 10),
537        b'A'..=b'F' => Some(b - b'A' + 10),
538        _ => None,
539    }
540}
541
542#[cfg(test)]
543mod tests {
544    use super::*;
545
546    fn tokenize(input: &[u8]) -> Vec<Token> {
547        let mut lexer = Lexer::new(input);
548        let mut tokens = Vec::new();
549        loop {
550            let tok = lexer.next_token().unwrap();
551            if tok == Token::Eof {
552                break;
553            }
554            tokens.push(tok);
555        }
556        tokens
557    }
558
559    #[test]
560    fn integers() {
561        assert_eq!(tokenize(b"42"), vec![Token::Int(42)]);
562        assert_eq!(tokenize(b"-7"), vec![Token::Int(-7)]);
563        assert_eq!(tokenize(b"+5"), vec![Token::Int(5)]);
564        assert_eq!(tokenize(b"0"), vec![Token::Int(0)]);
565    }
566
567    #[test]
568    fn reals() {
569        assert_eq!(tokenize(b"2.5"), vec![Token::Real(2.5)]);
570        assert_eq!(tokenize(b".5"), vec![Token::Real(0.5)]);
571        assert_eq!(tokenize(b"-2.0"), vec![Token::Real(-2.0)]);
572    }
573
574    #[test]
575    fn names() {
576        assert_eq!(tokenize(b"/Type"), vec![Token::Name(b"Type".to_vec())]);
577        assert_eq!(tokenize(b"/"), vec![Token::Name(b"".to_vec())]); // empty name
578        assert_eq!(tokenize(b"/A#20B"), vec![Token::Name(b"A B".to_vec())]); // hex escape
579    }
580
581    #[test]
582    fn strings() {
583        assert_eq!(
584            tokenize(b"(hello)"),
585            vec![Token::LitString(b"hello".to_vec())]
586        );
587        assert_eq!(
588            tokenize(b"(nested (parens))"),
589            vec![Token::LitString(b"nested (parens)".to_vec())]
590        );
591        assert_eq!(
592            tokenize(b"(line\\nfeed)"),
593            vec![Token::LitString(b"line\nfeed".to_vec())]
594        );
595        assert_eq!(
596            tokenize(b"(octal\\101)"),
597            vec![Token::LitString(b"octalA".to_vec())]
598        );
599    }
600
601    #[test]
602    fn hex_strings() {
603        assert_eq!(
604            tokenize(b"<48656C6C6F>"),
605            vec![Token::HexString(b"Hello".to_vec())]
606        );
607        // Odd digits: trailing 0
608        assert_eq!(tokenize(b"<ABC>"), vec![Token::HexString(vec![0xAB, 0xC0])]);
609        // Whitespace inside
610        assert_eq!(
611            tokenize(b"<48 65 6C>"),
612            vec![Token::HexString(b"Hel".to_vec())]
613        );
614    }
615
616    #[test]
617    fn booleans_and_null() {
618        assert_eq!(tokenize(b"true"), vec![Token::Bool(true)]);
619        assert_eq!(tokenize(b"false"), vec![Token::Bool(false)]);
620        let obj = parse_object(&mut Lexer::new(b"null")).unwrap();
621        assert_eq!(obj, PdfObj::Null);
622    }
623
624    #[test]
625    fn delimiters() {
626        let toks = tokenize(b"<< >> [ ]");
627        assert_eq!(
628            toks,
629            vec![
630                Token::DictBegin,
631                Token::DictEnd,
632                Token::ArrayBegin,
633                Token::ArrayEnd,
634            ]
635        );
636    }
637
638    #[test]
639    fn comments_skipped() {
640        assert_eq!(tokenize(b"% comment\n42"), vec![Token::Int(42)]);
641    }
642
643    #[test]
644    fn keywords() {
645        assert_eq!(
646            tokenize(b"obj endobj stream"),
647            vec![
648                Token::Keyword(b"obj".to_vec()),
649                Token::Keyword(b"endobj".to_vec()),
650                Token::Keyword(b"stream".to_vec()),
651            ]
652        );
653    }
654
655    #[test]
656    fn parse_array() {
657        let obj = parse_object(&mut Lexer::new(b"[1 2 /Name]")).unwrap();
658        assert_eq!(
659            obj,
660            PdfObj::Array(vec![
661                PdfObj::Int(1),
662                PdfObj::Int(2),
663                PdfObj::Name(b"Name".to_vec()),
664            ])
665        );
666    }
667
668    #[test]
669    fn parse_dict() {
670        let obj = parse_object(&mut Lexer::new(b"<< /Type /Page /Count 5 >>")).unwrap();
671        let dict = obj.as_dict().unwrap();
672        assert_eq!(dict.get_name(b"Type"), Some(b"Page".as_slice()));
673        assert_eq!(dict.get_int(b"Count"), Some(5));
674    }
675
676    #[test]
677    fn parse_indirect_ref() {
678        let obj = parse_object(&mut Lexer::new(b"10 0 R")).unwrap();
679        assert_eq!(obj, PdfObj::Ref(10, 0));
680    }
681
682    #[test]
683    fn parse_nested_dict() {
684        let obj = parse_object(&mut Lexer::new(
685            b"<< /Resources << /Font << /F1 5 0 R >> >> >>",
686        ))
687        .unwrap();
688        let dict = obj.as_dict().unwrap();
689        let res = dict.get_dict(b"Resources").unwrap();
690        let font = res.get_dict(b"Font").unwrap();
691        assert_eq!(font.get(b"F1"), Some(&PdfObj::Ref(5, 0)));
692    }
693
694    #[test]
695    fn int_not_ref_at_eof() {
696        // A lone integer should not be confused with a ref
697        let obj = parse_object(&mut Lexer::new(b"42")).unwrap();
698        assert_eq!(obj, PdfObj::Int(42));
699    }
700}
stet_pdf_reader/lexer.rs

stet_pdf_reader/
lexer.rs
