pdfrum_object/
string.rs

1//! PDF string objects (ISO 32000-1 §7.3.4) and the text codecs around them.
2//!
3//! A string object is raw bytes: the lexer has already resolved `\` escapes
4//! and paired hex digits, and no encoding is implied by the syntax. Text
5//! *meaning* is a separate question answered by [`decode_text`], which picks
6//! between `PDFDocEncoding` and three byte-order-marked Unicode encodings.
7
8use std::borrow::Cow;
9
10use crate::name::hex_pair;
11
12/// `PDFDocEncoding` (ISO 32000-1 Annex D.2) as a byte-to-code-point table.
13///
14/// Identity with Latin-1 except for the eight accent characters at
15/// `0x18..=0x1F`, the typographic block at `0x80..=0x9E`, the euro at `0xA0`,
16/// and three undefined positions (`0x7F`, `0x9F`, `0xAD`) that map to
17/// U+0000 rather than being dropped.
18#[rustfmt::skip]
19pub const PDF_DOC_ENCODING: [u16; 256] = [
20    0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
21    0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f,
22    0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017,
23    0x02d8, 0x02c7, 0x02c6, 0x02d9, 0x02dd, 0x02db, 0x02da, 0x02dc,
24    0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027,
25    0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f,
26    0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037,
27    0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f,
28    0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
29    0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f,
30    0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057,
31    0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f,
32    0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067,
33    0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f,
34    0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077,
35    0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x0000,
36    0x2022, 0x2020, 0x2021, 0x2026, 0x2014, 0x2013, 0x0192, 0x2044,
37    0x2039, 0x203a, 0x2212, 0x2030, 0x201e, 0x201c, 0x201d, 0x2018,
38    0x2019, 0x201a, 0x2122, 0xfb01, 0xfb02, 0x0141, 0x0152, 0x0160,
39    0x0178, 0x017d, 0x0131, 0x0142, 0x0153, 0x0161, 0x017e, 0x0000,
40    0x20ac, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7,
41    0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x0000, 0x00ae, 0x00af,
42    0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7,
43    0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf,
44    0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7,
45    0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf,
46    0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7,
47    0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df,
48    0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7,
49    0x00e8, 0x00e9, 0x00ea, 0x00eb, 0x00ec, 0x00ed, 0x00ee, 0x00ef,
50    0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7,
51    0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff,
52];
53
54/// How the string was spelled in the file.
55///
56/// PDF has two string syntaxes; a parsed string remembers which one it came
57/// from so that rewriting a file reproduces it. The choice carries no
58/// semantics — the bytes are identical either way.
59#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
60pub enum StringSyntax {
61    /// `(parenthesized)`, with backslash escapes.
62    Literal,
63    /// `<48657820>`, hexadecimal digit pairs.
64    Hex,
65}
66
67/// A PDF string object: raw bytes plus the syntax they were written in.
68///
69/// ```
70/// use pdfrum_object::PdfString;
71///
72/// let s = PdfString::literal(b"A simple test");
73/// assert_eq!(s.as_text(), "A simple test");
74/// assert!(!s.hex);
75///
76/// // A UTF-16BE byte-order mark selects the Unicode reading.
77/// let unicode = PdfString::hex(b"\xFE\xFF\x03\x30\x03\x31");
78/// assert_eq!(unicode.as_text(), "\u{0330}\u{0331}");
79/// assert!(unicode.hex);
80/// ```
81#[derive(Clone, PartialEq, Eq, Hash)]
82pub struct PdfString {
83    /// The string's bytes, escapes already resolved.
84    pub bytes: Box<[u8]>,
85    /// Whether the source syntax was `<hex>` rather than `(literal)`.
86    /// Round-tripped by the writer; never affects meaning.
87    pub hex: bool,
88}
89
90impl PdfString {
91    /// A string spelled `(like this)`.
92    #[must_use]
93    pub fn literal(bytes: impl AsRef<[u8]>) -> Self {
94        Self {
95            bytes: bytes.as_ref().into(),
96            hex: false,
97        }
98    }
99
100    /// A string spelled `<6C696B652074686973>`.
101    #[must_use]
102    pub fn hex(bytes: impl AsRef<[u8]>) -> Self {
103        Self {
104            bytes: bytes.as_ref().into(),
105            hex: true,
106        }
107    }
108
109    /// A string with an explicit syntax.
110    #[must_use]
111    pub fn new(bytes: impl AsRef<[u8]>, syntax: StringSyntax) -> Self {
112        Self {
113            bytes: bytes.as_ref().into(),
114            hex: matches!(syntax, StringSyntax::Hex),
115        }
116    }
117
118    /// Interpret the bytes as text — see [`decode_text`].
119    #[must_use]
120    pub fn as_text(&self) -> Cow<'_, str> {
121        decode_text(&self.bytes)
122    }
123
124    /// Re-spell the string in its original syntax, ready to write into a file.
125    #[must_use]
126    pub fn encode(&self) -> Vec<u8> {
127        if self.hex {
128            encode_string_hex(&self.bytes)
129        } else {
130            encode_string_literal(&self.bytes)
131        }
132    }
133}
134
135impl std::fmt::Debug for PdfString {
136    /// Prints the string the way a file spells it, so an object dump reads
137    /// like the PDF it came from rather than like a byte array.
138    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
139        write!(f, "{}", String::from_utf8_lossy(&self.encode()))
140    }
141}
142
143/// Drop the language-code regions ISO 32000-1 §7.9.2.2 allows inside text
144/// strings: U+001B opens a region and the next U+001B closes it; an
145/// unterminated region runs to the end.
146fn strip_language_codes(text: &str) -> Cow<'_, str> {
147    if !text.contains('\u{1b}') {
148        return Cow::Borrowed(text);
149    }
150    let mut out = String::with_capacity(text.len());
151    let mut inside = false;
152    for c in text.chars() {
153        if c == '\u{1b}' {
154            inside = !inside;
155        } else if !inside {
156            out.push(c);
157        }
158    }
159    Cow::Owned(out)
160}
161
162/// Decode UTF-16 code units, replacing unpaired surrogates with U+FFFD.
163fn decode_utf16(units: impl Iterator<Item = u16>) -> String {
164    char::decode_utf16(units)
165        .map(|r| r.unwrap_or(char::REPLACEMENT_CHARACTER))
166        .collect()
167}
168
169/// Read a text string's bytes as text (ISO 32000-1 §7.9.2.2).
170///
171/// A leading `FE FF` selects UTF-16BE and `EF BB BF` selects UTF-8; `FF FE`
172/// selects UTF-16LE, which is an extension beyond the specification.
173/// Anything else is `PDFDocEncoding`, one byte per character. In the marked
174/// encodings, language-code regions are stripped.
175///
176/// Unpaired surrogates and invalid UTF-8 become U+FFFD (Rust's `str` cannot
177/// carry either), and a trailing odd byte in a UTF-16 payload is ignored.
178///
179/// ```
180/// use pdfrum_object::decode_text;
181///
182/// assert_eq!(decode_text(b"the quick\tfox"), "the quick\tfox");
183/// assert_eq!(decode_text(b"\xFE\xFF\xD8\x3C\xDF\xA8"), "\u{1F3A8}");
184/// assert_eq!(decode_text(b"\xEF\xBB\xBF\xCC\xB0"), "\u{0330}");
185/// // 0x80 is a bullet in PDFDocEncoding, not a Latin-1 control.
186/// assert_eq!(decode_text(b"\x80"), "\u{2022}");
187/// ```
188#[must_use]
189pub fn decode_text(bytes: &[u8]) -> Cow<'_, str> {
190    if let Some(payload) = bytes.strip_prefix(b"\xFE\xFF") {
191        let units = payload
192            .as_chunks::<2>()
193            .0
194            .iter()
195            .map(|p| u16::from_be_bytes(*p));
196        return Cow::Owned(strip_language_codes(&decode_utf16(units)).into_owned());
197    }
198    if let Some(payload) = bytes.strip_prefix(b"\xFF\xFE") {
199        let units = payload
200            .as_chunks::<2>()
201            .0
202            .iter()
203            .map(|p| u16::from_le_bytes(*p));
204        return Cow::Owned(strip_language_codes(&decode_utf16(units)).into_owned());
205    }
206    if let Some(payload) = bytes.strip_prefix(b"\xEF\xBB\xBF") {
207        let text = String::from_utf8_lossy(payload);
208        return Cow::Owned(strip_language_codes(&text).into_owned());
209    }
210
211    // PDFDocEncoding. Pure-ASCII input (minus DEL, which the table redefines)
212    // is already its own decoding, so borrow it.
213    if bytes.iter().all(|b| *b < 0x18 || (0x20..0x7F).contains(b))
214        && let Ok(s) = std::str::from_utf8(bytes)
215    {
216        return Cow::Borrowed(s);
217    }
218    Cow::Owned(
219        bytes
220            .iter()
221            .map(|b| {
222                let cp = PDF_DOC_ENCODING.get(usize::from(*b)).copied().unwrap_or(0);
223                char::from_u32(u32::from(cp)).unwrap_or(char::REPLACEMENT_CHARACTER)
224            })
225            .collect(),
226    )
227}
228
229/// Write text back out as a text string's bytes (ISO 32000-1 §7.9.2.2).
230///
231/// `PDFDocEncoding` when every character has a byte in the table (the first
232/// matching byte wins, so U+0000 encodes as `0x00`), otherwise `FE FF`
233/// followed by UTF-16BE.
234///
235/// ```
236/// use pdfrum_object::encode_text;
237///
238/// assert_eq!(encode_text("the quick\tfox"), b"the quick\tfox");
239/// assert_eq!(encode_text("\u{0330}\u{0331}"), b"\xFE\xFF\x03\x30\x03\x31");
240/// ```
241#[must_use]
242pub fn encode_text(text: &str) -> Vec<u8> {
243    let mut pdf_doc = Vec::with_capacity(text.len());
244    let mut representable = true;
245    for c in text.chars() {
246        let Ok(cp) = u16::try_from(u32::from(c)) else {
247            representable = false;
248            break;
249        };
250        let Some(byte) = PDF_DOC_ENCODING
251            .iter()
252            .position(|e| *e == cp)
253            .and_then(|i| u8::try_from(i).ok())
254        else {
255            representable = false;
256            break;
257        };
258        pdf_doc.push(byte);
259    }
260    if representable {
261        return pdf_doc;
262    }
263
264    let mut out = vec![0xFE, 0xFF];
265    for unit in text.encode_utf16() {
266        out.extend_from_slice(&unit.to_be_bytes());
267    }
268    out
269}
270
271/// Spell bytes as a literal string: `(…)` with `\n`, `\r`, `\(`, `\)` and
272/// `\\` escaped and every other byte verbatim.
273///
274/// ```
275/// use pdfrum_object::encode_string_literal;
276///
277/// assert_eq!(encode_string_literal(b"a(b)c"), b"(a\\(b\\)c)");
278/// assert_eq!(encode_string_literal(b"line\n"), b"(line\\n)");
279/// ```
280#[must_use]
281pub fn encode_string_literal(bytes: &[u8]) -> Vec<u8> {
282    let mut out = Vec::with_capacity(bytes.len() + 2);
283    out.push(b'(');
284    for b in bytes {
285        match b {
286            b'\n' => out.extend_from_slice(b"\\n"),
287            b'\r' => out.extend_from_slice(b"\\r"),
288            b'(' | b')' | b'\\' => {
289                out.push(b'\\');
290                out.push(*b);
291            }
292            _ => out.push(*b),
293        }
294    }
295    out.push(b')');
296    out
297}
298
299/// Spell bytes as a hexadecimal string: `<…>` with uppercase digit pairs.
300///
301/// ```
302/// use pdfrum_object::encode_string_hex;
303///
304/// assert_eq!(encode_string_hex(b"\x12\xac"), b"<12AC>");
305/// assert_eq!(encode_string_hex(b""), b"<>");
306/// ```
307#[must_use]
308pub fn encode_string_hex(bytes: &[u8]) -> Vec<u8> {
309    let mut out = Vec::with_capacity(2 * bytes.len() + 2);
310    out.push(b'<');
311    for b in bytes {
312        out.extend_from_slice(&hex_pair(*b));
313    }
314    out.push(b'>');
315    out
316}
317
318#[cfg(test)]
319mod tests {
320    use super::{
321        PDF_DOC_ENCODING, PdfString, StringSyntax, decode_text, encode_string_hex,
322        encode_string_literal, encode_text,
323    };
324
325    // From fpdf_parser_decode_unittest.cpp:324-353.
326    #[test]
327    fn decode_text_picks_the_encoding_from_the_mark() {
328        assert_eq!(decode_text(b""), "");
329        assert_eq!(decode_text(b"the quick\tfox"), "the quick\tfox");
330        assert_eq!(
331            decode_text(b"\xEF\xBB\xBF\xCC\xB0\xCC\xB1"),
332            "\u{330}\u{331}"
333        );
334        assert_eq!(decode_text(b"\xFE\xFF\x03\x30\x03\x31"), "\u{330}\u{331}");
335        assert_eq!(
336            decode_text(
337                b"\xFE\xFF\x7F\x51\x98\x75\x00\x20\x56\xFE\x72\x47\x00\
338                  \x20\x8D\x44\x8B\xAF\x66\xF4\x59\x1A\x00\x20\x00\xBB"
339            ),
340            "\u{7F51}\u{9875}\u{20}\u{56FE}\u{7247}\u{20}\
341             \u{8D44}\u{8BAF}\u{66F4}\u{591A}\u{20}\u{BB}"
342        );
343        // Supplementary planes, through both marked encodings.
344        assert_eq!(decode_text(b"\xEF\xBB\xBF\xF0\x9F\x8E\xA8"), "\u{1F3A8}");
345        assert_eq!(decode_text(b"\xFE\xFF\xD8\x3C\xDF\xA8"), "\u{1F3A8}");
346    }
347
348    #[test]
349    fn decode_text_reads_utf16le_as_a_pdfium_extension() {
350        assert_eq!(decode_text(b"\xFF\xFE\x30\x03\x31\x03"), "\u{330}\u{331}");
351    }
352
353    // From fpdf_parser_decode_unittest.cpp:355-374.
354    #[test]
355    fn decode_text_strips_language_code_regions() {
356        assert_eq!(
357            decode_text(b"\xEF\xBB\xBF\x1B\x6A\x61\x1B\x20\xE5\x8D\xB0\xE5\x88\xB7"),
358            "\u{20}\u{5370}\u{5237}"
359        );
360        assert_eq!(
361            decode_text(b"\xFE\xFF\x00\x1B\x6A\x61\x00\x1B\x00\x20\x53\x70\x52\x37"),
362            "\u{20}\u{5370}\u{5237}"
363        );
364        // A trailing odd byte in the UTF-16 payload is ignored.
365        assert_eq!(
366            decode_text(b"\xFE\xFF\x00\x1B\x6A\x61\x00\x1B\x00\x20\x53\x70\x52\x37\x29"),
367            "\u{20}\u{5370}\u{5237}"
368        );
369        assert_eq!(
370            decode_text(b"\xFE\xFF\x00\x1B\x6A\x61\x4A\x50\x00\x1B\x00\x20\x53\x70\x52\x37"),
371            "\u{20}\u{5370}\u{5237}"
372        );
373        assert_eq!(
374            decode_text(b"\xFE\xFF\x00\x20\x00\x1B\x6A\x61\x4A\x50\x00\x1B\x52\x37"),
375            "\u{20}\u{5237}"
376        );
377    }
378
379    // From fpdf_parser_decode_unittest.cpp:376-384.
380    #[test]
381    fn decode_text_tolerates_broken_language_code_regions() {
382        assert_eq!(decode_text(b"\xEF\xBB\xBF\x1B\x1B"), "");
383        assert_eq!(decode_text(b"\xFE\xFF\x00\x1B\x00\x1B"), "");
384        // Unterminated region strips to the end.
385        assert_eq!(decode_text(b"\xFE\xFF\x00\x1B\x00\x1B\x20"), "");
386        assert_eq!(decode_text(b"\xEF\xBB\xBF\x1B\x1B\x20"), "\u{20}");
387        assert_eq!(decode_text(b"\xFE\xFF\x00\x1B\x00\x1B\x00\x20"), "\u{20}");
388    }
389
390    // From fpdf_parser_decode_unittest.cpp:386-395, adjusted for a stated
391    // divergence: Rust `str` cannot hold a lone surrogate, so each becomes
392    // U+FFFD where the oracle keeps the raw code unit.
393    #[test]
394    fn decode_text_replaces_unpaired_surrogates() {
395        assert_eq!(decode_text(b"\xFE\xFF\xD8\x00"), "\u{FFFD}");
396        assert_eq!(decode_text(b"\xFE\xFF\xDC\x00"), "\u{FFFD}");
397        assert_eq!(
398            decode_text(b"\xFE\xFF\xD8\x00\xD8\x3C\xDF\xA8"),
399            "\u{FFFD}\u{1F3A8}"
400        );
401        assert_eq!(
402            decode_text(b"\xFE\xFF\xD8\x3C\xDF\xA8\xDC\x00"),
403            "\u{1F3A8}\u{FFFD}"
404        );
405    }
406
407    #[test]
408    fn decode_text_uses_the_pdfdoc_table_without_a_mark() {
409        // The accent block and the typographic block are not Latin-1.
410        assert_eq!(decode_text(b"\x18\x19\x1A"), "\u{2D8}\u{2C7}\u{2C6}");
411        assert_eq!(decode_text(b"\x80\x8A\x9E"), "\u{2022}\u{2212}\u{17E}");
412        assert_eq!(decode_text(b"\xA0"), "\u{20AC}");
413        assert_eq!(decode_text(b"\xA1\xFF"), "\u{A1}\u{FF}");
414        // The three undefined positions become U+0000 and are kept.
415        assert_eq!(decode_text(b"\x7F\x9F\xAD"), "\0\0\0");
416        // Literal NULs survive, as in the C++ wide string.
417        assert_eq!(decode_text(b"a\0b"), "a\0b");
418    }
419
420    // From fpdf_parser_decode_unittest.cpp:397-416.
421    #[test]
422    fn encode_text_prefers_pdfdoc_then_falls_back_to_utf16() {
423        assert_eq!(encode_text(""), b"");
424        assert_eq!(encode_text("the quick\tfox"), b"the quick\tfox");
425        assert_eq!(encode_text("\u{330}\u{331}"), b"\xFE\xFF\x03\x30\x03\x31");
426        assert_eq!(
427            encode_text(
428                "\u{7F51}\u{9875}\u{20}\u{56FE}\u{7247}\u{20}\
429                 \u{8D44}\u{8BAF}\u{66F4}\u{591A}\u{20}\u{BB}"
430            ),
431            b"\xFE\xFF\x7F\x51\x98\x75\x00\x20\x56\xFE\x72\x47\x00\
432              \x20\x8D\x44\x8B\xAF\x66\xF4\x59\x1A\x00\x20\x00\xBB"
433        );
434        assert_eq!(encode_text("\u{1F3A8}"), b"\xFE\xFF\xD8\x3C\xDF\xA8");
435    }
436
437    // From fpdf_parser_decode_unittest.cpp:418-434: every byte round-trips
438    // except the three PDFDocEncoding leaves undefined, which collapse to NUL.
439    #[test]
440    fn text_round_trips_every_byte() {
441        for code in 0u16..256 {
442            #[expect(clippy::cast_possible_truncation, reason = "loop bound is 256")]
443            let original = [code as u8];
444            let reencoded = encode_text(&decode_text(&original));
445            match code {
446                0x7F | 0x9F | 0xAD => assert_eq!(reencoded, b"\0", "undefined at {code:#04x}"),
447                _ => assert_eq!(reencoded, original, "PDFDocEncoding {code:#04x}"),
448            }
449        }
450    }
451
452    #[test]
453    fn pdfdoc_table_is_the_annex_d_table() {
454        assert_eq!(PDF_DOC_ENCODING.len(), 256);
455        assert_eq!(PDF_DOC_ENCODING[0x41], 0x0041);
456        assert_eq!(PDF_DOC_ENCODING[0x18], 0x02D8);
457        assert_eq!(PDF_DOC_ENCODING[0x7F], 0x0000);
458        assert_eq!(PDF_DOC_ENCODING[0x80], 0x2022);
459        assert_eq!(PDF_DOC_ENCODING[0x9F], 0x0000);
460        assert_eq!(PDF_DOC_ENCODING[0xA0], 0x20AC);
461        assert_eq!(PDF_DOC_ENCODING[0xAD], 0x0000);
462        assert_eq!(PDF_DOC_ENCODING[0xFF], 0x00FF);
463    }
464
465    #[test]
466    fn string_syntax_round_trips_through_the_encoder() {
467        let literal = PdfString::new(b"a(b)\\c\n", StringSyntax::Literal);
468        assert!(!literal.hex);
469        assert_eq!(literal.encode(), b"(a\\(b\\)\\\\c\\n)");
470
471        let hex = PdfString::new(b"\x12\xAC", StringSyntax::Hex);
472        assert!(hex.hex);
473        assert_eq!(hex.encode(), b"<12AC>");
474    }
475
476    #[test]
477    fn string_escaping_leaves_other_control_bytes_alone() {
478        assert_eq!(encode_string_literal(b"\x00\x07\t"), b"(\x00\x07\t)");
479        assert_eq!(encode_string_literal(b""), b"()");
480        assert_eq!(encode_string_hex(b"\x00\xFF"), b"<00FF>");
481    }
482}
pdfrum_object/string.rs

pdfrum_object/
string.rs
