Skip to main content

flpdf/
pdf_string.rs

1//! Decode PDFDocEncoding, construct Unicode PDF strings, and serialize binary
2//! strings for flpdf consumers.
3//!
4//! qpdf correspondence: `libqpdf/QPDF_String.cc` PDF string semantics.
5//!
6//!
7
8/// PDFDocEncoding lookup table per ISO 32000-1 Annex D.3.
9const PDFDOC_ENCODING: [Option<char>; 256] = build_pdfdoc_table();
10
11// cov:ignore-start: const evaluation builds the lookup table before runtime; its assignments cannot execute under llvm-cov
12const fn build_pdfdoc_table() -> [Option<char>; 256] {
13    let mut table: [Option<char>; 256] = [None; 256];
14    table[0x08] = Some('\u{0008}');
15    table[0x09] = Some('\t');
16    table[0x0A] = Some('\n');
17    table[0x0B] = Some('\u{000B}');
18    table[0x0C] = Some('\u{000C}');
19    table[0x0D] = Some('\r');
20    table[0x18] = Some('\u{02D8}');
21    table[0x19] = Some('\u{02C7}');
22    table[0x1A] = Some('\u{02C6}');
23    table[0x1B] = Some('\u{02D9}');
24    table[0x1C] = Some('\u{02DD}');
25    table[0x1D] = Some('\u{02DB}');
26    table[0x1E] = Some('\u{02DA}');
27    table[0x1F] = Some('\u{02DC}');
28    let mut b = 0x20u8;
29    while b <= 0x7E {
30        table[b as usize] = Some(b as char);
31        b += 1;
32    }
33    table[0x80] = Some('\u{2022}');
34    table[0x81] = Some('\u{2020}');
35    table[0x82] = Some('\u{2021}');
36    table[0x83] = Some('\u{2026}');
37    table[0x84] = Some('\u{2014}');
38    table[0x85] = Some('\u{2013}');
39    table[0x86] = Some('\u{0192}');
40    table[0x87] = Some('\u{2044}');
41    table[0x88] = Some('\u{2039}');
42    table[0x89] = Some('\u{203A}');
43    table[0x8A] = Some('\u{2212}');
44    table[0x8B] = Some('\u{2030}');
45    table[0x8C] = Some('\u{201E}');
46    table[0x8D] = Some('\u{201C}');
47    table[0x8E] = Some('\u{201D}');
48    table[0x8F] = Some('\u{2018}');
49    table[0x90] = Some('\u{2019}');
50    table[0x91] = Some('\u{201A}');
51    table[0x92] = Some('\u{2122}');
52    table[0x93] = Some('\u{FB01}');
53    table[0x94] = Some('\u{FB02}');
54    table[0x95] = Some('\u{0141}');
55    table[0x96] = Some('\u{0152}');
56    table[0x97] = Some('\u{0160}');
57    table[0x98] = Some('\u{0178}');
58    table[0x99] = Some('\u{017D}');
59    table[0x9A] = Some('\u{0131}');
60    table[0x9B] = Some('\u{0142}');
61    table[0x9C] = Some('\u{0153}');
62    table[0x9D] = Some('\u{0161}');
63    table[0x9E] = Some('\u{017E}');
64    table[0xA0] = Some('\u{20AC}');
65    let mut b = 0xA1u8;
66    loop {
67        table[b as usize] = Some(b as char);
68        if b == 0xFF {
69            break;
70        }
71        b += 1;
72    }
73    table
74}
75// cov:ignore-end
76
77/// Return qpdf's UTF-8 view of one stored PDF string.
78pub fn utf8_value(bytes: &[u8]) -> Vec<u8> {
79    if let Some(rest) = bytes.strip_prefix(&[0xfe, 0xff]) {
80        return lossy_utf16_to_utf8(rest, false).into_bytes();
81    }
82    if let Some(rest) = bytes.strip_prefix(&[0xff, 0xfe]) {
83        return lossy_utf16_to_utf8(rest, true).into_bytes();
84    }
85    if let Some(rest) = bytes.strip_prefix(&[0xef, 0xbb, 0xbf]) {
86        return rest.to_vec();
87    }
88
89    bytes
90        .iter()
91        .map(|&byte| match byte {
92            0x7f | 0x9f | 0xad => '\u{fffd}',
93            _ => PDFDOC_ENCODING[byte as usize].unwrap_or(byte as char),
94        })
95        .collect::<String>()
96        .into_bytes()
97}
98
99/// Normalize UTF-8 using qpdf's error consumption and replacement rules.
100pub(crate) fn normalized_utf8_value(utf8: &[u8]) -> Vec<u8> {
101    normalize_utf8(utf8).0
102}
103
104fn normalize_utf8(utf8: &[u8]) -> (Vec<u8>, bool) {
105    let mut result = Vec::with_capacity(utf8.len());
106    let mut position = 0;
107    let mut had_error = false;
108    while position < utf8.len() {
109        let original_position = position;
110        let mut byte = utf8[position];
111        position += 1;
112
113        if byte < 0x80 {
114            result.push(byte);
115            continue;
116        }
117
118        let mut bytes_needed = 0;
119        let mut bit_check = 0x40;
120        let mut to_clear = 0x80;
121        while byte & bit_check != 0 {
122            bytes_needed += 1;
123            to_clear |= bit_check;
124            bit_check >>= 1;
125        }
126
127        let mut error = !(1..=5).contains(&bytes_needed) || position + bytes_needed > utf8.len();
128        let mut codepoint = 0xfffd;
129        if !error {
130            codepoint = u32::from(byte & !to_clear);
131            for _ in 0..bytes_needed {
132                byte = utf8[position];
133                position += 1;
134                if byte & 0xc0 != 0x80 {
135                    position -= 1;
136                    error = true;
137                    break;
138                }
139                codepoint = (codepoint << 6) + u32::from(byte & 0x3f);
140            }
141
142            if !error {
143                let lower_bounds = [0, 0, 1 << 7, 1 << 11, 1 << 16, 1 << 12, 1 << 26];
144                let lower_bound = lower_bounds[position - original_position];
145                if lower_bound > 0 && codepoint < lower_bound {
146                    error = true;
147                }
148            }
149        }
150
151        had_error |= error;
152        let scalar = if error {
153            '\u{fffd}'
154        } else {
155            char::from_u32(codepoint).unwrap_or('\u{fffd}')
156        };
157        let mut encoded = [0; 4];
158        result.extend_from_slice(scalar.encode_utf8(&mut encoded).as_bytes());
159    }
160    (result, had_error)
161}
162
163/// Construct the stored bytes for qpdf `newUnicodeString`.
164pub fn new_unicode_string(utf8: &[u8]) -> Vec<u8> {
165    let (normalized, had_error) = normalize_utf8(utf8);
166    let text = std::str::from_utf8(&normalized)
167        .expect("qpdf UTF-8 normalization always emits valid Unicode");
168    let mut pdfdoc = Vec::with_capacity(text.len());
169    let resembles_unicode_bom = utf8.starts_with("þÿ".as_bytes())
170        || utf8.starts_with("ÿþ".as_bytes())
171        || utf8.starts_with("".as_bytes());
172    if !had_error && !resembles_unicode_bom {
173        for character in text.chars() {
174            let mut encoded_character = [0; 4];
175            let encoded_character = character.encode_utf8(&mut encoded_character).as_bytes();
176            let encoded = (0_u16..=u16::from(u8::MAX))
177                .map(|byte| byte as u8)
178                .filter(|&byte| !matches!(byte, 0x7f | 0x9f | 0xad))
179                .find(|&byte| utf8_value(&[byte]) == encoded_character);
180            let Some(encoded) = encoded else {
181                return crate::filespec_helper::encode_utf16be(text);
182            };
183            pdfdoc.push(encoded);
184        }
185        return pdfdoc;
186    }
187    crate::filespec_helper::encode_utf16be(text)
188}
189
190/// Force a stored PDF string into qpdf's binary hexadecimal representation.
191pub fn unparse_binary(stored: &[u8]) -> Vec<u8> {
192    let mut output = Vec::with_capacity(stored.len().saturating_mul(2).saturating_add(2));
193    crate::writer::output::with_buffer_sink(&mut output, |out| {
194        crate::pdf_syntax::write_hex_string(out, stored)
195    })
196    .expect("writing a PDF string to a Vec cannot fail");
197    output
198}
199
200/// Decode a PDF text string into a Rust string using the canonical PDF string table.
201pub(crate) fn decode_pdf_text_string(bytes: &[u8]) -> Option<String> {
202    if let Some(rest) = bytes.strip_prefix(&[0xFE, 0xFF]) {
203        if rest.len() % 2 != 0 {
204            return None;
205        }
206        let units = rest
207            .chunks_exact(2)
208            .map(|c| u16::from_be_bytes([c[0], c[1]]));
209        return char::decode_utf16(units)
210            .collect::<Result<String, _>>()
211            .ok();
212    }
213    if let Some(rest) = bytes.strip_prefix(&[0xFF, 0xFE]) {
214        if rest.len() % 2 != 0 {
215            return None;
216        }
217        let units = rest
218            .chunks_exact(2)
219            .map(|c| u16::from_le_bytes([c[0], c[1]]));
220        return char::decode_utf16(units)
221            .collect::<Result<String, _>>()
222            .ok();
223    }
224    let mut out = String::with_capacity(bytes.len());
225    for &byte in bytes {
226        out.push(PDFDOC_ENCODING[byte as usize]?);
227    }
228    Some(out)
229}
230
231/// Lossy UTF-16 decoding matching qpdf's `QUtil::utf16_to_utf8`.
232pub(crate) fn lossy_utf16_to_utf8(bytes: &[u8], is_le: bool) -> String {
233    // bytes.len() is a sound capacity hint: each UTF-16 unit is 2 bytes and
234    // expands to 1–3 UTF-8 bytes (4 only for surrogate pairs, which consume
235    // 4 UTF-16 bytes). For ASCII-dominant inputs this slightly over-allocates;
236    // for BMP-heavy inputs it is roughly accurate.
237    let mut out = String::with_capacity(bytes.len());
238    let mut codepoint: u32 = 0;
239    let mut i = 0;
240    while i + 1 < bytes.len() {
241        let (msb_idx, lsb_idx) = if is_le { (i + 1, i) } else { (i, i + 1) };
242        let bits = (u16::from(bytes[msb_idx]) << 8) | u16::from(bytes[lsb_idx]);
243        match bits & 0xFC00 {
244            0xD800 => {
245                codepoint = 0x10000 + ((u32::from(bits) & 0x3FF) << 10);
246                i += 2;
247                continue;
248            }
249            0xDC00 => {
250                codepoint += u32::from(bits) & 0x3FF;
251            }
252            _ => {
253                codepoint = u32::from(bits);
254            }
255        }
256        if let Some(ch) = char::from_u32(codepoint) {
257            out.push(ch);
258        }
259        codepoint = 0;
260        i += 2;
261    }
262    out
263}
264
265#[cfg(test)]
266mod tests {
267    use super::{
268        decode_pdf_text_string, new_unicode_string, normalized_utf8_value, unparse_binary,
269        utf8_value,
270    };
271
272    #[test]
273    fn utf8_value_decodes_pdf_string_encodings() {
274        assert_eq!(utf8_value(b"plain"), b"plain");
275        assert_eq!(utf8_value(&[0x80]), "•".as_bytes());
276        assert_eq!(
277            utf8_value(&[0xfe, 0xff, 0x54, 0x0d, 0x52, 0x4d]),
278            "名前".as_bytes()
279        );
280        assert_eq!(
281            utf8_value(&[0xef, 0xbb, 0xbf, 0xe5, 0x90, 0x8d]),
282            "名".as_bytes()
283        );
284        assert_eq!(utf8_value(&[0x95]), "Ł".as_bytes());
285    }
286
287    #[test]
288    fn utf8_value_replaces_undefined_pdfdoc_bytes() {
289        assert_eq!(utf8_value(&[b'a', 0xad, b'b']), "a�b".as_bytes());
290    }
291
292    #[test]
293    fn utf8_value_preserves_invalid_explicit_utf8_bytes() {
294        assert_eq!(utf8_value(&[0xef, 0xbb, 0xbf, 0xff]), &[0xff]);
295    }
296
297    #[test]
298    fn new_unicode_string_uses_pdfdoc_encoding_when_lossless() {
299        assert_eq!(new_unicode_string(b"ASCII"), b"ASCII");
300        assert_eq!(new_unicode_string("þ".as_bytes()), b"\xfe");
301    }
302
303    #[test]
304    fn new_unicode_string_uses_utf16be_for_unrepresentable_text() {
305        assert_eq!(
306            new_unicode_string("🥔".as_bytes()),
307            b"\xfe\xff\xd8\x3e\xdd\x54"
308        );
309    }
310
311    #[test]
312    fn new_unicode_string_forces_unicode_bom_looking_inputs_to_utf16be() {
313        assert_eq!(
314            new_unicode_string("þÿ".as_bytes()),
315            b"\xfe\xff\x00\xfe\x00\xff"
316        );
317        assert_eq!(
318            new_unicode_string("ÿþ".as_bytes()),
319            b"\xfe\xff\x00\xff\x00\xfe"
320        );
321        assert_eq!(
322            new_unicode_string("".as_bytes()),
323            b"\xfe\xff\x00\xef\x00\xbb\x00\xbf"
324        );
325    }
326
327    #[test]
328    fn new_unicode_string_replaces_malformed_utf8_before_utf16be_encoding() {
329        assert_eq!(
330            new_unicode_string(b"\xfeafter"),
331            b"\xfe\xff\xff\xfd\x00a\x00f\x00t\x00e\x00r"
332        );
333    }
334
335    #[test]
336    fn normalized_utf8_value_matches_qpdf_error_consumption() {
337        assert_eq!(normalized_utf8_value(&[0xc2, b'A']), "�A".as_bytes());
338        assert_eq!(normalized_utf8_value(&[0xc0, 0x80]), "�".as_bytes());
339        assert_eq!(normalized_utf8_value(&[0xc2]), "�".as_bytes());
340        assert_eq!(normalized_utf8_value(&[0x80]), "�".as_bytes());
341        assert_eq!(
342            normalized_utf8_value(&[0xf8, 0x88, 0x80, 0x80, 0x80]),
343            "�".as_bytes()
344        );
345    }
346
347    #[test]
348    fn unparse_binary_uses_lowercase_hex() {
349        assert_eq!(unparse_binary(b"A\n\x80"), b"<410a80>");
350    }
351
352    #[test]
353    fn decode_pdf_text_string_handles_utf16_endianness_and_malformed_lengths() {
354        assert_eq!(
355            decode_pdf_text_string(&[0xfe, 0xff, 0x00, b'A']),
356            Some("A".into())
357        );
358        assert_eq!(
359            decode_pdf_text_string(&[0xff, 0xfe, b'A', 0x00]),
360            Some("A".into())
361        );
362        assert_eq!(decode_pdf_text_string(&[0xfe, 0xff, 0x00]), None);
363        assert_eq!(decode_pdf_text_string(&[0xff, 0xfe, 0x00]), None);
364        assert_eq!(decode_pdf_text_string(&[0xfe, 0xff, 0xd8, 0x00]), None);
365    }
366
367    #[test]
368    fn decode_pdf_text_string_falls_back_to_pdfdoc_encoding_without_a_bom() {
369        assert_eq!(
370            decode_pdf_text_string(b"plain ascii"),
371            Some("plain ascii".into())
372        );
373        // PDFDocEncoding (ISO 32000-1 Annex D.3) diverges from Latin-1 in the
374        // 0x18-0x1F range: 0x18 is BREVE, not the C0 control character U+0018.
375        assert_eq!(decode_pdf_text_string(&[0x18]), Some("\u{02D8}".into()));
376        // 0xA0..=0xFF otherwise follows ISO-8859-1, so 0xE9 decodes to 'é'.
377        assert_eq!(decode_pdf_text_string(&[0xe9]), Some("é".into()));
378        assert_eq!(decode_pdf_text_string(b"caf\xe9"), Some("café".into()));
379    }
380}