1const PDFDOC_ENCODING: [Option<char>; 256] = build_pdfdoc_table();
10
11const fn build_pdfdoc_table() -> [Option<char>; 256] {
13 let mut table: [Option<char>; 256] = [None; 256];
14 table[0x08] = Some('\u{0008}');
15 table[0x09] = Some('\t');
16 table[0x0A] = Some('\n');
17 table[0x0B] = Some('\u{000B}');
18 table[0x0C] = Some('\u{000C}');
19 table[0x0D] = Some('\r');
20 table[0x18] = Some('\u{02D8}');
21 table[0x19] = Some('\u{02C7}');
22 table[0x1A] = Some('\u{02C6}');
23 table[0x1B] = Some('\u{02D9}');
24 table[0x1C] = Some('\u{02DD}');
25 table[0x1D] = Some('\u{02DB}');
26 table[0x1E] = Some('\u{02DA}');
27 table[0x1F] = Some('\u{02DC}');
28 let mut b = 0x20u8;
29 while b <= 0x7E {
30 table[b as usize] = Some(b as char);
31 b += 1;
32 }
33 table[0x80] = Some('\u{2022}');
34 table[0x81] = Some('\u{2020}');
35 table[0x82] = Some('\u{2021}');
36 table[0x83] = Some('\u{2026}');
37 table[0x84] = Some('\u{2014}');
38 table[0x85] = Some('\u{2013}');
39 table[0x86] = Some('\u{0192}');
40 table[0x87] = Some('\u{2044}');
41 table[0x88] = Some('\u{2039}');
42 table[0x89] = Some('\u{203A}');
43 table[0x8A] = Some('\u{2212}');
44 table[0x8B] = Some('\u{2030}');
45 table[0x8C] = Some('\u{201E}');
46 table[0x8D] = Some('\u{201C}');
47 table[0x8E] = Some('\u{201D}');
48 table[0x8F] = Some('\u{2018}');
49 table[0x90] = Some('\u{2019}');
50 table[0x91] = Some('\u{201A}');
51 table[0x92] = Some('\u{2122}');
52 table[0x93] = Some('\u{FB01}');
53 table[0x94] = Some('\u{FB02}');
54 table[0x95] = Some('\u{0141}');
55 table[0x96] = Some('\u{0152}');
56 table[0x97] = Some('\u{0160}');
57 table[0x98] = Some('\u{0178}');
58 table[0x99] = Some('\u{017D}');
59 table[0x9A] = Some('\u{0131}');
60 table[0x9B] = Some('\u{0142}');
61 table[0x9C] = Some('\u{0153}');
62 table[0x9D] = Some('\u{0161}');
63 table[0x9E] = Some('\u{017E}');
64 table[0xA0] = Some('\u{20AC}');
65 let mut b = 0xA1u8;
66 loop {
67 table[b as usize] = Some(b as char);
68 if b == 0xFF {
69 break;
70 }
71 b += 1;
72 }
73 table
74}
75pub fn utf8_value(bytes: &[u8]) -> Vec<u8> {
79 if let Some(rest) = bytes.strip_prefix(&[0xfe, 0xff]) {
80 return lossy_utf16_to_utf8(rest, false).into_bytes();
81 }
82 if let Some(rest) = bytes.strip_prefix(&[0xff, 0xfe]) {
83 return lossy_utf16_to_utf8(rest, true).into_bytes();
84 }
85 if let Some(rest) = bytes.strip_prefix(&[0xef, 0xbb, 0xbf]) {
86 return rest.to_vec();
87 }
88
89 bytes
90 .iter()
91 .map(|&byte| match byte {
92 0x7f | 0x9f | 0xad => '\u{fffd}',
93 _ => PDFDOC_ENCODING[byte as usize].unwrap_or(byte as char),
94 })
95 .collect::<String>()
96 .into_bytes()
97}
98
99pub(crate) fn normalized_utf8_value(utf8: &[u8]) -> Vec<u8> {
101 normalize_utf8(utf8).0
102}
103
104fn normalize_utf8(utf8: &[u8]) -> (Vec<u8>, bool) {
105 let mut result = Vec::with_capacity(utf8.len());
106 let mut position = 0;
107 let mut had_error = false;
108 while position < utf8.len() {
109 let original_position = position;
110 let mut byte = utf8[position];
111 position += 1;
112
113 if byte < 0x80 {
114 result.push(byte);
115 continue;
116 }
117
118 let mut bytes_needed = 0;
119 let mut bit_check = 0x40;
120 let mut to_clear = 0x80;
121 while byte & bit_check != 0 {
122 bytes_needed += 1;
123 to_clear |= bit_check;
124 bit_check >>= 1;
125 }
126
127 let mut error = !(1..=5).contains(&bytes_needed) || position + bytes_needed > utf8.len();
128 let mut codepoint = 0xfffd;
129 if !error {
130 codepoint = u32::from(byte & !to_clear);
131 for _ in 0..bytes_needed {
132 byte = utf8[position];
133 position += 1;
134 if byte & 0xc0 != 0x80 {
135 position -= 1;
136 error = true;
137 break;
138 }
139 codepoint = (codepoint << 6) + u32::from(byte & 0x3f);
140 }
141
142 if !error {
143 let lower_bounds = [0, 0, 1 << 7, 1 << 11, 1 << 16, 1 << 12, 1 << 26];
144 let lower_bound = lower_bounds[position - original_position];
145 if lower_bound > 0 && codepoint < lower_bound {
146 error = true;
147 }
148 }
149 }
150
151 had_error |= error;
152 let scalar = if error {
153 '\u{fffd}'
154 } else {
155 char::from_u32(codepoint).unwrap_or('\u{fffd}')
156 };
157 let mut encoded = [0; 4];
158 result.extend_from_slice(scalar.encode_utf8(&mut encoded).as_bytes());
159 }
160 (result, had_error)
161}
162
163pub fn new_unicode_string(utf8: &[u8]) -> Vec<u8> {
165 let (normalized, had_error) = normalize_utf8(utf8);
166 let text = std::str::from_utf8(&normalized)
167 .expect("qpdf UTF-8 normalization always emits valid Unicode");
168 let mut pdfdoc = Vec::with_capacity(text.len());
169 let resembles_unicode_bom = utf8.starts_with("þÿ".as_bytes())
170 || utf8.starts_with("ÿþ".as_bytes())
171 || utf8.starts_with("".as_bytes());
172 if !had_error && !resembles_unicode_bom {
173 for character in text.chars() {
174 let mut encoded_character = [0; 4];
175 let encoded_character = character.encode_utf8(&mut encoded_character).as_bytes();
176 let encoded = (0_u16..=u16::from(u8::MAX))
177 .map(|byte| byte as u8)
178 .filter(|&byte| !matches!(byte, 0x7f | 0x9f | 0xad))
179 .find(|&byte| utf8_value(&[byte]) == encoded_character);
180 let Some(encoded) = encoded else {
181 return crate::filespec_helper::encode_utf16be(text);
182 };
183 pdfdoc.push(encoded);
184 }
185 return pdfdoc;
186 }
187 crate::filespec_helper::encode_utf16be(text)
188}
189
190pub fn unparse_binary(stored: &[u8]) -> Vec<u8> {
192 let mut output = Vec::with_capacity(stored.len().saturating_mul(2).saturating_add(2));
193 crate::writer::output::with_buffer_sink(&mut output, |out| {
194 crate::pdf_syntax::write_hex_string(out, stored)
195 })
196 .expect("writing a PDF string to a Vec cannot fail");
197 output
198}
199
200pub(crate) fn decode_pdf_text_string(bytes: &[u8]) -> Option<String> {
202 if let Some(rest) = bytes.strip_prefix(&[0xFE, 0xFF]) {
203 if rest.len() % 2 != 0 {
204 return None;
205 }
206 let units = rest
207 .chunks_exact(2)
208 .map(|c| u16::from_be_bytes([c[0], c[1]]));
209 return char::decode_utf16(units)
210 .collect::<Result<String, _>>()
211 .ok();
212 }
213 if let Some(rest) = bytes.strip_prefix(&[0xFF, 0xFE]) {
214 if rest.len() % 2 != 0 {
215 return None;
216 }
217 let units = rest
218 .chunks_exact(2)
219 .map(|c| u16::from_le_bytes([c[0], c[1]]));
220 return char::decode_utf16(units)
221 .collect::<Result<String, _>>()
222 .ok();
223 }
224 let mut out = String::with_capacity(bytes.len());
225 for &byte in bytes {
226 out.push(PDFDOC_ENCODING[byte as usize]?);
227 }
228 Some(out)
229}
230
231pub(crate) fn lossy_utf16_to_utf8(bytes: &[u8], is_le: bool) -> String {
233 let mut out = String::with_capacity(bytes.len());
238 let mut codepoint: u32 = 0;
239 let mut i = 0;
240 while i + 1 < bytes.len() {
241 let (msb_idx, lsb_idx) = if is_le { (i + 1, i) } else { (i, i + 1) };
242 let bits = (u16::from(bytes[msb_idx]) << 8) | u16::from(bytes[lsb_idx]);
243 match bits & 0xFC00 {
244 0xD800 => {
245 codepoint = 0x10000 + ((u32::from(bits) & 0x3FF) << 10);
246 i += 2;
247 continue;
248 }
249 0xDC00 => {
250 codepoint += u32::from(bits) & 0x3FF;
251 }
252 _ => {
253 codepoint = u32::from(bits);
254 }
255 }
256 if let Some(ch) = char::from_u32(codepoint) {
257 out.push(ch);
258 }
259 codepoint = 0;
260 i += 2;
261 }
262 out
263}
264
265#[cfg(test)]
266mod tests {
267 use super::{
268 decode_pdf_text_string, new_unicode_string, normalized_utf8_value, unparse_binary,
269 utf8_value,
270 };
271
272 #[test]
273 fn utf8_value_decodes_pdf_string_encodings() {
274 assert_eq!(utf8_value(b"plain"), b"plain");
275 assert_eq!(utf8_value(&[0x80]), "•".as_bytes());
276 assert_eq!(
277 utf8_value(&[0xfe, 0xff, 0x54, 0x0d, 0x52, 0x4d]),
278 "名前".as_bytes()
279 );
280 assert_eq!(
281 utf8_value(&[0xef, 0xbb, 0xbf, 0xe5, 0x90, 0x8d]),
282 "名".as_bytes()
283 );
284 assert_eq!(utf8_value(&[0x95]), "Ł".as_bytes());
285 }
286
287 #[test]
288 fn utf8_value_replaces_undefined_pdfdoc_bytes() {
289 assert_eq!(utf8_value(&[b'a', 0xad, b'b']), "a�b".as_bytes());
290 }
291
292 #[test]
293 fn utf8_value_preserves_invalid_explicit_utf8_bytes() {
294 assert_eq!(utf8_value(&[0xef, 0xbb, 0xbf, 0xff]), &[0xff]);
295 }
296
297 #[test]
298 fn new_unicode_string_uses_pdfdoc_encoding_when_lossless() {
299 assert_eq!(new_unicode_string(b"ASCII"), b"ASCII");
300 assert_eq!(new_unicode_string("þ".as_bytes()), b"\xfe");
301 }
302
303 #[test]
304 fn new_unicode_string_uses_utf16be_for_unrepresentable_text() {
305 assert_eq!(
306 new_unicode_string("🥔".as_bytes()),
307 b"\xfe\xff\xd8\x3e\xdd\x54"
308 );
309 }
310
311 #[test]
312 fn new_unicode_string_forces_unicode_bom_looking_inputs_to_utf16be() {
313 assert_eq!(
314 new_unicode_string("þÿ".as_bytes()),
315 b"\xfe\xff\x00\xfe\x00\xff"
316 );
317 assert_eq!(
318 new_unicode_string("ÿþ".as_bytes()),
319 b"\xfe\xff\x00\xff\x00\xfe"
320 );
321 assert_eq!(
322 new_unicode_string("".as_bytes()),
323 b"\xfe\xff\x00\xef\x00\xbb\x00\xbf"
324 );
325 }
326
327 #[test]
328 fn new_unicode_string_replaces_malformed_utf8_before_utf16be_encoding() {
329 assert_eq!(
330 new_unicode_string(b"\xfeafter"),
331 b"\xfe\xff\xff\xfd\x00a\x00f\x00t\x00e\x00r"
332 );
333 }
334
335 #[test]
336 fn normalized_utf8_value_matches_qpdf_error_consumption() {
337 assert_eq!(normalized_utf8_value(&[0xc2, b'A']), "�A".as_bytes());
338 assert_eq!(normalized_utf8_value(&[0xc0, 0x80]), "�".as_bytes());
339 assert_eq!(normalized_utf8_value(&[0xc2]), "�".as_bytes());
340 assert_eq!(normalized_utf8_value(&[0x80]), "�".as_bytes());
341 assert_eq!(
342 normalized_utf8_value(&[0xf8, 0x88, 0x80, 0x80, 0x80]),
343 "�".as_bytes()
344 );
345 }
346
347 #[test]
348 fn unparse_binary_uses_lowercase_hex() {
349 assert_eq!(unparse_binary(b"A\n\x80"), b"<410a80>");
350 }
351
352 #[test]
353 fn decode_pdf_text_string_handles_utf16_endianness_and_malformed_lengths() {
354 assert_eq!(
355 decode_pdf_text_string(&[0xfe, 0xff, 0x00, b'A']),
356 Some("A".into())
357 );
358 assert_eq!(
359 decode_pdf_text_string(&[0xff, 0xfe, b'A', 0x00]),
360 Some("A".into())
361 );
362 assert_eq!(decode_pdf_text_string(&[0xfe, 0xff, 0x00]), None);
363 assert_eq!(decode_pdf_text_string(&[0xff, 0xfe, 0x00]), None);
364 assert_eq!(decode_pdf_text_string(&[0xfe, 0xff, 0xd8, 0x00]), None);
365 }
366
367 #[test]
368 fn decode_pdf_text_string_falls_back_to_pdfdoc_encoding_without_a_bom() {
369 assert_eq!(
370 decode_pdf_text_string(b"plain ascii"),
371 Some("plain ascii".into())
372 );
373 assert_eq!(decode_pdf_text_string(&[0x18]), Some("\u{02D8}".into()));
376 assert_eq!(decode_pdf_text_string(&[0xe9]), Some("é".into()));
378 assert_eq!(decode_pdf_text_string(b"caf\xe9"), Some("café".into()));
379 }
380}