1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
use std::collections::HashMap;
/// Parse a ToUnicode CMap's `bfchar` / `bfrange` sections into code→string.
pub(crate) fn parse_tounicode(data: &[u8]) -> HashMap<u32, String> {
let text = String::from_utf8_lossy(data);
let mut map = HashMap::new();
let hex = |s: &str| -> Option<Vec<u16>> {
let s = s.trim();
if !s.starts_with('<') || !s.ends_with('>') {
return None;
}
let h = &s[1..s.len() - 1];
let bytes: Vec<u8> = (0..h.len())
.step_by(2)
.filter_map(|i| u8::from_str_radix(h.get(i..i + 2)?, 16).ok())
.collect();
Some(
bytes
.chunks(2)
.map(|c| {
if c.len() == 2 {
u16::from_be_bytes([c[0], c[1]])
} else {
c[0] as u16
}
})
.collect(),
)
};
// None for a lone surrogate, which leaves the code unmapped.
let u16s_to_string = |u: &[u16]| {
char::decode_utf16(u.iter().copied())
.collect::<Result<String, _>>()
.ok()
};
let code_of = |u: &[u16]| u.iter().fold(0u32, |acc, &x| (acc << 16) | x as u32);
// Tokenize by structure, not whitespace: CMap hex groups are often written
// back-to-back with no separators (`<21><21><0054>`), so scan for `<…>`
// groups, `[`/`]` brackets, and bareword keywords.
let tokens: Vec<String> = {
let bytes = text.as_bytes();
let mut toks = Vec::new();
let mut i = 0;
while i < bytes.len() {
let c = bytes[i];
if c.is_ascii_whitespace() {
i += 1;
} else if c == b'<' {
let start = i;
while i < bytes.len() && bytes[i] != b'>' {
i += 1;
}
i += 1; // include '>'
toks.push(String::from_utf8_lossy(&bytes[start..i.min(bytes.len())]).into_owned());
} else if c == b'[' || c == b']' {
toks.push((c as char).to_string());
i += 1;
} else {
let start = i;
while i < bytes.len()
&& !bytes[i].is_ascii_whitespace()
&& bytes[i] != b'<'
&& bytes[i] != b'['
&& bytes[i] != b']'
{
i += 1;
}
toks.push(String::from_utf8_lossy(&bytes[start..i]).into_owned());
}
}
toks
};
let tokens: Vec<&str> = tokens.iter().map(|s| s.as_str()).collect();
let mut i = 0;
while i < tokens.len() {
match tokens[i] {
"beginbfchar" => {
i += 1;
while i + 1 < tokens.len() && tokens[i] != "endbfchar" {
if let (Some(src), Some(s)) = (
hex(tokens[i]),
hex(tokens[i + 1]).and_then(|d| u16s_to_string(&d)),
) {
map.insert(code_of(&src), s);
}
i += 2;
}
}
"beginbfrange" => {
i += 1;
while i + 2 < tokens.len() && tokens[i] != "endbfrange" {
let (Some(lo), Some(hi)) = (hex(tokens[i]), hex(tokens[i + 1])) else {
i += 1;
continue;
};
let lo = code_of(&lo);
let hi = code_of(&hi);
if tokens[i + 2] == "[" {
// `<lo> <hi> [ <d0> <d1> ... ]`: one dst per code in the range.
let mut j = i + 3;
let mut code = lo;
while j < tokens.len() && tokens[j] != "]" {
if let Some(s) = hex(tokens[j]).and_then(|d| u16s_to_string(&d)) {
map.insert(code, s);
}
code += 1;
j += 1;
}
i = j + 1;
} else if let Some(mut dst) = hex(tokens[i + 2]) {
// Increment the last unit for at most 65,536 codes; the
// range ends where it overflows or stops being UTF-16.
for code in (lo..=hi).take(0x10000) {
let Some(s) = u16s_to_string(&dst) else { break };
map.insert(code, s);
match dst.last_mut() {
Some(unit) if *unit < u16::MAX => *unit += 1,
Some(_) => break,
None => {}
}
}
i += 3;
} else {
i += 1;
}
}
}
_ => i += 1,
}
}
map
}