Skip to main content

docling_pdf/render/font/
cmap.rs

1//! CMaps for composite fonts (ISO 32000-1, 9.7.5): the byte-length codespace
2//! ranges that split a string into codes, and the `cidchar` / `cidrange`
3//! entries that map codes to CIDs. Embedded CMap streams are read in full;
4//! of the predefined ones only the `Identity` and Unicode (`Uni*-UCS2`,
5//! `Uni*-UTF16`) families are known — the others (no `cmap-resources` in a
6//! Rust checkout) fall back to two-byte codes.
7
8use std::collections::HashMap;
9
10#[derive(Debug, Clone)]
11pub struct CMap {
12    /// `(byte length, low, high)` codespace ranges.
13    codespace: Vec<(usize, u32, u32)>,
14    single: HashMap<u32, u32>,
15    /// `(lo, hi, cid_of_lo)`.
16    ranges: Vec<(u32, u32, u32)>,
17    pub identity: bool,
18    /// The codes are Unicode (UCS-2 / UTF-16 BE) — a predefined `Uni*` CMap.
19    pub unicode_codes: bool,
20    pub vertical: bool,
21}
22
23impl CMap {
24    pub fn identity_h() -> CMap {
25        CMap {
26            codespace: vec![(2, 0, 0xFFFF)],
27            single: HashMap::new(),
28            ranges: Vec::new(),
29            identity: true,
30            unicode_codes: false,
31            vertical: false,
32        }
33    }
34
35    /// A predefined CMap by name.
36    pub fn predefined(name: &[u8]) -> CMap {
37        let s = String::from_utf8_lossy(name);
38        let vertical = s.ends_with("-V");
39        let mut c = CMap::identity_h();
40        c.vertical = vertical;
41        if s.starts_with("Identity") {
42            return c;
43        }
44        // CIDs remain unknown for non-Identity predefined CMaps, but the
45        // Unicode families tell us the codes and their byte lengths.
46        c.identity = false;
47        c.unicode_codes = s.contains("UCS2") || s.contains("UTF16");
48        if s.contains("UTF16") {
49            c.codespace = vec![
50                (2, 0, 0xd7ff),
51                (4, 0xd800dc00, 0xdbffdfff),
52                (2, 0xe000, 0xffff),
53            ];
54        }
55        c
56    }
57
58    /// Parse an embedded CMap stream.
59    pub fn parse(data: &[u8]) -> CMap {
60        let mut c = CMap {
61            codespace: Vec::new(),
62            single: HashMap::new(),
63            ranges: Vec::new(),
64            identity: false,
65            unicode_codes: false,
66            vertical: false,
67        };
68        let toks = tokenize(data);
69        let mut i = 0;
70        while i < toks.len() {
71            match &toks[i] {
72                Tok::Kw(k) if k == "begincodespacerange" => {
73                    i += 1;
74                    while i + 1 < toks.len() {
75                        match (&toks[i], &toks[i + 1]) {
76                            (Tok::Hex(lo), Tok::Hex(hi)) => {
77                                let n = lo.len().clamp(1, 4);
78                                c.codespace.push((n, be(lo), be(hi)));
79                                i += 2;
80                            }
81                            _ => break,
82                        }
83                    }
84                }
85                Tok::Kw(k) if k == "begincidrange" => {
86                    i += 1;
87                    while i + 2 < toks.len() {
88                        match (&toks[i], &toks[i + 1], &toks[i + 2]) {
89                            (Tok::Hex(lo), Tok::Hex(hi), Tok::Num(cid)) => {
90                                c.ranges.push((be(lo), be(hi), *cid as u32));
91                                if c.codespace.is_empty() {
92                                    c.codespace.push((lo.len().clamp(1, 4), 0, u32::MAX));
93                                }
94                                i += 3;
95                            }
96                            _ => break,
97                        }
98                    }
99                }
100                Tok::Kw(k) if k == "begincidchar" => {
101                    i += 1;
102                    while i + 1 < toks.len() {
103                        match (&toks[i], &toks[i + 1]) {
104                            (Tok::Hex(code), Tok::Num(cid)) => {
105                                c.single.insert(be(code), *cid as u32);
106                                if c.codespace.is_empty() {
107                                    c.codespace.push((code.len().clamp(1, 4), 0, u32::MAX));
108                                }
109                                i += 2;
110                            }
111                            _ => break,
112                        }
113                    }
114                }
115                Tok::Kw(k) if k == "usecmap" => {
116                    // `/Identity-H usecmap` and friends: the parent's codes.
117                    if let Some(Tok::Name(n)) = toks.get(i.wrapping_sub(1)) {
118                        if n.starts_with("Identity") {
119                            c.identity = c.single.is_empty() && c.ranges.is_empty();
120                            if c.codespace.is_empty() {
121                                c.codespace.push((2, 0, 0xFFFF));
122                            }
123                        }
124                    }
125                    i += 1;
126                }
127                Tok::Kw(k) if k == "def" => {
128                    if let (Some(Tok::Name(n)), Some(Tok::Num(v))) =
129                        (toks.get(i.wrapping_sub(2)), toks.get(i.wrapping_sub(1)))
130                    {
131                        if n == "WMode" && *v == 1.0 {
132                            c.vertical = true;
133                        }
134                    }
135                    i += 1;
136                }
137                _ => i += 1,
138            }
139        }
140        if c.codespace.is_empty() {
141            c.codespace.push((2, 0, 0xFFFF));
142        }
143        // Shortest byte length first, so a one-byte range wins over a
144        // two-byte one that happens to contain the same leading byte.
145        c.codespace.sort_by_key(|r| r.0);
146        c
147    }
148
149    /// Split `bytes` into `(code, byte length)` pairs.
150    pub fn split(&self, bytes: &[u8]) -> Vec<(u32, usize)> {
151        let mut out = Vec::with_capacity(bytes.len() / 2 + 1);
152        let mut i = 0;
153        while i < bytes.len() {
154            let mut taken = None;
155            for &(n, lo, hi) in &self.codespace {
156                if i + n > bytes.len() {
157                    continue;
158                }
159                let code = bytes[i..i + n]
160                    .iter()
161                    .fold(0u32, |a, &b| (a << 8) | u32::from(b));
162                if code >= lo
163                    && code <= hi
164                    && !(self.unicode_codes
165                        && n == 4
166                        && crate::textparse::unicode_cmap_char(code).is_none())
167                {
168                    taken = Some((code, n));
169                    break;
170                }
171            }
172            let (code, n) = taken.unwrap_or_else(|| {
173                // Not in any range: the shortest codespace length, per 9.7.6.3.
174                let n = self
175                    .codespace
176                    .first()
177                    .map(|r| r.0)
178                    .unwrap_or(1)
179                    .min(bytes.len() - i);
180                let code = bytes[i..i + n]
181                    .iter()
182                    .fold(0u32, |a, &b| (a << 8) | u32::from(b));
183                (code, n)
184            });
185            out.push((code, n));
186            i += n;
187        }
188        out
189    }
190
191    pub fn cid(&self, code: u32) -> u32 {
192        if self.identity {
193            return code;
194        }
195        if let Some(c) = self.single.get(&code) {
196            return *c;
197        }
198        for &(lo, hi, cid) in &self.ranges {
199            if code >= lo && code <= hi {
200                return cid + (code - lo);
201            }
202        }
203        if self.single.is_empty() && self.ranges.is_empty() {
204            code
205        } else {
206            0
207        }
208    }
209}
210
211fn be(bytes: &[u8]) -> u32 {
212    bytes
213        .iter()
214        .take(4)
215        .fold(0u32, |a, &b| (a << 8) | u32::from(b))
216}
217
218enum Tok {
219    Hex(Vec<u8>),
220    Num(f64),
221    Name(String),
222    Kw(String),
223}
224
225fn tokenize(data: &[u8]) -> Vec<Tok> {
226    let mut out = Vec::new();
227    let mut i = 0;
228    while i < data.len() {
229        let b = data[i];
230        match b {
231            b'%' => {
232                while i < data.len() && data[i] != b'\n' && data[i] != b'\r' {
233                    i += 1;
234                }
235            }
236            b'<' => {
237                if data.get(i + 1) == Some(&b'<') {
238                    i += 2;
239                    continue;
240                }
241                let end = data[i..]
242                    .iter()
243                    .position(|&c| c == b'>')
244                    .map(|p| i + p)
245                    .unwrap_or(data.len());
246                let mut bytes = Vec::new();
247                let mut hi: Option<u8> = None;
248                for &c in &data[i + 1..end] {
249                    let v = match c {
250                        b'0'..=b'9' => c - b'0',
251                        b'a'..=b'f' => c - b'a' + 10,
252                        b'A'..=b'F' => c - b'A' + 10,
253                        _ => continue,
254                    };
255                    match hi.take() {
256                        None => hi = Some(v),
257                        Some(h) => bytes.push((h << 4) | v),
258                    }
259                }
260                if let Some(h) = hi {
261                    bytes.push(h << 4);
262                }
263                out.push(Tok::Hex(bytes));
264                i = end + 1;
265            }
266            b'/' => {
267                let start = i + 1;
268                let mut j = start;
269                while j < data.len()
270                    && !data[j].is_ascii_whitespace()
271                    && !b"/<>[](){}%".contains(&data[j])
272                {
273                    j += 1;
274                }
275                out.push(Tok::Name(
276                    String::from_utf8_lossy(&data[start..j]).into_owned(),
277                ));
278                i = j;
279            }
280            b'[' | b']' | b'{' | b'}' | b'>' | b'(' | b')' => i += 1,
281            c if c.is_ascii_whitespace() => i += 1,
282            _ => {
283                let start = i;
284                while i < data.len()
285                    && !data[i].is_ascii_whitespace()
286                    && !b"/<>[](){}%".contains(&data[i])
287                {
288                    i += 1;
289                }
290                let s = String::from_utf8_lossy(&data[start..i]).into_owned();
291                match s.parse::<f64>() {
292                    Ok(v) => out.push(Tok::Num(v)),
293                    Err(_) => out.push(Tok::Kw(s)),
294                }
295            }
296        }
297    }
298    out
299}
300
301#[cfg(test)]
302mod tests {
303    use super::*;
304
305    #[test]
306    fn embedded_cmap_ranges_and_codespaces() {
307        let src = b"%!PS\n/CIDInit /ProcSet findresource begin 12 dict begin begincmap\n1 begincodespacerange <00> <80> <8140> <9ffc> endcodespacerange\n2 begincidrange <20> <7e> 1 <8140> <817e> 633 endcidrange\n1 begincidchar <80> 97 endcidchar\nendcmap";
308        let c = CMap::parse(src);
309        assert_eq!(
310            c.split(b"\x41\x81\x41\x80"),
311            vec![(0x41, 1), (0x8141, 2), (0x80, 1)]
312        );
313        assert_eq!(c.cid(0x41), 1 + 0x21);
314        assert_eq!(c.cid(0x8141), 634);
315        assert_eq!(c.cid(0x80), 97);
316    }
317
318    #[test]
319    fn identity_is_two_byte() {
320        let c = CMap::predefined(b"Identity-H");
321        assert_eq!(c.split(b"\x00\x41\x12\x34"), vec![(0x41, 2), (0x1234, 2)]);
322        assert_eq!(c.cid(0x1234), 0x1234);
323        assert!(CMap::predefined(b"UniGB-UCS2-H").unicode_codes);
324    }
325
326    #[test]
327    fn utf16_pairs_are_four_byte_codes_only_in_utf16_cmaps() {
328        let bytes = b"\x65\xe5\xd8\x42\xdf\xb7\x8a\x9e";
329        for name in [b"UniJIS-UTF16-H".as_slice(), b"UniJIS-UTF16-V"] {
330            assert_eq!(
331                CMap::predefined(name).split(bytes),
332                vec![(0x65e5, 2), (0xd842dfb7, 4), (0x8a9e, 2)]
333            );
334        }
335        for name in [b"Identity-H".as_slice(), b"UniJIS-UCS2-H"] {
336            assert_eq!(
337                CMap::predefined(name).split(bytes),
338                vec![(0x65e5, 2), (0xd842, 2), (0xdfb7, 2), (0x8a9e, 2)]
339            );
340        }
341        let c = CMap::predefined(b"UniJIS-UTF16-H");
342        assert_eq!(c.split(b"\xd8\x42\x65\xe5"), vec![(0xd842, 2), (0x65e5, 2)]);
343        assert_eq!(c.split(b"\xd8\x42"), vec![(0xd842, 2)]);
344    }
345}