pdfboss_core/cmap/
predefined.rs

1//! The predefined CJK CMaps (ISO 32000-1 Table 118, plus ISO 32000-2's
2//! UniAKR-UTF16-H): Adobe's BSD-licensed data files, packed verbatim into
3//! one zlib archive per character collection (`assets/cmaps/`, provenance
4//! and layout in the NOTICE there), decompressed once per collection and
5//! parsed once per CMap on first use. The archives are compiled in only
6//! behind the `predefined-cmaps` Cargo feature; without it every name but
7//! the two Identity mappings resolves to `None` and callers degrade to the
8//! Identity assumption they use today.
9
10use super::CidCmap;
11use std::sync::{Arc, OnceLock};
12
13/// The character collections with packed data, in archive order.
14#[cfg(feature = "predefined-cmaps")]
15#[derive(Clone, Copy)]
16enum Collection {
17    Japan1,
18    Gb1,
19    Cns1,
20    Korea1,
21    Kr,
22}
23
24/// Which collection owns a predefined CMap name; `None` for names ISO 32000
25/// does not predefine (an `/Encoding` naming one must embed it instead).
26///
27/// Covers ISO 32000-1 §9.7.5.2.
28#[cfg(feature = "predefined-cmaps")]
29fn collection_of(name: &str) -> Option<Collection> {
30    match name {
31        "83pv-RKSJ-H" | "90ms-RKSJ-H" | "90ms-RKSJ-V" | "90msp-RKSJ-H" | "90msp-RKSJ-V"
32        | "90pv-RKSJ-H" | "Add-RKSJ-H" | "Add-RKSJ-V" | "EUC-H" | "EUC-V" | "Ext-RKSJ-H"
33        | "Ext-RKSJ-V" | "H" | "V" | "UniJIS-UCS2-H" | "UniJIS-UCS2-V" | "UniJIS-UCS2-HW-H"
34        | "UniJIS-UCS2-HW-V" | "UniJIS-UTF16-H" | "UniJIS-UTF16-V" => Some(Collection::Japan1),
35        "GB-EUC-H" | "GB-EUC-V" | "GBpc-EUC-H" | "GBpc-EUC-V" | "GBK-EUC-H" | "GBK-EUC-V"
36        | "GBKp-EUC-H" | "GBKp-EUC-V" | "GBK2K-H" | "GBK2K-V" | "UniGB-UCS2-H" | "UniGB-UCS2-V"
37        | "UniGB-UTF16-H" | "UniGB-UTF16-V" => Some(Collection::Gb1),
38        "B5pc-H" | "B5pc-V" | "HKscs-B5-H" | "HKscs-B5-V" | "ETen-B5-H" | "ETen-B5-V"
39        | "ETenms-B5-H" | "ETenms-B5-V" | "CNS-EUC-H" | "CNS-EUC-V" | "UniCNS-UCS2-H"
40        | "UniCNS-UCS2-V" | "UniCNS-UTF16-H" | "UniCNS-UTF16-V" => Some(Collection::Cns1),
41        "KSC-EUC-H" | "KSC-EUC-V" | "KSCms-UHC-H" | "KSCms-UHC-V" | "KSCms-UHC-HW-H"
42        | "KSCms-UHC-HW-V" | "KSCpc-EUC-H" | "UniKS-UCS2-H" | "UniKS-UCS2-V" | "UniKS-UTF16-H"
43        | "UniKS-UTF16-V" => Some(Collection::Korea1),
44        "UniAKR-UTF16-H" => Some(Collection::Kr),
45        _ => None,
46    }
47}
48
49/// The predefined CMap called `name`, or `None` when ISO 32000 does not
50/// predefine it (or the `predefined-cmaps` feature left the data out).
51/// Parsed once per name; `usecmap` dependencies resolve within the set.
52///
53/// Covers ISO 32000-1 §9.7.5.2.
54pub fn predefined(name: &str) -> Option<Arc<CidCmap>> {
55    match name {
56        "Identity-H" => Some(Arc::clone(identity(false))),
57        "Identity-V" => Some(Arc::clone(identity(true))),
58        _ => load(name, 0),
59    }
60}
61
62fn identity(vertical: bool) -> &'static Arc<CidCmap> {
63    static H: OnceLock<Arc<CidCmap>> = OnceLock::new();
64    static V: OnceLock<Arc<CidCmap>> = OnceLock::new();
65    let cell = if vertical { &V } else { &H };
66    cell.get_or_init(|| Arc::new(CidCmap::identity(vertical)))
67}
68
69/// A CID-to-Unicode mapping, inverted from a collection's Uni*-UTF16-H
70/// CMap: that file maps UTF-16 code points to CIDs, and running it backward
71/// answers what a CID means when no `/ToUnicode` does. Where several code
72/// points share a CID the lowest wins — the unified ideograph rather than
73/// its compatibility duplicate.
74pub struct CidToUnicode {
75    map: crate::hash::FastMap<u32, char>,
76}
77
78impl CidToUnicode {
79    /// The Unicode scalar for `cid`, if the collection maps one.
80    pub fn lookup(&self, cid: u32) -> Option<char> {
81        self.map.get(&cid).copied()
82    }
83}
84
85/// The CID-to-Unicode mapping for a `/CIDSystemInfo` `/Ordering`, built on
86/// first use. `None` for unknown orderings (Identity included: its CIDs are
87/// font-private and mean nothing outside the font).
88///
89/// Japan1 also folds in UniJIS-UCS2-HW-H: the halfwidth-form CIDs it remaps
90/// Latin and digits onto (231-325) have no preimage in UniJIS-UTF16-H, and
91/// they are what every Shift-JIS ASCII run selects.
92///
93/// Covers ISO 32000-1 §9.7.3.
94pub fn cid_to_unicode(ordering: &str) -> Option<Arc<CidToUnicode>> {
95    let (slot, names): (usize, &[&str]) = match ordering {
96        "Japan1" => (0, &["UniJIS-UTF16-H", "UniJIS-UCS2-HW-H"]),
97        "GB1" => (1, &["UniGB-UTF16-H"]),
98        "CNS1" => (2, &["UniCNS-UTF16-H"]),
99        "Korea1" => (3, &["UniKS-UTF16-H"]),
100        "KR" => (4, &["UniAKR-UTF16-H"]),
101        _ => return None,
102    };
103    static INVERSES: [OnceLock<Option<Arc<CidToUnicode>>>; 5] = [const { OnceLock::new() }; 5];
104    INVERSES[slot]
105        .get_or_init(|| {
106            let cmaps: Vec<_> = names.iter().filter_map(|n| predefined(n)).collect();
107            (!cmaps.is_empty()).then(|| Arc::new(invert(&cmaps)))
108        })
109        .clone()
110}
111
112/// Runs code-to-CID CMaps backward, earlier sources winning. Codes arrive
113/// lowest first within each layer (see `CidCmap::mappings`), so the first
114/// insert for a CID is the lowest code point that reaches it — except that
115/// a radical-block code point (U+2E80..=U+2FDF) loses to the ideograph
116/// sharing its CID: 木 is what a page means, ⽊ is a dictionary artifact
117/// that happens to sort lower.
118fn invert(cmaps: &[Arc<CidCmap>]) -> CidToUnicode {
119    let radical = |c: char| ('\u{2E80}'..='\u{2FDF}').contains(&c);
120    let mut map = crate::hash::FastMap::default();
121    for cmap in cmaps {
122        cmap.mappings(&mut |len, lo, hi, cid| {
123            for offset in 0..=hi.saturating_sub(lo) {
124                let Some(c) = unicode_of(lo + offset, len) else {
125                    continue;
126                };
127                map.entry(cid.saturating_add(offset))
128                    .and_modify(|held| {
129                        if radical(*held) && !radical(c) {
130                            *held = c;
131                        }
132                    })
133                    .or_insert(c);
134            }
135        });
136    }
137    CidToUnicode { map }
138}
139
140/// Reads a code of `len` bytes as UTF-16BE: two bytes are one unit, four a
141/// surrogate pair. Unpaired surrogates answer `None`.
142fn unicode_of(code: u32, len: u8) -> Option<char> {
143    if len != 4 {
144        return char::from_u32(code);
145    }
146    let units = [(code >> 16) as u16, code as u16];
147    char::decode_utf16(units).next()?.ok()
148}
149
150#[cfg(feature = "predefined-cmaps")]
151mod packed {
152    use super::{collection_of, Collection};
153    use crate::cmap::CidCmap;
154    use crate::hash::FastMap;
155    use std::io::Read;
156    use std::ops::Range;
157    use std::sync::{Arc, Mutex, OnceLock};
158
159    static BLOBS: [&[u8]; 5] = [
160        include_bytes!("../../assets/cmaps/adobe-japan1.bin"),
161        include_bytes!("../../assets/cmaps/adobe-gb1.bin"),
162        include_bytes!("../../assets/cmaps/adobe-cns1.bin"),
163        include_bytes!("../../assets/cmaps/adobe-korea1.bin"),
164        include_bytes!("../../assets/cmaps/adobe-kr.bin"),
165    ];
166
167    /// One decompressed collection archive: the concatenated CMap files and
168    /// where each lives.
169    struct Archive {
170        data: Vec<u8>,
171        index: FastMap<String, Range<usize>>,
172    }
173
174    /// Decompresses and indexes a collection on first use. `None` sticks if
175    /// the compiled-in archive will not read, which only a corrupted build
176    /// could cause.
177    fn archive(collection: Collection) -> Option<&'static Archive> {
178        static ARCHIVES: [OnceLock<Option<Archive>>; 5] = [const { OnceLock::new() }; 5];
179        ARCHIVES[collection as usize]
180            .get_or_init(|| unpack(BLOBS[collection as usize]))
181            .as_ref()
182    }
183
184    /// Archive layout (documented in `assets/cmaps/NOTICE`): u32le count,
185    /// per entry a u16le name length + name + u32le data length, then the
186    /// payloads concatenated in entry order.
187    fn unpack(blob: &[u8]) -> Option<Archive> {
188        let mut data = Vec::new();
189        flate2::read::ZlibDecoder::new(blob)
190            .read_to_end(&mut data)
191            .ok()?;
192        let count = u32::from_le_bytes(data.get(..4)?.try_into().ok()?) as usize;
193        let mut pos = 4;
194        let mut sizes = Vec::with_capacity(count);
195        for _ in 0..count {
196            let name_len = u16::from_le_bytes(data.get(pos..pos + 2)?.try_into().ok()?) as usize;
197            let name = String::from_utf8(data.get(pos + 2..pos + 2 + name_len)?.to_vec()).ok()?;
198            pos += 2 + name_len;
199            let size = u32::from_le_bytes(data.get(pos..pos + 4)?.try_into().ok()?) as usize;
200            pos += 4;
201            sizes.push((name, size));
202        }
203        let mut index = FastMap::default();
204        for (name, size) in sizes {
205            let end = pos.checked_add(size)?;
206            data.get(pos..end)?;
207            index.insert(name, pos..end);
208            pos = end;
209        }
210        Some(Archive { data, index })
211    }
212
213    /// Loads and caches one packed CMap, resolving its `usecmap` chain
214    /// within the predefined set (bounded depth; the shipped chains are at
215    /// most one deep).
216    pub fn load(name: &str, depth: usize) -> Option<Arc<CidCmap>> {
217        if depth > 4 {
218            return None;
219        }
220        let collection = collection_of(name)?;
221        static PARSED: OnceLock<Mutex<FastMap<String, Arc<CidCmap>>>> = OnceLock::new();
222        let cache = PARSED.get_or_init(|| Mutex::new(FastMap::default()));
223        if let Some(cmap) = cache.lock().unwrap().get(name) {
224            return Some(Arc::clone(cmap));
225        }
226        let archive = archive(collection)?;
227        let span = archive.index.get(name)?.clone();
228        let mut resolve = |n: &str| super::predefined_at(n, depth + 1);
229        let cmap = Arc::new(CidCmap::parse_with(&archive.data[span], None, &mut resolve));
230        cache
231            .lock()
232            .unwrap()
233            .insert(name.to_owned(), Arc::clone(&cmap));
234        Some(cmap)
235    }
236}
237
238#[cfg(feature = "predefined-cmaps")]
239use packed::load;
240
241#[cfg(not(feature = "predefined-cmaps"))]
242fn load(_name: &str, _depth: usize) -> Option<Arc<CidCmap>> {
243    None
244}
245
246/// [`predefined`] with the `usecmap` recursion depth threaded through.
247#[cfg(feature = "predefined-cmaps")]
248fn predefined_at(name: &str, depth: usize) -> Option<Arc<CidCmap>> {
249    match name {
250        "Identity-H" => Some(Arc::clone(identity(false))),
251        "Identity-V" => Some(Arc::clone(identity(true))),
252        _ => load(name, depth),
253    }
254}
255
256#[cfg(all(test, feature = "predefined-cmaps"))]
257mod tests {
258    use super::*;
259
260    // Covers ISO 32000-1 §9.7.5.2.
261    #[test]
262    fn rksj_h_splits_and_maps_hiragana() {
263        let c = predefined("90ms-RKSJ-H").unwrap();
264        assert!(!c.vertical());
265        // あ is Shift-JIS <82A0>, inside the data file's range
266        // `<829f> <82f1> 842`, and 1-byte ASCII splits stay 1 byte.
267        let bytes = [0x41, 0x82, 0xA0];
268        assert_eq!(c.code_at(&bytes, 0), (0x41, 1));
269        assert_eq!(c.code_at(&bytes, 1), (0x82A0, 2));
270        assert_eq!(c.cid(0x82A0, 2), Some(843));
271        assert!(c.single_byte(0x20));
272    }
273
274    // Covers ISO 32000-1 §9.7.5.2.
275    #[test]
276    fn rksj_v_layers_vertical_variants_over_the_h_base() {
277        let v = predefined("90ms-RKSJ-V").unwrap();
278        assert!(v.vertical());
279        // Its own `<8141> <8142> 7887` beats the inherited horizontal 、.
280        assert_eq!(v.cid(0x8141, 2), Some(7887));
281        // Untouched codes fall through to 90ms-RKSJ-H.
282        assert_eq!(v.cid(0x82A0, 2), Some(843));
283        let h = v.parent().expect("usecmap parent");
284        assert_eq!(h.cid(0x8141, 2), Some(634));
285    }
286
287    // Covers ISO 32000-1 §9.7.5.2.
288    #[test]
289    fn every_shipped_name_loads_and_identity_needs_no_data() {
290        for name in [
291            "83pv-RKSJ-H",
292            "90msp-RKSJ-V",
293            "90pv-RKSJ-H",
294            "Add-RKSJ-V",
295            "EUC-H",
296            "Ext-RKSJ-V",
297            "H",
298            "V",
299            "UniJIS-UCS2-H",
300            "UniJIS-UCS2-HW-V",
301            "UniJIS-UTF16-V",
302            "GB-EUC-V",
303            "GBpc-EUC-H",
304            "GBK-EUC-H",
305            "GBKp-EUC-V",
306            "GBK2K-H",
307            "UniGB-UCS2-V",
308            "UniGB-UTF16-H",
309            "B5pc-V",
310            "HKscs-B5-H",
311            "ETen-B5-V",
312            "ETenms-B5-H",
313            "CNS-EUC-H",
314            "UniCNS-UCS2-V",
315            "UniCNS-UTF16-H",
316            "KSC-EUC-H",
317            "KSCms-UHC-V",
318            "KSCms-UHC-HW-H",
319            "KSCpc-EUC-H",
320            "UniKS-UCS2-V",
321            "UniKS-UTF16-H",
322            "UniAKR-UTF16-H",
323        ] {
324            let c = predefined(name).unwrap_or_else(|| panic!("{name} must load"));
325            assert!(!c.is_empty(), "{name} parsed to nothing");
326            assert_eq!(c.vertical(), name.ends_with("-V") || name == "V", "{name}");
327        }
328        assert!(predefined("Identity-H").is_some());
329        assert!(predefined("Identity-V").unwrap().vertical());
330        assert!(predefined("UniJIS2004-UTF16-H").is_none()); // not Table 118
331        assert!(predefined("WinAnsiEncoding").is_none());
332    }
333
334    // Covers ISO 32000-1 §9.10.2 and §9.7.3.
335    #[test]
336    fn japan1_cids_read_back_as_unicode() {
337        let inv = cid_to_unicode("Japan1").unwrap();
338        assert_eq!(inv.lookup(843), Some('あ'));
339        // CID 1 is the space in every Adobe collection.
340        assert_eq!(inv.lookup(1), Some(' '));
341        // The halfwidth-form digit 1 (CID 248, what Shift-JIS ASCII runs
342        // select): reachable only through UniJIS-UCS2-HW-H.
343        assert_eq!(inv.lookup(248), Some('1'));
344        // 木 (CID 3814): the Kangxi radical ⽊ shares the CID and sorts
345        // lower, but the ideograph is what a page means.
346        assert_eq!(inv.lookup(3814), Some('木'));
347        assert!(cid_to_unicode("Identity").is_none());
348        assert!(cid_to_unicode("Unknown").is_none());
349    }
350}
pdfboss_core/cmap/predefined.rs

pdfboss_core/cmap/
predefined.rs
