pdfboss_core/cmap/
predefined.rs

1//! The predefined CJK CMaps (ISO 32000-1 Table 118, plus ISO 32000-2's
2//! UniAKR-UTF16-H): Adobe's BSD-licensed data files, packed verbatim into
3//! one zlib archive per character collection (`assets/cmaps/`, provenance
4//! and layout in the NOTICE there), decompressed once per collection and
5//! parsed once per CMap on first use. The archives are compiled in only
6//! behind the `predefined-cmaps` Cargo feature; without it every name but
7//! the two Identity mappings resolves to `None` and callers degrade to the
8//! Identity assumption they use today.
9
10use super::CidCmap;
11use std::sync::{Arc, OnceLock};
12
13/// The character collections with packed data, in archive order.
14#[cfg(feature = "predefined-cmaps")]
15#[derive(Clone, Copy)]
16enum Collection {
17    Japan1,
18    Gb1,
19    Cns1,
20    Korea1,
21    Kr,
22}
23
24/// Which collection owns a predefined CMap name; `None` for names ISO 32000
25/// does not predefine (an `/Encoding` naming one must embed it instead).
26#[cfg(feature = "predefined-cmaps")]
27fn collection_of(name: &str) -> Option<Collection> {
28    match name {
29        "83pv-RKSJ-H" | "90ms-RKSJ-H" | "90ms-RKSJ-V" | "90msp-RKSJ-H" | "90msp-RKSJ-V"
30        | "90pv-RKSJ-H" | "Add-RKSJ-H" | "Add-RKSJ-V" | "EUC-H" | "EUC-V" | "Ext-RKSJ-H"
31        | "Ext-RKSJ-V" | "H" | "V" | "UniJIS-UCS2-H" | "UniJIS-UCS2-V" | "UniJIS-UCS2-HW-H"
32        | "UniJIS-UCS2-HW-V" | "UniJIS-UTF16-H" | "UniJIS-UTF16-V" => Some(Collection::Japan1),
33        "GB-EUC-H" | "GB-EUC-V" | "GBpc-EUC-H" | "GBpc-EUC-V" | "GBK-EUC-H" | "GBK-EUC-V"
34        | "GBKp-EUC-H" | "GBKp-EUC-V" | "GBK2K-H" | "GBK2K-V" | "UniGB-UCS2-H" | "UniGB-UCS2-V"
35        | "UniGB-UTF16-H" | "UniGB-UTF16-V" => Some(Collection::Gb1),
36        "B5pc-H" | "B5pc-V" | "HKscs-B5-H" | "HKscs-B5-V" | "ETen-B5-H" | "ETen-B5-V"
37        | "ETenms-B5-H" | "ETenms-B5-V" | "CNS-EUC-H" | "CNS-EUC-V" | "UniCNS-UCS2-H"
38        | "UniCNS-UCS2-V" | "UniCNS-UTF16-H" | "UniCNS-UTF16-V" => Some(Collection::Cns1),
39        "KSC-EUC-H" | "KSC-EUC-V" | "KSCms-UHC-H" | "KSCms-UHC-V" | "KSCms-UHC-HW-H"
40        | "KSCms-UHC-HW-V" | "KSCpc-EUC-H" | "UniKS-UCS2-H" | "UniKS-UCS2-V" | "UniKS-UTF16-H"
41        | "UniKS-UTF16-V" => Some(Collection::Korea1),
42        "UniAKR-UTF16-H" => Some(Collection::Kr),
43        _ => None,
44    }
45}
46
47/// The predefined CMap called `name`, or `None` when ISO 32000 does not
48/// predefine it (or the `predefined-cmaps` feature left the data out).
49/// Parsed once per name; `usecmap` dependencies resolve within the set.
50pub fn predefined(name: &str) -> Option<Arc<CidCmap>> {
51    match name {
52        "Identity-H" => Some(Arc::clone(identity(false))),
53        "Identity-V" => Some(Arc::clone(identity(true))),
54        _ => load(name, 0),
55    }
56}
57
58fn identity(vertical: bool) -> &'static Arc<CidCmap> {
59    static H: OnceLock<Arc<CidCmap>> = OnceLock::new();
60    static V: OnceLock<Arc<CidCmap>> = OnceLock::new();
61    let cell = if vertical { &V } else { &H };
62    cell.get_or_init(|| Arc::new(CidCmap::identity(vertical)))
63}
64
65/// A CID-to-Unicode mapping, inverted from a collection's Uni*-UTF16-H
66/// CMap: that file maps UTF-16 code points to CIDs, and running it backward
67/// answers what a CID means when no `/ToUnicode` does. Where several code
68/// points share a CID the lowest wins — the unified ideograph rather than
69/// its compatibility duplicate.
70pub struct CidToUnicode {
71    map: crate::hash::FastMap<u32, char>,
72}
73
74impl CidToUnicode {
75    /// The Unicode scalar for `cid`, if the collection maps one.
76    pub fn lookup(&self, cid: u32) -> Option<char> {
77        self.map.get(&cid).copied()
78    }
79}
80
81/// The CID-to-Unicode mapping for a `/CIDSystemInfo` `/Ordering`, built on
82/// first use. `None` for unknown orderings (Identity included: its CIDs are
83/// font-private and mean nothing outside the font).
84///
85/// Japan1 also folds in UniJIS-UCS2-HW-H: the halfwidth-form CIDs it remaps
86/// Latin and digits onto (231-325) have no preimage in UniJIS-UTF16-H, and
87/// they are what every Shift-JIS ASCII run selects.
88pub fn cid_to_unicode(ordering: &str) -> Option<Arc<CidToUnicode>> {
89    let (slot, names): (usize, &[&str]) = match ordering {
90        "Japan1" => (0, &["UniJIS-UTF16-H", "UniJIS-UCS2-HW-H"]),
91        "GB1" => (1, &["UniGB-UTF16-H"]),
92        "CNS1" => (2, &["UniCNS-UTF16-H"]),
93        "Korea1" => (3, &["UniKS-UTF16-H"]),
94        "KR" => (4, &["UniAKR-UTF16-H"]),
95        _ => return None,
96    };
97    static INVERSES: [OnceLock<Option<Arc<CidToUnicode>>>; 5] = [const { OnceLock::new() }; 5];
98    INVERSES[slot]
99        .get_or_init(|| {
100            let cmaps: Vec<_> = names.iter().filter_map(|n| predefined(n)).collect();
101            (!cmaps.is_empty()).then(|| Arc::new(invert(&cmaps)))
102        })
103        .clone()
104}
105
106/// Runs code-to-CID CMaps backward, earlier sources winning. Codes arrive
107/// lowest first within each layer (see `CidCmap::mappings`), so the first
108/// insert for a CID is the lowest code point that reaches it — except that
109/// a radical-block code point (U+2E80..=U+2FDF) loses to the ideograph
110/// sharing its CID: 木 is what a page means, ⽊ is a dictionary artifact
111/// that happens to sort lower.
112fn invert(cmaps: &[Arc<CidCmap>]) -> CidToUnicode {
113    let radical = |c: char| ('\u{2E80}'..='\u{2FDF}').contains(&c);
114    let mut map = crate::hash::FastMap::default();
115    for cmap in cmaps {
116        cmap.mappings(&mut |len, lo, hi, cid| {
117            for offset in 0..=hi.saturating_sub(lo) {
118                let Some(c) = unicode_of(lo + offset, len) else {
119                    continue;
120                };
121                map.entry(cid.saturating_add(offset))
122                    .and_modify(|held| {
123                        if radical(*held) && !radical(c) {
124                            *held = c;
125                        }
126                    })
127                    .or_insert(c);
128            }
129        });
130    }
131    CidToUnicode { map }
132}
133
134/// Reads a code of `len` bytes as UTF-16BE: two bytes are one unit, four a
135/// surrogate pair. Unpaired surrogates answer `None`.
136fn unicode_of(code: u32, len: u8) -> Option<char> {
137    if len != 4 {
138        return char::from_u32(code);
139    }
140    let units = [(code >> 16) as u16, code as u16];
141    char::decode_utf16(units).next()?.ok()
142}
143
144#[cfg(feature = "predefined-cmaps")]
145mod packed {
146    use super::{collection_of, Collection};
147    use crate::cmap::CidCmap;
148    use crate::hash::FastMap;
149    use std::io::Read;
150    use std::ops::Range;
151    use std::sync::{Arc, Mutex, OnceLock};
152
153    static BLOBS: [&[u8]; 5] = [
154        include_bytes!("../../assets/cmaps/adobe-japan1.bin"),
155        include_bytes!("../../assets/cmaps/adobe-gb1.bin"),
156        include_bytes!("../../assets/cmaps/adobe-cns1.bin"),
157        include_bytes!("../../assets/cmaps/adobe-korea1.bin"),
158        include_bytes!("../../assets/cmaps/adobe-kr.bin"),
159    ];
160
161    /// One decompressed collection archive: the concatenated CMap files and
162    /// where each lives.
163    struct Archive {
164        data: Vec<u8>,
165        index: FastMap<String, Range<usize>>,
166    }
167
168    /// Decompresses and indexes a collection on first use. `None` sticks if
169    /// the compiled-in archive will not read, which only a corrupted build
170    /// could cause.
171    fn archive(collection: Collection) -> Option<&'static Archive> {
172        static ARCHIVES: [OnceLock<Option<Archive>>; 5] = [const { OnceLock::new() }; 5];
173        ARCHIVES[collection as usize]
174            .get_or_init(|| unpack(BLOBS[collection as usize]))
175            .as_ref()
176    }
177
178    /// Archive layout (documented in `assets/cmaps/NOTICE`): u32le count,
179    /// per entry a u16le name length + name + u32le data length, then the
180    /// payloads concatenated in entry order.
181    fn unpack(blob: &[u8]) -> Option<Archive> {
182        let mut data = Vec::new();
183        flate2::read::ZlibDecoder::new(blob)
184            .read_to_end(&mut data)
185            .ok()?;
186        let count = u32::from_le_bytes(data.get(..4)?.try_into().ok()?) as usize;
187        let mut pos = 4;
188        let mut sizes = Vec::with_capacity(count);
189        for _ in 0..count {
190            let name_len = u16::from_le_bytes(data.get(pos..pos + 2)?.try_into().ok()?) as usize;
191            let name = String::from_utf8(data.get(pos + 2..pos + 2 + name_len)?.to_vec()).ok()?;
192            pos += 2 + name_len;
193            let size = u32::from_le_bytes(data.get(pos..pos + 4)?.try_into().ok()?) as usize;
194            pos += 4;
195            sizes.push((name, size));
196        }
197        let mut index = FastMap::default();
198        for (name, size) in sizes {
199            let end = pos.checked_add(size)?;
200            data.get(pos..end)?;
201            index.insert(name, pos..end);
202            pos = end;
203        }
204        Some(Archive { data, index })
205    }
206
207    /// Loads and caches one packed CMap, resolving its `usecmap` chain
208    /// within the predefined set (bounded depth; the shipped chains are at
209    /// most one deep).
210    pub fn load(name: &str, depth: usize) -> Option<Arc<CidCmap>> {
211        if depth > 4 {
212            return None;
213        }
214        let collection = collection_of(name)?;
215        static PARSED: OnceLock<Mutex<FastMap<String, Arc<CidCmap>>>> = OnceLock::new();
216        let cache = PARSED.get_or_init(|| Mutex::new(FastMap::default()));
217        if let Some(cmap) = cache.lock().unwrap().get(name) {
218            return Some(Arc::clone(cmap));
219        }
220        let archive = archive(collection)?;
221        let span = archive.index.get(name)?.clone();
222        let mut resolve = |n: &str| super::predefined_at(n, depth + 1);
223        let cmap = Arc::new(CidCmap::parse_with(&archive.data[span], None, &mut resolve));
224        cache
225            .lock()
226            .unwrap()
227            .insert(name.to_owned(), Arc::clone(&cmap));
228        Some(cmap)
229    }
230}
231
232#[cfg(feature = "predefined-cmaps")]
233use packed::load;
234
235#[cfg(not(feature = "predefined-cmaps"))]
236fn load(_name: &str, _depth: usize) -> Option<Arc<CidCmap>> {
237    None
238}
239
240/// [`predefined`] with the `usecmap` recursion depth threaded through.
241#[cfg(feature = "predefined-cmaps")]
242fn predefined_at(name: &str, depth: usize) -> Option<Arc<CidCmap>> {
243    match name {
244        "Identity-H" => Some(Arc::clone(identity(false))),
245        "Identity-V" => Some(Arc::clone(identity(true))),
246        _ => load(name, depth),
247    }
248}
249
250#[cfg(all(test, feature = "predefined-cmaps"))]
251mod tests {
252    use super::*;
253
254    #[test]
255    fn rksj_h_splits_and_maps_hiragana() {
256        let c = predefined("90ms-RKSJ-H").unwrap();
257        assert!(!c.vertical());
258        // あ is Shift-JIS <82A0>, inside the data file's range
259        // `<829f> <82f1> 842`, and 1-byte ASCII splits stay 1 byte.
260        let bytes = [0x41, 0x82, 0xA0];
261        assert_eq!(c.code_at(&bytes, 0), (0x41, 1));
262        assert_eq!(c.code_at(&bytes, 1), (0x82A0, 2));
263        assert_eq!(c.cid(0x82A0, 2), Some(843));
264        assert!(c.single_byte(0x20));
265    }
266
267    #[test]
268    fn rksj_v_layers_vertical_variants_over_the_h_base() {
269        let v = predefined("90ms-RKSJ-V").unwrap();
270        assert!(v.vertical());
271        // Its own `<8141> <8142> 7887` beats the inherited horizontal 、.
272        assert_eq!(v.cid(0x8141, 2), Some(7887));
273        // Untouched codes fall through to 90ms-RKSJ-H.
274        assert_eq!(v.cid(0x82A0, 2), Some(843));
275        let h = v.parent().expect("usecmap parent");
276        assert_eq!(h.cid(0x8141, 2), Some(634));
277    }
278
279    #[test]
280    fn every_shipped_name_loads_and_identity_needs_no_data() {
281        for name in [
282            "83pv-RKSJ-H",
283            "90msp-RKSJ-V",
284            "90pv-RKSJ-H",
285            "Add-RKSJ-V",
286            "EUC-H",
287            "Ext-RKSJ-V",
288            "H",
289            "V",
290            "UniJIS-UCS2-H",
291            "UniJIS-UCS2-HW-V",
292            "UniJIS-UTF16-V",
293            "GB-EUC-V",
294            "GBpc-EUC-H",
295            "GBK-EUC-H",
296            "GBKp-EUC-V",
297            "GBK2K-H",
298            "UniGB-UCS2-V",
299            "UniGB-UTF16-H",
300            "B5pc-V",
301            "HKscs-B5-H",
302            "ETen-B5-V",
303            "ETenms-B5-H",
304            "CNS-EUC-H",
305            "UniCNS-UCS2-V",
306            "UniCNS-UTF16-H",
307            "KSC-EUC-H",
308            "KSCms-UHC-V",
309            "KSCms-UHC-HW-H",
310            "KSCpc-EUC-H",
311            "UniKS-UCS2-V",
312            "UniKS-UTF16-H",
313            "UniAKR-UTF16-H",
314        ] {
315            let c = predefined(name).unwrap_or_else(|| panic!("{name} must load"));
316            assert!(!c.is_empty(), "{name} parsed to nothing");
317            assert_eq!(c.vertical(), name.ends_with("-V") || name == "V", "{name}");
318        }
319        assert!(predefined("Identity-H").is_some());
320        assert!(predefined("Identity-V").unwrap().vertical());
321        assert!(predefined("UniJIS2004-UTF16-H").is_none()); // not Table 118
322        assert!(predefined("WinAnsiEncoding").is_none());
323    }
324
325    #[test]
326    fn japan1_cids_read_back_as_unicode() {
327        let inv = cid_to_unicode("Japan1").unwrap();
328        assert_eq!(inv.lookup(843), Some('あ'));
329        // CID 1 is the space in every Adobe collection.
330        assert_eq!(inv.lookup(1), Some(' '));
331        // The halfwidth-form digit 1 (CID 248, what Shift-JIS ASCII runs
332        // select): reachable only through UniJIS-UCS2-HW-H.
333        assert_eq!(inv.lookup(248), Some('1'));
334        // 木 (CID 3814): the Kangxi radical ⽊ shares the CID and sorts
335        // lower, but the ideograph is what a page means.
336        assert_eq!(inv.lookup(3814), Some('木'));
337        assert!(cid_to_unicode("Identity").is_none());
338        assert!(cid_to_unicode("Unknown").is_none());
339    }
340}
pdfboss_core/cmap/predefined.rs

pdfboss_core/cmap/
predefined.rs
