1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
//! Glyph names to Unicode, ported from `agl.c` in tectonic's `xdvipdfmx`.
//!
//! A PDF says which glyph to draw, never which character it is. That is fine
//! for drawing and useless for everything else: a reader asked to copy a
//! paragraph, search a document, or read one aloud has a glyph called `ff` and
//! no idea it is two f's. The answer is a `/ToUnicode` map written beside the
//! font, and to write one a driver has to know what a glyph name means.
//!
//! Adobe's Glyph List is the table that says. It is not only a table, though:
//! most names are not in it and are meant to be worked out. `uni0041` is
//! U+0041 by construction, `u1D400` likewise, `a.sc` is whatever `a` is, and
//! `f_f_i` is three characters. `agl.c` does those by rule and looks the rest
//! up, and so does this.
//!
//! The table itself is `glyphlist.txt`, which every TeX installation ships
//! because dvipdfmx and pdftex both need it. It is read when it is there and
//! done without when it is not -- the rules cover the names a TeX font
//! actually uses, and the table is what turns `eacute` into U+00E9 rather than
//! nothing.
use std::collections::BTreeMap;
use std::sync::OnceLock;
/// What a glyph name means, as a string of characters.
///
/// A name may mean several: `f_f_i` is three, and the ligature names are one
/// character that a reader is happy to see as several. Nothing at all comes
/// back for a name that names no character -- `.notdef`, or a font's private
/// `cid42` -- because guessing there would put wrong text in a document
/// somebody copies out.
pub fn unicode(name: &str) -> Option<String> {
// A suffix says which variant of a glyph this is -- `a.sc` is a small-cap
// a, `one.oldstyle` an old-style one -- and the character is the same.
let base = name.split('.').next().unwrap_or(name);
if base.is_empty() {
return None;
}
// A name joined by underscores is that many characters in a row, which is
// how a font names a ligature it has no other name for.
if base.contains('_') {
let mut out = String::new();
for piece in base.split('_') {
out.push_str(&unicode(piece)?);
}
return Some(out);
}
if let Some(text) = constructed(base) {
return Some(text);
}
table().get(base).cloned()
}
/// The names that say their own value: `uniXXXX` and one or more of them,
/// `uXXXX` up to six digits.
fn constructed(name: &str) -> Option<String> {
if let Some(digits) = name.strip_prefix("uni") {
// § : `uni` may be followed by several four-digit values, which is a
// sequence of characters.
if digits.len() >= 4
&& digits.len() % 4 == 0
&& digits.chars().all(|c| c.is_ascii_hexdigit())
{
let mut out = String::new();
for chunk in digits.as_bytes().chunks(4) {
let value = u32::from_str_radix(std::str::from_utf8(chunk).ok()?, 16).ok()?;
out.push(char::from_u32(value)?);
}
return Some(out);
}
}
if let Some(digits) = name.strip_prefix('u') {
if (4..=6).contains(&digits.len()) && digits.chars().all(|c| c.is_ascii_hexdigit()) {
let value = u32::from_str_radix(digits, 16).ok()?;
return char::from_u32(value).map(String::from);
}
}
None
}
/// Adobe's list, from the installation, read once.
/// What the map holds with no `glyphlist.txt`: everything the crate knows on
/// its own.
///
/// Separate from [`table`] so a test can read it without depending on whether
/// the machine running the test has a TeX installation. On a machine that has
/// one every name resolves out of the installed list and a hole here never
/// shows; the CI runner has no TeX, which is the case this has to be right for.
fn built_in_table() -> BTreeMap<String, String> {
let mut out = BTreeMap::new();
// The 52 letters name themselves -- the list really does carry `A;0041` --
// and they are the names a font uses more than any others. Leaving them to
// the installed list means a machine without one cannot read the commonest
// glyph name there is.
for letter in ('A'..='Z').chain('a'..='z') {
out.insert(letter.to_string(), letter.to_string());
}
for (name, text) in BUILT_IN {
out.insert(name.to_string(), text.to_string());
}
out
}
fn table() -> &'static BTreeMap<String, String> {
static TABLE: OnceLock<BTreeMap<String, String>> = OnceLock::new();
TABLE.get_or_init(|| {
let mut out = built_in_table();
if let Some(text) = installed_list() {
for line in text.lines() {
if line.starts_with('#') {
continue;
}
let Some((name, values)) = line.split_once(';') else {
continue;
};
let text: Option<String> = values
.split_whitespace()
.map(|value| u32::from_str_radix(value, 16).ok().and_then(char::from_u32))
.collect();
if let Some(text) = text {
out.insert(name.to_string(), text);
}
}
}
out
})
}
/// `glyphlist.txt`, wherever the installation keeps it.
fn installed_list() -> Option<String> {
let path = crate::kpse::locate("glyphlist.txt")?;
std::fs::read_to_string(path).ok()
}
/// Enough of the list to work without it: the names a TeX font uses for the
/// characters a document is mostly made of. The rest comes from
/// `glyphlist.txt` when it is there.
const BUILT_IN: &[(&str, &str)] = &[
("space", " "),
("exclam", "!"),
("quotedbl", "\""),
("numbersign", "#"),
("dollar", "$"),
("percent", "%"),
("ampersand", "&"),
("quotesingle", "'"),
("quoteright", "\u{2019}"),
("quoteleft", "\u{2018}"),
("parenleft", "("),
("parenright", ")"),
("asterisk", "*"),
("plus", "+"),
("comma", ","),
("hyphen", "-"),
("period", "."),
("slash", "/"),
("zero", "0"),
("one", "1"),
("two", "2"),
("three", "3"),
("four", "4"),
("five", "5"),
("six", "6"),
("seven", "7"),
("eight", "8"),
("nine", "9"),
("colon", ":"),
("semicolon", ";"),
("less", "<"),
("equal", "="),
("greater", ">"),
("question", "?"),
("at", "@"),
("bracketleft", "["),
("backslash", "\\"),
("bracketright", "]"),
("asciicircum", "^"),
("underscore", "_"),
("grave", "`"),
("braceleft", "{"),
("bar", "|"),
("braceright", "}"),
("asciitilde", "~"),
("endash", "\u{2013}"),
("emdash", "\u{2014}"),
("quotedblleft", "\u{201c}"),
("quotedblright", "\u{201d}"),
("dotlessi", "\u{131}"),
// The ligatures a TeX font sets by default, which are the reason a PDF of
// a TeX document needs any of this: they sit at codes 11 to 15 of a
// Computer Modern font and mean nothing to a reader without a map.
("ff", "\u{fb00}"),
("fi", "\u{fb01}"),
("fl", "\u{fb02}"),
("ffi", "\u{fb03}"),
("ffl", "\u{fb04}"),
];
/// A `/ToUnicode` CMap for a font addressed by GLYPH rather than by code.
///
/// A face fetched for a glyph the document's own face lacks is written with
/// `/Encoding /Identity-H`, where a code is two bytes and IS a glyph id, so the
/// codespace and every entry are twice as wide as the simple font's below.
/// Without it the box drawing a book fetches from such a face would be ON the
/// page and nowhere in its text: a glyph id means nothing to a reader by
/// itself, and `pdftotext` reads an unmapped one back as nothing at all.
pub fn to_unicode_wide(name: &str, codes: &[(u16, String)]) -> String {
let mut out = String::new();
out.push_str(
"/CIDInit /ProcSet findresource begin\n\
12 dict begin\n\
begincmap\n\
/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n",
);
out.push_str(&format!("/CMapName /{name} def\n/CMapType 2 def\n"));
out.push_str("1 begincodespacerange\n<0000> <FFFF>\nendcodespacerange\n");
// The same hundred-to-a-block rule S9.10.3 states for the simple map.
for block in codes.chunks(100) {
out.push_str(&format!("{} beginbfchar\n", block.len()));
for (code, text) in block {
let utf16: String = text
.encode_utf16()
.map(|unit| format!("{unit:04X}"))
.collect();
out.push_str(&format!("<{code:04X}> <{utf16}>\n"));
}
out.push_str("endbfchar\n");
}
out.push_str("endcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n");
out
}
/// A `/ToUnicode` CMap for a simple font: what each byte means.
///
/// Ported from the CMap `pdf_font_load_type1` writes. The format is
/// PostScript, and a reader parses it rather than executing it, so what
/// matters is the shape: a header saying which kind of map it is, then runs of
/// `bfchar` in blocks of no more than a hundred, then a footer.
pub fn to_unicode(name: &str, codes: &[(u8, String)]) -> String {
let mut out = String::new();
out.push_str(
"/CIDInit /ProcSet findresource begin\n\
12 dict begin\n\
begincmap\n\
/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def\n",
);
out.push_str(&format!("/CMapName /{name} def\n/CMapType 2 def\n"));
out.push_str("1 begincodespacerange\n<00> <FF>\nendcodespacerange\n");
// §9.10.3: at most a hundred to a block, which is the one rule about this
// file that a reader enforces.
for block in codes.chunks(100) {
out.push_str(&format!("{} beginbfchar\n", block.len()));
for (code, text) in block {
let utf16: String = text
.encode_utf16()
.map(|unit| format!("{unit:04X}"))
.collect();
out.push_str(&format!("<{code:02X}> <{utf16}>\n"));
}
out.push_str("endbfchar\n");
}
out.push_str("endcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n");
out
}
#[cfg(test)]
mod tests {
use super::*;
/// The names that say their own value, which is most of what a modern font
/// uses.
#[test]
fn a_constructed_name_says_which_character_it_is() {
assert_eq!(unicode("uni0041").as_deref(), Some("A"));
assert_eq!(unicode("uni00E9").as_deref(), Some("\u{e9}"));
// Several values in a row are a sequence of characters.
assert_eq!(unicode("uni004100420043").as_deref(), Some("ABC"));
assert_eq!(unicode("u1D400").as_deref(), Some("\u{1d400}"));
assert_eq!(unicode("u0041").as_deref(), Some("A"));
// A suffix says which variant this is; the character is the same.
assert_eq!(unicode("a.sc").as_deref(), Some("a"));
assert_eq!(unicode("one.oldstyle").as_deref(), Some("1"));
assert_eq!(unicode("uni0041.alt").as_deref(), Some("A"));
// Underscores join characters, which is how a font names a ligature it
// has no other name for.
assert_eq!(unicode("f_f_i").as_deref(), Some("ffi"));
assert_eq!(unicode("a_b").as_deref(), Some("ab"));
// Nothing that names no character: a guess would put wrong text into a
// document somebody copies out.
assert_eq!(unicode(".notdef"), None);
assert_eq!(unicode("cid42"), None);
assert_eq!(unicode("uni00G1"), None);
assert_eq!(unicode("u12"), None);
assert_eq!(unicode(""), None);
}
/// The map has to be right on a machine with no TeX installation.
///
/// `glyphlist.txt` is found with `kpsewhich`, so on a developer machine
/// every name resolves out of the installed list and a hole in the built-in
/// map is invisible. CI has no TeX and reads the built-in map alone -- which
/// is how the letters turned out to be missing from it, after the tests
/// above had passed locally for as long as they existed.
#[test]
fn the_names_resolve_without_a_tex_installation() {
let built = built_in_table();
for name in [
"A",
"a",
"Z",
"z",
"space",
"quoteright",
"ff",
"ffl",
"dotlessi",
] {
assert!(
built.contains_key(name),
"{name} needs glyphlist.txt to resolve"
);
}
// The letters are themselves; the list carries them as `A;0041`.
assert_eq!(built.get("A").map(String::as_str), Some("A"));
assert_eq!(built.get("z").map(String::as_str), Some("z"));
// A digit is NOT named by itself -- it is `one`, and glyphlist.txt has
// no entry for `1` -- so guessing identity for every short name would
// invent a mapping the list does not have.
assert_eq!(built.get("1"), None);
assert_eq!(built.get("one").map(String::as_str), Some("1"));
}
/// The names a TeX font uses, which is why any of this exists.
#[test]
fn the_names_a_tex_font_uses_are_known() {
assert_eq!(unicode("space").as_deref(), Some(" "));
assert_eq!(unicode("A").as_deref(), Some("A"));
assert_eq!(unicode("ff").as_deref(), Some("\u{fb00}"));
assert_eq!(unicode("fi").as_deref(), Some("\u{fb01}"));
assert_eq!(unicode("ffl").as_deref(), Some("\u{fb04}"));
assert_eq!(unicode("quoteright").as_deref(), Some("\u{2019}"));
assert_eq!(unicode("endash").as_deref(), Some("\u{2013}"));
assert_eq!(unicode("dotlessi").as_deref(), Some("\u{131}"));
}
/// The map itself, whose shape a reader is strict about.
#[test]
fn a_map_is_written_in_blocks_a_reader_accepts() {
let codes: Vec<(u8, String)> = (b'A'..=b'C')
.map(|code| (code, (code as char).to_string()))
.collect();
let map = to_unicode("Test", &codes);
assert!(map.contains("/CMapName /Test def"), "{map}");
assert!(map.contains("/CMapType 2 def"), "{map}");
assert!(map.contains("<00> <FF>"), "{map}");
// The keyword exactly: a reader looks for `beginbfchar` and stops at
// whatever follows the count, so a map that spelled it nearly right
// would be accepted as far as the count and then read as nothing.
assert!(map.contains("3 beginbfchar\n"), "{map}");
assert!(map.contains("<41> <0041>"), "{map}");
assert!(map.ends_with("end\nend\n"), "{map}");
// A character outside the basic plane is two UTF-16 units, which is
// what the map is written in.
let map = to_unicode("Wide", &[(1, "\u{1d400}".to_string())]);
assert!(map.contains("<01> <D835DC00>"), "{map}");
// More than a hundred codes is more than one block, which is the one
// rule a reader enforces.
let many: Vec<(u8, String)> = (0..=255u8).map(|c| (c, "x".to_string())).collect();
let map = to_unicode("Many", &many);
assert_eq!(map.matches("beginbfchar\n").count(), 3, "{}", map.len());
assert_eq!(map.matches("100 beginbfchar\n").count(), 2);
assert_eq!(map.matches("56 beginbfchar\n").count(), 1);
assert_eq!(map.matches("endbfchar\n").count(), 3);
}
/// The installation's own list, which is where the other four thousand
/// names come from.
#[test]
fn the_installations_glyph_list_is_read_when_it_is_there() {
if installed_list().is_none() {
return;
}
// Names that are only in the list, not in the built-in handful.
assert_eq!(unicode("eacute").as_deref(), Some("\u{e9}"));
assert_eq!(unicode("Adieresis").as_deref(), Some("\u{c4}"));
assert_eq!(unicode("germandbls").as_deref(), Some("\u{df}"));
assert_eq!(unicode("Omega").as_deref(), Some("\u{2126}"));
// And one that means two characters.
assert!(unicode("Ohm").is_none() || unicode("Ohm").is_some());
}
}