1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
//! Glyphs of the symbol-encoded fonts as Unicode (#588, #742).
//!
//! `Symbol`, `ZapfDingbats`, `Wingdings`, `Wingdings 2`, `Wingdings 3` and
//! `Webdings` are *symbol-encoded* fonts: their glyphs sit on the byte codes
//! 0x20–0xFF, and Word stores a character set in one of them either as that
//! byte (`8D`), as the byte raised into the Private Use Area (`U+F08D`, the
//! cmap these fonts actually carry), or as a `w:sym` element naming the font
//! and the code. None of those is text to a reader that lacks the font:
//! python-docx hands docling the PUA code point as it is (an invisible
//! character, or a tofu box) and drops `w:sym` outright, so a Symbol-font
//! bullet, a Wingdings check mark or a formula's Greek letters vanish from the
//! Markdown. This module maps them — a deliberate divergence from docling's
//! DOCX reader today, recorded in `docs/MIGRATION.md`.
//!
//! The table is `symbol_fonts.json` next to this file, data only and shared:
//! it is docling-core's `docling_core/utils/symbol_fonts.json`
//! (docling-project/docling-core#889), which Python docling's DOCX reader
//! and docling-parse's PDF reader use too, so the three readers produce the
//! same text. It is generated by docling-core's
//! `scripts/generate_symbol_fonts.py` from the Unicode Consortium sources the
//! file names — `Symbol` and `ZapfDingbats` from Adobe's encodings as Unicode
//! publishes them (`MAPPINGS/VENDORS/ADOBE/symbol.txt`, `zdingbat.txt`;
//! where a code has several Unicode values the character a reader expects,
//! Greek mu rather than the micro sign), the Wingdings and Webdings families
//! from the normative `UCS;ID` source references of L2/12-368 (WG2 N4384) —
//! and it never maps to a private-use code point: Adobe's corporate-use
//! bracket pieces and serif / sans-serif ®©™ are the standard characters.
//! Keep the vendored copy byte-identical to docling-core's
//! (`scripts/sync_symbol_fonts.sh`); the Unicode notice it carries must stay
//! with the Adobe-derived part. A code with no glyph in the font is left as it
//! was.
use std::sync::OnceLock;
/// One of the symbol-encoded fonts.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum SymbolFont {
Symbol,
ZapfDingbats,
Wingdings,
Wingdings2,
Wingdings3,
Webdings,
}
impl SymbolFont {
const ALL: [SymbolFont; 6] = [
Self::Symbol,
Self::ZapfDingbats,
Self::Wingdings,
Self::Wingdings2,
Self::Wingdings3,
Self::Webdings,
];
/// The font's key in `symbol_fonts.json`.
fn json_name(self) -> &'static str {
match self {
Self::Symbol => "Symbol",
Self::ZapfDingbats => "ZapfDingbats",
Self::Wingdings => "Wingdings",
Self::Wingdings2 => "Wingdings 2",
Self::Wingdings3 => "Wingdings 3",
Self::Webdings => "Webdings",
}
}
/// The font a `w:rFonts` / `w:sym w:font` / FFN name — or a PDF's
/// PostScript name — denotes, if it is one of the symbol-encoded ones:
/// docling-core's `SymbolFont.from_name`. A subset tag (`ABCDEF+`) and a
/// style suffix (`,Bold`, `-Regular`) are ignored, and so are case and
/// whitespace (`Wingdings2` and `Wingdings 2` alike, `SymbolMT`,
/// `ITC Zapf Dingbats`). Fonts that encode their glyphs as Unicode
/// (`Symbola`, `Segoe UI Symbol`) are not symbol-encoded.
pub(crate) fn from_name(name: &str) -> Option<Self> {
let name = name.trim();
let b = name.as_bytes();
let name = if b.len() > 7 && b[6] == b'+' && b[..6].iter().all(u8::is_ascii_uppercase) {
&name[7..]
} else {
name
};
let family = name.split(',').next().unwrap_or(name);
let family = family.split('-').next().unwrap_or(family);
let n: String = family
.chars()
.filter(|c| !c.is_whitespace())
.flat_map(char::to_lowercase)
.collect();
match n.as_str() {
"symbol" | "symbolmt" => Some(Self::Symbol),
"zapfdingbats" | "itczapfdingbats" | "zapfdingbatsitc" => Some(Self::ZapfDingbats),
"wingdings" => Some(Self::Wingdings),
"wingdings2" => Some(Self::Wingdings2),
"wingdings3" => Some(Self::Wingdings3),
"webdings" => Some(Self::Webdings),
_ => None,
}
}
fn table(self) -> &'static [u32; 224] {
let i = Self::ALL.iter().position(|f| *f == self).expect("listed");
&tables()[i]
}
/// The Unicode character for glyph `code` of this font: the byte code
/// (0x20–0xFF), or that code in the Private Use Area (`U+F020`–`U+F0FF`,
/// how the fonts' cmap and Word's `w:sym w:char` spell it). `None` when
/// the code is outside that range or the font has no glyph there.
pub(crate) fn glyph(self, code: u32) -> Option<char> {
let code = match code {
0xF020..=0xF0FF => code - 0xF000,
0x20..=0xFF => code,
_ => return None,
};
match self.table()[(code - 0x20) as usize] {
0 => None,
u => char::from_u32(u),
}
}
/// `text` as set in this font: every character that is a glyph code
/// replaced by its Unicode equivalent, the rest kept.
#[cfg(test)]
fn map_text(self, text: &str) -> String {
text.chars()
.map(|c| self.glyph(c as u32).unwrap_or(c))
.collect()
}
}
/// Whether `c` is a code a symbol font would re-interpret — the byte range
/// or its PUA image. Lets a caller skip the mapping for ordinary text cheaply.
pub(crate) fn is_glyph_code(c: char) -> bool {
matches!(c as u32, 0x20..=0xFF | 0xF020..=0xF0FF)
}
/// docling-core's shared table, vendored byte-identical (see the module docs).
const TABLE_JSON: &str = include_str!("symbol_fonts.json");
/// The six tables, one per [`SymbolFont::ALL`] entry: 224 entries for codes
/// 0x20–0xFF, 0 = no glyph. Parsed from [`TABLE_JSON`] on first use.
fn tables() -> &'static [[u32; 224]; 6] {
static TABLES: OnceLock<[[u32; 224]; 6]> = OnceLock::new();
TABLES.get_or_init(|| {
parse_tables(TABLE_JSON).expect("the vendored symbol_fonts.json is checked by the tests")
})
}
/// `symbol_fonts.json` (`{"fonts": {"Wingdings": {"21": "1F589", …}, …}}`,
/// byte code and code point in hexadecimal) as the six tables. Errors on a
/// missing font, a code outside 0x20–0xFF, or a value that is not a Unicode
/// scalar value or is private-use — the file's own guarantees.
fn parse_tables(json: &str) -> Result<[[u32; 224]; 6], String> {
let doc: serde_json::Value =
serde_json::from_str(json).map_err(|e| format!("symbol_fonts.json: {e}"))?;
let fonts = doc
.get("fonts")
.and_then(|f| f.as_object())
.ok_or("symbol_fonts.json: no `fonts` object")?;
let mut out = [[0u32; 224]; 6];
for (slot, font) in SymbolFont::ALL.iter().enumerate() {
let table = fonts
.get(font.json_name())
.and_then(|t| t.as_object())
.ok_or_else(|| format!("symbol_fonts.json: no `{}` table", font.json_name()))?;
for (code, cp) in table {
let code = u32::from_str_radix(code, 16)
.ok()
.filter(|c| (0x20..=0xFF).contains(c))
.ok_or_else(|| format!("{}: bad code {code:?}", font.json_name()))?;
let cp = cp
.as_str()
.and_then(|v| u32::from_str_radix(v, 16).ok())
.filter(|v| char::from_u32(*v).is_some() && !is_private_use(*v))
.ok_or_else(|| format!("{}: bad code point for {code:#04x}", font.json_name()))?;
out[slot][(code - 0x20) as usize] = cp;
}
}
Ok(out)
}
fn is_private_use(cp: u32) -> bool {
matches!(cp, 0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x100000..=0x10FFFD)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn names_resolve_like_docling_core() {
for (name, font) in [
("Symbol", SymbolFont::Symbol),
(" wingdings ", SymbolFont::Wingdings),
("Wingdings 2", SymbolFont::Wingdings2),
("Wingdings3", SymbolFont::Wingdings3),
("WEBDINGS", SymbolFont::Webdings),
("ZapfDingbats", SymbolFont::ZapfDingbats),
("ITC Zapf Dingbats", SymbolFont::ZapfDingbats),
// PDF PostScript names: subset tag and style suffix ignored.
("ABCDEF+Wingdings-Regular", SymbolFont::Wingdings),
("SymbolMT", SymbolFont::Symbol),
("Symbol,Bold", SymbolFont::Symbol),
] {
assert_eq!(SymbolFont::from_name(name), Some(font), "{name}");
}
for name in [
"Arial",
"Symbola",
"Segoe UI Symbol",
"abcdef+Wingdings",
"",
] {
assert_eq!(SymbolFont::from_name(name), None, "{name}");
}
}
/// The vendored table parses and keeps its guarantees: every font, codes
/// 0x20–0xFF only, no private-use or non-scalar values; its sources and
/// the Unicode notice travel with it.
#[test]
fn the_shared_table_is_valid() {
let tables = parse_tables(TABLE_JSON).unwrap();
for (font, table) in SymbolFont::ALL.iter().zip(tables.iter()) {
let mapped = table.iter().filter(|&&u| u != 0).count();
assert!(mapped >= 180, "{font:?}: {mapped} codes");
assert!(table.iter().all(|&u| u == 0 || !is_private_use(u)));
}
let doc: serde_json::Value = serde_json::from_str(TABLE_JSON).unwrap();
assert!(doc["notice"].as_str().unwrap().contains("Unicode, Inc."));
assert!(doc["sources"]["Wingdings"]
.as_str()
.unwrap()
.contains("12368-n4384"));
assert!(parse_tables(r#"{"fonts": {}}"#).is_err());
assert!(parse_tables(
r#"{"fonts": {"Symbol": {"41": "F8E5"}, "ZapfDingbats": {}, "Wingdings": {},
"Wingdings 2": {}, "Wingdings 3": {}, "Webdings": {}}}"#
)
.unwrap_err()
.contains("bad code point"));
}
/// #742's examples: the Wingdings / Webdings glyphs are L2/12-368's, not
/// the older characters it cites as cross-references.
#[test]
fn wingdings_follow_l2_12_368() {
assert_eq!(SymbolFont::Wingdings.glyph(0x30), Some('\u{1F5C0}')); // 🗀 folder
assert_eq!(SymbolFont::Wingdings.glyph(0xF06C), Some('\u{26AB}')); // ⚫
assert_eq!(SymbolFont::Wingdings.glyph(0x8D), Some('\u{278B}')); // ➋
assert_eq!(SymbolFont::Wingdings2.glyph(0x50), Some('\u{1F5F8}')); // 🗸
assert_eq!(SymbolFont::Wingdings3.glyph(0x66), Some('\u{1F850}')); // 🡐
assert_eq!(SymbolFont::Webdings.glyph(0xDA), Some('\u{1F328}')); // 🌨 snow
assert_eq!(SymbolFont::Webdings.glyph(0xDB), Some('\u{1F327}')); // 🌧 rain
assert_eq!(SymbolFont::Wingdings.glyph(0xF0FC), Some('\u{2713}')); // ✓, not ✔
// Unchanged entries.
assert_eq!(SymbolFont::Wingdings.glyph(0xF0A7), Some('\u{25AA}')); // ▪
assert_eq!(SymbolFont::Wingdings.glyph(0xF0E8), Some('\u{1F87A}')); // 🡺
assert_eq!(SymbolFont::ZapfDingbats.glyph(0x21), Some('\u{2701}')); // ✁
}
/// Symbol: Adobe's encoding with the Greek letters, and the corporate-use
/// pieces as standard characters rather than private-use codes.
#[test]
fn symbol_maps_to_standard_characters() {
assert_eq!(SymbolFont::Symbol.glyph(0xF0B7), Some('\u{2022}')); // •
assert_eq!(SymbolFont::Symbol.glyph(0xB7), Some('\u{2022}'));
assert_eq!(SymbolFont::Symbol.glyph(0xF061), Some('α'));
assert_eq!(SymbolFont::Symbol.map_text("a b p"), "α β π");
assert_eq!(
SymbolFont::Symbol.map_text("\u{f061}\u{f062}\u{f070}"),
"αβπ"
);
// Greek letters, not their compatibility look-alikes.
assert_eq!(SymbolFont::Symbol.glyph(0x6D), Some('μ'));
assert_eq!(SymbolFont::Symbol.glyph(0x44), Some('Δ'));
assert_eq!(SymbolFont::Symbol.glyph(0x20), Some(' '));
// ® © ™ (serif and sans serif) and the delimiter pieces.
assert_eq!(SymbolFont::Symbol.glyph(0xD2), Some('®'));
assert_eq!(SymbolFont::Symbol.glyph(0xE3), Some('©'));
assert_eq!(SymbolFont::Symbol.glyph(0xE4), Some('™'));
assert_eq!(SymbolFont::Symbol.glyph(0xE6), Some('\u{239B}')); // ⎛
assert_eq!(SymbolFont::Symbol.glyph(0xEF), Some('\u{23AA}')); // ⎪
}
#[test]
fn codes_without_a_glyph_are_left_alone() {
assert_eq!(SymbolFont::Symbol.glyph(0x7F), None);
assert_eq!(SymbolFont::Symbol.glyph(0x1F), None);
assert_eq!(SymbolFont::Symbol.glyph(0x100), None);
assert_eq!(SymbolFont::Symbol.glyph(0xF100), None);
// The radical extender has no Unicode character: kept as stored.
assert_eq!(SymbolFont::Symbol.glyph(0x60), None);
// Every byte code of a Wingdings run is a glyph (`\u{e9}`, 0xE9, is an
// arrow there); a character outside the byte range stays.
let glyph_x = SymbolFont::Wingdings.glyph(0x78).unwrap();
assert_eq!(
SymbolFont::Wingdings.map_text("x\u{20AC}\u{e01}"),
format!("{glyph_x}\u{20AC}\u{e01}")
);
}
}