docling 2.82.1

DocumentConverter and format backends for docling.rs (a Rust port of docling).
Documentation
//! Glyphs of the symbol-encoded fonts as Unicode (#588, #742).
//!
//! `Symbol`, `ZapfDingbats`, `Wingdings`, `Wingdings 2`, `Wingdings 3` and
//! `Webdings` are *symbol-encoded* fonts: their glyphs sit on the byte codes
//! 0x20–0xFF, and Word stores a character set in one of them either as that
//! byte (`8D`), as the byte raised into the Private Use Area (`U+F08D`, the
//! cmap these fonts actually carry), or as a `w:sym` element naming the font
//! and the code. None of those is text to a reader that lacks the font:
//! python-docx hands docling the PUA code point as it is (an invisible
//! character, or a tofu box) and drops `w:sym` outright, so a Symbol-font
//! bullet, a Wingdings check mark or a formula's Greek letters vanish from the
//! Markdown. This module maps them — a deliberate divergence from docling's
//! DOCX reader today, recorded in `docs/MIGRATION.md`.
//!
//! The table is `symbol_fonts.json` next to this file, data only and shared:
//! it is docling-core's `docling_core/utils/symbol_fonts.json`
//! (docling-project/docling-core#889), which Python docling's DOCX reader
//! and docling-parse's PDF reader use too, so the three readers produce the
//! same text. It is generated by docling-core's
//! `scripts/generate_symbol_fonts.py` from the Unicode Consortium sources the
//! file names — `Symbol` and `ZapfDingbats` from Adobe's encodings as Unicode
//! publishes them (`MAPPINGS/VENDORS/ADOBE/symbol.txt`, `zdingbat.txt`;
//! where a code has several Unicode values the character a reader expects,
//! Greek mu rather than the micro sign), the Wingdings and Webdings families
//! from the normative `UCS;ID` source references of L2/12-368 (WG2 N4384) —
//! and it never maps to a private-use code point: Adobe's corporate-use
//! bracket pieces and serif / sans-serif ®©™ are the standard characters.
//! Keep the vendored copy byte-identical to docling-core's
//! (`scripts/sync_symbol_fonts.sh`); the Unicode notice it carries must stay
//! with the Adobe-derived part. A code with no glyph in the font is left as it
//! was.

use std::sync::OnceLock;

/// One of the symbol-encoded fonts.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum SymbolFont {
    Symbol,
    ZapfDingbats,
    Wingdings,
    Wingdings2,
    Wingdings3,
    Webdings,
}

impl SymbolFont {
    const ALL: [SymbolFont; 6] = [
        Self::Symbol,
        Self::ZapfDingbats,
        Self::Wingdings,
        Self::Wingdings2,
        Self::Wingdings3,
        Self::Webdings,
    ];

    /// The font's key in `symbol_fonts.json`.
    fn json_name(self) -> &'static str {
        match self {
            Self::Symbol => "Symbol",
            Self::ZapfDingbats => "ZapfDingbats",
            Self::Wingdings => "Wingdings",
            Self::Wingdings2 => "Wingdings 2",
            Self::Wingdings3 => "Wingdings 3",
            Self::Webdings => "Webdings",
        }
    }

    /// The font a `w:rFonts` / `w:sym w:font` / FFN name — or a PDF's
    /// PostScript name — denotes, if it is one of the symbol-encoded ones:
    /// docling-core's `SymbolFont.from_name`. A subset tag (`ABCDEF+`) and a
    /// style suffix (`,Bold`, `-Regular`) are ignored, and so are case and
    /// whitespace (`Wingdings2` and `Wingdings 2` alike, `SymbolMT`,
    /// `ITC Zapf Dingbats`). Fonts that encode their glyphs as Unicode
    /// (`Symbola`, `Segoe UI Symbol`) are not symbol-encoded.
    pub(crate) fn from_name(name: &str) -> Option<Self> {
        let name = name.trim();
        let b = name.as_bytes();
        let name = if b.len() > 7 && b[6] == b'+' && b[..6].iter().all(u8::is_ascii_uppercase) {
            &name[7..]
        } else {
            name
        };
        let family = name.split(',').next().unwrap_or(name);
        let family = family.split('-').next().unwrap_or(family);
        let n: String = family
            .chars()
            .filter(|c| !c.is_whitespace())
            .flat_map(char::to_lowercase)
            .collect();
        match n.as_str() {
            "symbol" | "symbolmt" => Some(Self::Symbol),
            "zapfdingbats" | "itczapfdingbats" | "zapfdingbatsitc" => Some(Self::ZapfDingbats),
            "wingdings" => Some(Self::Wingdings),
            "wingdings2" => Some(Self::Wingdings2),
            "wingdings3" => Some(Self::Wingdings3),
            "webdings" => Some(Self::Webdings),
            _ => None,
        }
    }

    fn table(self) -> &'static [u32; 224] {
        let i = Self::ALL.iter().position(|f| *f == self).expect("listed");
        &tables()[i]
    }

    /// The Unicode character for glyph `code` of this font: the byte code
    /// (0x20–0xFF), or that code in the Private Use Area (`U+F020`–`U+F0FF`,
    /// how the fonts' cmap and Word's `w:sym w:char` spell it). `None` when
    /// the code is outside that range or the font has no glyph there.
    pub(crate) fn glyph(self, code: u32) -> Option<char> {
        let code = match code {
            0xF020..=0xF0FF => code - 0xF000,
            0x20..=0xFF => code,
            _ => return None,
        };
        match self.table()[(code - 0x20) as usize] {
            0 => None,
            u => char::from_u32(u),
        }
    }

    /// `text` as set in this font: every character that is a glyph code
    /// replaced by its Unicode equivalent, the rest kept.
    #[cfg(test)]
    fn map_text(self, text: &str) -> String {
        text.chars()
            .map(|c| self.glyph(c as u32).unwrap_or(c))
            .collect()
    }
}

/// Whether `c` is a code a symbol font would re-interpret — the byte range
/// or its PUA image. Lets a caller skip the mapping for ordinary text cheaply.
pub(crate) fn is_glyph_code(c: char) -> bool {
    matches!(c as u32, 0x20..=0xFF | 0xF020..=0xF0FF)
}

/// docling-core's shared table, vendored byte-identical (see the module docs).
const TABLE_JSON: &str = include_str!("symbol_fonts.json");

/// The six tables, one per [`SymbolFont::ALL`] entry: 224 entries for codes
/// 0x20–0xFF, 0 = no glyph. Parsed from [`TABLE_JSON`] on first use.
fn tables() -> &'static [[u32; 224]; 6] {
    static TABLES: OnceLock<[[u32; 224]; 6]> = OnceLock::new();
    TABLES.get_or_init(|| {
        parse_tables(TABLE_JSON).expect("the vendored symbol_fonts.json is checked by the tests")
    })
}

/// `symbol_fonts.json` (`{"fonts": {"Wingdings": {"21": "1F589", …}, …}}`,
/// byte code and code point in hexadecimal) as the six tables. Errors on a
/// missing font, a code outside 0x20–0xFF, or a value that is not a Unicode
/// scalar value or is private-use — the file's own guarantees.
fn parse_tables(json: &str) -> Result<[[u32; 224]; 6], String> {
    let doc: serde_json::Value =
        serde_json::from_str(json).map_err(|e| format!("symbol_fonts.json: {e}"))?;
    let fonts = doc
        .get("fonts")
        .and_then(|f| f.as_object())
        .ok_or("symbol_fonts.json: no `fonts` object")?;
    let mut out = [[0u32; 224]; 6];
    for (slot, font) in SymbolFont::ALL.iter().enumerate() {
        let table = fonts
            .get(font.json_name())
            .and_then(|t| t.as_object())
            .ok_or_else(|| format!("symbol_fonts.json: no `{}` table", font.json_name()))?;
        for (code, cp) in table {
            let code = u32::from_str_radix(code, 16)
                .ok()
                .filter(|c| (0x20..=0xFF).contains(c))
                .ok_or_else(|| format!("{}: bad code {code:?}", font.json_name()))?;
            let cp = cp
                .as_str()
                .and_then(|v| u32::from_str_radix(v, 16).ok())
                .filter(|v| char::from_u32(*v).is_some() && !is_private_use(*v))
                .ok_or_else(|| format!("{}: bad code point for {code:#04x}", font.json_name()))?;
            out[slot][(code - 0x20) as usize] = cp;
        }
    }
    Ok(out)
}

fn is_private_use(cp: u32) -> bool {
    matches!(cp, 0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x100000..=0x10FFFD)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn names_resolve_like_docling_core() {
        for (name, font) in [
            ("Symbol", SymbolFont::Symbol),
            (" wingdings ", SymbolFont::Wingdings),
            ("Wingdings 2", SymbolFont::Wingdings2),
            ("Wingdings3", SymbolFont::Wingdings3),
            ("WEBDINGS", SymbolFont::Webdings),
            ("ZapfDingbats", SymbolFont::ZapfDingbats),
            ("ITC Zapf Dingbats", SymbolFont::ZapfDingbats),
            // PDF PostScript names: subset tag and style suffix ignored.
            ("ABCDEF+Wingdings-Regular", SymbolFont::Wingdings),
            ("SymbolMT", SymbolFont::Symbol),
            ("Symbol,Bold", SymbolFont::Symbol),
        ] {
            assert_eq!(SymbolFont::from_name(name), Some(font), "{name}");
        }
        for name in [
            "Arial",
            "Symbola",
            "Segoe UI Symbol",
            "abcdef+Wingdings",
            "",
        ] {
            assert_eq!(SymbolFont::from_name(name), None, "{name}");
        }
    }

    /// The vendored table parses and keeps its guarantees: every font, codes
    /// 0x20–0xFF only, no private-use or non-scalar values; its sources and
    /// the Unicode notice travel with it.
    #[test]
    fn the_shared_table_is_valid() {
        let tables = parse_tables(TABLE_JSON).unwrap();
        for (font, table) in SymbolFont::ALL.iter().zip(tables.iter()) {
            let mapped = table.iter().filter(|&&u| u != 0).count();
            assert!(mapped >= 180, "{font:?}: {mapped} codes");
            assert!(table.iter().all(|&u| u == 0 || !is_private_use(u)));
        }
        let doc: serde_json::Value = serde_json::from_str(TABLE_JSON).unwrap();
        assert!(doc["notice"].as_str().unwrap().contains("Unicode, Inc."));
        assert!(doc["sources"]["Wingdings"]
            .as_str()
            .unwrap()
            .contains("12368-n4384"));
        assert!(parse_tables(r#"{"fonts": {}}"#).is_err());
        assert!(parse_tables(
            r#"{"fonts": {"Symbol": {"41": "F8E5"}, "ZapfDingbats": {}, "Wingdings": {},
               "Wingdings 2": {}, "Wingdings 3": {}, "Webdings": {}}}"#
        )
        .unwrap_err()
        .contains("bad code point"));
    }

    /// #742's examples: the Wingdings / Webdings glyphs are L2/12-368's, not
    /// the older characters it cites as cross-references.
    #[test]
    fn wingdings_follow_l2_12_368() {
        assert_eq!(SymbolFont::Wingdings.glyph(0x30), Some('\u{1F5C0}')); // 🗀 folder
        assert_eq!(SymbolFont::Wingdings.glyph(0xF06C), Some('\u{26AB}')); // ⚫
        assert_eq!(SymbolFont::Wingdings.glyph(0x8D), Some('\u{278B}')); // ➋
        assert_eq!(SymbolFont::Wingdings2.glyph(0x50), Some('\u{1F5F8}')); // 🗸
        assert_eq!(SymbolFont::Wingdings3.glyph(0x66), Some('\u{1F850}')); // 🡐
        assert_eq!(SymbolFont::Webdings.glyph(0xDA), Some('\u{1F328}')); // 🌨 snow
        assert_eq!(SymbolFont::Webdings.glyph(0xDB), Some('\u{1F327}')); // 🌧 rain
        assert_eq!(SymbolFont::Wingdings.glyph(0xF0FC), Some('\u{2713}')); // ✓, not ✔
                                                                           // Unchanged entries.
        assert_eq!(SymbolFont::Wingdings.glyph(0xF0A7), Some('\u{25AA}')); // ▪
        assert_eq!(SymbolFont::Wingdings.glyph(0xF0E8), Some('\u{1F87A}')); // 🡺
        assert_eq!(SymbolFont::ZapfDingbats.glyph(0x21), Some('\u{2701}')); // ✁
    }

    /// Symbol: Adobe's encoding with the Greek letters, and the corporate-use
    /// pieces as standard characters rather than private-use codes.
    #[test]
    fn symbol_maps_to_standard_characters() {
        assert_eq!(SymbolFont::Symbol.glyph(0xF0B7), Some('\u{2022}')); // •
        assert_eq!(SymbolFont::Symbol.glyph(0xB7), Some('\u{2022}'));
        assert_eq!(SymbolFont::Symbol.glyph(0xF061), Some('α'));
        assert_eq!(SymbolFont::Symbol.map_text("a b p"), "α β π");
        assert_eq!(
            SymbolFont::Symbol.map_text("\u{f061}\u{f062}\u{f070}"),
            "αβπ"
        );
        // Greek letters, not their compatibility look-alikes.
        assert_eq!(SymbolFont::Symbol.glyph(0x6D), Some('μ'));
        assert_eq!(SymbolFont::Symbol.glyph(0x44), Some('Δ'));
        assert_eq!(SymbolFont::Symbol.glyph(0x20), Some(' '));
        // ® © ™ (serif and sans serif) and the delimiter pieces.
        assert_eq!(SymbolFont::Symbol.glyph(0xD2), Some('®'));
        assert_eq!(SymbolFont::Symbol.glyph(0xE3), Some('©'));
        assert_eq!(SymbolFont::Symbol.glyph(0xE4), Some('™'));
        assert_eq!(SymbolFont::Symbol.glyph(0xE6), Some('\u{239B}')); // ⎛
        assert_eq!(SymbolFont::Symbol.glyph(0xEF), Some('\u{23AA}')); // ⎪
    }

    #[test]
    fn codes_without_a_glyph_are_left_alone() {
        assert_eq!(SymbolFont::Symbol.glyph(0x7F), None);
        assert_eq!(SymbolFont::Symbol.glyph(0x1F), None);
        assert_eq!(SymbolFont::Symbol.glyph(0x100), None);
        assert_eq!(SymbolFont::Symbol.glyph(0xF100), None);
        // The radical extender has no Unicode character: kept as stored.
        assert_eq!(SymbolFont::Symbol.glyph(0x60), None);
        // Every byte code of a Wingdings run is a glyph (`\u{e9}`, 0xE9, is an
        // arrow there); a character outside the byte range stays.
        let glyph_x = SymbolFont::Wingdings.glyph(0x78).unwrap();
        assert_eq!(
            SymbolFont::Wingdings.map_text("x\u{20AC}\u{e01}"),
            format!("{glyph_x}\u{20AC}\u{e01}")
        );
    }
}