dxpdf 0.5.0

//! ICU4X plumbing shared by every i18n gap tracked in issue #124.
//!
//! Phase 0 (issue #127) of that effort. This module makes no behavior change
//! on its own — [`Locale`](crate::render::resolve::locale::Locale) still
//! answers the two questions it always has. What's here is the data-provider
//! infrastructure later phases (#128 regional decimal separators, #129
//! localized date names, #130 UAX #14 line breaking) build real features on
//! top of. #131 (UAX #9 bidi) is the one phase that needed none of it — see
//! [`bidi`], which carries its own tables.
//!
//! # Why a committed blob, not baked `compiled_data`
//!
//! ICU4X's default `compiled_data` Cargo feature bakes in *every* CLDR
//! locale — megabytes landing on the Python wheel, which #124 names as a
//! risk to measure and decide deliberately rather than discover after the
//! fact. Every `icu_*` leaf crate that ships locale data (`icu_decimal`
//! here; `icu_segmenter`/`icu_datetime` when later phases add them) is
//! therefore added to `Cargo.toml` with `default-features = false` — turning
//! `compiled_data` off is what makes the trimmed blob below the *only* copy
//! of the data instead of an addition on top of the default one.
//!
//! The trimmed data is generated by `scripts/make_icu_data.sh` into
//! `data/icu_data.blob`, which this module `include_bytes!`s and loads once
//! per thread (see `PROVIDER`'s doc below for why per-thread, not
//! process-wide), through [`icu_provider_blob::BlobDataProvider`]. This was
//! chosen over ICU4X's other trimming path — `--format baked` plus setting
//! `ICU4X_DATA_DIR` at `cargo build` time — because that path's correctness
//! depends on an environment variable being set identically across every CI
//! job that builds a wheel (`.github/workflows/ci.yml`'s `build-wheels`, 5
//! matrix legs, and `python.yml`'s release twin, another 5). That file
//! already documents, in its own comment, that `maturin-action` does *not*
//! forward arbitrary env vars into the Docker build it runs on Linux without
//! explicit `docker-options: -e ...` — and a missed leg wouldn't fail the
//! build, it would silently fall back to the full default `compiled_data`,
//! exactly the wheel-bloat risk this module exists to prevent. A committed
//! blob needs no such plumbing, in any workflow file, ever; the cost is
//! calling [`with_data_provider`] and using
//! `try_new_with_buffer_provider(provider, ...)` instead of the ergonomic
//! `try_new(...)` at each call site — a small, visible, permanent cost, not
//! a silent one.
//!
//! # Locale set
//!
//! Baked in: `und` (root, for ICU4X's locale-fallback chain), the 10 region
//! tags actually present in `test-files/`/`test-cases/`
//! (`ca-ES, de-AT, de-DE, en-CA, en-GB, en-US, fr-FR, it-IT, pl-PL, ru-RU`),
//! and — since #128 — `de-CH`, `en-ZA`, `es-MX`: the region-divergent
//! locales #124 names by name, each now with a real caller
//! (`de-CH` a fixture in `tests/document_locale.rs`, `en-ZA`/`es-MX`
//! [`decimal_separator_for_tag`]'s own test) rather than being spec-only.
//! `scripts/make_icu_data.sh` is the single source of truth for this list —
//! extend it there, not here, when a later phase needs a region this doesn't
//! cover yet (see the "Known simplification" comment on
//! [`Locale::from_tag`](crate::render::resolve::locale::Locale::from_tag)
//! for the ones still missing, e.g. `it-CH`).
//!
//! # No Cargo feature gate
//!
//! Deliberately always-on, like `bitflags`/`quick-xml`, not optional like
//! `subset-fonts`. `subset-fonts`'s motivation is output size, a trade-off a
//! caller might reasonably decline; this is correctness plumbing for
//! behavior that already exists today and is simply wrong for non-English
//! documents (a decimal tab in the wrong place isn't an optional feature).
//! A flag would fork correct output behind a switch most consumers never
//! see — there's one wheel per OS/arch already. If the locale set above
//! grows enough for wheel size to become a real problem, the right lever is
//! trimming it in the regen script, not forking behavior.

use icu_provider_blob::BlobDataProvider;

pub mod bidi;
pub mod segment;

static ICU_DATA: &[u8] = include_bytes!("data/icu_data.blob");

thread_local! {
    /// Per-thread cache: `BlobDataProvider`'s `Yoke<_, Option<Cart>>` carries
    /// an `Rc` in its `Cart` side regardless of which constructor built it
    /// (even the zero-copy `try_new_from_static_blob` path used here), so the
    /// type is `!Send`/`!Sync` and cannot live behind a process-wide `static`
    /// — confirmed by the compiler, not a design preference. `thread_local!`
    /// is the closest available reuse: one parse per thread instead of one
    /// per call, with no cross-thread sharing to prove sound.
    static PROVIDER: BlobDataProvider = BlobDataProvider::try_new_from_static_blob(ICU_DATA)
        .expect(
            "src/i18n/data/icu_data.blob is committed and regenerated by \
             scripts/make_icu_data.sh; a parse failure here means the blob \
             and the installed icu_provider_blob version have drifted",
        );
}

/// Run `f` with this thread's cached ICU4X data provider, loaded once per
/// thread from the blob committed at `src/i18n/data/icu_data.blob`
/// (regenerated by `scripts/make_icu_data.sh` — see the module doc for what's
/// baked in and why). ICU locale data is immutable reference data with no
/// mutation path anywhere in this codebase, unlike `FontRegistry`
/// (`crate::render::fonts`), which is deliberately *not* cached across
/// renders because the subset pass mutates typeface bytes in place after
/// layout — reuse here is safe on that front; see `PROVIDER`'s own doc for
/// why it's thread-local rather than process-wide.
pub fn with_data_provider<R>(f: impl FnOnce(&BlobDataProvider) -> R) -> R {
    PROVIDER.with(|provider| f(provider))
}

/// §17.18.85 (issue #128): the character a `decimal` tab zone aligns on for a
/// BCP-47 tag, resolved through real CLDR data instead of the primary-subtag
/// bucket `Locale::from_tag` uses — the bucket cannot represent a region that
/// diverges from its own language's answer (`de-CH` writes a point where
/// plain `de` writes a comma; see that function's "Known simplification"
/// doc). Returns `None` when `tag` doesn't parse as BCP-47, or when this
/// engine's baked data (`scripts/make_icu_data.sh`) has nothing for it even
/// after ICU4X's own locale fallback — the caller's job is deciding what to
/// do then, not this function's; `Locale::decimal_separator`'s hand-rolled
/// bucket is what every call site here falls back to.
pub fn decimal_separator_for_tag(tag: &str) -> Option<char> {
    use icu_decimal::options::DecimalFormatterOptions;
    use icu_decimal::DecimalFormatter;

    let locale: icu_locale_core::Locale = tag.parse().ok()?;
    with_data_provider(|provider| {
        let formatter = DecimalFormatter::try_new_with_buffer_provider(
            provider,
            locale.into(),
            DecimalFormatterOptions::default(),
        )
        .ok()?;
        // A single integer digit and a single fractional digit can never
        // trigger a grouping separator (that needs several digits on one
        // side), so whatever sits between "0" and "1" in the formatted
        // output is unambiguously the decimal separator alone — no risk of
        // reading `de-CH`'s grouping `'` where its decimal `.` belongs.
        let formatted = formatter.format_to_string(&"0.1".parse().ok()?);
        formatted
            .strip_prefix('0')?
            .strip_suffix('1')?
            .chars()
            .next()
    })
}

/// §17.16.4.2 (issue #129): a Gregorian month's name in `tag`'s language —
/// `MMMM` when `long`, `MMM` when not.
///
/// Only `date`'s month is read, so a caller holding just a month number can
/// build one with any valid day. OOXML date pictures are Gregorian, which is
/// why the calendar is fixed rather than resolved from the locale: a `w:lang`
/// of `th-TH` asks for Thai *names*, not Thai *era* reckoning.
///
/// **Stand-alone, not format, names.** CLDR carries two sets, and they differ
/// in more than case: `ru` writes "август" stand-alone against "августа"
/// (genitive) in format position, `pl` "sierpień" against "sierpnia", and
/// `ca` "agost" against "d’agost" — the format set carries the *preposition*.
/// §17.16.4.2 does not say which OOXML means, and the spec's own model
/// decides it: a picture is a caller-authored template whose tokens are
/// substituted one at a time, not a CLDR pattern. Substituting the format set
/// would inject CLDR's own connective glue into a template that already
/// supplies its own, doubling it — `\@ "d 'de' MMMM"` would render
/// "10 de d’agost". The stand-alone set is the bare name a token asks for.
/// **Word reference render**: unverified against Word, which is reported to
/// use one month-name table per language rather than distinguishing the two;
/// a Russian document whose picture is `d MMMM yyyy` would settle it, since
/// only that context makes the genitive correct.
///
/// `None` under the same two conditions [`decimal_separator_for_tag`]
/// documents — an unparseable tag, or one this engine's baked data has
/// nothing for even after ICU4X's own fallback. `field::format`'s hardcoded
/// English tables are what the call sites fall back to.
pub fn month_name_for_tag(
    date: &icu_calendar::Date<icu_calendar::Gregorian>,
    long: bool,
    tag: &str,
) -> Option<String> {
    use icu_datetime::fieldsets::M;
    use icu_datetime::pattern::{FixedCalendarDateTimeNames, MonthNameLength};

    let (length, pattern) = if long {
        (MonthNameLength::StandaloneWide, "LLLL")
    } else {
        (MonthNameLength::StandaloneAbbreviated, "LLL")
    };
    let locale: icu_locale_core::Locale = tag.parse().ok()?;
    with_data_provider(|provider| {
        let mut names =
            FixedCalendarDateTimeNames::<icu_calendar::Gregorian, M>::new_without_number_formatting(
                locale.into(),
            );
        names
            .load_month_names(&as_data_provider(provider), length)
            .ok()?;
        let pattern: icu_datetime::pattern::DateTimePattern = pattern.parse().ok()?;
        let formatter = names.with_pattern_unchecked(&pattern);
        write_field(formatter.format(date))
    })
}

/// §17.16.4.2 (issue #129): a weekday's name in `tag`'s language — `dddd`
/// when `long`, `ddd` when not.
///
/// Takes a bare [`Weekday`](icu_calendar::types::Weekday) rather than a date,
/// because that is all the underlying CLDR data is keyed by;
/// `icu_calendar::Date::weekday` is the route from a date to one. Unlike
/// month names, weekday-name data is calendar-agnostic — one CLDR marker
/// covers every calendar system.
///
/// Stand-alone names, for the reason [`month_name_for_tag`] gives. No locale
/// baked in today distinguishes the two sets for weekdays (`ru`, `pl` and
/// `ca` all repeat themselves where their months diverge), so this matches
/// month handling for consistency rather than because a case demands it.
///
/// `None` under the same conditions as [`month_name_for_tag`].
pub fn weekday_name_for_tag(
    weekday: icu_calendar::types::Weekday,
    long: bool,
    tag: &str,
) -> Option<String> {
    use icu_datetime::fieldsets::E;
    use icu_datetime::pattern::{FixedCalendarDateTimeNames, WeekdayNameLength};

    let (length, pattern) = if long {
        (WeekdayNameLength::StandaloneWide, "cccc")
    } else {
        (WeekdayNameLength::StandaloneAbbreviated, "ccc")
    };
    let locale: icu_locale_core::Locale = tag.parse().ok()?;
    with_data_provider(|provider| {
        let mut names =
            FixedCalendarDateTimeNames::<icu_calendar::Gregorian, E>::new_without_number_formatting(
                locale.into(),
            );
        names
            .load_weekday_names(&as_data_provider(provider), length)
            .ok()?;
        let pattern: icu_datetime::pattern::DateTimePattern = pattern.parse().ok()?;
        let formatter = names.with_pattern_unchecked(&pattern);
        write_field(formatter.format(&weekday))
    })
}

/// Deserialize a buffer provider into the `DataProvider` the `load_*_names`
/// calls above want — the same conversion ICU4X's own
/// `try_new_with_buffer_provider` constructors perform internally.
fn as_data_provider(
    provider: &BlobDataProvider,
) -> icu_provider::buf::DeserializingBufferProvider<'_, BlobDataProvider> {
    use icu_provider::buf::AsDeserializingBufferProvider;
    provider.as_deserializing()
}

/// Collect a formatted single-field pattern into a `String`.
///
/// The `TryWriteable` error side means "a field in the pattern had no loaded
/// names" — unreachable for the two literal single-field patterns above,
/// each of which formats exactly the field whose names were just loaded, but
/// answered as `None` rather than unwrapped so a future pattern that gets
/// this wrong degrades to the English fallback instead of panicking mid-render.
fn write_field(formatted: impl writeable::TryWriteable) -> Option<String> {
    formatted
        .try_write_to_string()
        .ok()
        .map(|written| written.into_owned())
}

#[cfg(test)]
mod tests {
    use super::*;
    use fixed_decimal::Decimal;
    use icu_decimal::options::DecimalFormatterOptions;
    use icu_decimal::DecimalFormatter;

    /// Phase 0 (#127) proof: one ICU4X component, formatting through the
    /// blob-provider path every later phase reuses. Covers every locale
    /// `scripts/make_icu_data.sh` bakes in (plus root `und`), not just one —
    /// a single passing locale wouldn't distinguish "the blob has real CLDR
    /// data for the whole set" from "the blob happens to have `de-DE`".
    /// Expected strings were read off this formatter's own real output
    /// (`cargo test -- --nocapture` against a throwaway probe), not guessed:
    /// `de-AT`/`ru-RU` group with U+00A0 NO-BREAK SPACE (not a point/comma
    /// like `de-DE`), `fr-FR` with U+202F NARROW NO-BREAK SPACE, and
    /// `it-IT`/`pl-PL` don't group a 4-digit number at all — real regional
    /// divergence already visible within this small a set, which is
    /// precisely the kind of fact a hand-rolled table (`Locale::from_tag`)
    /// can't be expected to get right without a source like this behind it.
    #[test]
    fn icu_decimal_formats_via_blob_provider() {
        const CASES: &[(&str, &str)] = &[
            ("und", "1,234"),
            ("ca-ES", "1.234"),
            ("de-AT", "1\u{a0}234"),
            ("de-DE", "1.234"),
            ("en-CA", "1,234"),
            ("en-GB", "1,234"),
            ("en-US", "1,234"),
            ("fr-FR", "1\u{202f}234"),
            ("it-IT", "1234"),
            ("pl-PL", "1234"),
            ("ru-RU", "1\u{a0}234"),
        ];
        for &(tag, expected) in CASES {
            let out = with_data_provider(|provider| {
                let formatter = DecimalFormatter::try_new_with_buffer_provider(
                    provider,
                    tag.parse::<icu_locale_core::Locale>()
                        .unwrap_or_else(|e| panic!("{tag:?} is a valid BCP-47 tag: {e}"))
                        .into(),
                    DecimalFormatterOptions::default(),
                )
                .unwrap_or_else(|e| panic!("{tag} is baked into src/i18n/data/icu_data.blob: {e}"));
                formatter.format_to_string(&Decimal::from(1234))
            });
            assert_eq!(out, expected, "locale {tag}");
        }
    }

    /// Issue #128's acceptance criterion, at the unit closest to the claim:
    /// `de-DE` and `de-CH` disagree, and `en-ZA`/`es-MX` — named in #128 as
    /// needing verification against real regional usage, not a guess — get
    /// their real CLDR answers too. All four values were read off this
    /// function's own output, not assumed: `de-CH` and `es-MX` behave like
    /// English (point) despite neither being English; `en-ZA` behaves like
    /// German (comma) despite not being German. Region, not primary subtag,
    /// decides — exactly the fact `Locale::from_tag` cannot represent.
    #[test]
    fn decimal_separator_for_tag_resolves_region_divergence() {
        assert_eq!(decimal_separator_for_tag("de-DE"), Some(','));
        assert_eq!(decimal_separator_for_tag("de-CH"), Some('.'));
        assert_eq!(decimal_separator_for_tag("en-ZA"), Some(','));
        assert_eq!(decimal_separator_for_tag("es-MX"), Some('.'));
    }

    #[test]
    fn decimal_separator_for_tag_is_none_for_a_tag_with_no_data() {
        assert_eq!(decimal_separator_for_tag("zz-ZZ"), None);
    }

    #[test]
    fn decimal_separator_for_tag_is_none_for_an_unparseable_tag() {
        assert_eq!(decimal_separator_for_tag(""), None);
        assert_eq!(decimal_separator_for_tag("not a bcp47 tag!"), None);
    }

    // ── §17.16.4.2 date-picture names (issue #129) ──────────────────────────

    /// 2026-08-10, a Monday — one date answering both lookups, so the month
    /// and weekday cases can't drift apart. Expected strings below were read
    /// off these functions' own output against the committed blob, not
    /// guessed from a table.
    fn reference_date() -> icu_calendar::Date<icu_calendar::Gregorian> {
        icu_calendar::Date::try_new_gregorian(2026, 8, 10).expect("2026-08-10 is a real date")
    }

    #[test]
    fn month_name_for_tag_localizes() {
        let d = reference_date();
        assert_eq!(
            month_name_for_tag(&d, true, "en-US").as_deref(),
            Some("August")
        );
        assert_eq!(
            month_name_for_tag(&d, false, "en-US").as_deref(),
            Some("Aug")
        );
        assert_eq!(
            month_name_for_tag(&d, true, "fr-FR").as_deref(),
            Some("août")
        );
        assert_eq!(
            month_name_for_tag(&d, true, "ru-RU").as_deref(),
            Some("август")
        );
        // German's August happens to be spelled like English's — included
        // deliberately so a reader doesn't mistake it for a failed lookup.
        assert_eq!(
            month_name_for_tag(&d, true, "de-DE").as_deref(),
            Some("August")
        );
    }

    #[test]
    fn weekday_name_for_tag_localizes() {
        let monday = reference_date().weekday();
        assert_eq!(
            weekday_name_for_tag(monday, true, "en-US").as_deref(),
            Some("Monday")
        );
        assert_eq!(
            weekday_name_for_tag(monday, false, "en-US").as_deref(),
            Some("Mon")
        );
        assert_eq!(
            weekday_name_for_tag(monday, true, "de-DE").as_deref(),
            Some("Montag")
        );
        assert_eq!(
            weekday_name_for_tag(monday, false, "de-DE").as_deref(),
            Some("Mo")
        );
        assert_eq!(
            weekday_name_for_tag(monday, true, "fr-FR").as_deref(),
            Some("lundi")
        );
    }

    /// The reference date really is a Monday — if this fails, every expected
    /// weekday string above is measuring the wrong day.
    #[test]
    fn the_reference_date_is_a_monday() {
        assert_eq!(
            reference_date().weekday(),
            icu_calendar::types::Weekday::Monday
        );
    }

    /// Locks in the stand-alone-vs-format choice `month_name_for_tag`'s doc
    /// argues for, in the three baked locales where CLDR's two sets actually
    /// differ. The format set would give "августа", "sierpnia" and "d’agost"
    /// — the last carrying a preposition that would double against a
    /// picture's own literal text. If this test starts failing, the choice
    /// was changed; re-read that doc before updating the expectations.
    #[test]
    fn month_names_are_the_standalone_set_not_the_format_set() {
        let d = reference_date();
        assert_eq!(
            month_name_for_tag(&d, true, "ru-RU").as_deref(),
            Some("август")
        );
        assert_eq!(
            month_name_for_tag(&d, true, "pl-PL").as_deref(),
            Some("sierpień")
        );
        assert_eq!(
            month_name_for_tag(&d, true, "ca-ES").as_deref(),
            Some("agost")
        );
    }

    /// Same two failure modes as `decimal_separator_for_tag`, so the caller's
    /// fallback (`field::format`'s English tables) is reachable identically.
    #[test]
    fn date_names_are_none_for_unusable_tags() {
        let d = reference_date();
        let monday = d.weekday();
        assert_eq!(month_name_for_tag(&d, true, "zz-ZZ"), None);
        assert_eq!(month_name_for_tag(&d, true, "not a bcp47 tag!"), None);
        assert_eq!(weekday_name_for_tag(monday, true, "zz-ZZ"), None);
        assert_eq!(weekday_name_for_tag(monday, true, ""), None);
    }
}