mf2-locale-data 2.0.0

//! `data/currencies.txt` and `data/units.txt` from every locale's
//! `currencies.json` / `units.json` in the `cargo xtask cldr-sync` cache
//! (feature `extract`). The records are CLDR's resolved ones; each locale's
//! lines keep the fields that differ from its parent's (the parent chain of
//! `data/numbers.txt`), and writing re-resolves every locale through the
//! text and requires CLDR's record exactly.
//!
//! Memory: 766 locales' resolved unit data is ~100 MB of strings, so the
//! records are interned (`(key, field, value)` ids), one file read at a time.

use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::fmt::Write as _;
use std::fs;
use std::path::Path;

use serde_json::Value;

use crate::blocks::{Block, Blocks, keys, resolve};
use crate::error::Error;
use crate::number::table::{Table, escape_tab};

use crate::currency::FIELDS as CURRENCY_FIELDS;
use crate::unit::{CATEGORIES, FIELDS as UNIT_FIELDS, WIDTHS};

const CURRENCY_HEADER: &str = "\
# mf2-locale-data: CLDR currency data for every CLDR locale (plans/05-tooling.md §7).
# Generated by `cargo xtask locale-data` from the `cargo xtask cldr-sync` cache (every
# locale's cldr-numbers-full/main/<locale>/currencies.json) and third_party/cldr-json
# (currencyData.json). Do not edit. TAB-separated lines: fraction <CODE|DEFAULT>
# digits=… [rounding=…] · currency <locale> <CODE> field=… — a field is written only where
# it differs from what the parents (those of data/numbers.txt) and CLDR's fallback give:
# narrow → symbol; name-<category> → name-other → name. Absent everywhere, symbol and
# name are the code. `\\u{…}` escapes invisible characters and `\\`. Format: src/currency.rs.
";

const UNIT_HEADER: &str = "\
# mf2-locale-data: CLDR unit data for every CLDR locale (plans/05-tooling.md §7).
# Generated by `cargo xtask locale-data` from the `cargo xtask cldr-sync` cache (every
# locale's cldr-units-full/main/<locale>/units.json). Do not edit. TAB-separated lines:
# unit-id <id> <category> (MF2's identifier is CLDR's key without its category) · per
# <locale> <width> pattern=… (the `per` compound pattern) · unit <locale> <width> <id>
# field=… (name, per = perUnitPattern, other … many = unitPattern-count-*) — a field is
# written only where it differs from what the parents (those of data/numbers.txt) and
# CLDR's fallback (a plural form → other) give. Case and gender forms are left out (MF2
# has no case option). `\\u{…}` escapes invisible characters and `\\`. Format: src/unit.rs.
";

fn assumption(message: String) -> Error {
    Error::Assumption(message)
}

fn read(path: &Path) -> Result<String, Error> {
    fs::read_to_string(path).map_err(|source| Error::Io {
        path: path.to_path_buf(),
        source,
    })
}

#[derive(Default)]
struct Interner {
    map: HashMap<String, u32>,
    strs: Vec<String>,
}

impl Interner {
    fn id(&mut self, s: &str) -> u32 {
        if let Some(&i) = self.map.get(s) {
            return i;
        }
        let i = u32::try_from(self.strs.len()).unwrap_or(u32::MAX);
        self.map.insert(s.to_owned(), i);
        self.strs.push(s.to_owned());
        i
    }

    fn get(&self, i: u32) -> &str {
        self.strs.get(i as usize).map_or("", String::as_str)
    }
}

/// A locale's record: `(key, field, value)` ids, sorted.
type Record = Vec<(u32, u32, u32)>;

struct Extracted {
    strings: Interner,
    records: BTreeMap<String, Record>,
}

impl Extracted {
    fn add(&mut self, locale: &str, raw: &Raw) {
        let mut rec = Record::new();
        for (key, fields) in raw {
            let k = self.strings.id(key);
            for (f, v) in fields {
                let f = self.strings.id(f);
                let v = self.strings.id(v);
                rec.push((k, f, v));
            }
        }
        rec.sort_unstable();
        self.records.insert(locale.to_owned(), rec);
    }
}

/// One locale's CLDR record: key (`currency USD`, `unit short kilometer`,
/// `per short`) → field → value, as its resolved JSON has them.
pub type Raw = BTreeMap<String, BTreeMap<&'static str, String>>;

/// The fields of a raw record's key as CLDR resolves them (a missing field
/// through `fallback`): what the table must resolve to.
pub fn effective<'r>(
    fields: &'r BTreeMap<&'static str, String>,
    order: &[&'static str],
    fallback: impl Fn(&str) -> Option<&'static str>,
) -> BTreeMap<&'static str, &'r str> {
    let value = |f: &'static str| {
        let mut f = Some(f);
        while let Some(x) = f {
            if let Some(v) = fields.get(x) {
                return Some(v.as_str());
            }
            f = fallback(x);
        }
        None
    };
    order.iter().filter_map(|f| Some((*f, value(f)?))).collect()
}

/// A locale's currency record from its `currencies.json`, every value
/// checked (known fields only; patterns parse; no control characters).
pub fn currency_raw(locale: &str, json: &str) -> Result<Raw, Error> {
    let v: Value = serde_json::from_str(json)?;
    let c = v
        .pointer(&format!("/main/{locale}/numbers/currencies"))
        .and_then(Value::as_object)
        .ok_or_else(|| Error::Shape(format!("{locale}: currencies")))?;
    let mut raw = Raw::new();
    for (code, d) in c {
        if code.len() != 3 || !code.bytes().all(|b| b.is_ascii_uppercase()) {
            return Err(assumption(format!("{locale}: currency code {code:?}")));
        }
        let d = d
            .as_object()
            .ok_or_else(|| Error::Shape(format!("{locale} {code}")))?;
        let mut fields = BTreeMap::new();
        for (k, v) in d {
            let v = v
                .as_str()
                .ok_or_else(|| Error::Shape(format!("{locale} {code} {k}")))?;
            let field: &'static str = match k.as_str() {
                "symbol" => "symbol",
                "symbol-alt-narrow" => "narrow",
                "displayName" => "name",
                // The currency's own standard pattern and separators
                // (`en-DE` EUR `¤#,##0.00`, `pt-CV` CVE decimal `$`).
                "pattern" => {
                    crate::number::pattern::parse(v)?;
                    "pattern"
                }
                "decimal" => "decimal",
                "group" => "group",
                // MF2 has no `variant` or `formal` display.
                "symbol-alt-variant" | "symbol-alt-formal" => continue,
                k => match k.strip_prefix("displayName-count-") {
                    Some(cat) => CURRENCY_FIELDS
                        .iter()
                        .copied()
                        .find(|f| f.strip_prefix("name-") == Some(cat))
                        .ok_or_else(|| assumption(format!("{locale} {code}: category {cat:?}")))?,
                    None => {
                        return Err(assumption(format!(
                            "{locale} {code}: unknown currency field {k:?}"
                        )));
                    }
                },
            };
            if v.is_empty() || v.len() > 255 || v.chars().any(char::is_control) {
                return Err(assumption(format!("{locale} {code} {k}: value {v:?}")));
            }
            fields.insert(field, v.to_owned());
        }
        if !fields.is_empty() {
            raw.insert(format!("currency {code}"), fields);
        }
    }
    Ok(raw)
}

/// A locale's unit record from its `units.json` (the `per` compound
/// patterns and every unit with data, by width), every value checked;
/// `categories` collects identifier → CLDR category and refuses an
/// identifier in two categories.
pub fn unit_raw(
    locale: &str,
    json: &str,
    categories: &mut BTreeMap<String, String>,
) -> Result<Raw, Error> {
    let doc: Value = serde_json::from_str(json)?;
    let units = doc
        .pointer(&format!("/main/{locale}/units"))
        .and_then(Value::as_object)
        .ok_or_else(|| Error::Shape(format!("{locale}: units")))?;
    let mut raw = Raw::new();
    for width in WIDTHS {
        let members = units
            .get(width)
            .and_then(Value::as_object)
            .ok_or_else(|| Error::Shape(format!("{locale}: units/{width}")))?;
        let per = members
            .get("per")
            .and_then(|p| p.get("compoundUnitPattern"))
            .and_then(Value::as_str)
            .ok_or_else(|| assumption(format!("{locale} {width}: no per pattern")))?;
        crate::template::parse(per, 2)?;
        raw.insert(
            format!("per {width}"),
            BTreeMap::from([("pattern", per.to_owned())]),
        );
        for (cldr_key, obj) in members {
            if compound_member(cldr_key) {
                continue;
            }
            let obj = obj
                .as_object()
                .ok_or_else(|| Error::Shape(format!("{locale} {width} {cldr_key}")))?;
            let (category, id) = cldr_key
                .split_once('-')
                .ok_or_else(|| assumption(format!("unit key {cldr_key:?} without category")))?;
            let mut fields: BTreeMap<&'static str, String> = BTreeMap::new();
            for (k, v) in obj {
                if ignored_unit_key(k) {
                    continue;
                }
                let v = v
                    .as_str()
                    .ok_or_else(|| Error::Shape(format!("{locale} {cldr_key} {k}")))?;
                let field: &'static str = match k.as_str() {
                    "displayName" => {
                        if v.contains(['{', '}']) || v.chars().any(char::is_control) {
                            return Err(assumption(format!("{locale} {cldr_key}: name {v:?}")));
                        }
                        "name"
                    }
                    "perUnitPattern" => {
                        crate::template::parse(v, 1)?;
                        "per"
                    }
                    k => match k.strip_prefix("unitPattern-count-") {
                        Some(cat) => {
                            let f = CATEGORIES.iter().copied().find(|c| *c == cat).ok_or_else(
                                || assumption(format!("{locale} {cldr_key}: category {cat:?}")),
                            )?;
                            crate::template::parse(v, 1)?;
                            f
                        }
                        None => {
                            return Err(assumption(format!(
                                "{locale} {cldr_key}: unknown unit field {k:?}"
                            )));
                        }
                    },
                };
                if v.len() > 255 {
                    return Err(assumption(format!(
                        "{locale} {cldr_key} {k}: over 255 bytes"
                    )));
                }
                fields.insert(field, v.to_owned());
            }
            if fields.is_empty() {
                continue;
            }
            let has_counts = fields.keys().any(|f| CATEGORIES.contains(f));
            if has_counts && !fields.contains_key("other") {
                return Err(assumption(format!(
                    "{locale} {width} {cldr_key}: patterns without other"
                )));
            }
            match categories.get(id) {
                Some(c) if c != category => {
                    return Err(assumption(format!(
                        "unit identifier {id} in two categories: {c} and {category}"
                    )));
                }
                Some(_) => {}
                None => {
                    categories.insert(id.to_owned(), category.to_owned());
                }
            }
            raw.insert(format!("unit {width} {id}"), fields);
        }
    }
    Ok(raw)
}

/// A locale's stored fields: key id → field → value id.
type Stored = BTreeMap<u32, BTreeMap<&'static str, u32>>;

/// What the table resolves for `(key, field)` through `chain` (the
/// locale's stored fields so far, then its ancestors'): the first that has
/// it, else the field's fallback.
fn table_value(
    chain: &[&Stored],
    key: u32,
    field: &'static str,
    fallback: &impl Fn(&str) -> Option<&'static str>,
) -> Option<u32> {
    for s in chain {
        if let Some(v) = s.get(&key).and_then(|m| m.get(field)) {
            return Some(*v);
        }
    }
    table_value(chain, key, fallback(field)?, fallback)
}

/// CLDR's value of `field` in a locale's raw record of one key: the field,
/// else its fallback's.
fn cldr_value(
    raw: &BTreeMap<&str, u32>,
    field: &'static str,
    fallback: &impl Fn(&str) -> Option<&'static str>,
) -> Option<u32> {
    raw.get(field)
        .copied()
        .or_else(|| cldr_value(raw, fallback(field)?, fallback))
}

/// Renders the table body and checks the round trip. A locale stores a
/// field exactly when CLDR's value differs from what the table would
/// resolve without it (its ancestors' stored fields, then the fallback), so
/// both parent inheritance and CLDR's own fallbacks (a plural form equal to
/// `other`, a narrow symbol equal to the symbol) cost nothing. `kinds` are
/// the locale-line kinds, `key_words(kind)` their key words, `order` every
/// field, fallback targets before the fields that fall back to them.
fn render(
    header: &str,
    ex: &Extracted,
    numbers: &Table<'_>,
    kinds: &[&str],
    key_words: impl Fn(&str) -> usize + Copy,
    order: &[&'static str],
    fallback: impl Fn(&str) -> Option<&'static str>,
) -> Result<String, Error> {
    let field_name = |id: u32| -> Result<&'static str, Error> {
        let name = ex.strings.get(id);
        order
            .iter()
            .copied()
            .find(|f| *f == name)
            .ok_or_else(|| assumption(format!("field {name:?} not in the table's field list")))
    };
    // Raw records as key → field → value.
    let mut raw: BTreeMap<&str, BTreeMap<u32, BTreeMap<&'static str, u32>>> = BTreeMap::new();
    for (locale, rec) in &ex.records {
        let mut m: BTreeMap<u32, BTreeMap<&'static str, u32>> = BTreeMap::new();
        for &(k, f, v) in rec {
            m.entry(k).or_default().insert(field_name(f)?, v);
        }
        raw.insert(locale.as_str(), m);
    }
    // Parents first.
    let mut todo: Vec<&str> = raw.keys().copied().collect();
    todo.sort_by_key(|l| (numbers.chain(l).len(), *l));
    let mut stored: HashMap<&str, Stored> = HashMap::new();
    let empty = BTreeMap::new();
    for locale in todo {
        let chain_names = numbers.chain(locale);
        let parent = chain_names.get(1).copied();
        let mine_raw = raw.get(locale).unwrap_or(&empty);
        // Resolved: every (key, field) of the parent is in the child.
        if let Some(p) = parent.and_then(|p| raw.get(p)) {
            for (k, fields) in p {
                for f in fields.keys() {
                    if mine_raw.get(k).and_then(|m| m.get(f)).is_none() {
                        return Err(assumption(format!(
                            "{locale}: no {} {f} although its parent {parent:?} has it (not resolved)",
                            ex.strings.get(*k)
                        )));
                    }
                }
            }
        }
        let mut mine = Stored::new();
        for (k, fields) in mine_raw {
            for f in order {
                let want = cldr_value(fields, f, &fallback);
                let ancestors: Vec<&Stored> = chain_names
                    .iter()
                    .skip(1)
                    .filter_map(|l| stored.get(l))
                    .collect();
                let mut chain: Vec<&Stored> = vec![&mine];
                chain.extend(ancestors);
                let have = table_value(&chain, *k, f, &fallback);
                if want != have {
                    let Some(v) = want else {
                        return Err(assumption(format!(
                            "{locale} {} {f}: CLDR has none, the table would inherit one",
                            ex.strings.get(*k)
                        )));
                    };
                    mine.entry(*k).or_default().insert(f, v);
                }
            }
        }
        stored.insert(locale, mine);
    }

    let mut out = header.to_owned();
    for locale in raw.keys() {
        let Some(mine) = stored.get(locale) else {
            continue;
        };
        let mut lines: Vec<(&str, &BTreeMap<&'static str, u32>)> =
            mine.iter().map(|(k, f)| (ex.strings.get(*k), f)).collect();
        lines.sort_by_key(|(k, _)| *k);
        for (key, fields) in lines {
            let (kind, rest) = key.split_once(' ').unwrap_or((key, ""));
            let _ = write!(out, "{kind}\t{locale}\t{}", rest.replace(' ', "\t"));
            for f in order {
                if let Some(v) = fields.get(f) {
                    let _ = write!(out, "\t{f}={}", escape_tab(ex.strings.get(*v)));
                }
            }
            out.push('\n');
        }
    }

    // The round trip, through the text: every field of every key, as CLDR
    // resolves it.
    let blocks = Blocks::index(&out, kinds)?;
    let mut cache: HashMap<&str, Block<'_>> = HashMap::new();
    for (locale, keys_raw) in &raw {
        let chain_names = numbers.chain(locale);
        for l in &chain_names {
            if !cache.contains_key(l) {
                cache.insert(l, blocks.block(l, key_words, |_, f| order.contains(&f))?);
            }
        }
        let chain: Vec<&Block<'_>> = chain_names.iter().filter_map(|l| cache.get(l)).collect();
        let mut have_keys = BTreeSet::new();
        for kind in kinds {
            have_keys.extend(keys(&chain, kind));
        }
        let want_keys: BTreeSet<&str> = keys_raw.keys().map(|k| ex.strings.get(*k)).collect();
        if have_keys != want_keys {
            return Err(assumption(format!(
                "{locale}: the table resolves {} keys, CLDR has {}",
                have_keys.len(),
                want_keys.len()
            )));
        }
        for (k, fields) in keys_raw {
            let key = ex.strings.get(*k);
            let want: BTreeMap<&str, &str> = order
                .iter()
                .filter_map(|f| Some((*f, ex.strings.get(cldr_value(fields, f, &fallback)?))))
                .collect();
            let have = resolve(&chain, key, order, &fallback);
            if have != want {
                return Err(assumption(format!(
                    "{locale} {key}: the table resolves to {have:?}, CLDR says {want:?}"
                )));
            }
        }
    }
    Ok(out)
}

/// The text of `data/currencies.txt`: CLDR's fraction digits and rounding
/// (`currencyData.json`), then per locale and currency the fields that differ
/// from its parent's. `main` is `cldr-numbers-full/main` in the cache;
/// `numbers` the text of `data/numbers.txt` (the parent chain).
pub fn currencies_table(
    main: &Path,
    locales: &[String],
    currency_data: &str,
    numbers: &str,
    cldr: &str,
) -> Result<String, Error> {
    let numbers = Table::parse(numbers)?;
    let mut ex = Extracted {
        strings: Interner::default(),
        records: BTreeMap::new(),
    };
    for locale in locales {
        let text = read(&main.join(locale).join("currencies.json"))?;
        let raw = currency_raw(locale, &text)?;
        ex.add(locale, &raw);
    }

    let cd: Value = serde_json::from_str(currency_data)?;
    let fractions = cd
        .pointer("/supplemental/currencyData/fractions")
        .and_then(Value::as_object)
        .ok_or_else(|| Error::Shape("currencyData/fractions".to_owned()))?;
    let mut header = String::new();
    header.push_str(CURRENCY_HEADER);
    let _ = writeln!(header, "cldr\t{cldr}");
    let mut codes: Vec<&String> = fractions.keys().collect();
    codes.sort_by_key(|c| (c.as_str() != "DEFAULT", c.as_str()));
    for code in codes {
        let f = fractions
            .get(code)
            .ok_or_else(|| Error::Shape(code.clone()))?;
        let num = |k: &str| {
            f.get(k)
                .and_then(Value::as_str)
                .and_then(|s| s.parse::<u32>().ok())
        };
        let (Some(digits), Some(rounding)) = (num("_digits"), num("_rounding")) else {
            return Err(assumption(format!(
                "fractions {code}: no _digits/_rounding"
            )));
        };
        if digits > 15 {
            return Err(assumption(format!("fractions {code}: {digits} digits")));
        }
        let _ = write!(header, "fraction\t{code}\tdigits={digits}");
        if rounding != 0 {
            let _ = write!(header, "\trounding={rounding}");
        }
        header.push('\n');
    }
    render(
        &header,
        &ex,
        &numbers,
        &["currency"],
        |_| 1,
        CURRENCY_FIELDS,
        crate::currency::fallback,
    )
}

/// Whether `k` is a key of a unit object that MF2 has no use for:
/// grammatical gender and case forms (MF2 has no case option).
fn ignored_unit_key(k: &str) -> bool {
    k == "gender"
        || k.split_once("-count-")
            .is_some_and(|(case, cat)| case != "unitPattern" && CATEGORIES.contains(&cat))
}

/// Whether a width member is a compound or prefix pattern, not a unit.
fn compound_member(k: &str) -> bool {
    matches!(k, "per" | "times" | "power2" | "power3" | "coordinateUnit")
        || k.strip_prefix("10p").is_some_and(|n| {
            n.trim_start_matches('-')
                .bytes()
                .all(|b| b.is_ascii_digit())
        })
        || k.strip_prefix("1024p")
            .is_some_and(|n| n.bytes().all(|b| b.is_ascii_digit()))
}

/// The text of `data/units.txt`: the identifier map (`unit-id <id>
/// <category>`: MF2's unit identifiers are CLDR's without the category
/// prefix, checked to be unique), then per locale the `per` compound
/// pattern of each width and per width and unit the fields that differ
/// from the parent's. `main` is `cldr-units-full/main` in the cache.
pub fn units_table(
    main: &Path,
    locales: &[String],
    numbers: &str,
    cldr: &str,
) -> Result<String, Error> {
    let numbers = Table::parse(numbers)?;
    let mut ex = Extracted {
        strings: Interner::default(),
        records: BTreeMap::new(),
    };
    let mut categories: BTreeMap<String, String> = BTreeMap::new();
    for locale in locales {
        let text = read(&main.join(locale).join("units.json"))?;
        let raw = unit_raw(locale, &text, &mut categories)?;
        ex.add(locale, &raw);
    }
    let mut header = String::new();
    header.push_str(UNIT_HEADER);
    let _ = writeln!(header, "cldr\t{cldr}");
    for (id, category) in &categories {
        let _ = writeln!(header, "unit-id\t{id}\t{category}");
    }
    let order: Vec<&'static str> = std::iter::once("pattern")
        .chain(UNIT_FIELDS.iter().copied())
        .collect();
    render(
        &header,
        &ex,
        &numbers,
        &["per", "unit"],
        |kind| if kind == "unit" { 2 } else { 1 },
        &order,
        crate::unit::fallback,
    )
}