use std::collections::{BTreeMap, BTreeSet};
use std::fmt::Write as _;
use serde_json::Value;
use crate::error::Error;
use crate::number::Record;
mod measure;
use crate::plural::{Category, PluralKind};
pub use measure::{Raw, currencies_table, currency_raw, effective, unit_raw, units_table};
pub fn plurals_table(plurals_json: &str, ordinals_json: &str, cldr: &str) -> Result<String, Error> {
let mut out = String::new();
out.push_str(
"# mf2-locale-data: CLDR plural rules, cardinal and ordinal, for every CLDR locale.\n",
);
out.push_str(
"# Generated by `cargo xtask locale-data` from third_party/cldr-json. Do not edit.\n",
);
out.push_str("# Line: kind locale category rule (UTS #35 syntax, samples included).\n");
let _ = writeln!(out, "cldr {cldr}");
for (kind, json) in [
(PluralKind::Cardinal, plurals_json),
(PluralKind::Ordinal, ordinals_json),
] {
let v: Value = serde_json::from_str(json)?;
let key = match kind {
PluralKind::Cardinal => "plurals-type-cardinal",
PluralKind::Ordinal => "plurals-type-ordinal",
};
let locales = v
.pointer(&format!("/supplemental/{key}"))
.and_then(Value::as_object)
.ok_or_else(|| Error::Shape(key.to_owned()))?;
let mut names: Vec<&String> = locales.keys().collect();
names.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
for locale in names {
let rules = locales
.get(locale)
.and_then(Value::as_object)
.ok_or_else(|| Error::Shape(locale.clone()))?;
let mut lines = Vec::new();
for (name, text) in rules {
let category = name
.strip_prefix("pluralRule-count-")
.ok_or_else(|| Error::Shape(name.clone()))?;
let category = Category::from_name(category)?;
let text = text.as_str().ok_or_else(|| Error::Shape(name.clone()))?;
lines.push((category, text.trim().to_owned()));
}
lines.sort_by_key(|(c, _)| *c);
for (category, text) in lines {
let _ = writeln!(out, "{} {locale} {} {text}", kind.name(), category.as_str());
}
}
}
Ok(out)
}
pub fn directions_table(
likely_json: &str,
scripts_json: &str,
cldr: &str,
) -> Result<String, Error> {
let scripts: Value = serde_json::from_str(scripts_json)?;
let rtl_scripts: BTreeSet<&str> = scripts
.pointer("/scriptMetadata")
.and_then(Value::as_object)
.ok_or_else(|| Error::Shape("scriptMetadata".to_owned()))?
.iter()
.filter(|(_, v)| v.get("rtl").and_then(Value::as_str) == Some("YES"))
.map(|(k, _)| k.as_str())
.collect();
let likely: Value = serde_json::from_str(likely_json)?;
let likely = likely
.pointer("/supplemental/likelySubtags")
.and_then(Value::as_object)
.ok_or_else(|| Error::Shape("likelySubtags".to_owned()))?;
let script_of = |full: &str| full.split('-').nth(1).map(str::to_owned);
let is_rtl = |full: &str| script_of(full).is_some_and(|s| rtl_scripts.contains(s.as_str()));
let mut languages = BTreeSet::new();
let mut exceptions: BTreeMap<bool, BTreeSet<&str>> = BTreeMap::new();
for (key, full) in likely {
let full = full.as_str().ok_or_else(|| Error::Shape(key.clone()))?;
let parts: Vec<&str> = key.split('-').collect();
match parts.as_slice() {
["und", ..] => {}
[_] => {
if is_rtl(full) {
languages.insert(key.as_str());
}
}
[lang, region]
if region.len() == 2 && region.bytes().all(|b| b.is_ascii_uppercase())
|| region.len() == 3 && region.bytes().all(|b| b.is_ascii_digit()) =>
{
let base = likely
.get(*lang)
.and_then(Value::as_str)
.is_some_and(is_rtl);
let here = is_rtl(full);
if here != base {
exceptions.entry(here).or_default().insert(key.as_str());
}
}
_ => {}
}
}
let mut out = String::new();
out.push_str("# mf2-locale-data: the text direction of every CLDR locale.\n");
out.push_str(
"# Generated by `cargo xtask locale-data` from third_party/cldr-json. Do not edit.\n",
);
out.push_str("# A script subtag decides; else lang-REGION's exception; else the language.\n");
let _ = writeln!(out, "cldr {cldr}");
let join = |set: &BTreeSet<&str>| set.iter().copied().collect::<Vec<_>>().join(" ");
let _ = writeln!(out, "rtl-scripts {}", join(&rtl_scripts));
let _ = writeln!(out, "rtl-languages {}", join(&languages));
for (rtl, tags) in &exceptions {
let _ = writeln!(out, "{} {}", if *rtl { "rtl" } else { "ltr" }, join(tags));
}
Ok(out)
}
#[derive(Clone, Copy, Debug)]
pub struct NumberInputs<'a> {
pub locales: &'a [(String, String)],
pub parent_locales: &'a str,
pub likely_subtags: &'a str,
pub numbering_systems: &'a str,
pub available_locales: &'a str,
pub default_content: &'a str,
}
const SYSTEM_SOURCES: &[(&str, &str, &str, bool, bool)] = &[
("decimal", "symbols", "decimal", true, false),
("group", "symbols", "group", true, false),
("minus", "symbols", "minusSign", true, false),
("plus", "symbols", "plusSign", true, false),
("percent", "symbols", "percentSign", true, false),
("decimal-pattern", "decimalFormats", "standard", true, true),
("percent-pattern", "percentFormats", "standard", true, true),
("currency", "currencyFormats", "standard", true, true),
(
"currency-alpha",
"currencyFormats",
"standard-alphaNextToNumber",
false,
true,
),
(
"currency-none",
"currencyFormats",
"standard-noCurrency",
true,
true,
),
("accounting", "currencyFormats", "accounting", true, true),
(
"accounting-alpha",
"currencyFormats",
"accounting-alphaNextToNumber",
false,
true,
),
(
"accounting-none",
"currencyFormats",
"accounting-noCurrency",
true,
true,
),
(
"currency-name-zero",
"currencyFormats",
"unitPattern-count-zero",
false,
false,
),
(
"currency-name-one",
"currencyFormats",
"unitPattern-count-one",
false,
false,
),
(
"currency-name-two",
"currencyFormats",
"unitPattern-count-two",
false,
false,
),
(
"currency-name-few",
"currencyFormats",
"unitPattern-count-few",
false,
false,
),
(
"currency-name-many",
"currencyFormats",
"unitPattern-count-many",
false,
false,
),
(
"currency-name-other",
"currencyFormats",
"unitPattern-count-other",
false,
false,
),
];
const CURRENCY_SPACING: (&str, &str, &str) = ("[[:^S:]&[:^Z:]]", "[:digit:]", "\u{a0}");
fn assumption(message: String) -> Error {
Error::Assumption(message)
}
pub fn numeric_systems(numbering_systems_json: &str) -> Result<BTreeMap<String, String>, Error> {
let v: Value = serde_json::from_str(numbering_systems_json)?;
let systems = v
.pointer("/supplemental/numberingSystems")
.and_then(Value::as_object)
.ok_or_else(|| Error::Shape("numberingSystems".to_owned()))?;
let mut out = BTreeMap::new();
for (nu, s) in systems {
if s.get("_type").and_then(Value::as_str) != Some("numeric") {
continue;
}
let digits = s
.get("_digits")
.and_then(Value::as_str)
.ok_or_else(|| Error::Shape(format!("digits of {nu}")))?;
let mut widths = digits.chars().map(char::len_utf8);
let w = widths.next();
if digits.chars().count() != 10 || widths.any(|x| Some(x) != w) {
return Err(assumption(format!(
"{nu}: digits are not ten of one UTF-8 width"
)));
}
if nu == "latn" {
if digits != "0123456789" {
return Err(assumption("latn digits are not ASCII".to_owned()));
}
continue;
}
out.insert(nu.clone(), digits.to_owned());
}
Ok(out)
}
pub fn locale_record(
locale: &str,
numbers_json: &str,
numeric: &BTreeMap<String, String>,
) -> Result<Record, Error> {
let v: Value = serde_json::from_str(numbers_json)?;
let n = v
.pointer(&format!("/main/{locale}/numbers"))
.and_then(Value::as_object)
.ok_or_else(|| Error::Shape(format!("main/{locale}/numbers")))?;
let text = |k: &str| n.get(k).and_then(Value::as_str);
let mut r = Record::default();
let default = text("defaultNumberingSystem")
.ok_or_else(|| Error::Shape(format!("{locale}: defaultNumberingSystem")))?;
r.fields.insert("default".to_owned(), default.to_owned());
let others = n.get("otherNumberingSystems").and_then(Value::as_object);
let other = |k: &str| others.and_then(|o| o.get(k)).and_then(Value::as_str);
if let Some(native) = other("native") {
r.fields.insert("native".to_owned(), native.to_owned());
}
for k in ["traditional", "finance"] {
if let Some(nu) = other(k)
&& (nu == "latn" || numeric.contains_key(nu))
{
return Err(assumption(format!(
"{locale}: {k} system {nu} is numeric (`-u-nu-traditio`/`finance` would need it)"
)));
}
}
let min = text("minimumGroupingDigits")
.ok_or_else(|| Error::Shape(format!("{locale}: minimumGroupingDigits")))?;
if !matches!(min.parse::<u8>(), Ok(1..=15)) {
return Err(assumption(format!("{locale}: minimumGroupingDigits {min}")));
}
r.fields.insert("min-grouping".to_owned(), min.to_owned());
let mut expect: Vec<&str> = vec!["latn", default];
expect.extend(other("native"));
expect.sort_unstable();
expect.dedup();
let mut found: Vec<&str> = n
.keys()
.filter_map(|k| k.strip_prefix("symbols-numberSystem-"))
.collect();
found.sort_unstable();
if found != expect {
return Err(assumption(format!(
"{locale}: symbols for {found:?}, expected latn, default and native {expect:?}"
)));
}
for nu in found {
if nu != "latn" && !numeric.contains_key(nu) {
return Err(assumption(format!("{locale}: system {nu} is not numeric")));
}
let mut s = BTreeMap::new();
for (name, object, key, required, number_pattern) in SYSTEM_SOURCES {
let value = n
.get(&format!("{object}-numberSystem-{nu}"))
.and_then(|o| o.get(*key))
.and_then(Value::as_str);
match value {
Some(value) => {
if *number_pattern {
crate::number::pattern::parse(value)?;
} else if name.starts_with("currency-name-") {
crate::template::parse(value, 2)?;
}
s.insert((*name).to_owned(), value.to_owned());
}
None if *required => {
return Err(assumption(format!("{locale} {nu}: no {object}/{key}")));
}
None => {}
}
}
let spacing = n
.get(&format!("currencyFormats-numberSystem-{nu}"))
.and_then(|o| o.get("currencySpacing"));
for side in ["beforeCurrency", "afterCurrency"] {
let get = |k: &str| {
spacing
.and_then(|sp| sp.get(side))
.and_then(|x| x.get(k))
.and_then(Value::as_str)
};
let seen = (
get("currencyMatch"),
get("surroundingMatch"),
get("insertBetween"),
);
let want = (
Some(CURRENCY_SPACING.0),
Some(CURRENCY_SPACING.1),
Some(CURRENCY_SPACING.2),
);
if seen != want {
return Err(assumption(format!(
"{locale} {nu}: currency spacing {side} {seen:?} is not CLDR's constant one"
)));
}
}
r.systems.insert(nu.to_owned(), s);
}
Ok(r)
}
fn likely_lines(
likely_json: &str,
locales: &BTreeSet<&str>,
) -> Result<BTreeMap<String, String>, Error> {
let v: Value = serde_json::from_str(likely_json)?;
let likely = v
.pointer("/supplemental/likelySubtags")
.and_then(Value::as_object)
.ok_or_else(|| Error::Shape("likelySubtags".to_owned()))?;
let script_of = |key: &str| {
likely
.get(key)
.and_then(Value::as_str)
.and_then(|full| full.split('-').nth(1))
.filter(|s| s.len() == 4)
};
let mut out = BTreeMap::new();
let languages: BTreeSet<&str> = locales
.iter()
.filter_map(|l| l.split('-').next())
.filter(|l| *l != crate::number::table::ROOT)
.collect();
for lang in &languages {
if let Some(s) = script_of(lang) {
out.insert((*lang).to_owned(), s.to_owned());
}
}
for key in likely.keys() {
let Some((lang, region)) = key.split_once('-') else {
continue;
};
let is_region = (region.len() == 2 && region.bytes().all(|b| b.is_ascii_uppercase()))
|| (region.len() == 3 && region.bytes().all(|b| b.is_ascii_digit()));
if !is_region || !languages.contains(lang) {
continue;
}
let (Some(here), Some(base)) = (script_of(key), script_of(lang)) else {
continue;
};
let prefix = format!("{lang}-{here}");
let exists = locales
.iter()
.any(|l| *l == prefix || l.starts_with(&format!("{prefix}-")));
if here != base && exists {
out.insert(key.clone(), here.to_owned());
}
}
Ok(out)
}
pub fn numbers_table(inputs: &NumberInputs<'_>, cldr: &str) -> Result<String, Error> {
use crate::number::table::{LOCALE_FIELDS, ROOT, SYSTEM_FIELDS, Table, escape};
let numeric = numeric_systems(inputs.numbering_systems)?;
let mut records: BTreeMap<&str, Record> = BTreeMap::new();
for (locale, json) in inputs.locales {
records.insert(locale.as_str(), locale_record(locale, json, &numeric)?);
}
let names: BTreeSet<&str> = records.keys().copied().collect();
let available: Value = serde_json::from_str(inputs.available_locales)?;
let full: BTreeSet<&str> = available
.pointer("/availableLocales/full")
.and_then(Value::as_array)
.ok_or_else(|| Error::Shape("availableLocales/full".to_owned()))?
.iter()
.filter_map(Value::as_str)
.collect();
if full != names {
return Err(assumption(format!(
"availableLocales/full ({}) differs from the numbers files ({})",
full.len(),
names.len()
)));
}
let dc: Value = serde_json::from_str(inputs.default_content)?;
let defaults = dc
.pointer("/defaultContent")
.and_then(Value::as_array)
.ok_or_else(|| Error::Shape("defaultContent".to_owned()))?;
if let Some(d) = defaults
.iter()
.filter_map(Value::as_str)
.find(|d| names.contains(d))
{
return Err(assumption(format!(
"default-content locale {d} has its own files (the lookup assumes truncation reaches its parent's)"
)));
}
if !names.contains(ROOT) {
return Err(assumption("no root (`und`) locale".to_owned()));
}
let pl: Value = serde_json::from_str(inputs.parent_locales)?;
let pl = pl
.pointer("/supplemental/parentLocales")
.ok_or_else(|| Error::Shape("parentLocales".to_owned()))?;
if pl
.pointer("/_localeRules/parentLocale/nonlikelyScript")
.and_then(Value::as_str)
!= Some("root")
{
return Err(assumption(
"parentLocales _localeRules: nonlikelyScript is not root".to_owned(),
));
}
let parents: BTreeMap<String, String> = pl
.get("parentLocale")
.and_then(Value::as_object)
.ok_or_else(|| Error::Shape("parentLocales/parentLocale".to_owned()))?
.iter()
.filter_map(|(k, v)| Some((k.clone(), v.as_str()?.to_owned())))
.collect();
let likely = likely_lines(inputs.likely_subtags, &names)?;
let mut skeleton = Table::default();
for (k, v) in &parents {
skeleton.parents.insert(k, v);
}
for (k, v) in &likely {
skeleton.likely.insert(k, v);
}
for l in &names {
skeleton.locales.insert(l, BTreeMap::new());
}
let mut out = String::new();
out.push_str(
"# mf2-locale-data: CLDR number data for every CLDR locale (plans/05-tooling.md §7).\n",
);
out.push_str(
"# Generated by `cargo xtask locale-data` from the `cargo xtask cldr-sync` cache (every\n",
);
out.push_str(
"# locale's cldr-numbers-full/main/<locale>/numbers.json, parentLocales.json) and\n",
);
out.push_str(
"# third_party/cldr-json (numberingSystems.json, likelySubtags.json). Do not edit.\n",
);
out.push_str(
"# Lines: digits <nu> <0–9> · likely <lang[-REGION]> <Script> · parent <locale> <parent>\n",
);
out.push_str(
"# · locale <tag> key=value… · system <tag> <nu> key=value… — a locale's fields that\n",
);
out.push_str(
"# differ from its parent's (CLDR's resolved data, deduplicated). `\\u{…}` escapes\n",
);
out.push_str(
"# spaces, controls, format characters, `=` and `\\`. Format: src/number/table.rs.\n",
);
let _ = writeln!(out, "cldr {cldr}");
for (nu, d) in &numeric {
let _ = writeln!(out, "digits {nu} {}", escape(d));
}
for (k, s) in &likely {
let _ = writeln!(out, "likely {k} {s}");
}
for (k, p) in &parents {
let _ = writeln!(out, "parent {k} {p}");
}
let empty = Record::default();
for (locale, rec) in &records {
let parent = skeleton.chain(locale).get(1).copied();
let prec = parent.and_then(|p| records.get(p)).unwrap_or(&empty);
let mut line = format!("locale {locale}");
for f in LOCALE_FIELDS {
let (mine, theirs) = (rec.fields.get(*f), prec.fields.get(*f));
if mine.is_none() && theirs.is_some() {
return Err(assumption(format!(
"{locale}: no {f} although its parent {parent:?} has one (not resolved)"
)));
}
if let Some(v) = mine
&& Some(v) != theirs
{
let _ = write!(line, " {f}={}", escape(v));
}
}
out.push_str(&line);
out.push('\n');
for (nu, fields) in &rec.systems {
let theirs = prec.systems.get(nu);
let mut line = format!("system {locale} {nu}");
let mut any = false;
for f in SYSTEM_FIELDS {
let (mine, base) = (fields.get(*f), theirs.and_then(|t| t.get(*f)));
if mine.is_none() && base.is_some() {
return Err(assumption(format!(
"{locale} {nu}: no {f} although its parent {parent:?} has one (not resolved)"
)));
}
if let Some(v) = mine
&& Some(v) != base
{
let _ = write!(line, " {f}={}", escape(v));
any = true;
}
}
if any {
out.push_str(&line);
out.push('\n');
}
}
}
let parsed = Table::parse(&out)?;
for (locale, rec) in &records {
let back = parsed.resolved(locale);
if back != *rec {
return Err(assumption(format!(
"{locale}: the table resolves to {back:?}, CLDR says {rec:?}"
)));
}
}
Ok(out)
}
pub fn matching_table(
likely_json: &str,
matching_json: &str,
containment_json: &str,
cldr: &str,
) -> Result<String, Error> {
use crate::matching::{Matching, Rule, Variable};
let shape = |what: &str| Error::Shape(what.to_owned());
let assume = |what: String| Error::Assumption(what);
let likely: Value = serde_json::from_str(likely_json)?;
let likely = likely
.pointer("/supplemental/likelySubtags")
.and_then(Value::as_object)
.ok_or_else(|| shape("likelySubtags"))?;
let mut table = BTreeMap::new();
for (from, to) in likely {
if from == "und" || from.starts_with("und-") {
continue;
}
let to = to.as_str().ok_or_else(|| shape(from))?;
if to.split('-').count() != 3 {
return Err(assume(format!(
"likely {from} → {to} is not language-script-region"
)));
}
table.insert(from.clone(), to.to_owned());
}
let matching: Value = serde_json::from_str(matching_json)?;
let written = matching
.pointer("/supplemental/languageMatching/written-new")
.ok_or_else(|| shape("languageMatching/written-new"))?;
let paradigms = written
.pointer("/paradigmLocales/_locales")
.and_then(Value::as_array)
.ok_or_else(|| shape("paradigmLocales"))?
.iter()
.map(|v| {
v.as_str()
.map(str::to_owned)
.ok_or_else(|| shape("a paradigm locale"))
})
.collect::<Result<Vec<_>, _>>()?;
let containment: Value = serde_json::from_str(containment_json)?;
let containment = containment
.pointer("/supplemental/territoryContainment")
.and_then(Value::as_object)
.ok_or_else(|| shape("territoryContainment"))?;
let contents = |code: &str| -> Option<Vec<String>> {
containment
.get(code)?
.get("_contains")?
.as_array()
.map(|a| {
a.iter()
.filter_map(Value::as_str)
.map(str::to_owned)
.collect()
})
};
let macros: Vec<&String> = containment.keys().filter(|k| !k.contains('-')).collect();
let mut variables = Vec::new();
let names = written
.get("matchVariables")
.and_then(Value::as_object)
.ok_or_else(|| shape("matchVariables"))?;
for (name, v) in names {
let value = v
.get("_value")
.and_then(Value::as_str)
.ok_or_else(|| shape(name))?;
let mut set = BTreeSet::new();
let mut op = '+';
let mut token = String::new();
for ch in value.chars().chain(core::iter::once('+')) {
if ch == '+' || ch == '-' {
let part = leaves(&token, &contents, &mut BTreeSet::new());
if op == '+' {
set.extend(part);
} else {
set.retain(|r| !part.contains(r));
}
op = ch;
token.clear();
} else {
token.push(ch);
}
}
let inside: Vec<String> = macros
.iter()
.filter(|m| {
let l = leaves(m, &contents, &mut BTreeSet::new());
!l.is_empty() && l.is_subset(&set)
})
.map(|m| (*m).clone())
.collect();
set.extend(inside);
variables.push(Variable {
name: name.clone(),
value: value.to_owned(),
regions: set.into_iter().collect(),
});
}
let mut rules = Vec::new();
for r in written
.get("languageMatch")
.and_then(Value::as_array)
.ok_or_else(|| shape("languageMatch"))?
{
let field = |k: &str| r.get(k).and_then(Value::as_str).ok_or_else(|| shape(k));
let desired = field("_desired")?.to_owned();
let supported = field("_supported")?.to_owned();
let distance = r
.get("_distance")
.and_then(Value::as_u64)
.and_then(|d| u8::try_from(d).ok())
.ok_or_else(|| shape("_distance"))?;
let oneway = r.get("_oneway").and_then(Value::as_bool).unwrap_or(false);
if desired.split('-').count() != supported.split('-').count() {
return Err(assume(format!(
"rule {desired} {supported}: sides of two levels"
)));
}
rules.push(Rule {
desired,
supported,
distance,
oneway,
});
}
let m = Matching {
cldr: cldr.to_owned(),
paradigms,
variables,
rules,
likely: table,
};
m.encode().map_err(|e| assume(e.to_string()))?;
Ok(m.text())
}
fn leaves(
code: &str,
contents: &dyn Fn(&str) -> Option<Vec<String>>,
seen: &mut BTreeSet<String>,
) -> BTreeSet<String> {
match contents(code) {
None => BTreeSet::from([code.to_owned()]),
Some(children) => {
let mut out = BTreeSet::new();
for child in children {
if seen.insert(child.clone()) {
out.extend(leaves(&child, contents, seen));
}
}
out
}
}
}