use std::collections::HashMap;
use std::fmt;
use std::sync::{Arc, OnceLock};
use serde::de::{MapAccess, Visitor};
use serde::{Deserialize, Deserializer, Serialize};
use crate::error::Result;
const BUNDLED_LANGUAGE_CODES: &str = include_str!(concat!(env!("OUT_DIR"), "/language_codes.json"));
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LanguageCode {
pub tag: String,
pub iso639_3: String,
#[serde(default)]
pub region: Option<String>,
pub name: String,
#[serde(default)]
pub scope: Option<String>,
#[serde(default, rename = "type")]
pub kind: Option<String>,
#[serde(default)]
pub aliases: Vec<String>,
#[serde(default)]
pub resolution: Option<String>,
}
#[derive(Debug, Clone)]
pub struct LanguageRegistry {
inner: Arc<RegistryInner>,
}
#[derive(Debug)]
struct RegistryInner {
version: u32,
codes: Vec<LanguageCode>,
by_tag: HashMap<String, usize>,
by_code: HashMap<String, Vec<String>>,
by_alias: HashMap<String, String>,
}
#[derive(Debug, Deserialize)]
struct RawRegistry {
#[serde(default)]
version: u32,
#[serde(deserialize_with = "ordered_codes")]
codes: Vec<LanguageCode>,
#[serde(default)]
name_to_tag: HashMap<String, String>,
}
fn ordered_codes<'de, D>(deserializer: D) -> std::result::Result<Vec<LanguageCode>, D::Error>
where
D: Deserializer<'de>,
{
struct CodesVisitor;
impl<'de> Visitor<'de> for CodesVisitor {
type Value = Vec<LanguageCode>;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("a map of language tag to registry entry")
}
fn visit_map<M>(self, mut access: M) -> std::result::Result<Self::Value, M::Error>
where
M: MapAccess<'de>,
{
let mut out = Vec::with_capacity(access.size_hint().unwrap_or(0));
while let Some((_, value)) = access.next_entry::<String, LanguageCode>()? {
out.push(value);
}
Ok(out)
}
}
deserializer.deserialize_map(CodesVisitor)
}
impl LanguageRegistry {
pub fn bundled() -> Result<Self> {
Ok(Self::shared().clone())
}
pub(crate) fn shared() -> &'static Self {
static SHARED: OnceLock<LanguageRegistry> = OnceLock::new();
SHARED.get_or_init(|| {
Self::from_json(BUNDLED_LANGUAGE_CODES).expect("bundled language_codes.json parses")
})
}
pub fn from_json(raw: &str) -> Result<Self> {
let doc: RawRegistry = serde_json::from_str(raw)?;
Ok(Self::from_raw(doc))
}
fn from_raw(doc: RawRegistry) -> Self {
let mut by_tag = HashMap::with_capacity(doc.codes.len());
let mut by_code: HashMap<String, Vec<String>> = HashMap::new();
let mut by_alias: HashMap<String, String> = HashMap::new();
for (i, code) in doc.codes.iter().enumerate() {
by_tag.insert(code.tag.clone(), i);
by_code
.entry(code.iso639_3.clone())
.or_default()
.push(code.tag.clone());
}
for code in &doc.codes {
add_alias(&mut by_alias, &code.name, &code.tag);
for alias in &code.aliases {
add_alias(&mut by_alias, alias, &code.tag);
}
}
let mut extra: Vec<(&String, &String)> = doc.name_to_tag.iter().collect();
extra.sort_unstable();
for (name, tag) in extra {
if !tag.is_empty() && by_tag.contains_key(tag) {
add_alias(&mut by_alias, name, tag);
}
}
Self {
inner: Arc::new(RegistryInner {
version: doc.version,
codes: doc.codes,
by_tag,
by_code,
by_alias,
}),
}
}
pub fn version(&self) -> u32 {
self.inner.version
}
pub fn codes(&self) -> &[LanguageCode] {
&self.inner.codes
}
pub fn tags(&self) -> impl Iterator<Item = &str> {
self.inner.codes.iter().map(|c| c.tag.as_str())
}
pub fn get(&self, tag: &str) -> Option<&LanguageCode> {
let canonical = canonicalise_tag(tag)?;
self.inner
.by_tag
.get(&canonical)
.map(|i| &self.inner.codes[*i])
}
pub fn tags_for_code(&self, code: &str) -> &[String] {
self.inner
.by_code
.get(&code.trim().to_lowercase())
.map_or(&[][..], Vec::as_slice)
}
pub fn resolve(&self, value: &str, include_varieties: bool) -> Vec<String> {
let raw = value.trim();
if raw.is_empty() {
return Vec::new();
}
if let Some(tag) = canonicalise_tag(raw) {
let varieties: &[String] = if tag.contains('-') {
&[]
} else {
self.tags_for_code(&tag)
};
if self.inner.by_tag.contains_key(&tag) {
if !include_varieties || varieties.is_empty() {
return vec![tag];
}
let mut out = vec![tag.clone()];
out.extend(varieties.iter().filter(|t| **t != tag).cloned());
return out;
}
if !varieties.is_empty() {
return varieties.to_vec();
}
}
let Some(tag) = self
.inner
.by_alias
.get(&raw.to_lowercase())
.or_else(|| self.inner.by_alias.get(&slugify(raw)))
.cloned()
else {
return Vec::new();
};
if include_varieties {
if let Some(entry) = self.get(&tag) {
if entry.region.is_none() {
let varieties = self.tags_for_code(&entry.iso639_3);
if varieties.len() > 1 {
let mut out = vec![tag.clone()];
out.extend(varieties.iter().filter(|t| **t != tag).cloned());
return out;
}
}
}
}
vec![tag]
}
pub fn resolve_all<I, S>(&self, values: I, include_varieties: bool) -> Vec<String>
where
I: IntoIterator<Item = S>,
S: AsRef<str>,
{
let mut out: Vec<String> = Vec::new();
for value in values {
for tag in self.resolve(value.as_ref(), include_varieties) {
if !out.contains(&tag) {
out.push(tag);
}
}
}
out
}
}
fn add_alias(map: &mut HashMap<String, String>, name: &str, tag: &str) {
let lower = name.trim().to_lowercase();
if lower.is_empty() {
return;
}
map.entry(lower.clone()).or_insert_with(|| tag.to_string());
let slug = slugify(&lower);
if !slug.is_empty() {
map.entry(slug).or_insert_with(|| tag.to_string());
}
}
pub fn canonicalise_tag(value: &str) -> Option<String> {
let trimmed = value.trim();
let (primary, region) = match trimmed.split_once('-') {
Some((p, r)) => (p, Some(r)),
None => (trimmed, None),
};
if primary.len() != 3 || !primary.chars().all(|c| c.is_ascii_alphabetic()) {
return None;
}
let primary = primary.to_ascii_lowercase();
match region {
None => Some(primary),
Some(r) if r.len() == 2 && r.chars().all(|c| c.is_ascii_alphabetic()) => {
Some(format!("{primary}-{}", r.to_ascii_uppercase()))
}
Some(r) if r.len() == 3 && r.chars().all(|c| c.is_ascii_digit()) => {
Some(format!("{primary}-{r}"))
}
Some(_) => None,
}
}
fn slugify(value: &str) -> String {
let mut out = String::with_capacity(value.len());
let mut pending_hyphen = false;
for c in value.chars().flat_map(fold_char) {
if c.is_ascii_alphanumeric() {
if pending_hyphen && !out.is_empty() {
out.push('-');
}
pending_hyphen = false;
out.push(c.to_ascii_lowercase());
} else {
pending_hyphen = true;
}
}
out
}
const FOLD_FROM: &str =
"ÀÁÂÃÄÅàáâãäåÈÉÊËèéêëÌÍÎÏìíîïÒÓÔÕÖØòóôõöøÙÚÛÜùúûüÇçÐðÑñÝýÿŠšŽžĀāĒēĪīŌōŪūŴŵŶŷ";
const FOLD_TO: &str = "AAAAAAaaaaaaEEEEeeeeIIIIiiiiOOOOOOooooooUUUUuuuuCcDdNnYyySsZzAaEeIiOoUuWwYy";
fn fold_char(c: char) -> Vec<char> {
if c.is_ascii() {
return vec![c];
}
if ('\u{0300}'..='\u{036f}').contains(&c) {
return Vec::new();
}
match c {
'Æ' => return vec!['A', 'E'],
'æ' => return vec!['a', 'e'],
'ß' => return vec!['s', 's'],
_ => {}
}
match FOLD_FROM.chars().position(|f| f == c) {
Some(i) => FOLD_TO.chars().nth(i).map(|f| vec![f]).unwrap_or_default(),
None => vec![c],
}
}
#[cfg(test)]
mod tests {
use super::*;
fn registry() -> &'static LanguageRegistry {
LanguageRegistry::shared()
}
#[test]
fn the_fold_table_lines_up() {
assert_eq!(FOLD_FROM.chars().count(), FOLD_TO.chars().count());
}
#[test]
fn slugs_strip_accents_and_punctuation() {
assert_eq!(slugify("Sängö"), "sango");
assert_eq!(slugify("N'Ko (script)"), "n-ko-script");
assert_eq!(slugify("isiZulu"), "isizulu");
}
#[test]
fn tags_are_canonicalised_case_and_shape() {
assert_eq!(canonicalise_tag("sna").as_deref(), Some("sna"));
assert_eq!(canonicalise_tag("SNA").as_deref(), Some("sna"));
assert_eq!(canonicalise_tag("eng-ng").as_deref(), Some("eng-NG"));
assert_eq!(canonicalise_tag(" ENG-NG ").as_deref(), Some("eng-NG"));
assert_eq!(canonicalise_tag("und-419").as_deref(), Some("und-419"));
assert_eq!(canonicalise_tag("en"), None);
assert_eq!(canonicalise_tag("english"), None);
assert_eq!(canonicalise_tag("eng-NGA"), None);
}
#[test]
fn the_registry_is_read_in_document_order() {
let reg = registry();
let english = reg.tags_for_code("eng");
assert_eq!(english.first().map(String::as_str), Some("eng-NG"));
assert!(english.contains(&"eng".to_string()));
let widened = reg.resolve("eng", true);
assert_eq!(widened.first().map(String::as_str), Some("eng"));
assert_eq!(widened.len(), english.len());
assert!(reg.codes().len() >= 300);
assert_eq!(reg.version(), 1);
}
#[test]
fn a_bare_code_that_is_only_regional_resolves_to_its_varieties() {
let reg = registry();
assert!(reg.get("deu").is_none());
assert_eq!(reg.resolve("deu", false), vec!["deu-NA".to_string()]);
}
}