language_matcher/
lib.rs

1//! A language matcher with CLDR.
2//!
3//! The "sync" feature of `icu_provider` is enabled because we like Sync.
4
5#![warn(missing_docs)]
6#![deny(unsafe_code)]
7
8use icu_locale::{LanguageIdentifier, LocaleExpander};
9use std::collections::{HashMap, HashSet};
10
11include!(concat!(env!("OUT_DIR"), "/language_info.rs"));
12
13type Variables = HashMap<&'static str, HashSet<&'static str>>;
14
15trait Rule<T> {
16    fn matches(self, tag: T, vars: &Variables) -> bool;
17}
18
19#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
20enum SubTagRule {
21    Str(&'static str),
22    Var(&'static str),
23    VarExclude(&'static str),
24    All,
25}
26
27impl Rule<&'_ str> for &'_ SubTagRule {
28    fn matches(self, tag: &str, vars: &Variables) -> bool {
29        match self {
30            SubTagRule::Str(s) => *s == tag,
31            SubTagRule::Var(key) => vars[key].contains(tag),
32            SubTagRule::VarExclude(key) => !vars[key].contains(tag),
33            SubTagRule::All => true,
34        }
35    }
36}
37
38impl Rule<Option<&'_ str>> for Option<&'_ SubTagRule> {
39    fn matches(self, tag: Option<&str>, vars: &Variables) -> bool {
40        match (self, tag) {
41            (None, None) | (Some(SubTagRule::All), _) => true,
42            (Some(s), Some(tag)) => s.matches(tag, vars),
43            _ => false,
44        }
45    }
46}
47
48#[derive(Debug, PartialEq)]
49struct LanguageIdentifierRule {
50    pub language: SubTagRule,
51    pub script: Option<SubTagRule>,
52    pub region: Option<SubTagRule>,
53}
54
55impl Rule<&'_ LanguageIdentifier> for &'_ LanguageIdentifierRule {
56    fn matches(self, lang: &LanguageIdentifier, vars: &Variables) -> bool {
57        self.language.matches(lang.language.as_str(), vars)
58            && self
59                .script
60                .as_ref()
61                .matches(lang.script.as_ref().map(|s| s.as_str()), vars)
62            && self
63                .region
64                .as_ref()
65                .matches(lang.region.as_ref().map(|s| s.as_str()), vars)
66    }
67}
68
69#[derive(Debug, PartialEq)]
70struct ParadigmLocales {
71    pub locales: &'static [LanguageIdentifier],
72}
73
74#[derive(Debug, PartialEq)]
75struct MatchVariable {
76    pub id: &'static str,
77    pub value: &'static [&'static str],
78}
79
80#[derive(Debug, PartialEq)]
81struct LanguageMatch {
82    pub desired: LanguageIdentifierRule,
83    pub supported: LanguageIdentifierRule,
84    pub distance: u16,
85    pub oneway: bool,
86}
87
88#[derive(Debug, PartialEq)]
89struct LanguageMatches {
90    pub paradigm_locales: ParadigmLocales,
91    pub match_variable: &'static [MatchVariable],
92    pub language_match: &'static [LanguageMatch],
93}
94
95#[derive(Debug, PartialEq)]
96struct LanguageMatching {
97    pub language_matches: LanguageMatches,
98}
99
100#[derive(Debug, PartialEq)]
101struct SupplementalData {
102    pub language_matching: LanguageMatching,
103}
104
105/// This is a language matcher.
106/// The distance of two languages are calculated by the algorithm of [CLDR].
107/// The value of distance is multiplied by 10, because we need to consider the paradigm locales.
108///
109/// [CLDR]: https://www.unicode.org/reports/tr35/tr35.html#EnhancedLanguageMatching
110///
111/// # Examples
112///
113/// ```
114/// use icu_locale::langid;
115/// use language_matcher::LanguageMatcher;
116///
117/// let matcher = LanguageMatcher::new();
118/// assert_eq!(matcher.distance(langid!("zh-CN"), langid!("zh-Hans")), 0);
119/// assert_eq!(matcher.distance(langid!("zh-HK"), langid!("zh-MO")), 40);
120/// assert_eq!(matcher.distance(langid!("en-US"), langid!("en-GB")), 50);
121/// assert_eq!(matcher.distance(langid!("en-US"), langid!("en-CA")), 39);
122/// ```
123///
124/// With the distance, you can choose the nearst language from a set of languages:
125///
126/// ```
127/// use icu_locale::langid;
128/// use language_matcher::LanguageMatcher;
129///
130/// let matcher = LanguageMatcher::new();
131/// let accepts = [
132///     langid!("en"),
133///     langid!("ja"),
134///     langid!("zh-Hans"),
135///     langid!("zh-Hant"),
136/// ];
137///
138/// assert_eq!(matcher.matches(langid!("zh-CN"), &accepts),Some((&langid!("zh-Hans"), 0)));
139/// ```
140pub struct LanguageMatcher {
141    paradigm: HashSet<&'static LanguageIdentifier>,
142    vars: Variables,
143    rules: &'static [LanguageMatch],
144    expander: LocaleExpander,
145}
146
147impl From<&SupplementalData> for LanguageMatcher {
148    fn from(data: &SupplementalData) -> Self {
149        let expander = LocaleExpander::new_extended();
150
151        let matches = &data.language_matching.language_matches;
152
153        let paradigm = HashSet::from_iter(matches.paradigm_locales.locales);
154        let vars = matches
155            .match_variable
156            .iter()
157            .map(|MatchVariable { id, value }| (*id, HashSet::from_iter(value.iter().copied())))
158            .collect::<HashMap<_, _>>();
159        Self {
160            paradigm,
161            vars,
162            rules: matches.language_match,
163            expander,
164        }
165    }
166}
167
168impl LanguageMatcher {
169    /// Creates an instance of [`LanguageMatcher`].
170    pub fn new() -> Self {
171        Self::from(&LANGUAGE_INFO)
172    }
173
174    /// Choose the nearst language of desired language from the supported language collection.
175    /// Returns the chosen language and the distance.
176    ///
177    /// `None` will be returned if no language gives the distance less than 1000.
178    /// That usually means no language matches the desired one.
179    pub fn matches<'a>(
180        &self,
181        mut desired: LanguageIdentifier,
182        supported: impl IntoIterator<Item = &'a LanguageIdentifier>,
183    ) -> Option<(&'a LanguageIdentifier, u16)> {
184        self.expander.maximize(&mut desired);
185        supported
186            .into_iter()
187            .map(|s| {
188                let mut max_s = s.clone();
189                self.expander.maximize(&mut max_s);
190                (s, self.distance_impl(desired.clone(), max_s))
191            })
192            .min_by_key(|(_, dis)| *dis)
193            .filter(|(_, dis)| *dis < 1000)
194    }
195
196    /// Calculate the distance of the two language.
197    /// Some rule in CLDR is one way. Be careful about the parameters order.
198    ///
199    /// The return value is multiplied by 10, and if only one is paradigm locale,
200    /// the value is substructed by 1.
201    pub fn distance(
202        &self,
203        mut desired: LanguageIdentifier,
204        mut supported: LanguageIdentifier,
205    ) -> u16 {
206        self.expander.maximize(&mut desired);
207        self.expander.maximize(&mut supported);
208        self.distance_impl(desired, supported)
209    }
210
211    fn distance_impl(
212        &self,
213        mut desired: LanguageIdentifier,
214        mut supported: LanguageIdentifier,
215    ) -> u16 {
216        debug_assert!(desired.region.is_some());
217        debug_assert!(desired.script.is_some());
218        debug_assert!(supported.region.is_some());
219        debug_assert!(supported.script.is_some());
220
221        let mut distance = 0;
222
223        if desired.region != supported.region {
224            distance += self.distance_match(&desired, &supported);
225        }
226        desired.region = None;
227        supported.region = None;
228
229        if desired.script != supported.script {
230            distance += self.distance_match(&desired, &supported);
231        }
232        desired.script = None;
233        supported.script = None;
234
235        if desired.language != supported.language {
236            distance += self.distance_match(&desired, &supported);
237        }
238
239        distance
240    }
241
242    fn distance_match(&self, desired: &LanguageIdentifier, supported: &LanguageIdentifier) -> u16 {
243        for rule in self.rules {
244            let mut matches = rule.desired.matches(desired, &self.vars)
245                && rule.supported.matches(supported, &self.vars);
246            if !rule.oneway && !matches {
247                matches = rule.supported.matches(desired, &self.vars)
248                    && rule.desired.matches(supported, &self.vars);
249            }
250            if matches {
251                let mut distance = rule.distance * 10;
252                if self.is_paradigm(desired) ^ self.is_paradigm(supported) {
253                    distance -= 1
254                }
255                return distance;
256            }
257        }
258        unreachable!()
259    }
260
261    fn is_paradigm(&self, lang: &LanguageIdentifier) -> bool {
262        self.paradigm.contains(lang)
263    }
264}
265
266impl Default for LanguageMatcher {
267    fn default() -> Self {
268        Self::new()
269    }
270}
271
272#[cfg(test)]
273mod test {
274    use crate::LanguageMatcher;
275    use icu_locale::langid;
276
277    #[test]
278    fn distance() {
279        let matcher = LanguageMatcher::new();
280
281        assert_eq!(matcher.distance(langid!("zh-CN"), langid!("zh-Hans")), 0);
282        assert_eq!(matcher.distance(langid!("zh-TW"), langid!("zh-Hant")), 0);
283        assert_eq!(matcher.distance(langid!("zh-HK"), langid!("zh-MO")), 40);
284        assert_eq!(matcher.distance(langid!("zh-HK"), langid!("zh-Hant")), 50);
285    }
286
287    #[test]
288    fn matcher() {
289        let matcher = LanguageMatcher::new();
290
291        let accepts = [
292            langid!("en"),
293            langid!("ja"),
294            langid!("zh-Hans"),
295            langid!("zh-Hant"),
296        ];
297        assert_eq!(
298            matcher.matches(langid!("zh-CN"), &accepts),
299            Some((&langid!("zh-Hans"), 0))
300        );
301        assert_eq!(
302            matcher.matches(langid!("zh-TW"), &accepts),
303            Some((&langid!("zh-Hant"), 0))
304        );
305    }
306}
language_matcher/lib.rs

language_matcher/
lib.rs
