poly_l10n/
lib.rs

1//! `poly_l10n`: Handle locali(s|z)ations the correct way
2//!
3//! ## Intentions
4//!
5//! See <https://blog.fyralabs.com/advice-on-internationalization/#language-fallbacks>.
6//!
7//! In short, this crate handles language fallbacks and detect system languages *the correct way*.
8//!
9//! Get started by [`LocaleFallbackSolver`], [`system_want_langids()`] and [`langid!`].
10//!
11//! ## 📃 License
12//!
13//! `GPL-3.0-or-later`
14//!
15//!    Copyright (C) 2025  madonuko <mado@fyralabs.com> <madonuko@outlook.com>
16//!
17//!    This program is free software: you can redistribute it and/or modify
18//!    it under the terms of the GNU General Public License as published by
19//!    the Free Software Foundation, either version 3 of the License, or
20//!    (at your option) any later version.
21//!
22//!    This program is distributed in the hope that it will be useful,
23//!    but WITHOUT ANY WARRANTY; without even the implied warranty of
24//!    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
25//!    GNU General Public License for more details.
26//!
27//!    You should have received a copy of the GNU General Public License
28//!    along with this program.  If not, see <https://www.gnu.org/licenses/>.
29
30mod default_rulebook;
31#[cfg(feature = "getlang")]
32pub mod getlang;
33pub mod macros;
34#[cfg(feature = "per_lang_default_rules")]
35pub mod per_lang_default_rules;
36
37use std::{rc::Rc, sync::Arc};
38
39#[cfg(feature = "getlang")]
40pub use getlang::system_want_langids;
41use itertools::Itertools;
42pub use unic_langid::{self, LanguageIdentifier};
43
44/// Entry point of `poly_l10n`.
45///
46/// A solver requires a [`Rulebook`] or [`ARulebook`] to process and solve locales. The latter is
47/// used by [`Self::default`].
48///
49/// # Examples
50/// ```
51/// let solver = poly_l10n::LocaleFallbackSolver::<poly_l10n::ARulebook>::default();
52/// # #[cfg(feature = "per_lang_default_rules")]
53/// assert_eq!(solver.solve_locale(poly_l10n::langid!("arb")), poly_l10n::langid!["arb", "ar-AE", "ara-AE", "arb-AE", "ar", "ara"]);
54/// ```
55#[derive(Clone, Copy, Debug, Default)]
56pub struct LocaleFallbackSolver<R: for<'a> PolyL10nRulebook<'a> = ARulebook> {
57    pub rulebook: R,
58}
59
60impl<R: for<'a> PolyL10nRulebook<'a>> LocaleFallbackSolver<R> {
61    /// Find alternative fallbacks for the given `locale` as specified by the `rulebook`. This
62    /// operation is recursive and expensive.
63    ///
64    /// ```
65    /// let solver = poly_l10n::LocaleFallbackSolver::<poly_l10n::Rulebook>::default();
66    /// # #[cfg(feature = "per_lang_default_rules")]
67    /// assert_eq!(solver.solve_locale(poly_l10n::langid!("arb")), poly_l10n::langid!["arb", "ar-AE", "ara-AE", "arb-AE", "ar", "ara"]);
68    /// ```
69    pub fn solve_locale<L: AsRef<LanguageIdentifier>>(&self, locale: L) -> Vec<LanguageIdentifier> {
70        use std::hash::{Hash, Hasher};
71        let locale = locale.as_ref();
72        let mut locales = self.rulebook.find_fallback_locale(locale).collect_vec();
73        let h = |l: &LanguageIdentifier| {
74            let mut hasher = std::hash::DefaultHasher::default();
75            l.hash(&mut hasher);
76            hasher.finish()
77        };
78        let mut locale_hashes = locales.iter().map(h).collect_vec();
79        let mut old_len = 0;
80        while old_len != locales.len() {
81            #[allow(clippy::indexing_slicing)]
82            let new_locales = locales[old_len..]
83                .iter()
84                .flat_map(|locale| {
85                    self.rulebook.find_fallback_locale(locale).chain(
86                        self.rulebook
87                            .find_fallback_locale_ref(locale)
88                            .map(Clone::clone),
89                    )
90                })
91                .filter(|l| !locale_hashes.contains(&h(l)))
92                .unique()
93                .collect_vec();
94            old_len = locales.len();
95            locales.extend_from_slice(&new_locales);
96            locale_hashes.extend(new_locales.iter().map(h));
97        }
98        locales.into_iter().unique().collect_vec()
99    }
100}
101
102/// Rulebook trait.
103///
104/// A rulebook is a set of rules for [`LocaleFallbackSolver`]. The solver obtains the list of
105/// fallback locales from the rules in the solver's rulebook.
106///
107/// The default rulebook is [`ARulebook`] and you may create a solver with it using:
108///
109/// ```
110/// poly_l10n::LocaleFallbackSolver::<poly_l10n::ARulebook>::default()
111/// # ;
112/// ```
113///
114/// With that being said, a custom tailor-made rulebook is possible by implementing this trait for
115/// a new struct.
116///
117/// # Implementation
118/// Only one of [`PolyL10nRulebook::find_fallback_locale`] and
119/// [`PolyL10nRulebook::find_fallback_locale_ref`] SHOULD be implemented. Note that for the latter,
120/// [`LocaleFallbackSolver`] will clone the items in the returned iterator, so there are virtually
121/// no performance difference between the two.
122///
123/// If both functions are implemented, the solver will [`Iterator::chain`] them together.
124pub trait PolyL10nRulebook<'s> {
125    fn find_fallback_locale(
126        &self,
127        _: &LanguageIdentifier,
128    ) -> impl Iterator<Item = LanguageIdentifier> {
129        std::iter::empty()
130    }
131
132    fn find_fallback_locale_ref(
133        &'s self,
134        _: &LanguageIdentifier,
135    ) -> impl Iterator<Item = &'s LanguageIdentifier> {
136        std::iter::empty()
137    }
138}
139
140// NOTE: rust disallows multiple blanket impls, so unfortunately we need to choose one
141/*
142impl<'s, M> PolyL10nRulebook<'s> for M
143where
144    M: for<'a> std::ops::Index<&'a LanguageIdentifier, Output = LanguageIdentifier>,
145{
146    fn find_fallback_locale(
147        &'s self,
148        locale: &LanguageIdentifier,
149    ) -> impl Iterator<Item = &'s LanguageIdentifier> {
150        std::iter::once(&self[locale])
151    }
152}
153*/
154
155impl<'s, M, LS: 's> PolyL10nRulebook<'s> for M
156where
157    M: for<'a> std::ops::Index<&'a LanguageIdentifier, Output = LS>,
158    &'s LS: IntoIterator<Item = &'s LanguageIdentifier>,
159{
160    fn find_fallback_locale_ref(
161        &'s self,
162        locale: &LanguageIdentifier,
163    ) -> impl Iterator<Item = &'s LanguageIdentifier> {
164        (&self[locale]).into_iter()
165    }
166}
167
168pub type FnRules = Vec<Box<dyn Fn(&LanguageIdentifier) -> Vec<LanguageIdentifier>>>;
169
170/// A set of rules that govern how [`LocaleFallbackSolver`] should handle fallbacks.
171///
172/// For the thread-safe version, see [`ARulebook<A>`].
173///
174/// [`Rulebook<A>`], regardless of type `A`, stores the rules as [`FnRules`], a vector of boxed
175/// `dyn Fn(&LanguageIdentifier) -> Vec<LanguageIdentifier>`. Therefore, the actual correct name of
176/// this struct should be something along the lines of `FnsRulebook`.
177///
178/// Obviously this rulebook can be used with the solver because it implements [`PolyL10nRulebook`].
179///
180/// In addition, the default rulebook [`Rulebook::default()`] can and probably should be used for
181/// most situations you ever need to deal with.
182pub struct Rulebook<A = ()> {
183    pub rules: FnRules,
184    pub owned_values: A,
185}
186
187impl<A: std::fmt::Debug> std::fmt::Debug for Rulebook<A> {
188    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
189        f.debug_struct("Rulebook")
190            .field("owned_values", &self.owned_values)
191            .field("rules", &PseudoFnRules::from(&self.rules))
192            .finish_non_exhaustive()
193    }
194}
195/// Used for implementing [`Debug`] for [`Rulebook`].
196struct PseudoFnRules {
197    len: usize,
198}
199impl From<&FnRules> for PseudoFnRules {
200    fn from(value: &FnRules) -> Self {
201        Self { len: value.len() }
202    }
203}
204impl std::fmt::Debug for PseudoFnRules {
205    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
206        f.debug_struct("FnRules")
207            .field("len", &self.len)
208            .finish_non_exhaustive()
209    }
210}
211
212impl<A> PolyL10nRulebook<'_> for Rulebook<A> {
213    fn find_fallback_locale(
214        &self,
215        locale: &LanguageIdentifier,
216    ) -> impl Iterator<Item = LanguageIdentifier> {
217        self.rules.iter().flat_map(|f| f(locale))
218    }
219}
220
221impl Rulebook<Rc<Vec<Rulebook>>> {
222    /// Combine multiple rulebooks into one.
223    ///
224    /// See also: [`Self::from_ref_rulebooks`].
225    ///
226    /// # Examples
227    /// ```
228    /// let rb1 = poly_l10n::Rulebook::from_fn(|l| {
229    ///   let mut l = l.clone();
230    ///   l.script = None;
231    ///   vec![l]
232    /// });
233    /// let rb2 = poly_l10n::Rulebook::from_fn(|l| {
234    ///   let mut l = l.clone();
235    ///   l.region = None;
236    ///   vec![l]
237    /// });
238    /// let rulebook = poly_l10n::Rulebook::from_rulebooks([rb1, rb2].into_iter());
239    /// let solv = poly_l10n::LocaleFallbackSolver { rulebook };
240    ///
241    /// assert_eq!(
242    ///   solv.solve_locale(poly_l10n::langid!["zh-Hant-HK"]),
243    ///   poly_l10n::langid!["zh-HK", "zh-Hant", "zh"]
244    /// );
245    /// ```
246    pub fn from_rulebooks<I: Iterator<Item = Rulebook>>(rulebooks: I) -> Self {
247        let mut new = Self {
248            owned_values: Rc::new(rulebooks.collect_vec()),
249            rules: vec![],
250        };
251        let owned_values = Rc::clone(&new.owned_values);
252        new.rules = vec![Box::new(move |l: &LanguageIdentifier| {
253            owned_values
254                .iter()
255                .flat_map(|rulebook| rulebook.find_fallback_locale(l).collect_vec())
256                .collect()
257        })];
258        new
259    }
260}
261impl<RR, R> Rulebook<(Rc<Vec<RR>>, std::marker::PhantomData<R>)>
262where
263    RR: AsRef<Rulebook<R>> + 'static,
264{
265    /// Combine multiple rulebooks into one. Each given rulebook `r` must implement
266    /// [`AsRef::as_ref`].
267    ///
268    /// For the owned version, see [`Self::from_rulebooks`].
269    ///
270    /// # Examples
271    ///
272    /// ```
273    /// # use std::rc::Rc;
274    /// let rb1 = poly_l10n::Rulebook::from_fn(|l| {
275    ///   let mut l = l.clone();
276    ///   l.script = None;
277    ///   vec![l]
278    /// });
279    /// let rb2 = poly_l10n::Rulebook::from_fn(|l| {
280    ///   let mut l = l.clone();
281    ///   l.region = None;
282    ///   vec![l]
283    /// });
284    /// let (rb1, rb2) = (Rc::new(rb1), Rc::new(rb2));
285    /// let rulebook = poly_l10n::Rulebook::from_ref_rulebooks([rb1, rb2].iter().cloned());
286    /// let solv = poly_l10n::LocaleFallbackSolver { rulebook };
287    ///
288    /// assert_eq!(
289    ///   solv.solve_locale(poly_l10n::langid!["zh-Hant-HK"]),
290    ///   poly_l10n::langid!["zh-HK", "zh-Hant", "zh"]
291    /// );
292    /// ```
293    pub fn from_ref_rulebooks<I: Iterator<Item = RR>>(rulebooks: I) -> Self {
294        let mut new = Self {
295            owned_values: (Rc::new(rulebooks.collect_vec()), std::marker::PhantomData),
296            rules: vec![],
297        };
298        let owned_values = Rc::clone(&new.owned_values.0);
299        new.rules = vec![Box::new(move |l: &LanguageIdentifier| {
300            (owned_values.iter())
301                .flat_map(|rulebook| rulebook.as_ref().find_fallback_locale(l).collect_vec())
302                .collect()
303        })];
304        new
305    }
306}
307
308impl Rulebook {
309    #[must_use]
310    pub fn from_fn<F: Fn(&LanguageIdentifier) -> Vec<LanguageIdentifier> + 'static>(f: F) -> Self {
311        Self {
312            rules: vec![Box::new(f)],
313            owned_values: (),
314        }
315    }
316    #[must_use]
317    pub const fn from_fns(rules: FnRules) -> Self {
318        Self {
319            rules,
320            owned_values: (),
321        }
322    }
323    /// Convert a map (or anything that impl [`std::ops::Index<&LanguageIdentifier>`]) into
324    /// a rulebook.
325    ///
326    /// The output of the map must implement [`IntoIterator<Item = &LanguageIdentifier>`].
327    ///
328    /// While any valid arguments to this constructor are guaranteed to satisfy the trait
329    /// [`PolyL10nRulebook`], it could be useful to convert them to rulebooks, e.g. to combine
330    /// multiple rulebooks using [`Self::from_rulebooks`].
331    pub fn from_map<M, LS>(map: M) -> Self
332    where
333        M: for<'a> std::ops::Index<&'a LanguageIdentifier, Output = LS> + 'static,
334        for<'b> &'b LS: IntoIterator<Item = &'b LanguageIdentifier>,
335    {
336        Self::from_fn(move |l| map[l].into_iter().cloned().collect())
337    }
338}
339
340// TODO: rules?
341impl Default for Rulebook {
342    fn default() -> Self {
343        Self::from_fn(default_rulebook::default_rulebook)
344    }
345}
346
347pub type AFnRules = Vec<Box<dyn Fn(&LanguageIdentifier) -> Vec<LanguageIdentifier> + Send + Sync>>;
348
349/// A set of rules that govern how [`LocaleFallbackSolver`] should handle fallbacks.
350///
351/// This is the thread-safe version of [`Rulebook`].
352///
353/// [`ARulebook<A>`], regardless of type `A`, stores the rules as [`AFnRules`], a vector of boxed
354/// `dyn Fn(&LanguageIdentifier) -> Vec<LanguageIdentifier> + Send + Sync`. Therefore, the actual
355/// correct name of this struct should be something along the lines of `AFnsRulebook`.
356///
357/// Obviously this rulebook can be used with the solver because it implements [`PolyL10nRulebook`].
358///
359/// In addition, the default rulebook [`ARulebook::default()`] can and probably should be used for
360/// most situations you ever need to deal with.
361pub struct ARulebook<A = ()> {
362    pub rules: AFnRules,
363    pub owned_values: A,
364}
365
366impl<A: std::fmt::Debug> std::fmt::Debug for ARulebook<A> {
367    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
368        f.debug_struct("ARulebook")
369            .field("owned_values", &self.owned_values)
370            .field("rules", &APseudoFnRules::from(&self.rules))
371            .finish_non_exhaustive()
372    }
373}
374/// Used for implementing [`Debug`] for [`ARulebook`].
375struct APseudoFnRules {
376    len: usize,
377}
378impl From<&AFnRules> for APseudoFnRules {
379    fn from(value: &AFnRules) -> Self {
380        Self { len: value.len() }
381    }
382}
383impl std::fmt::Debug for APseudoFnRules {
384    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
385        f.debug_struct("AFnRules")
386            .field("len", &self.len)
387            .finish_non_exhaustive()
388    }
389}
390
391impl<A> PolyL10nRulebook<'_> for ARulebook<A> {
392    fn find_fallback_locale(
393        &self,
394        locale: &LanguageIdentifier,
395    ) -> impl Iterator<Item = LanguageIdentifier> {
396        self.rules.iter().flat_map(|f| f(locale))
397    }
398}
399
400impl ARulebook<Arc<Vec<ARulebook>>> {
401    /// Combine multiple rulebooks into one.
402    ///
403    /// See also: [`Self::from_ref_rulebooks`].
404    ///
405    /// # Examples
406    /// ```
407    /// let rb1 = poly_l10n::ARulebook::from_fn(|l| {
408    ///   let mut l = l.clone();
409    ///   l.script = None;
410    ///   vec![l]
411    /// });
412    /// let rb2 = poly_l10n::ARulebook::from_fn(|l| {
413    ///   let mut l = l.clone();
414    ///   l.region = None;
415    ///   vec![l]
416    /// });
417    /// let rulebook = poly_l10n::ARulebook::from_rulebooks([rb1, rb2].into_iter());
418    /// let solv = poly_l10n::LocaleFallbackSolver { rulebook };
419    ///
420    /// assert_eq!(
421    ///   solv.solve_locale(poly_l10n::langid!["zh-Hant-HK"]),
422    ///   poly_l10n::langid!["zh-HK", "zh-Hant", "zh"]
423    /// );
424    /// ```
425    pub fn from_rulebooks<I: Iterator<Item = ARulebook>>(rulebooks: I) -> Self {
426        let mut new = Self {
427            owned_values: Arc::new(rulebooks.collect_vec()),
428            rules: vec![],
429        };
430        let owned_values = Arc::clone(&new.owned_values);
431        new.rules = vec![Box::new(move |l: &LanguageIdentifier| {
432            owned_values
433                .iter()
434                .flat_map(|rulebook| rulebook.find_fallback_locale(l).collect_vec())
435                .collect()
436        })];
437        new
438    }
439}
440impl<RR, R> ARulebook<(Arc<Vec<RR>>, std::marker::PhantomData<R>)>
441where
442    RR: AsRef<ARulebook<R>> + 'static + Send + Sync,
443{
444    /// Combine multiple rulebooks into one. Each given rulebook `r` must implement
445    /// [`AsRef::as_ref`].
446    ///
447    /// For the owned version, see [`Self::from_rulebooks`].
448    ///
449    /// # Examples
450    ///
451    /// ```
452    /// # use std::sync::Arc;
453    /// let rb1 = poly_l10n::ARulebook::from_fn(|l| {
454    ///   let mut l = l.clone();
455    ///   l.script = None;
456    ///   vec![l]
457    /// });
458    /// let rb2 = poly_l10n::ARulebook::from_fn(|l| {
459    ///   let mut l = l.clone();
460    ///   l.region = None;
461    ///   vec![l]
462    /// });
463    /// let (rb1, rb2) = (Arc::new(rb1), Arc::new(rb2));
464    /// let rulebook = poly_l10n::ARulebook::from_ref_rulebooks([rb1, rb2].iter().cloned());
465    /// let solv = poly_l10n::LocaleFallbackSolver { rulebook };
466    ///
467    /// assert_eq!(
468    ///   solv.solve_locale(poly_l10n::langid!["zh-Hant-HK"]),
469    ///   poly_l10n::langid!["zh-HK", "zh-Hant", "zh"]
470    /// );
471    /// ```
472    pub fn from_ref_rulebooks<I: Iterator<Item = RR>>(rulebooks: I) -> Self {
473        let mut new = Self {
474            owned_values: (Arc::new(rulebooks.collect_vec()), std::marker::PhantomData),
475            rules: vec![],
476        };
477        let owned_values = Arc::clone(&new.owned_values.0);
478        new.rules = vec![Box::new(move |l: &LanguageIdentifier| {
479            (owned_values.iter())
480                .flat_map(|rulebook| rulebook.as_ref().find_fallback_locale(l).collect_vec())
481                .collect()
482        })];
483        new
484    }
485}
486
487impl ARulebook {
488    #[must_use]
489    pub fn from_fn<
490        F: Fn(&LanguageIdentifier) -> Vec<LanguageIdentifier> + 'static + Send + Sync,
491    >(
492        f: F,
493    ) -> Self {
494        Self {
495            rules: vec![Box::new(f)],
496            owned_values: (),
497        }
498    }
499    #[must_use]
500    pub const fn from_fns(rules: AFnRules) -> Self {
501        Self {
502            rules,
503            owned_values: (),
504        }
505    }
506    /// Convert a map (or anything that impl [`std::ops::Index<&LanguageIdentifier>`]) into
507    /// a rulebook.
508    ///
509    /// The output of the map must implement [`IntoIterator<Item = &LanguageIdentifier>`].
510    ///
511    /// While any valid arguments to this constructor are guaranteed to satisfy the trait
512    /// [`PolyL10nRulebook`], it could be useful to convert them to rulebooks, e.g. to combine
513    /// multiple rulebooks using [`Self::from_rulebooks`].
514    pub fn from_map<M, LS>(map: M) -> Self
515    where
516        M: for<'a> std::ops::Index<&'a LanguageIdentifier, Output = LS> + 'static + Send + Sync,
517        for<'b> &'b LS: IntoIterator<Item = &'b LanguageIdentifier>,
518    {
519        Self::from_fn(move |l| map[l].into_iter().cloned().collect())
520    }
521}
522
523// TODO: rules?
524impl Default for ARulebook {
525    fn default() -> Self {
526        Self::from_fn(default_rulebook::default_rulebook)
527    }
528}
poly_l10n/lib.rs

poly_l10n/
lib.rs
