rust_i18n_support/
backend.rs

1use std::borrow::Cow;
2use std::collections::HashMap;
3
4/// A view of another backend restricted to a single namespace.
5pub struct NamespacedBackend {
6    backend: &'static dyn Backend,
7    namespace: &'static str,
8}
9
10impl NamespacedBackend {
11    /// Create a backend that exposes only keys below `namespace`.
12    pub fn new(backend: &'static dyn Backend, namespace: &'static str) -> Self {
13        Self { backend, namespace }
14    }
15
16    fn namespaced_key(&self, key: &str) -> String {
17        format!("{}.{key}", self.namespace)
18    }
19}
20
21impl Backend for NamespacedBackend {
22    fn available_locales(&self) -> Vec<Cow<'_, str>> {
23        self.backend
24            .available_locales()
25            .into_iter()
26            .filter(|locale| self.messages_for_locale(locale).is_some())
27            .collect()
28    }
29
30    fn translate(&self, locale: &str, key: &str) -> Option<Cow<'_, str>> {
31        self.backend.translate(locale, &self.namespaced_key(key))
32    }
33
34    fn messages_for_locale(&self, locale: &str) -> Option<Vec<(Cow<'_, str>, Cow<'_, str>)>> {
35        let prefix = format!("{}.", self.namespace);
36        let messages = self
37            .backend
38            .messages_for_locale(locale)?
39            .into_iter()
40            .filter_map(|(key, value)| {
41                key.strip_prefix(&prefix)
42                    .map(|key| (Cow::Owned(key.to_string()), value))
43            })
44            .collect::<Vec<_>>();
45
46        (!messages.is_empty()).then_some(messages)
47    }
48}
49
50/// I18n backend trait
51pub trait Backend: Send + Sync + 'static {
52    /// Return the available locales
53    fn available_locales(&self) -> Vec<Cow<'_, str>>;
54    /// Get the translation for the given locale and key
55    fn translate(&self, locale: &str, key: &str) -> Option<Cow<'_, str>>;
56    /// Get all translations for the given locale
57    fn messages_for_locale(&self, locale: &str) -> Option<Vec<(Cow<'_, str>, Cow<'_, str>)>>;
58}
59
60pub trait BackendExt: Backend {
61    /// Extend backend to add more translations
62    fn extend<T: Backend>(self, other: T) -> CombinedBackend<Self, T>
63    where
64        Self: Sized,
65    {
66        CombinedBackend(self, other)
67    }
68}
69
70pub struct CombinedBackend<A, B>(A, B);
71
72impl<A, B> Backend for CombinedBackend<A, B>
73where
74    A: Backend,
75    B: Backend,
76{
77    fn available_locales(&self) -> Vec<Cow<'_, str>> {
78        let mut available_locales = self.0.available_locales();
79        for locale in self.1.available_locales() {
80            if !available_locales.contains(&locale) {
81                available_locales.push(locale);
82            }
83        }
84        available_locales
85    }
86
87    #[inline]
88    fn translate(&self, locale: &str, key: &str) -> Option<Cow<'_, str>> {
89        self.1
90            .translate(locale, key)
91            .or_else(|| self.0.translate(locale, key))
92    }
93
94    fn messages_for_locale(&self, locale: &str) -> Option<Vec<(Cow<'_, str>, Cow<'_, str>)>> {
95        match (
96            self.1.messages_for_locale(locale),
97            self.0.messages_for_locale(locale),
98        ) {
99            (None, None) => None,
100            (None, a) => a,
101            (b, None) => b,
102            (Some(b), Some(a)) => Some(
103                b.into_iter()
104                    .chain(
105                        a.into_iter()
106                            .filter(|(k, _)| self.1.translate(locale, k).is_none()),
107                    )
108                    .collect(),
109            ),
110        }
111    }
112}
113
114/// Simple KeyValue storage backend
115pub struct SimpleBackend {
116    /// All translations key is flatten key, like `en.hello.world`
117    translations: LocaleTranslations,
118}
119
120// Keys come from the application's own translation files, so the lookup
121// tables do not need SipHash's resistance to adversarial keys.
122type FastMap<K, V> = HashMap<K, V, foldhash::fast::RandomState>;
123type Messages = FastMap<Cow<'static, str>, Cow<'static, str>>;
124type InputMessages = HashMap<Cow<'static, str>, Cow<'static, str>>;
125
126// Comparing cached two-byte prefixes avoids hashing the locale before hashing
127// the message key in small catalogs. Use a hash map for larger catalogs or
128// many regional variants sharing a prefix, bounding full-string comparisons.
129const SMALL_LOCALE_LIMIT: usize = 16;
130const SMALL_PREFIX_LIMIT: usize = 4;
131
132struct SmallLocale {
133    prefix: u16,
134    locale: Cow<'static, str>,
135    messages: Messages,
136}
137
138impl SmallLocale {
139    fn new(locale: Cow<'static, str>, messages: Messages) -> Self {
140        Self {
141            prefix: locale_prefix(&locale),
142            locale,
143            messages,
144        }
145    }
146}
147
148// A prefix is only an index. Empty, short, NUL-containing, and UTF-8 locale
149// names can share it, so every candidate is also compared in full.
150fn locale_prefix(locale: &str) -> u16 {
151    let bytes = locale.as_bytes();
152    u16::from_be_bytes([*bytes.first().unwrap_or(&0), *bytes.get(1).unwrap_or(&0)])
153}
154
155enum LocaleTranslations {
156    Small(Vec<SmallLocale>),
157    Large(FastMap<Cow<'static, str>, Messages>),
158}
159
160impl LocaleTranslations {
161    fn from_map(map: HashMap<Cow<'static, str>, InputMessages>) -> Self {
162        let map = map
163            .into_iter()
164            .map(|(locale, messages)| (locale, messages.into_iter().collect::<Messages>()))
165            .collect::<FastMap<_, _>>();
166        if map.len() <= SMALL_LOCALE_LIMIT {
167            let mut entries = map
168                .into_iter()
169                .map(|(locale, messages)| SmallLocale::new(locale, messages))
170                .collect::<Vec<_>>();
171            entries.sort_by(|a, b| {
172                a.prefix
173                    .cmp(&b.prefix)
174                    .then_with(|| a.locale.cmp(&b.locale))
175            });
176            if entries
177                .windows(SMALL_PREFIX_LIMIT + 1)
178                .any(|group| group.first().unwrap().prefix == group.last().unwrap().prefix)
179            {
180                return Self::Large(
181                    entries
182                        .into_iter()
183                        .map(|entry| (entry.locale, entry.messages))
184                        .collect(),
185                );
186            }
187            Self::Small(entries)
188        } else {
189            Self::Large(map)
190        }
191    }
192
193    fn get(&self, locale: &str) -> Option<&Messages> {
194        match self {
195            Self::Small(entries) => {
196                let prefix = locale_prefix(locale);
197                entries
198                    .iter()
199                    .find(|entry| entry.prefix == prefix && entry.locale.as_ref() == locale)
200                    .map(|entry| &entry.messages)
201            }
202            Self::Large(entries) => entries.get(locale),
203        }
204    }
205
206    fn add(&mut self, locale: Cow<'static, str>, data: InputMessages) {
207        let data = data.into_iter().collect::<Messages>();
208        match self {
209            Self::Small(entries) => {
210                let prefix = locale_prefix(&locale);
211                let start = entries.partition_point(|entry| entry.prefix < prefix);
212                let existing = entries[start..]
213                    .iter()
214                    .take_while(|entry| entry.prefix == prefix)
215                    .position(|entry| entry.locale.as_ref() == locale.as_ref());
216                if let Some(index) = existing {
217                    entries[start + index].messages.extend(data);
218                } else if entries.len() < SMALL_LOCALE_LIMIT
219                    && entries[start..]
220                        .iter()
221                        .take_while(|entry| entry.prefix == prefix)
222                        .count()
223                        < SMALL_PREFIX_LIMIT
224                {
225                    entries.insert(start, SmallLocale::new(locale, data));
226                } else {
227                    let mut map = std::mem::take(entries)
228                        .into_iter()
229                        .map(|entry| (entry.locale, entry.messages))
230                        .collect::<FastMap<_, _>>();
231                    map.insert(locale, data);
232                    *self = Self::Large(map);
233                }
234            }
235            Self::Large(entries) => entries.entry(locale).or_default().extend(data),
236        }
237    }
238
239    fn available_locales(&self) -> Vec<Cow<'_, str>> {
240        let mut locales: Vec<_> = match self {
241            Self::Small(entries) => entries.iter().map(|entry| entry.locale.clone()).collect(),
242            Self::Large(entries) => entries.keys().cloned().collect(),
243        };
244        locales.sort();
245        locales
246    }
247}
248
249impl
250    FromIterator<(
251        Cow<'static, str>,
252        HashMap<Cow<'static, str>, Cow<'static, str>>,
253    )> for SimpleBackend
254{
255    fn from_iter<
256        I: IntoIterator<
257            Item = (
258                Cow<'static, str>,
259                HashMap<Cow<'static, str>, Cow<'static, str>>,
260            ),
261        >,
262    >(
263        iter: I,
264    ) -> Self {
265        Self {
266            // HashMap::collect retains the existing last-wins behavior for
267            // duplicate locale entries.
268            translations: LocaleTranslations::from_map(iter.into_iter().collect()),
269        }
270    }
271}
272
273impl SimpleBackend {
274    /// Create a new SimpleBackend.
275    pub fn new() -> Self {
276        SimpleBackend {
277            translations: LocaleTranslations::Small(Vec::new()),
278        }
279    }
280
281    /// Add more translations for the given locale.
282    ///
283    /// ```no_run
284    /// # use std::collections::HashMap;
285    /// # use rust_i18n_support::SimpleBackend;
286    /// # let mut backend = SimpleBackend::new();
287    /// let mut trs = HashMap::new();
288    /// trs.insert("hello".into(), "Hello".into());
289    /// trs.insert("foo".into(), "Foo bar".into());
290    /// backend.add_translations("en".into(), trs);
291    /// ```
292    pub fn add_translations(
293        &mut self,
294        locale: Cow<'static, str>,
295        data: HashMap<Cow<'static, str>, Cow<'static, str>>,
296    ) {
297        self.translations.add(locale, data);
298    }
299}
300
301impl Backend for SimpleBackend {
302    fn available_locales(&self) -> Vec<Cow<'_, str>> {
303        self.translations.available_locales()
304    }
305
306    fn translate(&self, locale: &str, key: &str) -> Option<Cow<'_, str>> {
307        if let Some(trs) = self.translations.get(locale) {
308            return trs.get(key).cloned();
309        }
310
311        None
312    }
313
314    fn messages_for_locale(&self, locale: &str) -> Option<Vec<(Cow<'_, str>, Cow<'_, str>)>> {
315        self.translations
316            .get(locale)
317            .map(|trs| trs.iter().map(|(k, v)| (k.clone(), v.clone())).collect())
318    }
319}
320
321impl BackendExt for SimpleBackend {}
322
323impl Default for SimpleBackend {
324    fn default() -> Self {
325        Self::new()
326    }
327}
328
329#[cfg(test)]
330mod tests {
331    use std::borrow::Cow;
332    use std::collections::HashMap;
333
334    use super::SimpleBackend;
335    use super::{Backend, BackendExt, LocaleTranslations, NamespacedBackend, SMALL_LOCALE_LIMIT};
336
337    #[test]
338    fn small_locale_storage_transitions_and_keeps_merged_messages() {
339        let mut backend = SimpleBackend::new();
340        for index in 0..SMALL_LOCALE_LIMIT {
341            let locale = format!("{index:02}-locale");
342            backend.add_translations(
343                Cow::Owned(locale),
344                HashMap::from([(Cow::Borrowed("first"), Cow::Borrowed("value"))]),
345            );
346        }
347        assert!(matches!(
348            &backend.translations,
349            LocaleTranslations::Small(_)
350        ));
351
352        backend.add_translations(
353            Cow::Borrowed("00-locale"),
354            HashMap::from([(Cow::Borrowed("second"), Cow::Borrowed("another"))]),
355        );
356        assert_eq!(
357            backend.translate("00-locale", "first"),
358            Some(Cow::Borrowed("value"))
359        );
360        assert_eq!(
361            backend.translate("00-locale", "second"),
362            Some(Cow::Borrowed("another"))
363        );
364
365        backend.add_translations(
366            Cow::Borrowed("extra"),
367            HashMap::from([(Cow::Borrowed("first"), Cow::Borrowed("extra value"))]),
368        );
369        assert!(matches!(
370            &backend.translations,
371            LocaleTranslations::Large(_)
372        ));
373        assert_eq!(
374            backend.translate("00-locale", "second"),
375            Some(Cow::Borrowed("another"))
376        );
377        assert_eq!(
378            backend.translate("extra", "first"),
379            Some(Cow::Borrowed("extra value"))
380        );
381        assert_eq!(backend.messages_for_locale("missing"), None);
382        assert_eq!(backend.available_locales().len(), SMALL_LOCALE_LIMIT + 1);
383    }
384
385    #[test]
386    fn from_iterator_duplicate_locale_uses_last_entry() {
387        let first = HashMap::from([(Cow::Borrowed("key"), Cow::Borrowed("old"))]);
388        let second = HashMap::from([(Cow::Borrowed("key"), Cow::Borrowed("new"))]);
389        let backend =
390            SimpleBackend::from_iter([(Cow::Borrowed("en"), first), (Cow::Borrowed("en"), second)]);
391
392        assert!(matches!(
393            &backend.translations,
394            LocaleTranslations::Small(_)
395        ));
396        assert_eq!(backend.available_locales(), vec!["en"]);
397        assert_eq!(backend.translate("en", "key"), Some(Cow::Borrowed("new")));
398    }
399
400    #[test]
401    fn small_locale_index_checks_full_name_after_prefix_collision() {
402        let mut backend = SimpleBackend::new();
403        for (locale, value) in [
404            ("", "empty"),
405            ("\0", "nul"),
406            ("a", "short"),
407            ("a\0", "short nul"),
408            ("zh", "chinese"),
409            ("zh-CN", "simplified"),
410            ("é", "accent"),
411            ("é-x", "accent extended"),
412        ] {
413            backend.add_translations(
414                Cow::Borrowed(locale),
415                HashMap::from([(Cow::Borrowed("key"), Cow::Borrowed(value))]),
416            );
417        }
418
419        assert!(matches!(
420            &backend.translations,
421            LocaleTranslations::Small(_)
422        ));
423        for (locale, expected) in [
424            ("", "empty"),
425            ("\0", "nul"),
426            ("a", "short"),
427            ("a\0", "short nul"),
428            ("zh", "chinese"),
429            ("zh-CN", "simplified"),
430            ("é", "accent"),
431            ("é-x", "accent extended"),
432        ] {
433            assert_eq!(
434                backend.translate(locale, "key"),
435                Some(Cow::Borrowed(expected))
436            );
437        }
438        for missing in ["\0\0", "a\0x", "zh-TW", "é-y"] {
439            assert_eq!(backend.translate(missing, "key"), None);
440        }
441    }
442
443    #[test]
444    fn adding_fifth_shared_prefix_promotes_to_hash_map() {
445        let mut backend = SimpleBackend::new();
446        for index in 0..4 {
447            backend.add_translations(
448                Cow::Owned(format!("en-{index}")),
449                HashMap::from([(Cow::Borrowed("key"), Cow::Owned(index.to_string()))]),
450            );
451        }
452        assert!(matches!(
453            &backend.translations,
454            LocaleTranslations::Small(_)
455        ));
456
457        backend.add_translations(
458            Cow::Borrowed("en-0"),
459            HashMap::from([(Cow::Borrowed("extra"), Cow::Borrowed("merged"))]),
460        );
461        assert!(matches!(
462            &backend.translations,
463            LocaleTranslations::Small(_)
464        ));
465
466        backend.add_translations(
467            Cow::Borrowed("en-4"),
468            HashMap::from([(Cow::Borrowed("key"), Cow::Borrowed("four"))]),
469        );
470        assert!(matches!(
471            &backend.translations,
472            LocaleTranslations::Large(_)
473        ));
474        assert_eq!(
475            backend.translate("en-0", "extra"),
476            Some(Cow::Borrowed("merged"))
477        );
478        assert_eq!(
479            backend.translate("en-4", "key"),
480            Some(Cow::Borrowed("four"))
481        );
482        assert_eq!(backend.translate("en-missing", "key"), None);
483        assert_eq!(backend.available_locales().len(), 5);
484    }
485
486    #[test]
487    fn from_iterator_with_five_shared_prefixes_uses_hash_map() {
488        let backend = SimpleBackend::from_iter((0..5).map(|index| {
489            (
490                Cow::Owned(format!("en-{index}")),
491                HashMap::from([(Cow::Borrowed("key"), Cow::Owned(index.to_string()))]),
492            )
493        }));
494
495        assert!(matches!(
496            &backend.translations,
497            LocaleTranslations::Large(_)
498        ));
499        assert_eq!(backend.translate("en-4", "key"), Some(Cow::Borrowed("4")));
500        assert_eq!(backend.translate("en-missing", "key"), None);
501    }
502
503    #[test]
504    fn test_simple_backend() {
505        let mut backend = SimpleBackend::new();
506        let mut data = HashMap::new();
507        data.insert("hello".into(), "Hello".into());
508        data.insert("foo".into(), "Foo bar".into());
509        backend.add_translations("en".into(), data);
510
511        let mut data_cn = HashMap::new();
512        data_cn.insert("hello".into(), "你好".into());
513        data_cn.insert("foo".into(), "Foo 测试".into());
514        backend.add_translations("zh-CN".into(), data_cn);
515
516        assert_eq!(backend.translate("en", "hello"), Some(Cow::from("Hello")));
517        assert_eq!(backend.translate("en", "foo"), Some(Cow::from("Foo bar")));
518        assert_eq!(backend.translate("zh-CN", "hello"), Some(Cow::from("你好")));
519        assert_eq!(
520            backend.translate("zh-CN", "foo"),
521            Some(Cow::from("Foo 测试"))
522        );
523
524        assert_eq!(backend.available_locales(), vec!["en", "zh-CN"]);
525    }
526
527    #[test]
528    fn test_combined_backend() {
529        let mut backend = SimpleBackend::new();
530        let mut data = HashMap::new();
531        data.insert("hello".into(), "Hello".into());
532        data.insert("foo".into(), "Foo bar".into());
533        backend.add_translations("en".into(), data);
534
535        let mut data_cn = HashMap::new();
536        data_cn.insert("hello".into(), "你好".into());
537        data_cn.insert("foo".into(), "Foo 测试".into());
538        backend.add_translations("zh-CN".into(), data_cn);
539
540        let mut backend2 = SimpleBackend::new();
541        let mut data2 = HashMap::new();
542        data2.insert("hello".into(), "Hello2".into());
543        backend2.add_translations("en".into(), data2);
544
545        let mut data_cn2 = HashMap::new();
546        data_cn2.insert("hello".into(), "你好2".into());
547        backend2.add_translations("zh-CN".into(), data_cn2);
548
549        let combined = backend.extend(backend2);
550        assert_eq!(combined.translate("en", "hello"), Some(Cow::from("Hello2")));
551        assert_eq!(
552            combined.translate("zh-CN", "hello"),
553            Some(Cow::from("你好2"))
554        );
555
556        assert_eq!(combined.available_locales(), vec!["en", "zh-CN"]);
557    }
558
559    #[test]
560    fn test_namespaced_backend() {
561        let mut backend = SimpleBackend::new();
562        let mut data = HashMap::new();
563        data.insert("ui_component.title".into(), "Custom title".into());
564        data.insert("title".into(), "Unrelated title".into());
565        backend.add_translations("en".into(), data);
566
567        let backend = Box::leak(Box::new(backend));
568        let namespaced = NamespacedBackend::new(backend, "ui_component");
569
570        assert_eq!(
571            namespaced.translate("en", "title"),
572            Some(Cow::Borrowed("Custom title"))
573        );
574        assert_eq!(
575            namespaced.messages_for_locale("en"),
576            Some(vec![(
577                Cow::Owned("title".to_string()),
578                Cow::Borrowed("Custom title")
579            )])
580        );
581    }
582}
rust_i18n_support/backend.rs

rust_i18n_support/
backend.rs
