language_text_analysis/
language.rs

1//! Supported analysis languages and BCP-47 tag resolution.
2
3use rust_stemmers::Algorithm;
4
5use crate::stopwords::{english, german};
6
7/// A language the analyzer can stem and stop-word-filter for.
8///
9/// Selected from a BCP-47 language tag (`de`, `en-US`) when present;
10/// untagged text uses the analyzer's configured default. The set mirrors
11/// the supported Snowball stemmer algorithms and is `#[non_exhaustive]`
12/// so more can be added without breaking callers.
13#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
14#[non_exhaustive]
15pub enum Language {
16    /// English.
17    English,
18    /// German.
19    German,
20    /// French.
21    French,
22    /// Spanish.
23    Spanish,
24    /// Italian.
25    Italian,
26    /// Dutch.
27    Dutch,
28    /// Portuguese.
29    Portuguese,
30    /// Swedish.
31    Swedish,
32    /// Norwegian.
33    Norwegian,
34    /// Danish.
35    Danish,
36    /// Finnish.
37    Finnish,
38    /// Russian.
39    Russian,
40}
41
42impl Language {
43    /// Resolve a BCP-47 language tag to a supported [`Language`].
44    ///
45    /// Only the primary subtag is significant: `en`, `en-US`, and
46    /// `en_GB` all resolve to [`Language::English`]. An unsupported or
47    /// malformed tag yields `None`, leaving the caller to fall back to
48    /// its configured default.
49    #[must_use]
50    pub fn from_tag(tag: &str) -> Option<Self> {
51        let primary = tag.split(['-', '_']).next().unwrap_or(tag);
52        match primary.to_ascii_lowercase().as_str() {
53            "en" => Some(Self::English),
54            "de" => Some(Self::German),
55            "fr" => Some(Self::French),
56            "es" => Some(Self::Spanish),
57            "it" => Some(Self::Italian),
58            "nl" => Some(Self::Dutch),
59            "pt" => Some(Self::Portuguese),
60            "sv" => Some(Self::Swedish),
61            "no" | "nb" | "nn" => Some(Self::Norwegian),
62            "da" => Some(Self::Danish),
63            "fi" => Some(Self::Finnish),
64            "ru" => Some(Self::Russian),
65            _ => None,
66        }
67    }
68
69    /// The Snowball stemmer algorithm for this language.
70    pub(crate) fn stemmer_algorithm(self) -> Algorithm {
71        match self {
72            Self::English => Algorithm::English,
73            Self::German => Algorithm::German,
74            Self::French => Algorithm::French,
75            Self::Spanish => Algorithm::Spanish,
76            Self::Italian => Algorithm::Italian,
77            Self::Dutch => Algorithm::Dutch,
78            Self::Portuguese => Algorithm::Portuguese,
79            Self::Swedish => Algorithm::Swedish,
80            Self::Norwegian => Algorithm::Norwegian,
81            Self::Danish => Algorithm::Danish,
82            Self::Finnish => Algorithm::Finnish,
83            Self::Russian => Algorithm::Russian,
84        }
85    }
86
87    /// The default stop-word list for this language, already normalized
88    /// to the post-[`normalize`](crate) token form. Languages without a
89    /// bundled list return an empty slice (no removal).
90    pub(crate) fn stop_words(self) -> &'static [&'static str] {
91        match self {
92            Self::English => english::WORDS,
93            Self::German => german::WORDS,
94            _ => &[],
95        }
96    }
97}
language_text_analysis/language.rs

language_text_analysis/
language.rs
