language_text_analysis/
language.rs

1//! Supported analysis languages and BCP-47 tag resolution.
2
3use rust_stemmers::Algorithm;
4
5use crate::stopwords::{
6    english,
7    german,
8};
9
10/// A language the analyzer can stem and stop-word-filter for.
11///
12/// Selected from a BCP-47 language tag (`de`, `en-US`) when present;
13/// untagged text uses the analyzer's configured default. The set mirrors
14/// the supported Snowball stemmer algorithms and is `#[non_exhaustive]`
15/// so more can be added without breaking callers.
16#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
17#[non_exhaustive]
18pub enum Language {
19    /// English.
20    English,
21    /// German.
22    German,
23    /// French.
24    French,
25    /// Spanish.
26    Spanish,
27    /// Italian.
28    Italian,
29    /// Dutch.
30    Dutch,
31    /// Portuguese.
32    Portuguese,
33    /// Swedish.
34    Swedish,
35    /// Norwegian.
36    Norwegian,
37    /// Danish.
38    Danish,
39    /// Finnish.
40    Finnish,
41    /// Russian.
42    Russian,
43}
44
45impl Language {
46    /// Resolve a BCP-47 language tag to a supported [`Language`].
47    ///
48    /// Only the primary subtag is significant: `en`, `en-US`, and
49    /// `en_GB` all resolve to [`Language::English`]. An unsupported or
50    /// malformed tag yields `None`, leaving the caller to fall back to
51    /// its configured default.
52    #[must_use]
53    pub fn from_tag(tag: &str) -> Option<Self> {
54        let primary = tag.split(['-', '_']).next().unwrap_or(tag);
55        match primary.to_ascii_lowercase().as_str() {
56            "en" => Some(Self::English),
57            "de" => Some(Self::German),
58            "fr" => Some(Self::French),
59            "es" => Some(Self::Spanish),
60            "it" => Some(Self::Italian),
61            "nl" => Some(Self::Dutch),
62            "pt" => Some(Self::Portuguese),
63            "sv" => Some(Self::Swedish),
64            "no" | "nb" | "nn" => Some(Self::Norwegian),
65            "da" => Some(Self::Danish),
66            "fi" => Some(Self::Finnish),
67            "ru" => Some(Self::Russian),
68            _ => None,
69        }
70    }
71
72    /// The Snowball stemmer algorithm for this language.
73    pub(crate) fn stemmer_algorithm(self) -> Algorithm {
74        match self {
75            Self::English => Algorithm::English,
76            Self::German => Algorithm::German,
77            Self::French => Algorithm::French,
78            Self::Spanish => Algorithm::Spanish,
79            Self::Italian => Algorithm::Italian,
80            Self::Dutch => Algorithm::Dutch,
81            Self::Portuguese => Algorithm::Portuguese,
82            Self::Swedish => Algorithm::Swedish,
83            Self::Norwegian => Algorithm::Norwegian,
84            Self::Danish => Algorithm::Danish,
85            Self::Finnish => Algorithm::Finnish,
86            Self::Russian => Algorithm::Russian,
87        }
88    }
89
90    /// The default stop-word list for this language, already normalized
91    /// to the post-[`normalize`](crate) token form. Languages without a
92    /// bundled list return an empty slice (no removal).
93    pub(crate) fn stop_words(self) -> &'static [&'static str] {
94        match self {
95            Self::English => english::WORDS,
96            Self::German => german::WORDS,
97            _ => &[],
98        }
99    }
100}
language_text_analysis/language.rs

language_text_analysis/
language.rs
