qql_embed/bm25_lang.rs
1//! BM25 text-processing languages, mirroring Qdrant's `Language` names.
2//!
3//! The canonical spellings are Qdrant's `snake_case` serializations
4//! (`"english"`, `"spanish"`, …); [`Language::parse`] additionally accepts
5//! Qdrant's two-letter aliases (`"en"`, `"es"`, …) case-insensitively.
6//! Unknown names fail closed with `QQL-VALIDATION-CONFIG` — like Qdrant's
7//! edge builder, which rejects unsupported languages instead of silently
8//! disabling stemming and stopwords.
9
10use qql_core::error::QqlError;
11use rust_stemmers::Algorithm;
12
13/// Text-processing language for BM25 tokenization. One variant per Qdrant
14/// `Language`; serializations match Qdrant's `snake_case` names.
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
16pub enum Language {
17 /// Arabic (`"ar"`). Snowball stemming + stopwords.
18 Arabic,
19 /// Azerbaijani (`"az"`). No default stemmer; stopwords only.
20 Azerbaijani,
21 /// Basque (`"eu"`). No default stemmer; stopwords only.
22 Basque,
23 /// Bengali (`"bn"`). No default stemmer; stopwords only.
24 Bengali,
25 /// Catalan (`"ca"`). No default stemmer; stopwords only.
26 Catalan,
27 /// Chinese (`"zh"`). No default stemmer; stopwords only.
28 Chinese,
29 /// Danish (`"da"`). Snowball stemming + stopwords.
30 Danish,
31 /// Dutch (`"nl"`). Snowball stemming + stopwords.
32 Dutch,
33 /// English (`"en"`). Snowball stemming + stopwords.
34 English,
35 /// Finnish (`"fi"`). Snowball stemming + stopwords.
36 Finnish,
37 /// French (`"fr"`). Snowball stemming + stopwords.
38 French,
39 /// German (`"de"`). Snowball stemming + stopwords.
40 German,
41 /// Greek (`"el"`). Snowball stemming + stopwords.
42 Greek,
43 /// Hebrew (`"he"`). No default stemmer; stopwords only.
44 Hebrew,
45 /// Hinglish (`"hi-en"`). No default stemmer; stopwords only.
46 Hinglish,
47 /// Hungarian (`"hu"`). Snowball stemming + stopwords.
48 Hungarian,
49 /// Indonesian (`"id"`). No default stemmer; stopwords only.
50 Indonesian,
51 /// Italian (`"it"`). Snowball stemming + stopwords.
52 Italian,
53 /// Japanese (`"jp"`). No default stemmer; stopwords only.
54 Japanese,
55 /// Kazakh (`"kk"`). No default stemmer; stopwords only.
56 Kazakh,
57 /// Nepali (`"ne"`). No default stemmer; stopwords only.
58 Nepali,
59 /// Norwegian (`"no"`). Snowball stemming + stopwords.
60 Norwegian,
61 /// Portuguese (`"pt"`). Snowball stemming + stopwords.
62 Portuguese,
63 /// Romanian (`"ro"`). Snowball stemming + stopwords.
64 Romanian,
65 /// Russian (`"ru"`). Snowball stemming + stopwords.
66 Russian,
67 /// Slovene (`"sl"`). No default stemmer; stopwords only.
68 Slovene,
69 /// Spanish (`"es"`). Snowball stemming + stopwords.
70 Spanish,
71 /// Swedish (`"sv"`). Snowball stemming + stopwords.
72 Swedish,
73 /// Tajik (`"tg"`). No default stemmer; stopwords only.
74 Tajik,
75 /// Turkish (`"tr"`). Snowball stemming + stopwords.
76 Turkish,
77}
78
79impl Default for Language {
80 /// Qdrant's BM25 default: English stemming and stopwords.
81 fn default() -> Self {
82 Self::English
83 }
84}
85
86impl Language {
87 /// Canonical Qdrant spelling (`snake_case`, as Qdrant serializes it).
88 pub fn name(self) -> &'static str {
89 match self {
90 Self::Arabic => "arabic",
91 Self::Azerbaijani => "azerbaijani",
92 Self::Basque => "basque",
93 Self::Bengali => "bengali",
94 Self::Catalan => "catalan",
95 Self::Chinese => "chinese",
96 Self::Danish => "danish",
97 Self::Dutch => "dutch",
98 Self::English => "english",
99 Self::Finnish => "finnish",
100 Self::French => "french",
101 Self::German => "german",
102 Self::Greek => "greek",
103 Self::Hebrew => "hebrew",
104 Self::Hinglish => "hinglish",
105 Self::Hungarian => "hungarian",
106 Self::Indonesian => "indonesian",
107 Self::Italian => "italian",
108 Self::Japanese => "japanese",
109 Self::Kazakh => "kazakh",
110 Self::Nepali => "nepali",
111 Self::Norwegian => "norwegian",
112 Self::Portuguese => "portuguese",
113 Self::Romanian => "romanian",
114 Self::Russian => "russian",
115 Self::Slovene => "slovene",
116 Self::Spanish => "spanish",
117 Self::Swedish => "swedish",
118 Self::Tajik => "tajik",
119 Self::Turkish => "turkish",
120 }
121 }
122
123 /// Parse a language name or Qdrant two-letter alias (`"es"`, `"zh"`,
124 /// `"hi-en"`, …), ASCII-case-insensitively. The case-folding is a
125 /// deliberate superset of Qdrant's case-sensitive serde (same accepted
126 /// set, friendlier spelling). Anything else fails closed.
127 pub fn parse(name: &str) -> Result<Self, QqlError> {
128 // `hi-en` is the only alias outside `[a-z]`; compare lowercased.
129 let lower = name.to_ascii_lowercase();
130 let language = match lower.as_str() {
131 "arabic" | "ar" => Self::Arabic,
132 "azerbaijani" | "az" => Self::Azerbaijani,
133 "basque" | "eu" => Self::Basque,
134 "bengali" | "bn" => Self::Bengali,
135 "catalan" | "ca" => Self::Catalan,
136 "chinese" | "zh" => Self::Chinese,
137 "danish" | "da" => Self::Danish,
138 "dutch" | "nl" => Self::Dutch,
139 "english" | "en" => Self::English,
140 "finnish" | "fi" => Self::Finnish,
141 "french" | "fr" => Self::French,
142 "german" | "de" => Self::German,
143 "greek" | "el" => Self::Greek,
144 "hebrew" | "he" => Self::Hebrew,
145 "hinglish" | "hi-en" => Self::Hinglish,
146 "hungarian" | "hu" => Self::Hungarian,
147 "indonesian" | "id" => Self::Indonesian,
148 "italian" | "it" => Self::Italian,
149 "japanese" | "jp" => Self::Japanese,
150 "kazakh" | "kk" => Self::Kazakh,
151 "nepali" | "ne" => Self::Nepali,
152 "norwegian" | "no" => Self::Norwegian,
153 "portuguese" | "pt" => Self::Portuguese,
154 "romanian" | "ro" => Self::Romanian,
155 "russian" | "ru" => Self::Russian,
156 "slovene" | "sl" => Self::Slovene,
157 "spanish" | "es" => Self::Spanish,
158 "swedish" | "sv" => Self::Swedish,
159 "tajik" | "tg" => Self::Tajik,
160 "turkish" | "tr" => Self::Turkish,
161 _ => {
162 return Err(QqlError::validation(
163 "QQL-VALIDATION-CONFIG",
164 format!("unsupported bm25 language: {name:?}"),
165 None,
166 ));
167 }
168 };
169 Ok(language)
170 }
171
172 /// Snowball stemmer for this language, if Qdrant defines one.
173 ///
174 /// Mirrors Qdrant's `Stemmer::try_default_from_language`: exactly the 17
175 /// Snowball languages reachable via `language` stem (Armenian and Tamil
176 /// exist in Qdrant's `SnowballLanguage` but have no `Language` variant,
177 /// so they are explicit-stemmer-only — see [`Stemmer`](super::bm25_text::Stemmer)).
178 /// The rest (Chinese, Japanese, Hebrew, …) have no default stemmer and
179 /// pass tokens through unstemmed. `qdrant-rust-stemmers` is the same
180 /// crate family Qdrant uses, so the stems are identical.
181 pub fn stem_algorithm(self) -> Option<Algorithm> {
182 match self {
183 Self::Arabic => Some(Algorithm::Arabic),
184 Self::Danish => Some(Algorithm::Danish),
185 Self::Dutch => Some(Algorithm::Dutch),
186 Self::English => Some(Algorithm::English),
187 Self::Finnish => Some(Algorithm::Finnish),
188 Self::French => Some(Algorithm::French),
189 Self::German => Some(Algorithm::German),
190 Self::Greek => Some(Algorithm::Greek),
191 Self::Hungarian => Some(Algorithm::Hungarian),
192 Self::Italian => Some(Algorithm::Italian),
193 Self::Norwegian => Some(Algorithm::Norwegian),
194 Self::Portuguese => Some(Algorithm::Portuguese),
195 Self::Romanian => Some(Algorithm::Romanian),
196 Self::Russian => Some(Algorithm::Russian),
197 Self::Spanish => Some(Algorithm::Spanish),
198 Self::Swedish => Some(Algorithm::Swedish),
199 Self::Turkish => Some(Algorithm::Turkish),
200 _ => None,
201 }
202 }
203}