snowball_stemmers_rs/
lib.rs

1//! This library provides rust implementations for some stemmer algorithms
2//! written in the [snowball language](https://snowballstem.org/).
3//!
4//!
5//! All algorithms expect the input to already be lowercased.
6//!
7//! # Usage
8//! ```toml
9//! [dependencies]
10//! rust-stemmers = "^1.0"
11//! ```
12//!
13//! ```rust
14//! extern crate snowball_stemmers_rs;
15//!
16//! use snowball_stemmers_rs::{Algorithm, Stemmer};
17//!
18//! fn main() {
19//!    let en_stemmer = Stemmer::create(Algorithm::English);
20//!    assert_eq!(en_stemmer.stem("fishing"), "fish");
21//! }
22//! ```
23extern crate serde;
24#[macro_use]
25extern crate serde_derive;
26
27use std::borrow::Cow;
28
29mod snowball;
30
31use snowball::algorithms;
32use snowball::SnowballEnv;
33
34/// Enum of all supported algorithms.
35/// Check the [Snowball-Website](https://snowballstem.org/) for details.
36#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Copy, Clone)]
37pub enum Algorithm {
38    Arabic,
39    Armenian,
40    Basque,
41    Catalan,
42    Czech,
43    Danish,
44    Dutch,
45    DutchPorter,
46    English,
47    Esperanto,
48    Estonian,
49    Finnish,
50    French,
51    German,
52    Greek,
53    Hindi,
54    Hungarian,
55    Indonesian,
56    Irish,
57    Italian,
58    Lithuanian,
59    Lovins,
60    Nepali,
61    Norwegian,
62    Persian,
63    Polish,
64    Porter,
65    Portuguese,
66    Romanian,
67    Russian,
68    Serbian,
69    Sesotho,
70    Spanish,
71    Swedish,
72    Tamil,
73    Turkish,
74    Ukrainian,
75    Yiddish,
76}
77
78/// Interface around the Snowball stemmer implementation
79pub struct Stemmer {
80    stemmer: fn(&mut SnowballEnv) -> bool,
81}
82
83impl Stemmer {
84    /// Create a new stemmer from an algorithm
85    pub fn create(lang: Algorithm) -> Self {
86        match lang {
87            Algorithm::Arabic => Stemmer {
88                stemmer: algorithms::arabic::stem,
89            },
90            Algorithm::Armenian => Stemmer {
91                stemmer: algorithms::armenian::stem,
92            },
93            Algorithm::Basque => Stemmer {
94                stemmer: algorithms::basque::stem,
95            },
96            Algorithm::Catalan => Stemmer {
97                stemmer: algorithms::catalan::stem,
98            },
99            Algorithm::Czech => Stemmer {
100                stemmer: algorithms::czech::stem,
101            },
102            Algorithm::Danish => Stemmer {
103                stemmer: algorithms::danish::stem,
104            },
105            Algorithm::Dutch => Stemmer {
106                stemmer: algorithms::dutch::stem,
107            },
108            Algorithm::DutchPorter => Stemmer {
109                stemmer: algorithms::dutch_porter::stem,
110            },
111            Algorithm::English => Stemmer {
112                stemmer: algorithms::english::stem,
113            },
114            Algorithm::Esperanto => Stemmer {
115                stemmer: algorithms::esperanto::stem,
116            },
117            Algorithm::Estonian => Stemmer {
118                stemmer: algorithms::estonian::stem,
119            },
120            Algorithm::Finnish => Stemmer {
121                stemmer: algorithms::finnish::stem,
122            },
123            Algorithm::French => Stemmer {
124                stemmer: algorithms::french::stem,
125            },
126            Algorithm::German => Stemmer {
127                stemmer: algorithms::german::stem,
128            },
129            Algorithm::Greek => Stemmer {
130                stemmer: algorithms::greek::stem,
131            },
132            Algorithm::Hindi => Stemmer {
133                stemmer: algorithms::hindi::stem,
134            },
135            Algorithm::Hungarian => Stemmer {
136                stemmer: algorithms::hungarian::stem,
137            },
138            Algorithm::Indonesian => Stemmer {
139                stemmer: algorithms::indonesian::stem,
140            },
141            Algorithm::Irish => Stemmer {
142                stemmer: algorithms::irish::stem,
143            },
144            Algorithm::Italian => Stemmer {
145                stemmer: algorithms::italian::stem,
146            },
147            Algorithm::Lithuanian => Stemmer {
148                stemmer: algorithms::lithuanian::stem,
149            },
150            Algorithm::Lovins => Stemmer {
151                stemmer: algorithms::lovins::stem,
152            },
153            Algorithm::Nepali => Stemmer {
154                stemmer: algorithms::nepali::stem,
155            },
156            Algorithm::Norwegian => Stemmer {
157                stemmer: algorithms::norwegian::stem,
158            },
159            Algorithm::Persian => Stemmer {
160                stemmer: algorithms::persian::stem,
161            },
162            Algorithm::Polish => Stemmer {
163                stemmer: algorithms::polish::stem,
164            },
165            Algorithm::Porter => Stemmer {
166                stemmer: algorithms::porter::stem,
167            },
168            Algorithm::Portuguese => Stemmer {
169                stemmer: algorithms::portuguese::stem,
170            },
171            Algorithm::Romanian => Stemmer {
172                stemmer: algorithms::romanian::stem,
173            },
174            Algorithm::Russian => Stemmer {
175                stemmer: algorithms::russian::stem,
176            },
177            Algorithm::Serbian => Stemmer {
178                stemmer: algorithms::serbian::stem,
179            },
180            Algorithm::Sesotho => Stemmer {
181                stemmer: algorithms::sesotho::stem,
182            },
183            Algorithm::Spanish => Stemmer {
184                stemmer: algorithms::spanish::stem,
185            },
186            Algorithm::Swedish => Stemmer {
187                stemmer: algorithms::swedish::stem,
188            },
189            Algorithm::Tamil => Stemmer {
190                stemmer: algorithms::tamil::stem,
191            },
192            Algorithm::Turkish => Stemmer {
193                stemmer: algorithms::turkish::stem,
194            },
195            Algorithm::Ukrainian => Stemmer {
196                stemmer: algorithms::ukrainian::stem,
197            },
198            Algorithm::Yiddish => Stemmer {
199                stemmer: algorithms::yiddish::stem,
200            },
201        }
202    }
203
204    /// Stem a single word, the input is expected to be in lowercase.
205    pub fn stem<'a>(&self, input: &'a str) -> Cow<'a, str> {
206        let mut env = SnowballEnv::create(input);
207        (self.stemmer)(&mut env);
208        env.get_current()
209    }
210}
211
212#[cfg(test)]
213mod tests {
214    use super::{Algorithm, Stemmer};
215
216    #[test]
217    fn english_test() {
218        let vocabulary = vec![
219            ("fishing", "fish"),
220            ("running", "run"),
221            ("quickly", "quick"),
222            ("connection", "connect"),
223            ("cared", "care"),
224            ("jumped", "jump"),
225            ("skies", "sky"),
226        ];
227        let stemmer = Stemmer::create(Algorithm::English);
228
229        for (voc, res) in vocabulary {
230            assert_eq!(stemmer.stem(voc), res);
231        }
232    }
233
234    #[test]
235    fn polish_test() {
236        let vocabulary = vec![
237            ("samochód", "samochód"),
238            ("samochodu", "samochod"),
239            ("samochodowi", "samochod"),
240            ("samochodem", "samochod"),
241            ("samochodzie", "samochodz"),
242            ("samochody", "samochod"),
243            ("samochodów", "samochod"),
244            ("samochodom", "samochod"),
245            ("samochodami", "samochod"),
246            ("samochodach", "samochod"),
247            ("stołowi", "stoł"),
248            ("słoniowi", "słon"),
249        ];
250        let stemmer = Stemmer::create(Algorithm::Polish);
251
252        for (voc, res) in vocabulary {
253            assert_eq!(stemmer.stem(voc), res);
254        }
255    }
256}
snowball_stemmers_rs/lib.rs

snowball_stemmers_rs/
lib.rs
