langdetect-rs 0.2.3

Language detection in Rust. Port of Mimino666's langdetect.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
use std::fs;
use std::path::Path;
use serde_json;
use std::collections::HashMap;
use crate::utils::lang_profile::LangProfile;
use crate::detector::{Detector, DetectorError};
use crate::language::Language;
use crate::utils::lang_profile::LangProfileJson;

/// Errors that can occur when working with DetectorFactory.
#[derive(Debug, Clone)]
pub enum DetectorFactoryError {
    /// Attempted to add a language profile that already exists.
    DuplicatedLanguage(String),
    /// At least 2 languages are required for detection.
    NotEnoughProfiles,
}

impl std::fmt::Display for DetectorFactoryError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            DetectorFactoryError::DuplicatedLanguage(lang) => {
                write!(f, "Duplicated language profile: {}", lang)
            }
            DetectorFactoryError::NotEnoughProfiles => {
                write!(f, "Two languages at least are required")
            }
        }
    }
}

/// Factory for creating language detectors with pre-loaded language profiles.
///
/// The DetectorFactory manages a collection of language profiles and provides
/// methods to create Detector instances for language identification.
///
/// # Examples
///
/// ```rust
/// use langdetect_rs::detector_factory::DetectorFactory;
///
/// // Create factory with built-in profiles
/// let factory = DetectorFactory::default().build();
///
/// // Create a detector
/// let detector = factory.create(None);
/// ```
#[derive(Clone)]
pub struct DetectorFactory {
    /// Word-to-language probability mapping for all loaded languages.
    pub word_lang_prob_map: HashMap<String, Vec<f64>>,
    /// List of language identifiers in the same order as probability vectors.
    pub langlist: Vec<String>,
    /// Optional seed for reproducible randomization.
    pub seed: Option<u64>,
}

impl DetectorFactory {
    /// Creates a new DetectorFactory builder.
    ///
    /// Use the builder pattern to configure the factory before calling `build()`.
    ///
    /// # Examples
    ///
    /// ```rust
    /// use langdetect_rs::detector_factory::DetectorFactory;
    ///
    /// let factory = DetectorFactory::new()
    ///     .with_seed(Some(42))
    ///     .build();
    /// ```
    pub fn new() -> DetectorFactoryBuilder {
        DetectorFactoryBuilder {
            factory: DetectorFactory {
                word_lang_prob_map: HashMap::new(),
                langlist: Vec::new(),
                seed: None,
            },
        }
    }

    /// Creates a DetectorFactoryBuilder with all built-in language profiles loaded.
    ///
    /// This method loads the 55 built-in language profiles from the crate's
    /// profiles directory and returns a builder that can be further re-configured.
    /// The profiles are cached for performance.
    ///
    /// # Example
    ///
    /// ```rust
    /// use langdetect_rs::detector_factory::DetectorFactory;
    ///
    /// let factory = DetectorFactory::default()
    ///     .with_seed(Some(42))
    ///     .build();
    /// ```
    pub fn default() -> DetectorFactoryBuilder {
        use std::sync::Mutex;
        use lazy_static::lazy_static;
        lazy_static! {
            static ref DEFAULT_FACTORY: Mutex<Option<DetectorFactory>> = Mutex::new(None);
        }
        {
            let factory_guard = DEFAULT_FACTORY.lock().unwrap();
            if let Some(factory) = &*factory_guard {
                return DetectorFactoryBuilder { factory: factory.clone() };
            }
        }
        let mut factory = DetectorFactory::new().build();
        // Try to load profiles from crate-level "profiles" folder
        let crate_profiles = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("profiles");

        let _ = factory.load_profile(&crate_profiles);
        // Cache the factory for future use
        let mut factory_guard = DEFAULT_FACTORY.lock().unwrap();
        *factory_guard = Some(factory.clone());
        DetectorFactoryBuilder { factory }
    }

    /// Returns the path to the default language profiles directory.
    ///
    /// This method provides the path to the built-in language profile files that ship
    /// with the crate. End-users can use this path to load default profiles when
    /// extending or customizing the factory.
    ///
    /// Note: This path is only accessible when the crate is used as a source dependency
    /// or when running from the crate's directory. When used as a published dependency,
    /// the profiles may not be available as filesystem files.
    ///
    /// # Returns
    /// A PathBuf pointing to the default profiles directory.
    ///
    /// # Example
    ///
    /// ```rust
    /// use langdetect_rs::detector_factory::DetectorFactory;
    /// use langdetect_rs::utils::lang_profile::{LangProfileJson, LangProfile};
    ///
    /// // Get path to default profiles
    /// let profiles_path = DetectorFactory::get_default_profiles_path();
    /// println!("Default profiles are located at: {:?}", profiles_path);
    ///
    /// // Load a specific profile
    /// let en_profile = LangProfileJson::new_from_file(profiles_path.join("en")).unwrap();
    /// let profile = LangProfile::from_json(en_profile).unwrap();
    ///
    /// // Add to custom factory
    /// let mut factory = DetectorFactory::new().build();
    /// factory.add_profile(profile, 0, 1).unwrap();
    /// ```
    pub fn get_default_profiles_path() -> std::path::PathBuf {
        std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("profiles")
    }

    /// Clears all loaded language profiles and mappings.
    pub fn clear(&mut self) {
        self.langlist.clear();
        self.word_lang_prob_map.clear();
    }

    /// Sets the randomization seed for reproducible results.
    ///
    /// # Arguments
    /// * `seed` - The seed value to use for randomization.
    pub fn set_seed(&mut self, seed: u64) {
        self.seed = Some(seed);
    }

    /// Returns a list of all loaded language identifiers.
    ///
    /// # Returns
    /// A vector of language codes (ISO 639-1) in the order they were loaded.
    pub fn get_lang_list(&self) -> Vec<String> {
        self.langlist.clone()
    }

    /// Creates a new Detector instance with the current profiles.
    ///
    /// # Arguments
    /// * `alpha` - Optional alpha smoothing parameter (default: 0.5).
    ///
    /// # Returns
    /// A configured Detector ready for language detection.
    pub fn create(&self, alpha: Option<f64>) -> Detector {
        let mut detector = Detector::new(
            self.word_lang_prob_map.clone(),
            self.langlist.clone(),
            self.seed,
        );
        if let Some(a) = alpha {
            detector.alpha = a;
        }
        detector
    }

    /// Overrides an existing language profile at the specified index.
    ///
    /// This is an internal method used during profile loading.
    ///
    /// # Arguments
    /// * `profile` - The language profile to add.
    /// * `index` - The index in the language list.
    /// * `langsize` - Total number of languages.
    pub fn override_profile(&mut self, profile: LangProfile, index: usize, langsize: usize) -> Result<(), DetectorFactoryError> {
        let lang = profile.name.clone().unwrap();
        self.langlist.push(lang.clone());
        for (word, &count) in profile.freq.iter() {
            if !self.word_lang_prob_map.contains_key(word) {
                self.word_lang_prob_map.insert(word.clone(), vec![0.0; langsize]);
            }
            let length = word.chars().count();
            if length >= 1 && length <= 3 {
                let prob = count as f64 / profile.n_words[length - 1] as f64;
                if let Some(vec) = self.word_lang_prob_map.get_mut(word) {
                    vec[index] = prob;
                }
            }
        }
        Ok(())
    }

    /// Adds a new language profile to the factory.
    ///
    /// # Arguments
    /// * `profile` - The language profile to add.
    /// * `index` - The index position for this language.
    /// * `langsize` - Total number of languages in the profile set.
    ///
    /// # Errors
    /// Returns `DetectorFactoryError::DuplicatedLanguage` if the language already exists.
    pub fn add_profile(&mut self, profile: LangProfile, index: usize, langsize: usize) -> Result<(), DetectorFactoryError> {
        let lang = profile.name.clone().unwrap();
        if self.langlist.contains(&lang) {
            return Err(DetectorFactoryError::DuplicatedLanguage(lang));
        }
        self.override_profile(profile, index, langsize)
    }

    /// Removes a language profile from the factory.
    ///
    /// # Arguments
    /// * `lang` - The language code to remove.
    ///
    /// # Errors
    /// Returns `DetectorFactoryError::DuplicatedLanguage` if the language doesn't exist.
    pub fn delete_profile(&mut self, lang: &str) -> Result<(), DetectorFactoryError> {
        let pos = self.langlist.iter().position(|l| l == lang);
        if let Some(index) = pos {
            self.langlist.remove(index);
            // Remove the language's probabilities from word_lang_prob_map
            for vec in self.word_lang_prob_map.values_mut() {
                if vec.len() > index {
                    vec.remove(index);
                }
            }
            Ok(())
        } else {
            Err(DetectorFactoryError::DuplicatedLanguage(lang.to_string()))
        }
    }

    /// Loads language profiles from JSON strings.
    ///
    /// # Arguments
    /// * `json_profiles` - Array of JSON strings representing language profiles.
    ///
    /// # Errors
    /// Returns `DetectorFactoryError::NotEnoughProfiles` if fewer than 2 profiles provided.
    pub fn load_json_profile(&mut self, json_profiles: &[&str]) -> Result<(), DetectorFactoryError> {
        let langsize = json_profiles.len();
        if langsize < 2 {
            return Err(DetectorFactoryError::NotEnoughProfiles);
        }
        let mut index = 0;
        for json_profile in json_profiles {
            let json_data: LangProfileJson = serde_json::from_str(json_profile)
                .map_err(|_| DetectorFactoryError::NotEnoughProfiles)?;
            let profile = LangProfile {
                name: Some(json_data.name),
                freq: json_data.freq,
                n_words: {
                    let mut arr = [0; 3];
                    for (i, v) in json_data.n_words.iter().enumerate().take(3) {
                        arr[i] = *v;
                    }
                    arr
                },
            };
            self.add_profile(profile, index, langsize)?;
            index += 1;
        }
        Ok(())
    }

    /// Shortcut method to detect language from text in one call.
    ///
    /// # Arguments
    /// * `text` - The text to analyze.
    /// * `alpha` - Optional alpha smoothing parameter.
    ///
    /// # Returns
    /// The detected language code or an error.
    ///
    /// # Example
    ///
    /// ```rust
    /// use langdetect_rs::detector_factory::DetectorFactory;
    ///
    /// let factory = DetectorFactory::default().build();
    /// let result = factory.detect("Hello world!", None);
    /// ```
    pub fn detect(&self, text: &str, alpha: Option<f64>) -> Result<String, DetectorError> {
        let mut detector = self.create(alpha);
        detector.append(text);
        detector.detect()
    }

    /// Shortcut method to get language probabilities from text in one call.
    ///
    /// # Arguments
    /// * `text` - The text to analyze.
    /// * `alpha` - Optional alpha smoothing parameter.
    ///
    /// # Returns
    /// A vector of languages with their probabilities, sorted by probability descending.
    ///
    /// # Example
    ///
    /// ```rust
    /// use langdetect_rs::detector_factory::DetectorFactory;
    ///
    /// let factory = DetectorFactory::default().build();
    /// let result = factory.get_probabilities("Hello world!", None);
    /// ```
    pub fn get_probabilities(&self, text: &str, alpha: Option<f64>) -> Result<Vec<Language>, DetectorError> {
        let mut detector = self.create(alpha);
        detector.append(text);
        detector.get_probabilities()
    }

    /// Loads all language profiles from a directory of JSON files.
    ///
    /// # Arguments
    /// * `profile_directory` - Path to directory containing JSON profile files.
    ///
    /// # Returns
    /// Ok(()) on success, or an error string on failure.
    ///
    /// # Example
    ///
    /// ```rust
    /// use langdetect_rs::detector_factory::DetectorFactory;
    ///
    /// let mut factory = DetectorFactory::new().build();
    /// factory.load_profile("profiles/").unwrap();
    /// ```
    pub fn load_profile<P: AsRef<Path>>(&mut self, profile_directory: P) -> Result<(), String> {
        let dir = profile_directory.as_ref();
        let entries = fs::read_dir(dir).map_err(|e| format!("Failed to read profile directory: {}", e))?;
        let mut json_profiles = Vec::new();
        for entry in entries {
            let entry = entry.map_err(|e| format!("Failed to read entry: {}", e))?;
            let path = entry.path();
            if path.is_file() {
                let content = fs::read_to_string(&path)
                    .map_err(|e| format!("Failed to read file {:?}: {}", path, e))?;
                json_profiles.push(content);
            }
        }
        let json_refs: Vec<&str> = json_profiles.iter().map(|s| s.as_str()).collect();
        self.load_json_profile(&json_refs)
            .map_err(|e| format!("Failed to parse JSON profiles: {:?}", e))?;
        Ok(())
    }
}

/// Builder for `DetectorFactory` with fluent setters.
///
/// Provides a convenient way to configure a DetectorFactory before building it.
///
/// # Examples
///
/// ```rust
/// use langdetect_rs::detector_factory::DetectorFactory;
/// use std::collections::HashMap;
///
/// let factory = DetectorFactory::new()
///     .with_langlist(vec!["en".to_string(), "fr".to_string()])
///     .with_seed(Some(42))
///     .build();
/// ```
pub struct DetectorFactoryBuilder {
    factory: DetectorFactory,
}

impl DetectorFactoryBuilder {
    /// Set the word language probability map.
    ///
    /// # Arguments
    /// * `word_lang_prob_map` - A HashMap of word to language probabilities.
    ///
    /// # Example
    /// ```
    /// use std::collections::HashMap;
    /// use langdetect_rs::detector_factory::DetectorFactory;
    /// let mut word_lang_prob_map = HashMap::new();
    /// word_lang_prob_map.insert("hello".to_string(), vec![0.5, 0.3]);
    /// let builder = DetectorFactory::new().with_word_lang_prob_map(word_lang_prob_map);
    /// ```
    pub fn with_word_lang_prob_map(mut self, word_lang_prob_map: HashMap<String, Vec<f64>>) -> Self {
        self.factory.word_lang_prob_map = word_lang_prob_map;
        self
    }

    /// Set the language list.
    ///
    /// # Arguments
    /// * `langlist` - A vector of language names.
    ///
    /// # Example
    /// ```
    /// use langdetect_rs::detector_factory::DetectorFactory;
    /// let builder = DetectorFactory::new().with_langlist(vec!["en".to_string(), "fr".to_string()]);
    /// ```
    pub fn with_langlist(mut self, langlist: Vec<String>) -> Self {
        self.factory.langlist = langlist;
        self
    }

    /// Set the seed for randomization.
    ///
    /// # Arguments
    /// * `seed` - An optional u64 seed value.
    ///
    /// # Example
    /// ```
    /// use langdetect_rs::detector_factory::DetectorFactory;
    /// let builder = DetectorFactory::new().with_seed(Some(42));
    /// ```
    pub fn with_seed(mut self, seed: Option<u64>) -> Self {
        self.factory.seed = seed;
        self
    }

    /// Builds the final `DetectorFactory` object with the configured properties.
    ///
    /// # Returns
    /// The fully constructed `DetectorFactory` object.
    ///
    /// # Example
    /// ```
    /// use langdetect_rs::detector_factory::DetectorFactory;
    /// let factory = DetectorFactory::new().with_seed(Some(123)).build();
    /// ```
    pub fn build(self) -> DetectorFactory {
        self.factory
    }
}