charabia 0.9.9

A simple library to detect the language, tokenize the text and normalize the tokens
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

pub use script_language::{Language, Script};
use whatlang::Detector;

// file copy pasted from whatlang.
#[allow(dead_code)]
mod chars;
mod script_language;

pub struct StrDetection<'o, 'al> {
    inner: &'o str,
    pub script: Option<Script>,
    pub language: Option<Language>,
    allow_list: Option<&'al [Language]>,
}

impl<'o, 'al> StrDetection<'o, 'al> {
    pub fn new(inner: &'o str, allow_list: Option<&'al [Language]>) -> Self {
        Self { inner, script: None, language: None, allow_list }
    }

    pub fn script(&mut self) -> Script {
        let inner = self.inner;
        *self.script.get_or_insert_with(|| Self::detect_script(inner))
    }

    pub fn language(&mut self) -> Option<Language> {
        let inner = self.inner;
        self.language = match self.language.take() {
            Some(lang) => Some(lang),
            None => match self.allow_list {
                Some([unique_language]) => Some(*unique_language),
                None if Self::detect_script(inner) == Script::Latin => None,
                _otherwise => Self::detect_lang(inner, self.allow_list),
            },
        };

        self.language
    }

    /// detect script with whatlang,
    /// if no script is detected, return Script::Other
    fn detect_script(text: &str) -> Script {
        whatlang::detect_script(text).map(Script::from).unwrap_or_default()
    }

    /// detect lang with whatlang
    /// if no language is detected, return Language::Other
    fn detect_lang(text: &str, allow_list: Option<&[Language]>) -> Option<Language> {
        let detector = allow_list
            .map(|allow_list| allow_list.iter().map(|lang| (*lang).into()).collect())
            .map(Detector::with_allowlist)
            .unwrap_or_default();

        detector.detect_lang(text).map(Language::from)
    }
}

pub trait Detect<'o, 'al> {
    fn detect(&'o self, allow_list: Option<&'al [Language]>) -> StrDetection<'o, 'al>;
}

impl<'o, 'al> Detect<'o, 'al> for &str {
    fn detect(&'o self, allow_list: Option<&'al [Language]>) -> StrDetection<'o, 'al> {
        StrDetection::new(self, allow_list)
    }
}