alphabet_detector 0.12.2

Natural language alphabet detection library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

use crate::{
    filter_max, filter_with_margin, filter_with_margin_sorted, slang_arr_default,
    words::{self, WordBuf},
    ScriptLanguage, ScriptLanguageArr, Word,
};
use debug_unsafe::slice::SliceGetter;

/// All words detection summed up.
pub fn fulltext<B: WordBuf>(
    char_indices: impl Iterator<Item = (usize, char)>,
) -> (Vec<Word<B>>, ScriptLanguageArr<u32>) {
    let mut words = Vec::new();
    let mut langs_count: ScriptLanguageArr<u32> = slang_arr_default();

    let found_words = words::from_ch_ind(char_indices);
    for wld in found_words {
        // let (langs, count_max) = filter_max(wld.langs_cnt); // worse at detecting
        for (lang, cnt) in wld.langs_cnt.iter().enumerate() {
            *langs_count.get_safe_unchecked_mut(lang) += cnt;
        }
        words.push(wld);
    }

    (words, langs_count)
}

/// All words detection summed up, then filtered by max ([`filter_max`](fn.filter_max.html)).
pub fn fulltext_filter_max<B: WordBuf>(
    char_indices: impl Iterator<Item = (usize, char)>,
) -> (
    Vec<Word<B>>,
    impl Iterator<Item = ScriptLanguage> + Clone,
    u32,
) {
    let (words, langs_count) = fulltext(char_indices);
    let (langs, cnt) = filter_max(langs_count);

    (words, langs, cnt)
}

/// All words detection summed up, then filtered with margin percent
/// ([`filter_with_margin`](fn.filter_with_margin.html)).
///
/// Less then (100 - `PERCENT`)% margin for an error.
/// `PERCENT` = 95 is recommended.
///
/// Recommended.
pub fn fulltext_filter_with_margin<B: WordBuf, const PERCENT: u32>(
    char_indices: impl Iterator<Item = (usize, char)>,
) -> (
    Vec<Word<B>>,
    impl Iterator<Item = (ScriptLanguage, u32)> + Clone,
    u32,
) {
    let (words, langs_count) = fulltext(char_indices);
    let (langs, langs_count_margin) = filter_with_margin::<PERCENT>(langs_count);

    (words, langs, langs_count_margin)
}

/// All words detection summed up, then filtered with margin percent
/// ([`filter_with_margin_sorted`](fn.filter_with_margin_sorted.html)), then sorted.
///
/// Less then (100 - `PERCENT`)% margin for an error.
/// `PERCENT` = 95 is recommended.
pub fn fulltext_filter_with_margin_sorted<B: WordBuf, const PERCENT: u32>(
    char_indices: impl Iterator<Item = (usize, char)>,
) -> (Vec<Word<B>>, Vec<(ScriptLanguage, u32)>, u32) {
    let (words, langs_count) = fulltext(char_indices);
    let (langs, langs_count_margin) = filter_with_margin_sorted::<PERCENT>(langs_count);

    (words, langs, langs_count_margin)
}