thefuzz-parity 0.1.0

Unicode fuzzy scoring and ranked choice extraction. Not affiliated with SeatGeek or the thefuzz maintainers. Parity is a tested claim against thefuzz 0.22.1.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

//! Text processing used by composite scorers.

use crate::unicode_tables::ALNUM_RANGES;

/// Replaces characters outside Unicode letter and number categories with
/// spaces, lowercases the result, and trims the ends.
///
/// With `force_ascii`, code points from U+0080 through U+00FF are deleted
/// before classification.
#[must_use]
pub fn full_process(value: &str, force_ascii: bool) -> String {
    let mut processed = String::new();
    for ch in value.chars() {
        let code = u32::from(ch);
        if force_ascii && (0x80..=0xff).contains(&code) {
            continue;
        }
        if is_letter_or_number(code) {
            if ch == '\u{130}' {
                processed.push('i');
            } else {
                processed.extend(ch.to_lowercase());
            }
        } else {
            processed.push(' ');
        }
    }
    processed.trim().to_owned()
}

fn is_letter_or_number(code: u32) -> bool {
    ALNUM_RANGES
        .binary_search_by(|&(start, end)| {
            if end < code {
                std::cmp::Ordering::Less
            } else if start > code {
                std::cmp::Ordering::Greater
            } else {
                std::cmp::Ordering::Equal
            }
        })
        .is_ok()
}