txtfp 0.2.0

Text fingerprinting: MinHash + LSH, SimHash, and ONNX semantic embeddings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

//! UTS #39 confusable skeleton, behind the `security` feature.
//!
//! Maps visually similar codepoints (Cyrillic 'а', Latin 'a', Greek 'α',
//! …) to a common skeleton string. Use in security-sensitive matching:
//! domain names, usernames, filename collisions.

use alloc::string::String;
use unicode_security::GeneralSecurityProfile;

/// Map a string through the confusable skeleton.
///
/// Returns the skeleton form of `input` per UTS #39 §4.
pub(super) fn skeleton(input: &str) -> String {
    // The `unicode-security` crate exposes the skeleton via the
    // `confusable_detection::skeleton` iterator on `&str`. The result
    // is a `String`-collectable iterator of `char`.
    use unicode_security::confusable_detection::skeleton;
    skeleton(input).collect()
}

/// Inert helper that lets the `security`-feature module pull
/// [`GeneralSecurityProfile`] into scope when callers want stricter
/// validation. Currently unused but kept for forward-compatibility.
#[allow(dead_code)]
fn _identifier_status_witness(c: char) -> bool {
    c.identifier_allowed()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn cyrillic_a_collapses_to_latin_a() {
        // Cyrillic small letter a (U+0430) should skeletonize to ascii 'a'.
        let cyr = skeleton("а");
        let lat = skeleton("a");
        assert_eq!(cyr, lat);
    }

    #[test]
    fn paypal_homograph() {
        // "раураl" (Cyrillic а, у) versus "paypal".
        let s1 = skeleton("раураl");
        let s2 = skeleton("paypal");
        assert_eq!(s1, s2);
    }

    #[test]
    fn ascii_passthrough() {
        assert_eq!(skeleton("hello"), "hello");
    }
}