evoc 0.0.1

Embedding Vector Oriented Clustering — fast clustering of high-dimensional embedding vectors (Rust port of EVōC)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123

//! Hashed bag-of-words features for text clustering examples.

use std::hash::{Hash, Hasher};

pub const DEFAULT_FEATURE_DIM: usize = 16_384;

pub fn normalize_body(raw: &str) -> String {
    raw.replace('\r', "")
        .split_whitespace()
        .collect::<Vec<_>>()
        .join(" ")
}

/// Strip common Usenet headers/footers (20 Newsgroups).
pub fn strip_newsgroup_text(raw: &str) -> String {
    let mut text = raw.replace('\r', "");
    if let Some(pos) = text.find("\n\n") {
        text = text[pos + 2..].to_string();
    }
    if let Some(pos) = text.rfind("\n-- \n") {
        text.truncate(pos);
    }
    if let Some(pos) = text.rfind("\n--\n") {
        text.truncate(pos);
    }
    normalize_body(&text)
}

fn is_stopword(word: &str) -> bool {
    matches!(
        word,
        "a" | "an"
            | "and"
            | "are"
            | "as"
            | "at"
            | "be"
            | "been"
            | "but"
            | "by"
            | "can"
            | "did"
            | "do"
            | "for"
            | "from"
            | "had"
            | "has"
            | "have"
            | "he"
            | "her"
            | "him"
            | "his"
            | "if"
            | "in"
            | "into"
            | "is"
            | "it"
            | "its"
            | "may"
            | "more"
            | "most"
            | "not"
            | "of"
            | "on"
            | "or"
            | "our"
            | "out"
            | "she"
            | "so"
            | "such"
            | "than"
            | "that"
            | "the"
            | "their"
            | "them"
            | "then"
            | "there"
            | "these"
            | "they"
            | "this"
            | "to"
            | "up"
            | "was"
            | "we"
            | "were"
            | "what"
            | "when"
            | "which"
            | "who"
            | "will"
            | "with"
            | "would"
            | "you"
            | "your"
    )
}

fn normalize_token(raw: &str) -> Option<String> {
    let t: String = raw
        .chars()
        .filter(|c| c.is_ascii_alphanumeric())
        .flat_map(|c| c.to_lowercase())
        .collect();
    if t.len() < 3 || is_stopword(&t) {
        None
    } else {
        Some(t)
    }
}

pub fn hash_bow_into_row(text: &str, mut row: ndarray::ArrayViewMut1<f32>) {
    let dim = row.len();
    for token in text.split_whitespace() {
        let Some(t) = normalize_token(token) else {
            continue;
        };
        let mut hasher = std::collections::hash_map::DefaultHasher::new();
        t.hash(&mut hasher);
        let idx = (hasher.finish() as usize) % dim;
        row[idx] += 1.0;
    }
    row.mapv_inplace(|v| if v > 0.0 { (1.0 + v).ln() } else { 0.0 });
}