simstring_rust 0.1.0

A native Rust implementation of the SimString algorithm
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

use super::FeatureExtractor;
use std::collections::HashMap;

pub struct WordNGrams {
    pub n: usize,
    pub splitter: String,
    pub padder: String,
}

impl FeatureExtractor for WordNGrams {
    fn extract(&self, s: &str) -> Vec<(String, i32)> {
        let mut words: Vec<String> = s.split(&self.splitter).map(|w| w.to_string()).collect();
        pad_vector(&mut words, &self.padder);

        let ngrams = init_word_ngrams(&words, self.n);
        count_word_ngrams(ngrams)
    }
}

fn pad_vector(vec: &mut Vec<String>, padder: &str) {
    vec.insert(0, padder.to_string());
    vec.push(padder.to_string());
}

fn init_word_ngrams(words: &[String], n: usize) -> Vec<String> {
    let mut ngrams = Vec::new();
    let len = words.len();
    if len < n {
        return ngrams;
    }
    for i in 0..=len - n {
        let ngram_words = &words[i..i + n];
        let ngram = ngram_words.join(" ");
        ngrams.push(ngram);
    }
    ngrams
}

fn count_word_ngrams(ngrams: Vec<String>) -> Vec<(String, i32)> {
    let mut counter = HashMap::new();
    let mut result = Vec::with_capacity(ngrams.len());

    for ngram in ngrams {
        let count = counter.entry(ngram.clone()).or_insert(0);
        *count += 1;
        result.push((ngram, *count));
    }

    result
}