simstring_rust 0.3.4

A native Rust implementation of the SimString algorithm
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

use crate::database::{Database, StringId};
use crate::extractors::FeatureExtractor;
use lasso::{Rodeo, Spur};
use rustc_hash::{FxHashMap, FxHashSet};
use std::fmt;
use std::sync::{Arc, Mutex};

pub struct HashDb {
    feature_extractor: Arc<dyn FeatureExtractor>,
    pub strings: Vec<String>,
    string_features: Vec<Vec<Spur>>,
    feature_map: FxHashMap<usize, FxHashMap<Spur, FxHashSet<StringId>>>,
    interner: Arc<Mutex<Rodeo>>,
}

impl fmt::Debug for HashDb {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let total_unique_features: usize = self
            .feature_map
            .values()
            .map(|size_map| size_map.len())
            .sum();
        let interner = self.interner.lock().unwrap();

        f.debug_struct("HashDb")
            .field("num_strings", &self.strings.len())
            .field("num_feature_size_buckets", &self.feature_map.len())
            .field("total_unique_features_interned", &interner.len())
            .field("total_unique_features_indexed", &total_unique_features)
            .finish()
    }
}

impl HashDb {
    pub fn new(feature_extractor: Arc<dyn FeatureExtractor>) -> Self {
        Self {
            feature_extractor,
            strings: Vec::new(),
            string_features: Vec::new(),
            feature_map: FxHashMap::default(),
            interner: Arc::new(Mutex::new(Rodeo::default())),
        }
    }

    pub fn insert(&mut self, text: String) {
        Database::insert(self, text);
    }

    pub fn clear(&mut self) {
        Database::clear(self);
    }
}

impl Database for HashDb {
    fn insert(&mut self, text: String) {
        let mut interner = self.interner.lock().unwrap();
        let features = self.feature_extractor.features(&text, &mut interner);
        let size = features.len();
        let string_id = self.strings.len();

        self.strings.push(text);
        self.string_features.push(features.clone());

        let size_map = self.feature_map.entry(size).or_default();
        for feature in features {
            size_map.entry(feature).or_default().insert(string_id);
        }
    }

    fn clear(&mut self) {
        self.strings.clear();
        self.string_features.clear();
        self.feature_map.clear();
        // clear the interner to release memory
        self.interner.lock().unwrap().clear();
    }

    fn lookup_strings(&self, size: usize, feature: Spur) -> Option<&FxHashSet<StringId>> {
        self.feature_map.get(&size)?.get(&feature)
    }

    fn get_string(&self, id: StringId) -> Option<&str> {
        self.strings.get(id).map(AsRef::as_ref)
    }

    fn get_features(&self, id: StringId) -> Option<&Vec<Spur>> {
        self.string_features.get(id)
    }

    fn feature_extractor(&self) -> &dyn FeatureExtractor {
        &*self.feature_extractor
    }

    fn max_feature_len(&self) -> usize {
        self.feature_map.keys().max().copied().unwrap_or(0)
    }

    fn interner(&self) -> Arc<Mutex<Rodeo>> {
        Arc::clone(&self.interner)
    }

    fn total_strings(&self) -> usize {
        self.strings.len()
    }
}