tf-idf-vectorizer 0.10.4

A simple search and analyze engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

use std::ops::{BitAnd, BitOr, BitXor, Not};

pub struct BitMap {
    pub buckets: Vec<L2Bits>,
    pub size: usize,
}

#[derive(Clone, Debug, Default)]
pub struct L2Bits {
    pub bucket_map: u64,
    pub buckets: Box<[L1Bits]>,
}

impl BitAnd for &L2Bits {
    type Output = L2Bits;

    fn bitand(self, rhs: Self) -> Self::Output {
        let mut result_buckets = Vec::new();
        let mut result_map = 0u64;

        let mut si = 0usize;
        let mut ri = 0usize;

        for idx in 0..64 {
            let mask = 1u64 << idx;
            let sh = (self.bucket_map & mask) != 0;
            let rh = (rhs.bucket_map & mask) != 0;

            if sh && rh {
                result_map |= mask;
                result_buckets.push(&self.buckets[si] & &rhs.buckets[ri]);
            }
            if sh { si += 1; }
            if rh { ri += 1; }
        }

        L2Bits { bucket_map: result_map, buckets: result_buckets.into_boxed_slice() }
    }
}


/// 8 x 64 = 512bits bucket
#[derive(Clone, Copy, Debug, Default)]
#[repr(align(64))]
pub struct L1Bits {
    bucket: [u64; 8],
}

impl BitAnd for &L1Bits {
    type Output = L1Bits;

    fn bitand(self, rhs: Self) -> Self::Output {
        let mut result = L1Bits::default();
        for i in 0..8 {
            result.bucket[i] = self.bucket[i] & rhs.bucket[i];
        }
        result
    }
}

impl BitOr for &L1Bits {
    type Output = L1Bits;

    fn bitor(self, rhs: Self) -> Self::Output {
        let mut result = L1Bits::default();
        for i in 0..8 {
            result.bucket[i] = self.bucket[i] | rhs.bucket[i];
        }
        result
    }
}

impl BitXor for &L1Bits {
    type Output = L1Bits;

    fn bitxor(self, rhs: Self) -> Self::Output {
        let mut result = L1Bits::default();
        for i in 0..8 {
            result.bucket[i] = self.bucket[i] ^ rhs.bucket[i];
        }
        result
    }
}

impl Not for &L1Bits {
    type Output = L1Bits;

    fn not(self) -> Self::Output {
        let mut result = L1Bits::default();
        for i in 0..8 {
            result.bucket[i] = !self.bucket[i];
        }
        result
    }
}