ld-lucivy 0.26.1

BM25 search engine with cross-token fuzzy matching, substring search, regex, and highlights
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

#[inline]
pub fn compress_sorted<'a>(input: &[u32], output: &'a mut [u8], mut offset: u32) -> &'a [u8] {
    let mut byte_written = 0;
    for &v in input {
        let mut to_encode: u32 = v - offset;
        offset = v;
        loop {
            let next_byte: u8 = (to_encode % 128u32) as u8;
            to_encode /= 128u32;
            if to_encode == 0u32 {
                output[byte_written] = next_byte | 128u8;
                byte_written += 1;
                break;
            } else {
                output[byte_written] = next_byte;
                byte_written += 1;
            }
        }
    }
    &output[..byte_written]
}

#[inline]
pub(crate) fn compress_unsorted<'a>(input: &[u32], output: &'a mut [u8]) -> &'a [u8] {
    let mut byte_written = 0;
    for &v in input {
        let mut to_encode: u32 = v;
        loop {
            let next_byte: u8 = (to_encode % 128u32) as u8;
            to_encode /= 128u32;
            if to_encode == 0u32 {
                output[byte_written] = next_byte | 128u8;
                byte_written += 1;
                break;
            } else {
                output[byte_written] = next_byte;
                byte_written += 1;
            }
        }
    }
    &output[..byte_written]
}

#[inline]
pub fn uncompress_sorted(compressed_data: &[u8], output: &mut [u32], offset: u32) -> usize {
    let mut read_byte = 0;
    let mut result = offset;
    for output_mut in output.iter_mut() {
        let mut shift = 0u32;
        loop {
            let cur_byte = compressed_data[read_byte];
            read_byte += 1;
            result += u32::from(cur_byte % 128u8) << shift;
            if cur_byte & 128u8 != 0u8 {
                break;
            }
            shift += 7;
        }
        *output_mut = result;
    }
    read_byte
}

#[inline]
pub(crate) fn uncompress_unsorted(compressed_data: &[u8], output_arr: &mut [u32]) -> usize {
    let mut num_read_bytes = 0;
    for output_mut in output_arr.iter_mut() {
        let mut result = 0u32;
        let mut shift = 0u32;
        loop {
            let cur_byte = compressed_data[num_read_bytes];
            num_read_bytes += 1;
            result += u32::from(cur_byte % 128u8) << shift;
            if cur_byte & 128u8 != 0u8 {
                break;
            }
            shift += 7;
        }
        *output_mut = result;
    }
    num_read_bytes
}

#[inline]
pub(crate) fn uncompress_unsorted_until_end(
    compressed_data: &[u8],
    output_arr: &mut [u32],
) -> usize {
    let mut num_read_bytes = 0;
    for (num_ints_written, output_mut) in output_arr.iter_mut().enumerate() {
        if compressed_data.len() == num_read_bytes {
            return num_ints_written;
        }
        let mut result = 0u32;
        let mut shift = 0u32;
        loop {
            let cur_byte = compressed_data[num_read_bytes];
            num_read_bytes += 1;
            result += u32::from(cur_byte % 128u8) << shift;
            if cur_byte & 128u8 != 0u8 {
                break;
            }
            shift += 7;
        }
        *output_mut = result;
    }
    output_arr.len()
}