nvs-core 0.1.0

Native Vector Store (Rust) core: bundle format, mmap reader, vector/BM25/hybrid search
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

use super::pack::TmpChunk;
use super::TokenCounter;

pub fn split_oversized(
    chunks: Vec<TmpChunk>,
    max_tokens: usize,
    tokenizer: &dyn TokenCounter,
) -> Vec<TmpChunk> {
    let mut out = Vec::new();
    for ch in chunks.into_iter() {
        if ch.tokens <= max_tokens {
            out.push(ch);
            continue;
        }
        let mut cur = TmpChunk {
            text: String::new(),
            tokens: 0,
            start_page: ch.start_page,
            end_page: ch.end_page,
            has_major_heading: ch.has_major_heading,
            min_heading_level: ch.min_heading_level,
        };
        for line in ch.text.split('\n') {
            let tst = if cur.text.is_empty() {
                format!("{}\n", line)
            } else {
                format!("{}\n{}\n", cur.text.trim_end_matches('\n'), line)
            };
            let tokens = tokenizer.count_tokens(&tst);
            if !cur.text.is_empty() && tokens > max_tokens {
                // finalize current
                cur.tokens = tokenizer.count_tokens(&cur.text);
                out.push(cur);
                cur = TmpChunk {
                    text: String::new(),
                    tokens: 0,
                    start_page: ch.start_page,
                    end_page: ch.end_page,
                    has_major_heading: ch.has_major_heading,
                    min_heading_level: ch.min_heading_level,
                };
                cur.text = format!("{}\n", line);
                cur.tokens = tokenizer.count_tokens(&cur.text);
            } else {
                cur.text = tst;
                cur.tokens = tokens;
            }
        }
        if !cur.text.is_empty() {
            cur.tokens = tokenizer.count_tokens(&cur.text);
            out.push(cur);
        }
    }
    out
}