vectus 0.1.37

A vector database implemented in Rust for learning purposes.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

use crate::document::{DocBuilder, Document};

pub async fn recursive_character_text_split(
    text: &str,
    max_length: Option<usize>,
    overlap: Option<usize>,
) -> Vec<Document> {
    let max_length = max_length.unwrap_or(1000);
    let overlap = overlap.unwrap_or(0);

    let separators = vec!["\n\n", "\n", ".", " ", ""];

    let mut result = Vec::new();
    let mut start = 0;
    let text_len = text.len();

    while start < text_len {
        let mut end = start + max_length;

        if end >= text_len {
            end = text_len;
        } else {
            while !text.is_char_boundary(end) {
                end -= 1;
            }
        }

        let mut found_separator = false;

        for separator in &separators {
            if separator.is_empty() {
                continue;
            }

            if let Some(index) = text[start..end].rfind(separator) {
                end = start + index + separator.len();
                found_separator = true;
                break;
            }
        }

        if !found_separator {
            end = start + max_length;
            if end < text_len {
                while !text.is_char_boundary(end) {
                    end -= 1;
                }
            }
        }

        result.push(
            DocBuilder::new()
                .with_page_content(&text[start..end].to_string())
                .build(),
        );

        start = if end >= text_len {
            text_len
        } else {
            end.saturating_sub(overlap)
        };
    }

    result
}