vectus 0.1.36

A vector database implemented in Rust for learning purposes.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

use crate::document::{DocBuilder, Document};

pub async fn recursive_character_text_split(
    text: &str,
    max_length: Option<usize>,
    overlap: Option<usize>,
) -> Vec<Document> {
    let max_length = max_length.unwrap_or(1000);
    let overlap = overlap.unwrap_or(0);

    let separators = vec!["\n\n", "\n", ".", " ", ""];

    let mut result = Vec::new();
    let mut start = 0;
    let text_len = text.len();

    while start < text_len {
        if start + max_length >= text_len {
            result.push(
                DocBuilder::new()
                    .with_page_content(&text[start..].to_string())
                    .build(),
            );
            break;
        }

        let mut end = start + max_length;
        let mut found_separator = false;

        for separator in &separators {
            if separator.is_empty() {
                continue;
            }

            if let Some(index) = text[start..end].rfind(separator) {
                end = start + index + separator.len();
                found_separator = true;
                break;
            }
        }

        if !found_separator {
            end = start + max_length;
        }

        result.push(
            DocBuilder::new()
                .with_page_content(&text[start..end].to_string())
                .build(),
        );

        start = if end >= text_len {
            text_len
        } else {
            end.saturating_sub(overlap)
        };
    }

    result
}