vectus 0.1.38

A vector database implemented in Rust for learning purposes.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

use crate::document::{DocBuilder, Document};
use text_splitter::{ChunkConfig, TextSplitter};

pub async fn recursive_character_text_split(
    text: &str,
    max_length: Option<usize>,
    overlap: Option<usize>,
) -> Vec<Document> {
    let config = ChunkConfig::new(max_length.unwrap_or(1000))
        .with_overlap(overlap.unwrap_or(0))
        .unwrap();
    let splitter = TextSplitter::new(config);
    let chunks = splitter.chunks(text);

    let mut docs = Vec::new();
    for chunk in chunks {
        let doc = DocBuilder::new().with_page_content(chunk).build();
        docs.push(doc);
    }

    docs
}