nvs-core 0.1.0

Native Vector Store (Rust) core: bundle format, mmap reader, vector/BM25/hybrid search
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

use super::TokenCounter;

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum LineType {
    Normal,
    MajorHeading,
    MinorHeading,
    ListItem,
    Blank,
    CodeBlock,
}

#[derive(Clone, Debug)]
pub struct AnnotatedLine {
    pub text: String,
    pub line_type: LineType,
    pub tokens: usize,
    pub page: i32,
    pub heading_level: i32,
}

fn detect_line_type(line: &str) -> (LineType, i32) {
    let s = line.trim();
    if s.is_empty() {
        return (LineType::Blank, 0);
    }
    // markdown-style headings
    if let Some(stripped) = s.strip_prefix('#') {
        let mut level = 1;
        let mut rest = stripped;
        while let Some(r) = rest.strip_prefix('#') {
            level += 1;
            rest = r;
        }
        if level <= 2 {
            return (LineType::MajorHeading, level as i32);
        }
        return (LineType::MinorHeading, level as i32);
    }
    // list items
    if s.starts_with('-') || s.starts_with('*') || s.starts_with('+') {
        return (LineType::ListItem, 0);
    }
    if s.chars().all(|c| c == '`') {
        return (LineType::CodeBlock, 0);
    }
    (LineType::Normal, 0)
}

pub fn annotate_lines(pages: &[(String, i32)], tokenizer: &dyn TokenCounter) -> Vec<AnnotatedLine> {
    let mut out = Vec::new();
    for (text, page) in pages {
        for line in text.split('\n') {
            let (lt, lvl) = detect_line_type(line);
            let tokens = tokenizer.count_tokens(line);
            out.push(AnnotatedLine {
                text: line.to_string(),
                line_type: lt,
                tokens,
                page: *page,
                heading_level: lvl,
            });
        }
    }
    out
}