vantadb 0.1.4

VantaDB: An embedded persistent memory and vector retrieval engine for local-first AI applications.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

#![allow(dead_code)]

use std::fs::File;
use std::io::{BufReader, Read};
use std::path::Path;

/// Parses an `.fvecs` file into a `Vec<Vec<f32>>`.
/// Format: The file consists of sequences of (d, v_1, ..., v_d).
/// Where `d` is the dimension (i32) and `v_i` are vector elements (f32).
pub fn read_fvecs<P: AsRef<Path>>(path: P) -> std::io::Result<Vec<Vec<f32>>> {
    let mut file = BufReader::new(File::open(path)?);
    let mut results = Vec::new();
    let mut d_buf = [0u8; 4];

    loop {
        if let Err(e) = file.read_exact(&mut d_buf) {
            if e.kind() == std::io::ErrorKind::UnexpectedEof {
                // Natural end of file
                break;
            } else {
                return Err(e);
            }
        }

        let d = i32::from_le_bytes(d_buf) as usize;
        let mut vector_buf = vec![0u8; d * 4];
        file.read_exact(&mut vector_buf)?;

        let mut vector = Vec::with_capacity(d);
        for i in 0..d {
            let val_bytes = [
                vector_buf[i * 4],
                vector_buf[i * 4 + 1],
                vector_buf[i * 4 + 2],
                vector_buf[i * 4 + 3],
            ];
            vector.push(f32::from_le_bytes(val_bytes));
        }

        results.push(vector);
    }

    Ok(results)
}

/// Parses an `.ivecs` file into a `Vec<Vec<usize>>`.
/// Same structure but values are i32 instead of f32.
pub fn read_ivecs<P: AsRef<Path>>(path: P) -> std::io::Result<Vec<Vec<usize>>> {
    let mut file = BufReader::new(File::open(path)?);
    let mut results = Vec::new();
    let mut d_buf = [0u8; 4];

    loop {
        if let Err(e) = file.read_exact(&mut d_buf) {
            if e.kind() == std::io::ErrorKind::UnexpectedEof {
                break;
            } else {
                return Err(e);
            }
        }

        let d = i32::from_le_bytes(d_buf) as usize;
        let mut vector_buf = vec![0u8; d * 4];
        file.read_exact(&mut vector_buf)?;

        let mut vector = Vec::with_capacity(d);
        for i in 0..d {
            let val_bytes = [
                vector_buf[i * 4],
                vector_buf[i * 4 + 1],
                vector_buf[i * 4 + 2],
                vector_buf[i * 4 + 3],
            ];
            let val = i32::from_le_bytes(val_bytes) as usize;
            vector.push(val);
        }

        results.push(vector);
    }

    Ok(results)
}