qdrant-edge 0.7.2

A lightweight, in-process vector search engine designed for embedded devices, autonomous systems, and mobile agents.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

use std::borrow::Cow;

use crate::common::types::PointOffsetType;
use crate::common::universal_io::MmapFile;
use rand::{RngExt, SeedableRng};
use tempfile::TempDir;

use crate::sparse::common::sparse_vector::RemappedSparseVector;
use crate::sparse::common::types::{DimOffset, Weight};
use crate::sparse::index::inverted_index::InvertedIndex;
use crate::sparse::index::inverted_index::inverted_index_compressed_mmap::InvertedIndexCompressedMmap;
use crate::sparse::index::inverted_index::inverted_index_ram::InvertedIndexRam;
use crate::sparse::index::inverted_index::inverted_index_ram_builder::InvertedIndexBuilder;

pub struct TestIndex<I: InvertedIndex> {
    pub index: I,
    _temp_dir: TempDir,
}

impl<I: InvertedIndex> TestIndex<I> {
    fn from_ram(ram_index: InvertedIndexRam) -> Self {
        let temp_dir = tempfile::Builder::new()
            .prefix("test_index_dir")
            .tempdir()
            .unwrap();
        TestIndex {
            index: I::from_ram_index(Cow::Owned(ram_index), &temp_dir).unwrap(),
            _temp_dir: temp_dir,
        }
    }
}

pub fn random_sparse_vector<R: rand::Rng>(
    rnd_gen: &mut R,
    density: usize,
    vocab1: usize,
    vocab2: usize,
) -> RemappedSparseVector {
    let mut indices = vec![];
    let mut values = vec![];

    let value_range = -0.0..2.0;

    for _ in 0..density {
        loop {
            let index = if rnd_gen.random_bool(0.5) {
                rnd_gen.random_range(0..vocab1)
            } else {
                rnd_gen.random_range(vocab1..(vocab1 + vocab2))
            };

            if indices.contains(&(index as DimOffset)) {
                continue;
            }

            let value = rnd_gen.random_range(value_range.clone());
            indices.push(index as DimOffset);
            values.push(value);
            break;
        }
    }

    RemappedSparseVector { indices, values }
}

pub fn generate_sparse_index<W, R>(
    rnd: &mut R,
    count: usize,
    density: usize,
    vocab1: usize,
    vocab2: usize,
) -> TestIndex<InvertedIndexCompressedMmap<W, MmapFile>>
where
    W: Weight + 'static,
    R: rand::Rng,
{
    let mut builder = InvertedIndexBuilder::new();

    for i in 0..count {
        let vector = random_sparse_vector(rnd, density, vocab1, vocab2);
        builder.add(i as PointOffsetType, vector);
    }

    TestIndex::from_ram(builder.build())
}

pub fn build_index<W>(
    count: usize,
    density: usize,
    vocab1: usize,
    vocab2: usize,
) -> TestIndex<InvertedIndexCompressedMmap<W, MmapFile>>
where
    W: Weight + 'static,
{
    let seed = 42;
    let mut rnd_gen = rand::rngs::StdRng::seed_from_u64(seed);
    generate_sparse_index::<W, _>(&mut rnd_gen, count, density, vocab1, vocab2)
}

pub fn match_all(_p: PointOffsetType) -> bool {
    true
}