qdrant-edge 0.7.1

A lightweight, in-process vector search engine designed for embedded devices, autonomous systems, and mobile agents.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

use serde::{Deserialize, Serialize};

/// Expect JSON values to have roughly 3–5 fields with mostly small values.
/// For 1M values, this would require 128MB of memory.
pub const DEFAULT_BLOCK_SIZE_BYTES: usize = 128;

/// Default page size used when not specified
pub const DEFAULT_PAGE_SIZE_BYTES: usize = 32 * 1024 * 1024; // 32MB

pub const DEFAULT_REGION_SIZE_BLOCKS: usize = 8_192;

pub const DEFAULT_USE_COMPRESSION: bool = true;

#[derive(Debug, Copy, Clone, Serialize, Deserialize, Default)]
pub enum Compression {
    None,
    #[default]
    LZ4,
}

/// Configuration options for the storage
#[derive(Debug, Default)]
pub struct StorageOptions {
    /// Size of a page in bytes. Must be a multiple of (`block_size_bytes` * `region_size_blocks`).
    ///
    /// Default is 32MB
    pub page_size_bytes: Option<usize>,

    /// Size of a block in bytes
    ///
    /// Default is 128 bytes
    pub block_size_bytes: Option<usize>,

    /// Size of a region in blocks
    ///
    /// Default is 8192 blocks
    pub region_size_blocks: Option<u16>,

    /// Use compression
    ///
    /// Default is LZ4
    pub compression: Option<Compression>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct StorageConfig {
    /// Size of a page in bytes
    ///
    /// Default is 32MB
    pub page_size_bytes: usize,

    /// Size of a block in bytes
    ///
    /// Default is 128 bytes
    pub block_size_bytes: usize,

    /// Size of a region in blocks
    ///
    /// Default is 8192 blocks
    pub region_size_blocks: usize,

    /// Use compression
    ///
    /// Default is true
    #[serde(default)]
    pub compression: Compression,
}

impl TryFrom<StorageOptions> for StorageConfig {
    type Error = &'static str;

    fn try_from(options: StorageOptions) -> Result<Self, Self::Error> {
        let page_size_bytes = options.page_size_bytes.unwrap_or(DEFAULT_PAGE_SIZE_BYTES);
        let block_size_bytes = options.block_size_bytes.unwrap_or(DEFAULT_BLOCK_SIZE_BYTES);
        let region_size_blocks = options
            .region_size_blocks
            .map(|x| x as usize)
            .unwrap_or(DEFAULT_REGION_SIZE_BLOCKS);

        if block_size_bytes == 0 {
            return Err("Block size must be greater than 0");
        }

        if region_size_blocks == 0 {
            return Err("Region size must be greater than 0");
        }

        if page_size_bytes == 0 {
            return Err("Page size must be greater than 0");
        }

        let region_size_bytes = block_size_bytes * region_size_blocks;

        if page_size_bytes < region_size_bytes {
            return Err("Page size must be greater than or equal to (block size * region size)");
        }

        if !page_size_bytes.is_multiple_of(region_size_bytes) {
            return Err("Page size must be a multiple of (block size * region size)");
        }

        Ok(Self {
            page_size_bytes,
            block_size_bytes,
            region_size_blocks,
            compression: options.compression.unwrap_or_default(),
        })
    }
}

impl From<&StorageConfig> for StorageOptions {
    fn from(config: &StorageConfig) -> Self {
        Self {
            page_size_bytes: Some(config.page_size_bytes),
            block_size_bytes: Some(config.block_size_bytes),
            region_size_blocks: Some(config.region_size_blocks as u16),
            compression: Some(config.compression),
        }
    }
}