weavatrix-search-vector 0.3.1

Persistent, mutable, bounded vector candidate search for Rust and Weavatrix
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

mod decoder;
mod encoder;
mod format;
mod header;
mod index_io;
mod mapped_access;
mod mapped_batch;
mod mapped_exact;
mod mapped_graph;
mod mapped_owned;
mod mapped_search;
mod mapped_validation;
mod validation;
mod writer;

use crate::config::IndexConfig;
use crate::mmap::Mapping;
use crate::simd::DistanceKernel;

pub(super) const MAGIC: &[u8; 8] = b"WVSVEC02";
pub(super) const FORMAT_VERSION: u32 = 1;
pub(super) const HEADER_LEN: usize = 192;
pub(super) const GRAPH_HEADER_LEN: usize = 40;
pub(super) const NONE_ENTRY: u64 = u64::MAX;
pub(super) const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325;
pub(super) const FNV_PRIME: u64 = 0x0000_0100_0000_01b3;

/// Integrity policy used while opening a memory-mapped snapshot.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum SnapshotValidation {
    /// Validate structure, finite vectors, and the full payload checksum.
    #[default]
    Full,
    /// Validate all ranges and graph references without scanning the checksum.
    StructureOnly,
}

/// Read-only HNSW index backed directly by a versioned memory-mapped snapshot.
#[derive(Debug)]
pub struct MappedVectorIndex {
    pub(super) mapping: Mapping,
    pub(super) header: Header,
    pub(super) graphs: Vec<MappedGraph>,
    pub(super) distance_kernel: DistanceKernel,
}

/// Header-only snapshot description.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SnapshotMetadata {
    pub format_version: u32,
    pub vector_count: usize,
    pub serialized_bytes: usize,
    pub config: IndexConfig,
}

#[derive(Debug)]
pub(super) struct MappedGraph {
    pub(super) entry: Option<usize>,
    pub(super) max_level: usize,
    pub(super) node_count: usize,
    pub(super) node_layers_offset: usize,
    pub(super) total_layers: usize,
    pub(super) layer_neighbors_offset: usize,
    pub(super) neighbors_offset: usize,
    pub(super) neighbor_count: usize,
}

#[derive(Debug, Clone)]
pub(super) struct Header {
    pub(super) checksum: u64,
    pub(super) file_len: usize,
    pub(super) count: usize,
    pub(super) config: IndexConfig,
    pub(super) keys_offset: usize,
    pub(super) vectors_offset: usize,
    pub(super) routing_codes_offset: usize,
    pub(super) routing_nodes_offset: usize,
    pub(super) graphs_offset: usize,
}