nodedb_vector/collection/
segment.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! Segment types for the VectorCollection lifecycle.
4
5use crate::collection::tier::StorageTier;
6use crate::flat::FlatIndex;
7use crate::hnsw::{HnswIndex, HnswParams};
8use crate::mmap_segment::MmapVectorSegment;
9use crate::quantize::pq::PqCodec;
10use crate::quantize::sq8::Sq8Codec;
11
12/// Default threshold for sealing the growing segment.
13/// 64K vectors × 768 dims × 4 bytes = ~192 MiB per segment.
14pub const DEFAULT_SEAL_THRESHOLD: usize = 65_536;
15
16/// What a finished build replaces.
17#[derive(Debug, Clone, Copy, PartialEq, Eq)]
18pub enum BuildKind {
19    /// Promote building segment `segment_id` to sealed.
20    Seal,
21    /// Replace the sealed segment at `base_id`, which held `len` nodes when
22    /// its vectors were read. A segment that no longer holds `len` nodes
23    /// (compaction renumbered it) refuses the result.
24    Rebuild { base_id: u32, len: usize },
25}
26
27/// Request to build an HNSW index (sent to the builder thread).
28///
29/// `vectors` holds one vector per local node id, soft-deleted nodes
30/// included, so the built graph keeps every id. The owning core applies
31/// the tombstones when it installs the result.
32pub struct BuildRequest {
33    pub key: String,
34    pub segment_id: u32,
35    pub kind: BuildKind,
36    pub vectors: Vec<Vec<f32>>,
37    pub dim: usize,
38    pub params: HnswParams,
39}
40
41/// Completed HNSW build (sent back from builder thread).
42pub struct BuildComplete {
43    pub key: String,
44    pub segment_id: u32,
45    pub kind: BuildKind,
46    /// The built index, or the error that stopped the build. A failed build
47    /// leaves the segment as it was.
48    pub result: Result<HnswIndex, crate::error::VectorError>,
49}
50
51/// A sealed segment whose HNSW index is being built in background.
52pub struct BuildingSegment {
53    /// Flat index for brute-force search while HNSW is building.
54    pub flat: FlatIndex,
55    /// Base ID offset: vectors have global IDs [base_id .. base_id + count).
56    pub base_id: u32,
57    /// Unique segment identifier (for matching with BuildComplete).
58    pub segment_id: u32,
59}
60
61/// A sealed segment with a completed HNSW index.
62pub struct SealedSegment {
63    /// Built HNSW index (immutable after construction).
64    pub index: HnswIndex,
65    /// Base ID offset.
66    pub base_id: u32,
67    /// Optional SQ8 quantized vectors for accelerated traversal.
68    pub sq8: Option<(Sq8Codec, Vec<u8>)>,
69    /// Optional PQ-compressed codes (for HnswPq-configured indexes).
70    pub pq: Option<(PqCodec, Vec<u8>)>,
71    /// Storage tier: L0Ram = FP32 in HNSW nodes, L1Nvme = FP32 in mmap file.
72    pub tier: StorageTier,
73    /// mmap-backed vector segment for L1 NVMe tier.
74    pub mmap_vectors: Option<MmapVectorSegment>,
75}
nodedb_vector/collection/segment.rs

nodedb_vector/collection/
segment.rs
