nodedb_vector/hnsw/graph/index/
backing.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! Attaching an external vector segment to a graph-only index.
4//!
5//! Lite's restore path loads the graph topology from a B+ tree blob while the
6//! vectors stay in a pagedb segment, so the index arrives with empty per-node
7//! storage and has to be pointed at that segment before it can answer anything.
8//!
9//! The attachment is where a bad segment has to be caught. Once attached, an
10//! unserviceable segment is indistinguishable from a healthy one until a query
11//! scores a node that has no vector — by which time the collection is already
12//! published and serving.
13
14#[cfg(not(target_arch = "wasm32"))]
15use std::sync::Arc;
16
17use crate::error::VectorError;
18
19use super::state::HnswIndex;
20
21impl HnswIndex {
22    /// Attach a [`crate::segment_backing::VectorSegmentBacking`] to this index.
23    ///
24    /// After this succeeds, `dist_to_node` falls back to the backing whenever an
25    /// F32 node's local vector storage is empty.  This is used by Lite's
26    /// graph-checkpoint-only restore path: the graph topology is loaded from the
27    /// B+ tree blob, but vector data lives in a pagedb segment.
28    ///
29    /// Origin never calls this — its node arenas are always populated.
30    ///
31    /// # Errors
32    ///
33    /// The backing is validated against this index BEFORE being attached, and is
34    /// not attached at all when it cannot serve every node whose local storage
35    /// is empty. An attached-but-unserviceable backing is the worst outcome
36    /// available: the graph looks healthy, so search proceeds and then finds no
37    /// vector for a node it must score. Callers should treat an error as "this
38    /// segment is unusable — rebuild the index from the authoritative vectors,
39    /// or leave the collection unloaded", never as something to ignore.
40    ///
41    /// - [`VectorError::StoredDimensionMismatch`] if the backing's `dim()` is not this
42    ///   index's `dim`.
43    /// - [`VectorError::VectorUnavailable`] if the backing holds fewer vectors
44    ///   than the index has nodes, or if a node that needs the backing has no
45    ///   correctly-sized vector in it.
46    #[cfg(not(target_arch = "wasm32"))]
47    pub fn with_backing(
48        &mut self,
49        b: Arc<dyn crate::segment_backing::VectorSegmentBacking>,
50    ) -> Result<&mut Self, VectorError> {
51        if b.dim() != self.dim {
52            return Err(VectorError::StoredDimensionMismatch {
53                expected: self.dim,
54                got: b.dim(),
55            });
56        }
57        if b.len() < self.nodes.len() {
58            // A segment written from an index that had no vectors to give
59            // declares its real count in the header but carries no payload;
60            // this is where that lie becomes visible. `b.len()` is also the id
61            // of the first node the backing cannot serve.
62            return Err(VectorError::VectorUnavailable { id: b.len() as u32 });
63        }
64        // Every node that will actually read through the backing must resolve to
65        // a correctly-sized vector. Nodes with local storage are unaffected.
66        for (id, node) in self.nodes.iter().enumerate() {
67            if !node.storage.as_bytes().is_empty() {
68                continue;
69            }
70            let id = id as u32;
71            match b.get_vector(id) {
72                Some(v) if v.len() == self.dim => {}
73                _ => return Err(VectorError::VectorUnavailable { id }),
74            }
75        }
76        self.backing = Some(b);
77        Ok(self)
78    }
79
80    /// Fetch node `id`'s vector from the attached segment backing.
81    #[cfg(not(target_arch = "wasm32"))]
82    pub(super) fn backing_vector(&self, id: u32) -> Result<Vec<f32>, VectorError> {
83        self.backing
84            .as_ref()
85            .and_then(|b| b.get_vector(id))
86            .map(<[f32]>::to_vec)
87            .ok_or(VectorError::VectorUnavailable { id })
88    }
89
90    /// WASM targets have no segment backing (it requires mmap).
91    #[cfg(target_arch = "wasm32")]
92    pub(super) fn backing_vector(&self, id: u32) -> Result<Vec<f32>, VectorError> {
93        Err(VectorError::VectorUnavailable { id })
94    }
95}
nodedb_vector/hnsw/graph/index/backing.rs

nodedb_vector/hnsw/graph/index/
backing.rs
