nodedb_physical/physical_plan/vector.rs
1// SPDX-License-Identifier: Apache-2.0
2
3//! Vector engine operations dispatched to the Data Plane.
4
5use nodedb_types::{Surrogate, SurrogateBitmap, vector_distance::DistanceMetric};
6
7use crate::physical_plan::PhysicalPlan;
8
9/// Vector engine physical operations.
10#[derive(
11 Debug,
12 Clone,
13 PartialEq,
14 serde::Serialize,
15 serde::Deserialize,
16 zerompk::ToMessagePack,
17 zerompk::FromMessagePack,
18)]
19pub enum VectorOp {
20 /// Vector similarity search.
21 Search {
22 collection: String,
23 query_vector: Vec<f32>,
24 top_k: usize,
25 /// Optional search beam width override. If 0, uses default `4 * top_k`.
26 ef_search: usize,
27 /// Per-query distance metric override. Overrides the collection-default
28 /// metric configured at index creation time.
29 metric: DistanceMetric,
30 /// Pre-computed bitmap of eligible surrogates (from filter evaluation).
31 filter_bitmap: Option<SurrogateBitmap>,
32 /// Named vector field to search. Empty string = default field.
33 field_name: String,
34 /// RLS post-candidate filters (serialized `Vec<ScanFilter>`).
35 /// Applied after HNSW returns candidates, before returning to client.
36 /// Result count may be less than requested `top_k`.
37 rls_filters: Vec<u8>,
38 /// Optional sub-plan whose output rows materialize a `SurrogateBitmap`
39 /// used as an additional prefilter (intersected with `filter_bitmap`).
40 ///
41 /// Set by the planner when fusing `Vector ORDER BY ... + JOIN
42 /// ARRAY_SLICE(...) ON v.id = s.surrogate` into a single op:
43 /// the array slice runs first, its surrogate column becomes the
44 /// vector engine's pre-filter, then HNSW search runs against the
45 /// reduced candidate set. Mirrors `HashJoin`'s `inline_*_bitmap`
46 /// mechanism so the Data Plane composition is uniform.
47 inline_prefilter_plan: Option<Box<PhysicalPlan>>,
48 /// ANN tuning knobs from the SQL caller. Defaults to no overrides.
49 ann_options: nodedb_types::VectorAnnOptions,
50 /// When `true`, the SELECT projection contains only `id` and/or
51 /// `vector_distance(...)` — no payload fields. The handler skips
52 /// the document-body fetch. Ignored (treated as `false`) when RLS
53 /// filters are active, as body fetch is required for RLS evaluation.
54 skip_payload_fetch: bool,
55 /// Optional payload bitmap pre-filter for vector-primary collections.
56 /// Each atom is `Eq` / `In` / `Range` — the handler ANDs all atoms
57 /// and intersects the resulting bitmap with the HNSW candidate set
58 /// before walking. The planner only emits atoms whose field has a
59 /// registered payload index of the matching kind. Empty means no
60 /// payload pre-filter.
61 payload_filters: Vec<nodedb_types::PayloadAtom>,
62 },
63
64 /// Insert a vector into the HNSW index (write path).
65 Insert {
66 collection: String,
67 vector: Vec<f32>,
68 dim: usize,
69 /// Named vector field. Empty string = default (unnamed) field.
70 field_name: String,
71 /// Global surrogate identifying this row. Allocated by the
72 /// Control Plane via `SurrogateAssigner` before dispatch; the
73 /// engine binds the HNSW node id to this surrogate.
74 surrogate: Surrogate,
75 /// User PK bytes (UTF-8 of the document id) when the insert
76 /// originates from a PK-bearing path; `None` for headless
77 /// inserts. Carried on the replication wire so followers bind
78 /// the leader-assigned surrogate to this exact key rather than
79 /// re-allocating a divergent one.
80 #[serde(default)]
81 pk_bytes: Option<Vec<u8>>,
82 /// Sync provenance: identifies the originating peer and sequence for idempotency.
83 #[serde(default)]
84 provenance: Option<nodedb_types::sync::wire::SyncProvenance>,
85 },
86
87 /// Batch insert vectors into the HNSW index.
88 BatchInsert {
89 collection: String,
90 vectors: Vec<Vec<f32>>,
91 dim: usize,
92 /// One surrogate per inserted vector, parallel to `vectors`.
93 /// Empty vector = headless batch (no PK binding).
94 surrogates: Vec<Surrogate>,
95 },
96
97 /// Multi-vector search: query across all named vector fields, fuse via RRF.
98 MultiSearch {
99 collection: String,
100 query_vector: Vec<f32>,
101 top_k: usize,
102 ef_search: usize,
103 filter_bitmap: Option<SurrogateBitmap>,
104 /// RLS post-candidate filters.
105 rls_filters: Vec<u8>,
106 },
107
108 /// Soft-delete a vector by internal node ID.
109 Delete { collection: String, vector_id: u32 },
110
111 /// Soft-delete a vector by surrogate (sync inbound path).
112 ///
113 /// The Data Plane resolves `surrogate → HNSW node_id` via the
114 /// `surrogate_to_local` map and tombstones the node. If the surrogate
115 /// is not found the op is a no-op (idempotent).
116 DeleteBySurrogate {
117 collection: String,
118 surrogate: nodedb_types::Surrogate,
119 /// Named vector field; empty = default field.
120 field_name: String,
121 /// Sync provenance: identifies the originating peer and sequence for idempotency.
122 #[serde(default)]
123 provenance: Option<nodedb_types::sync::wire::SyncProvenance>,
124 },
125
126 /// Set vector index parameters for a collection.
127 SetParams {
128 collection: String,
129 /// Named vector field this config applies to. Empty string = the
130 /// default (unnamed) vector field. Lets one collection carry
131 /// multiple vector indexes, one per `VECTOR` / embedding column.
132 field_name: String,
133 m: usize,
134 ef_construction: usize,
135 metric: String,
136 /// Index type: "hnsw" (default), "hnsw_pq", or "ivf_pq".
137 index_type: String,
138 /// PQ subvectors (for hnsw_pq and ivf_pq). Default: 8.
139 pq_m: usize,
140 /// IVF cells (for ivf_pq only). Default: 256.
141 ivf_cells: usize,
142 /// IVF probe count (for ivf_pq only). Default: 16.
143 ivf_nprobe: usize,
144 },
145
146 /// Query live vector index statistics. Returns `VectorIndexStats` as payload.
147 QueryStats {
148 collection: String,
149 /// Named vector field. Empty string = default (unnamed) field.
150 field_name: String,
151 },
152
153 /// Force-seal the growing segment, triggering background HNSW build.
154 Seal {
155 collection: String,
156 /// Named vector field. Empty string = default field.
157 field_name: String,
158 },
159
160 /// Force tombstone compaction on sealed segments.
161 CompactIndex {
162 collection: String,
163 /// Named vector field. Empty string = default field.
164 field_name: String,
165 },
166
167 /// Rebuild sealed segments with new HNSW parameters.
168 /// Old index serves queries until rebuild completes, then swaps atomically.
169 Rebuild {
170 collection: String,
171 /// Named vector field. Empty string = default field.
172 field_name: String,
173 /// New M parameter. 0 = keep current.
174 m: usize,
175 /// New M0 parameter. 0 = keep current.
176 m0: usize,
177 /// New ef_construction. 0 = keep current.
178 ef_construction: usize,
179 },
180
181 /// Insert a sparse vector into the inverted index.
182 SparseInsert {
183 collection: String,
184 /// Named sparse vector field.
185 field_name: String,
186 /// Document ID to associate with this sparse vector.
187 doc_id: String,
188 /// Sparse vector entries as `(dimension, weight)` pairs.
189 entries: Vec<(u32, f32)>,
190 },
191
192 /// Search the sparse inverted index via dot-product scoring.
193 SparseSearch {
194 collection: String,
195 /// Named sparse vector field.
196 field_name: String,
197 /// Query sparse vector entries.
198 query_entries: Vec<(u32, f32)>,
199 /// Maximum results to return.
200 top_k: usize,
201 },
202
203 /// Delete a document from the sparse inverted index.
204 SparseDelete {
205 collection: String,
206 /// Named sparse vector field.
207 field_name: String,
208 /// Document ID to remove.
209 doc_id: String,
210 },
211
212 /// Insert multiple vectors for a single document (ColBERT-style).
213 /// All vectors are inserted as separate HNSW nodes sharing the
214 /// same `document_surrogate`.
215 MultiVectorInsert {
216 collection: String,
217 /// Named vector field. Empty = default.
218 field_name: String,
219 /// Surrogate shared by all vectors of the document.
220 document_surrogate: Surrogate,
221 /// Flat vector data: count × dim f32 values.
222 vectors: Vec<f32>,
223 /// Number of vectors.
224 count: usize,
225 /// Dimensionality of each vector.
226 dim: usize,
227 },
228
229 /// Delete all vectors for a document from the multi-vector index.
230 MultiVectorDelete {
231 collection: String,
232 /// Named vector field. Empty = default.
233 field_name: String,
234 /// Document surrogate whose vectors should be tombstoned.
235 document_surrogate: Surrogate,
236 },
237
238 /// Search with multi-vector aggregated scoring (MaxSim, AvgSim, SumSim).
239 /// Over-fetches from HNSW, groups by doc_id, aggregates, deduplicates.
240 MultiVectorScoreSearch {
241 collection: String,
242 /// Named vector field. Empty = default.
243 field_name: String,
244 /// Query vector.
245 query_vector: Vec<f32>,
246 /// Maximum documents to return.
247 top_k: usize,
248 /// HNSW ef_search override. 0 = auto.
249 ef_search: usize,
250 /// Aggregation mode: "max_sim", "avg_sim", "sum_sim".
251 mode: String,
252 },
253
254 /// Direct vector upsert for vector-primary collections.
255 ///
256 /// Bypasses MessagePack document encoding — the Data Plane inserts the
257 /// vector into HNSW directly and updates payload bitmap indexes from
258 /// `payload`. No full-document blob is written.
259 ///
260 /// Ordering invariant (enforced by the handler):
261 /// 1. Validate dim.
262 /// 2. Decode `payload` bytes.
263 /// 3. Insert into HNSW (surrogate bound).
264 /// 4. Update payload bitmap indexes.
265 ///
266 /// If step 3 fails, step 4 is not reached — no partial state.
267 /// If step 4 fails (should not happen — pure in-memory), the handler
268 /// attempts to delete the just-inserted HNSW node and returns an error.
269 DirectUpsert {
270 collection: String,
271 /// Vector column name. Used to compute the vector index key so the
272 /// SELECT path (which keys by `(tid, collection, field)`) finds the
273 /// same index this insert wrote into.
274 field: String,
275 /// Global surrogate allocated by the Control Plane.
276 surrogate: Surrogate,
277 /// FP32 vector values.
278 vector: Vec<f32>,
279 /// Pre-encoded MessagePack of only the payload-indexed fields.
280 /// Empty when the collection has no payload indexes configured.
281 payload: Vec<u8>,
282 /// Collection-level quantization. Applied via `set_quantization` on
283 /// first insert so subsequent seals trigger codec-dispatch rebuilds
284 /// against the configured codec (RaBitQ / BBQ).
285 quantization: nodedb_types::VectorQuantization,
286 /// Native storage dtype for vector values (F32 / F16 / BF16). The
287 /// executor uses this to decide whether to transcode incoming f32
288 /// components before writing them into the HNSW index backing store.
289 storage_dtype: nodedb_types::VectorStorageDtype,
290 /// Payload field bitmap indexes (name + kind). Registered via
291 /// `payload.add_index` on the first insert into a new collection.
292 payload_indexes: Vec<(String, nodedb_types::PayloadIndexKind)>,
293 },
294}