1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
// SPDX-License-Identifier: BUSL-1.1
//! Vector engine operations dispatched to the Data Plane.
use nodedb_types::{Surrogate, SurrogateBitmap, vector_distance::DistanceMetric};
use crate::bridge::envelope::PhysicalPlan;
/// Vector engine physical operations.
#[derive(
Debug,
Clone,
PartialEq,
serde::Serialize,
serde::Deserialize,
zerompk::ToMessagePack,
zerompk::FromMessagePack,
)]
pub enum VectorOp {
/// Vector similarity search.
Search {
collection: String,
query_vector: Vec<f32>,
top_k: usize,
/// Optional search beam width override. If 0, uses default `4 * top_k`.
ef_search: usize,
/// Per-query distance metric override. Overrides the collection-default
/// metric configured at index creation time.
metric: DistanceMetric,
/// Pre-computed bitmap of eligible surrogates (from filter evaluation).
filter_bitmap: Option<SurrogateBitmap>,
/// Named vector field to search. Empty string = default field.
field_name: String,
/// RLS post-candidate filters (serialized `Vec<ScanFilter>`).
/// Applied after HNSW returns candidates, before returning to client.
/// Result count may be less than requested `top_k`.
rls_filters: Vec<u8>,
/// Optional sub-plan whose output rows materialize a `SurrogateBitmap`
/// used as an additional prefilter (intersected with `filter_bitmap`).
///
/// Set by the planner when fusing `Vector ORDER BY ... + JOIN
/// ARRAY_SLICE(...) ON v.id = s.surrogate` into a single op:
/// the array slice runs first, its surrogate column becomes the
/// vector engine's pre-filter, then HNSW search runs against the
/// reduced candidate set. Mirrors `HashJoin`'s `inline_*_bitmap`
/// mechanism so the Data Plane composition is uniform.
inline_prefilter_plan: Option<Box<PhysicalPlan>>,
/// ANN tuning knobs from the SQL caller. Defaults to no overrides.
ann_options: nodedb_types::VectorAnnOptions,
/// When `true`, the SELECT projection contains only `id` and/or
/// `vector_distance(...)` — no payload fields. The handler skips
/// the document-body fetch. Ignored (treated as `false`) when RLS
/// filters are active, as body fetch is required for RLS evaluation.
skip_payload_fetch: bool,
/// Optional payload bitmap pre-filter for vector-primary collections.
/// Each atom is `Eq` / `In` / `Range` — the handler ANDs all atoms
/// and intersects the resulting bitmap with the HNSW candidate set
/// before walking. The planner only emits atoms whose field has a
/// registered payload index of the matching kind. Empty means no
/// payload pre-filter.
payload_filters: Vec<nodedb_types::PayloadAtom>,
},
/// Insert a vector into the HNSW index (write path).
Insert {
collection: String,
vector: Vec<f32>,
dim: usize,
/// Named vector field. Empty string = default (unnamed) field.
field_name: String,
/// Global surrogate identifying this row. Allocated by the
/// Control Plane via `SurrogateAssigner` before dispatch; the
/// engine binds the HNSW node id to this surrogate.
surrogate: Surrogate,
},
/// Batch insert vectors into the HNSW index.
BatchInsert {
collection: String,
vectors: Vec<Vec<f32>>,
dim: usize,
/// One surrogate per inserted vector, parallel to `vectors`.
/// Empty vector = headless batch (no PK binding).
surrogates: Vec<Surrogate>,
},
/// Multi-vector search: query across all named vector fields, fuse via RRF.
MultiSearch {
collection: String,
query_vector: Vec<f32>,
top_k: usize,
ef_search: usize,
filter_bitmap: Option<SurrogateBitmap>,
/// RLS post-candidate filters.
rls_filters: Vec<u8>,
},
/// Soft-delete a vector by internal node ID.
Delete { collection: String, vector_id: u32 },
/// Set vector index parameters for a collection.
SetParams {
collection: String,
/// Named vector field this config applies to. Empty string = the
/// default (unnamed) vector field. Lets one collection carry
/// multiple vector indexes, one per `VECTOR` / embedding column.
field_name: String,
m: usize,
ef_construction: usize,
metric: String,
/// Index type: "hnsw" (default), "hnsw_pq", or "ivf_pq".
index_type: String,
/// PQ subvectors (for hnsw_pq and ivf_pq). Default: 8.
pq_m: usize,
/// IVF cells (for ivf_pq only). Default: 256.
ivf_cells: usize,
/// IVF probe count (for ivf_pq only). Default: 16.
ivf_nprobe: usize,
},
/// Query live vector index statistics. Returns `VectorIndexStats` as payload.
QueryStats {
collection: String,
/// Named vector field. Empty string = default (unnamed) field.
field_name: String,
},
/// Force-seal the growing segment, triggering background HNSW build.
Seal {
collection: String,
/// Named vector field. Empty string = default field.
field_name: String,
},
/// Force tombstone compaction on sealed segments.
CompactIndex {
collection: String,
/// Named vector field. Empty string = default field.
field_name: String,
},
/// Rebuild sealed segments with new HNSW parameters.
/// Old index serves queries until rebuild completes, then swaps atomically.
Rebuild {
collection: String,
/// Named vector field. Empty string = default field.
field_name: String,
/// New M parameter. 0 = keep current.
m: usize,
/// New M0 parameter. 0 = keep current.
m0: usize,
/// New ef_construction. 0 = keep current.
ef_construction: usize,
},
/// Insert a sparse vector into the inverted index.
SparseInsert {
collection: String,
/// Named sparse vector field.
field_name: String,
/// Document ID to associate with this sparse vector.
doc_id: String,
/// Sparse vector entries as `(dimension, weight)` pairs.
entries: Vec<(u32, f32)>,
},
/// Search the sparse inverted index via dot-product scoring.
SparseSearch {
collection: String,
/// Named sparse vector field.
field_name: String,
/// Query sparse vector entries.
query_entries: Vec<(u32, f32)>,
/// Maximum results to return.
top_k: usize,
},
/// Delete a document from the sparse inverted index.
SparseDelete {
collection: String,
/// Named sparse vector field.
field_name: String,
/// Document ID to remove.
doc_id: String,
},
/// Insert multiple vectors for a single document (ColBERT-style).
/// All vectors are inserted as separate HNSW nodes sharing the
/// same `document_surrogate`.
MultiVectorInsert {
collection: String,
/// Named vector field. Empty = default.
field_name: String,
/// Surrogate shared by all vectors of the document.
document_surrogate: Surrogate,
/// Flat vector data: count × dim f32 values.
vectors: Vec<f32>,
/// Number of vectors.
count: usize,
/// Dimensionality of each vector.
dim: usize,
},
/// Delete all vectors for a document from the multi-vector index.
MultiVectorDelete {
collection: String,
/// Named vector field. Empty = default.
field_name: String,
/// Document surrogate whose vectors should be tombstoned.
document_surrogate: Surrogate,
},
/// Search with multi-vector aggregated scoring (MaxSim, AvgSim, SumSim).
/// Over-fetches from HNSW, groups by doc_id, aggregates, deduplicates.
MultiVectorScoreSearch {
collection: String,
/// Named vector field. Empty = default.
field_name: String,
/// Query vector.
query_vector: Vec<f32>,
/// Maximum documents to return.
top_k: usize,
/// HNSW ef_search override. 0 = auto.
ef_search: usize,
/// Aggregation mode: "max_sim", "avg_sim", "sum_sim".
mode: String,
},
/// Direct vector upsert for vector-primary collections.
///
/// Bypasses MessagePack document encoding — the Data Plane inserts the
/// vector into HNSW directly and updates payload bitmap indexes from
/// `payload`. No full-document blob is written.
///
/// Ordering invariant (enforced by the handler):
/// 1. Validate dim.
/// 2. Decode `payload` bytes.
/// 3. Insert into HNSW (surrogate bound).
/// 4. Update payload bitmap indexes.
///
/// If step 3 fails, step 4 is not reached — no partial state.
/// If step 4 fails (should not happen — pure in-memory), the handler
/// attempts to delete the just-inserted HNSW node and returns an error.
DirectUpsert {
collection: String,
/// Vector column name. Used to compute the vector index key so the
/// SELECT path (which keys by `(tid, collection, field)`) finds the
/// same index this insert wrote into.
field: String,
/// Global surrogate allocated by the Control Plane.
surrogate: Surrogate,
/// FP32 vector values.
vector: Vec<f32>,
/// Pre-encoded MessagePack of only the payload-indexed fields.
/// Empty when the collection has no payload indexes configured.
payload: Vec<u8>,
/// Collection-level quantization. Applied via `set_quantization` on
/// first insert so subsequent seals trigger codec-dispatch rebuilds
/// against the configured codec (RaBitQ / BBQ).
quantization: nodedb_types::VectorQuantization,
/// Payload field bitmap indexes (name + kind). Registered via
/// `payload.add_index` on the first insert into a new collection.
payload_indexes: Vec<(String, nodedb_types::PayloadIndexKind)>,
},
}