nodedb_types/
collection_config.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! Vector-primary collection configuration types, plus partition-strategy
4//! metadata.
5//!
6//! `PrimaryEngine` is a parallel attribute to `CollectionType` that tells the
7//! planner which engine is the primary access path for a collection.
8//! Vectors remain an index — not a collection type — but a `primary = 'vector'`
9//! attribute means the vector index is the hot path and the document store is
10//! a metadata sidecar.
11//!
12//! `PartitionStrategy` records HOW a collection is distributed across vShards.
13//! It is the single authoritative source for partition metadata; future routing
14//! and resharding layers read this field instead of inferring from engine type.
15
16use crate::collection::CollectionType;
17use crate::columnar::{ColumnarProfile, DocumentMode};
18use crate::vector_ann::VectorQuantization;
19use crate::vector_distance::DistanceMetric;
20use crate::vector_dtype::VectorStorageDtype;
21
22/// Which engine serves as the primary access path for a collection.
23///
24/// This is independent of `CollectionType` — it is an optimizer hint that
25/// instructs the planner and executor to use the named engine as the hot path.
26/// The default is inferred from `CollectionType` so existing collections need
27/// no migration.
28#[repr(u8)]
29#[derive(
30    Debug,
31    Clone,
32    Copy,
33    Default,
34    PartialEq,
35    Eq,
36    Hash,
37    serde::Serialize,
38    serde::Deserialize,
39    zerompk::ToMessagePack,
40    zerompk::FromMessagePack,
41)]
42#[non_exhaustive]
43pub enum PrimaryEngine {
44    /// Schemaless document (MessagePack). The historic default.
45    #[default]
46    Document = 0,
47    /// Strict document (Binary Tuples).
48    Strict = 1,
49    /// Key-Value hash store.
50    KeyValue = 2,
51    /// Columnar / plain-analytics.
52    Columnar = 3,
53    /// Columnar with spatial profile.
54    Spatial = 4,
55    /// Vector-primary: HNSW is the hot path; document store is a metadata sidecar.
56    Vector = 10,
57}
58
59impl PrimaryEngine {
60    /// Infer the primary engine from a `CollectionType`.
61    ///
62    /// Used when reading catalog entries that predate the `primary` field —
63    /// guarantees that existing collections behave as before.
64    pub fn infer_from_collection_type(ct: &CollectionType) -> Self {
65        match ct {
66            CollectionType::Document(DocumentMode::Schemaless) => Self::Document,
67            CollectionType::Document(DocumentMode::Strict(_)) => Self::Strict,
68            CollectionType::Columnar(ColumnarProfile::Plain) => Self::Columnar,
69            CollectionType::Columnar(ColumnarProfile::Timeseries { .. }) => Self::Columnar,
70            CollectionType::Columnar(ColumnarProfile::Spatial { .. }) => Self::Spatial,
71            CollectionType::KeyValue(_) => Self::KeyValue,
72        }
73    }
74}
75
76/// Configuration for a vector-primary collection.
77///
78/// Stored in `StoredCollection::vector_primary` when `primary == PrimaryEngine::Vector`.
79/// All options correspond to HNSW construction parameters and codec selection for the
80/// primary vector index.
81#[derive(
82    Debug,
83    Clone,
84    PartialEq,
85    serde::Serialize,
86    serde::Deserialize,
87    zerompk::ToMessagePack,
88    zerompk::FromMessagePack,
89)]
90pub struct VectorPrimaryConfig {
91    /// The name of the column that holds vector data (must be of type VECTOR(n)).
92    pub vector_field: String,
93    /// Vector dimensionality.
94    pub dim: u32,
95    /// Quantization codec for the primary HNSW index.
96    pub quantization: VectorQuantization,
97    /// HNSW `M` parameter (number of connections per node).
98    pub m: u8,
99    /// HNSW `ef_construction` parameter (beam width during index construction).
100    pub ef_construction: u16,
101    /// Distance metric used for similarity search.
102    pub metric: DistanceMetric,
103    /// Native storage dtype for vector values. Controls whether incoming
104    /// f32 components are stored as-is (F32), downsized to half precision
105    /// (F16), or brain-float (BF16) to halve memory at the cost of reduced
106    /// mantissa precision. Quantization codecs (RaBitQ, BBQ, SQ8 …) apply
107    /// on top and are orthogonal to this setting.
108    pub storage_dtype: VectorStorageDtype,
109    /// Payload field names that receive in-memory bitmap indexes for fast
110    /// pre-filtering, paired with the storage kind (Equality / Range /
111    /// Boolean). The DDL handler infers the kind from the column type:
112    /// numeric / timestamp / decimal → Range; bool → Boolean; everything
113    /// else → Equality.
114    pub payload_indexes: Vec<(String, PayloadIndexKind)>,
115}
116
117impl Default for VectorPrimaryConfig {
118    fn default() -> Self {
119        Self {
120            vector_field: String::new(),
121            dim: 0,
122            quantization: VectorQuantization::default(),
123            m: 16,
124            ef_construction: 200,
125            metric: DistanceMetric::Cosine,
126            storage_dtype: VectorStorageDtype::F32,
127            payload_indexes: Vec::new(),
128        }
129    }
130}
131
132/// Storage kind for a payload bitmap index. Equality fields use a
133/// `HashMap<key, bitmap>` (O(1) lookup); Range fields use a `BTreeMap`
134/// for sorted range scans; Boolean is a low-cardinality equality variant.
135#[derive(
136    Debug,
137    Clone,
138    Copy,
139    Default,
140    PartialEq,
141    Eq,
142    serde::Serialize,
143    serde::Deserialize,
144    zerompk::ToMessagePack,
145    zerompk::FromMessagePack,
146)]
147#[non_exhaustive]
148pub enum PayloadIndexKind {
149    #[default]
150    Equality,
151    Range,
152    Boolean,
153}
154
155/// A single payload-bitmap predicate atom emitted by the SQL planner and
156/// consumed by the vector search handler. The handler ANDs all atoms in
157/// `VectorOp::Search::payload_filters`; each atom may itself be a
158/// disjunction (`In`).
159#[derive(
160    Debug,
161    Clone,
162    PartialEq,
163    serde::Serialize,
164    serde::Deserialize,
165    zerompk::ToMessagePack,
166    zerompk::FromMessagePack,
167)]
168#[non_exhaustive]
169pub enum PayloadAtom {
170    /// `field = value` — single equality bitmap lookup.
171    Eq(String, crate::Value),
172    /// `field IN (v1, v2, ...)` — union of per-value bitmaps.
173    In(String, Vec<crate::Value>),
174    /// `field >= low AND field <= high` — sorted range scan over a
175    /// `PayloadIndexKind::Range` index. Either bound being `None` means
176    /// open on that side.
177    Range {
178        field: String,
179        low: Option<crate::Value>,
180        low_inclusive: bool,
181        high: Option<crate::Value>,
182        high_inclusive: bool,
183    },
184}
185
186/// Names WHAT a key-partitioned collection hashes to derive its vShard.
187///
188/// Defined as substrate for future routing and array layers; not yet populated
189/// at create time for those paths — only `PartitionStrategy::CollectionHomed`
190/// is used at create time in this release.
191#[derive(
192    Debug,
193    Clone,
194    PartialEq,
195    Eq,
196    serde::Serialize,
197    serde::Deserialize,
198    zerompk::ToMessagePack,
199    zerompk::FromMessagePack,
200)]
201pub enum KeySpec {
202    /// Graph edge endpoint node-id string (hashed via VShardId::from_key).
203    NodeId,
204    /// Array name ‖ tile-id (array tile routing).
205    ArrayTile,
206}
207
208/// How a collection's rows are distributed across vShards.
209///
210/// This is the authoritative per-collection partition metadata. Future routing,
211/// Calvin, and resharding layers read this field instead of inferring
212/// distribution from engine type.
213#[derive(
214    Debug,
215    Clone,
216    Default,
217    PartialEq,
218    Eq,
219    serde::Serialize,
220    serde::Deserialize,
221    zerompk::ToMessagePack,
222    zerompk::FromMessagePack,
223)]
224pub enum PartitionStrategy {
225    /// All rows live on one owning vShard derived from (db_id, collection).
226    ///
227    /// Every base collection is collection-homed today: Document (schemaless
228    /// and strict), Columnar (plain, timeseries, spatial), and Key-Value all
229    /// route to a single vShard that owns the collection. Graph edge-key
230    /// partitioning and Array tile partitioning are operation-level concerns
231    /// handled separately and are NOT reflected here at create time.
232    #[default]
233    CollectionHomed,
234    /// Rows routed by hashing a key field. Reserved for future key-partitioned
235    /// collections; not populated at create time in this release.
236    KeyPartitioned { key: KeySpec },
237}
238
239impl PartitionStrategy {
240    /// Derive the create-time default strategy from a [`CollectionType`].
241    ///
242    /// Every base engine is collection-homed today:
243    /// - `Document` (schemaless and strict): single-vShard B-tree ownership.
244    /// - `Columnar` (plain, timeseries, spatial): single-vShard segment
245    ///   ownership; timeseries append partitioning is a write-path concern,
246    ///   not a metadata-level partition strategy.
247    /// - `KeyValue`: single-vShard hash-index ownership; per-key routing is
248    ///   internal to the engine, not exposed at the collection layer.
249    ///
250    /// Graph edge-key and Array tile partitioning are operation-level and are
251    /// handled separately — they are NOT set here at collection create time.
252    pub fn default_for_collection_type(ct: &CollectionType) -> Self {
253        match ct {
254            CollectionType::Document(DocumentMode::Schemaless) => Self::CollectionHomed,
255            CollectionType::Document(DocumentMode::Strict(_)) => Self::CollectionHomed,
256            CollectionType::Columnar(ColumnarProfile::Plain) => Self::CollectionHomed,
257            CollectionType::Columnar(ColumnarProfile::Timeseries { .. }) => Self::CollectionHomed,
258            CollectionType::Columnar(ColumnarProfile::Spatial { .. }) => Self::CollectionHomed,
259            CollectionType::KeyValue(_) => Self::CollectionHomed,
260        }
261    }
262
263    /// Stable lowercase tag for catalog and SHOW output.
264    pub fn as_str(&self) -> &'static str {
265        match self {
266            Self::CollectionHomed => "collection_homed",
267            Self::KeyPartitioned { .. } => "key_partitioned",
268        }
269    }
270}
271
272#[cfg(test)]
273mod tests {
274    use super::*;
275
276    #[test]
277    fn primary_engine_default_is_document() {
278        assert_eq!(PrimaryEngine::default(), PrimaryEngine::Document);
279    }
280
281    #[test]
282    fn infer_from_collection_type_document_schemaless() {
283        let ct = CollectionType::document();
284        assert_eq!(
285            PrimaryEngine::infer_from_collection_type(&ct),
286            PrimaryEngine::Document
287        );
288    }
289
290    #[test]
291    fn infer_from_collection_type_document_strict() {
292        use crate::columnar::{ColumnDef, ColumnType, StrictSchema};
293        let schema = StrictSchema::new(vec![
294            ColumnDef::required("id", ColumnType::Int64).with_primary_key(),
295        ])
296        .unwrap();
297        let ct = CollectionType::strict(schema);
298        assert_eq!(
299            PrimaryEngine::infer_from_collection_type(&ct),
300            PrimaryEngine::Strict
301        );
302    }
303
304    #[test]
305    fn infer_from_collection_type_columnar_plain() {
306        let ct = CollectionType::columnar();
307        assert_eq!(
308            PrimaryEngine::infer_from_collection_type(&ct),
309            PrimaryEngine::Columnar
310        );
311    }
312
313    #[test]
314    fn infer_from_collection_type_columnar_timeseries() {
315        let ct = CollectionType::timeseries("ts", "1h");
316        assert_eq!(
317            PrimaryEngine::infer_from_collection_type(&ct),
318            PrimaryEngine::Columnar
319        );
320    }
321
322    #[test]
323    fn infer_from_collection_type_columnar_spatial() {
324        let ct = CollectionType::spatial("geom");
325        assert_eq!(
326            PrimaryEngine::infer_from_collection_type(&ct),
327            PrimaryEngine::Spatial
328        );
329    }
330
331    #[test]
332    fn infer_from_collection_type_kv() {
333        use crate::columnar::{ColumnDef, ColumnType, StrictSchema};
334        let schema = StrictSchema::new(vec![
335            ColumnDef::required("k", ColumnType::String).with_primary_key(),
336        ])
337        .unwrap();
338        let ct = CollectionType::kv(schema);
339        assert_eq!(
340            PrimaryEngine::infer_from_collection_type(&ct),
341            PrimaryEngine::KeyValue
342        );
343    }
344
345    #[test]
346    fn primary_engine_serde_roundtrip() {
347        for variant in [
348            PrimaryEngine::Document,
349            PrimaryEngine::Strict,
350            PrimaryEngine::KeyValue,
351            PrimaryEngine::Columnar,
352            PrimaryEngine::Spatial,
353            PrimaryEngine::Vector,
354        ] {
355            let json = sonic_rs::to_string(&variant).unwrap();
356            let back: PrimaryEngine = sonic_rs::from_str(&json).unwrap();
357            assert_eq!(back, variant);
358        }
359    }
360
361    #[test]
362    fn primary_engine_msgpack_roundtrip() {
363        for variant in [
364            PrimaryEngine::Document,
365            PrimaryEngine::Strict,
366            PrimaryEngine::KeyValue,
367            PrimaryEngine::Columnar,
368            PrimaryEngine::Spatial,
369            PrimaryEngine::Vector,
370        ] {
371            let bytes = zerompk::to_msgpack_vec(&variant).unwrap();
372            let back: PrimaryEngine = zerompk::from_msgpack(&bytes).unwrap();
373            assert_eq!(back, variant);
374        }
375    }
376
377    #[test]
378    fn vector_primary_config_serde_roundtrip() {
379        let cfg = VectorPrimaryConfig {
380            vector_field: "embedding".to_string(),
381            dim: 1024,
382            quantization: VectorQuantization::RaBitQ,
383            m: 32,
384            ef_construction: 200,
385            metric: DistanceMetric::Cosine,
386            storage_dtype: VectorStorageDtype::F32,
387            payload_indexes: vec![
388                ("category".to_string(), PayloadIndexKind::Equality),
389                ("timestamp".to_string(), PayloadIndexKind::Range),
390            ],
391        };
392        let json = sonic_rs::to_string(&cfg).unwrap();
393        let back: VectorPrimaryConfig = sonic_rs::from_str(&json).unwrap();
394        assert_eq!(back, cfg);
395    }
396
397    #[test]
398    fn vector_primary_config_msgpack_roundtrip() {
399        let cfg = VectorPrimaryConfig {
400            vector_field: "vec".to_string(),
401            dim: 512,
402            quantization: VectorQuantization::Bbq,
403            m: 16,
404            ef_construction: 100,
405            metric: DistanceMetric::L2,
406            storage_dtype: VectorStorageDtype::F32,
407            payload_indexes: vec![],
408        };
409        let bytes = zerompk::to_msgpack_vec(&cfg).unwrap();
410        let back: VectorPrimaryConfig = zerompk::from_msgpack(&bytes).unwrap();
411        assert_eq!(back, cfg);
412    }
413}
nodedb_types/collection_config.rs

nodedb_types/
collection_config.rs
