kevy_index/
catalog.rs

1//! [`Catalog`] — the index registry: declarations, states, and the
2//! compiled prefix matcher the write-path hook consults.
3
4use crate::value::IndexValue;
5
6/// Declared scalar type of an index (`TYPE i64|f64|str`).
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub enum ValType {
9    /// v2.8: f32 LE vector blob (ANN kinds parse the field
10    /// themselves — never coerced through IndexValue).
11    Vector,
12    /// Signed 64-bit integer.
13    I64,
14    /// Finite 64-bit float (NaN coerce-fails).
15    F64,
16    /// Raw bytes, memcmp order.
17    Str,
18}
19
20impl ValType {
21    /// Wire tag (catalog sidecar + IDX.LIST).
22    pub fn tag(self) -> &'static str {
23        match self {
24            ValType::I64 => "i64",
25            ValType::F64 => "f64",
26            ValType::Str => "str",
27            ValType::Vector => "vector",
28        }
29    }
30
31    /// Parse a wire tag.
32    pub fn parse(raw: &[u8]) -> Option<ValType> {
33        if raw.eq_ignore_ascii_case(b"i64") {
34            Some(ValType::I64)
35        } else if raw.eq_ignore_ascii_case(b"f64") {
36            Some(ValType::F64)
37        } else if raw.eq_ignore_ascii_case(b"str") {
38            Some(ValType::Str)
39        } else if raw.eq_ignore_ascii_case(b"vector") {
40            Some(ValType::Vector)
41        } else {
42            None
43        }
44    }
45}
46
47/// Index kind (`KIND range|unique`).
48#[derive(Debug, Clone, Copy, PartialEq, Eq)]
49pub enum IndexKind {
50    /// Ordered scan over `(value, key)` pairs.
51    Range,
52    /// Point lookup by value; duplicates recorded (declarative fence —
53    /// RFC D3: uniqueness is verified, not write-enforced).
54    Unique,
55    /// v2.7 full-text: the field tokenizes into an inverted segment
56    /// (kevy-text); queried with `MATCH`, BM25-ranked.
57    Text,
58    /// v2.8 ANN: the field holds an f32 LE vector indexed in an HNSW
59    /// graph (kevy-vector); queried with `KNN`, distance-ranked.
60    Ann,
61    /// v3.1 aggregate: per-group count/sum/min/max of the field,
62    /// grouped by `IndexSpec::group_by`; queried with `GROUP`/`GROUPS`.
63    Agg,
64}
65
66impl IndexKind {
67    /// Wire tag.
68    pub fn tag(self) -> &'static str {
69        match self {
70            IndexKind::Range => "range",
71            IndexKind::Unique => "unique",
72            IndexKind::Text => "text",
73            IndexKind::Ann => "ann",
74            IndexKind::Agg => "agg",
75        }
76    }
77
78    /// Parse a wire tag.
79    pub fn parse(raw: &[u8]) -> Option<IndexKind> {
80        if raw.eq_ignore_ascii_case(b"range") {
81            Some(IndexKind::Range)
82        } else if raw.eq_ignore_ascii_case(b"unique") {
83            Some(IndexKind::Unique)
84        } else if raw.eq_ignore_ascii_case(b"text") {
85            Some(IndexKind::Text)
86        } else if raw.eq_ignore_ascii_case(b"ann") {
87            Some(IndexKind::Ann)
88        } else if raw.eq_ignore_ascii_case(b"agg") {
89            Some(IndexKind::Agg)
90        } else {
91            None
92        }
93    }
94}
95
96/// Lifecycle state (RFC D5).
97#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98pub enum IndexState {
99    /// Backfill in progress; queries answer `-INDEXBUILDING`.
100    Building,
101    /// Serving.
102    Ready,
103    /// Build aborted over budget (RFC D7); queries answer an error.
104    FailedOverBudget,
105}
106
107/// One declared index.
108#[derive(Debug, Clone, PartialEq, Eq)]
109pub struct IndexSpec {
110    /// Unique catalog name.
111    pub name: Vec<u8>,
112    /// Key-prefix domain (`ON PREFIX user:`).
113    pub prefix: Vec<u8>,
114    /// Hash field the value comes from.
115    pub field: Vec<u8>,
116    /// Declared scalar type.
117    pub ty: ValType,
118    /// Range or unique.
119    pub kind: IndexKind,
120    /// Optional per-index byte budget (`MAXMEM`); 0 = unlimited.
121    pub max_bytes: u64,
122    /// v2.8: ANN parameters (`Some` iff kind == Ann).
123    pub ann: Option<AnnSpec>,
124    /// v3.1: grouping field (`Some` iff kind == Agg).
125    pub group_by: Option<Vec<u8>>,
126}
127
128/// v2.8 — HNSW declaration (immutable once created; RFC D2).
129#[derive(Debug, Clone, Copy, PartialEq, Eq)]
130pub struct AnnSpec {
131    /// Vector dimensionality (field bytes must be dim×4 f32 LE).
132    pub dim: u32,
133    /// 0=cosine 1=l2 2=ip (kevy-vector's Distance tags).
134    pub distance: u8,
135    /// Max links per node per layer.
136    pub m: u16,
137    /// Construction beam width.
138    pub ef: u16,
139}
140
141/// Hard cap on declared indexes (RFC D1).
142pub const MAX_INDEXES: usize = 64;
143
144/// The registry. The runtime holds one per process behind an RCU-style
145/// swap; shards read their clone lock-free.
146#[derive(Debug, Clone, Default)]
147pub struct Catalog {
148    specs: Vec<(IndexSpec, IndexState)>,
149}
150
151impl Catalog {
152    /// Empty catalog.
153    pub fn new() -> Self {
154        Self::default()
155    }
156
157    /// Register a new index. Errors on duplicate name / cap.
158    pub fn create(&mut self, spec: IndexSpec) -> Result<(), &'static str> {
159        if self.specs.len() >= MAX_INDEXES {
160            return Err("ERR index limit reached (64)");
161        }
162        if self.specs.iter().any(|(s, _)| s.name == spec.name) {
163            return Err("ERR index already exists");
164        }
165        self.specs.push((spec, IndexState::Building));
166        Ok(())
167    }
168
169    /// Drop by name; `false` if absent.
170    pub fn drop_index(&mut self, name: &[u8]) -> bool {
171        let before = self.specs.len();
172        self.specs.retain(|(s, _)| s.name != name);
173        self.specs.len() != before
174    }
175
176    /// Set an index's lifecycle state; `false` if absent.
177    pub fn set_state(&mut self, name: &[u8], state: IndexState) -> bool {
178        for (s, st) in &mut self.specs {
179            if s.name == name {
180                *st = state;
181                return true;
182            }
183        }
184        false
185    }
186
187    /// Look up by name.
188    pub fn get(&self, name: &[u8]) -> Option<(&IndexSpec, IndexState)> {
189        self.specs.iter().find(|(s, _)| s.name == name).map(|(s, st)| (s, *st))
190    }
191
192    /// All specs with states, declaration order.
193    pub fn iter(&self) -> impl Iterator<Item = (&IndexSpec, IndexState)> {
194        self.specs.iter().map(|(s, st)| (s, *st))
195    }
196
197    /// Number of declared indexes.
198    pub fn len(&self) -> usize {
199        self.specs.len()
200    }
201
202    /// Whether no indexes are declared (the write hook's fast path).
203    pub fn is_empty(&self) -> bool {
204        self.specs.is_empty()
205    }
206
207    /// The write-path matcher: indexes whose prefix domain contains
208    /// `key`. Linear over ≤64 specs with a memcmp each — the compiled
209    /// trie of the RFC becomes worthwhile only past this cap, so the
210    /// simple form IS the fast form at our scale.
211    pub fn matching<'a>(&'a self, key: &'a [u8]) -> impl Iterator<Item = (&'a IndexSpec, IndexState)> {
212        self.specs
213            .iter()
214            .filter(move |(s, _)| key.starts_with(&s.prefix))
215            .map(|(s, st)| (s, *st))
216    }
217
218    /// Coerce a raw field value for `spec` (convenience passthrough).
219    pub fn coerce(spec: &IndexSpec, raw: &[u8]) -> Option<IndexValue> {
220        IndexValue::coerce(spec.ty, raw)
221    }
222
223    /// Serialize to the sidecar text form (one line per index:
224    /// `name<TAB>prefix<TAB>field<TAB>ty<TAB>kind<TAB>max_bytes[<TAB>ann]`,
225    /// fields hex-escaped for tabs/newlines via `%XX`; the 7th column
226    /// is `dim,distance,m,ef` for ANN kinds).
227    pub fn to_sidecar(&self) -> String {
228        let mut out = String::from("kevy-index-catalog v1\n");
229        for (s, _) in &self.specs {
230            out.push_str(&format!(
231                "{}\t{}\t{}\t{}\t{}\t{}",
232                esc(&s.name),
233                esc(&s.prefix),
234                esc(&s.field),
235                s.ty.tag(),
236                s.kind.tag(),
237                s.max_bytes
238            ));
239            // 7th column is kind-interpreted: ann params for Ann,
240            // escaped group field for Agg.
241            if let Some(a) = &s.ann {
242                out.push_str(&format!("\t{},{},{},{}", a.dim, a.distance, a.m, a.ef));
243            } else if let Some(g) = &s.group_by {
244                out.push_str(&format!("\t{}", esc(g)));
245            }
246            out.push('\n');
247        }
248        out
249    }
250
251    /// Parse the sidecar text form; all indexes load as `Building`
252    /// (boot rebuild, RFC D5). `None` on malformed input.
253    pub fn from_sidecar(text: &str) -> Option<Catalog> {
254        let mut lines = text.lines();
255        if lines.next()? != "kevy-index-catalog v1" {
256            return None;
257        }
258        let mut c = Catalog::new();
259        for line in lines {
260            if line.is_empty() {
261                continue;
262            }
263            let parts: Vec<&str> = line.split('\t').collect();
264            if !(parts.len() == 6 || parts.len() == 7) {
265                return None;
266            }
267            let kind = IndexKind::parse(parts[4].as_bytes())?;
268            let (ann, group_by) = if parts.len() == 7 {
269                match kind {
270                    IndexKind::Ann => {
271                        let nums: Vec<&str> = parts[6].split(',').collect();
272                        if nums.len() != 4 {
273                            return None;
274                        }
275                        (
276                            Some(AnnSpec {
277                                dim: nums[0].parse().ok()?,
278                                distance: nums[1].parse().ok()?,
279                                m: nums[2].parse().ok()?,
280                                ef: nums[3].parse().ok()?,
281                            }),
282                            None,
283                        )
284                    }
285                    IndexKind::Agg => (None, Some(unesc(parts[6])?)),
286                    _ => return None,
287                }
288            } else {
289                (None, None)
290            };
291            let spec = IndexSpec {
292                name: unesc(parts[0])?,
293                prefix: unesc(parts[1])?,
294                field: unesc(parts[2])?,
295                ty: ValType::parse(parts[3].as_bytes())?,
296                kind,
297                max_bytes: parts[5].parse().ok()?,
298                ann,
299                group_by,
300            };
301            c.create(spec).ok()?;
302        }
303        Some(c)
304    }
305}
306
307fn esc(b: &[u8]) -> String {
308    let mut out = String::with_capacity(b.len());
309    for &c in b {
310        if c == b'\t' || c == b'\n' || c == b'%' || !(32..127).contains(&c) {
311            out.push_str(&format!("%{c:02X}"));
312        } else {
313            out.push(c as char);
314        }
315    }
316    out
317}
318
319fn unesc(s: &str) -> Option<Vec<u8>> {
320    let mut out = Vec::with_capacity(s.len());
321    let bytes = s.as_bytes();
322    let mut i = 0;
323    while i < bytes.len() {
324        if bytes[i] == b'%' {
325            let hex = s.get(i + 1..i + 3)?;
326            out.push(u8::from_str_radix(hex, 16).ok()?);
327            i += 3;
328        } else {
329            out.push(bytes[i]);
330            i += 1;
331        }
332    }
333    Some(out)
334}
335
336#[cfg(test)]
337mod tests {
338    use super::*;
339
340    fn spec(name: &str, prefix: &str) -> IndexSpec {
341        IndexSpec {
342            name: name.into(),
343            prefix: prefix.into(),
344            field: b"age".to_vec(),
345            ty: ValType::I64,
346            kind: IndexKind::Range,
347            ann: None,
348            max_bytes: 0,
349            group_by: None,
350        }
351    }
352
353    #[test]
354    fn create_drop_match_lifecycle() {
355        let mut c = Catalog::new();
356        c.create(spec("a", "user:")).unwrap();
357        c.create(spec("b", "sess:")).unwrap();
358        assert!(c.create(spec("a", "x:")).is_err(), "dup name");
359        assert_eq!(c.matching(b"user:42").count(), 1);
360        assert_eq!(c.matching(b"other:1").count(), 0);
361        assert_eq!(c.get(b"a").unwrap().1, IndexState::Building);
362        assert!(c.set_state(b"a", IndexState::Ready));
363        assert_eq!(c.get(b"a").unwrap().1, IndexState::Ready);
364        assert!(c.drop_index(b"b"));
365        assert!(!c.drop_index(b"b"));
366        assert_eq!(c.len(), 1);
367    }
368
369    #[test]
370    fn sidecar_roundtrip_with_escapes() {
371        let mut c = Catalog::new();
372        let mut s = spec("weird", "pre\tfix:");
373        s.field = b"f%\n".to_vec();
374        s.max_bytes = 1024;
375        c.create(s).unwrap();
376        let text = c.to_sidecar();
377        let c2 = Catalog::from_sidecar(&text).unwrap();
378        let (got, st) = c2.get(b"weird").unwrap();
379        assert_eq!(got.prefix, b"pre\tfix:".to_vec());
380        assert_eq!(got.field, b"f%\n".to_vec());
381        assert_eq!(got.max_bytes, 1024);
382        assert_eq!(st, IndexState::Building, "boot loads as Building");
383        assert!(Catalog::from_sidecar("bogus").is_none());
384    }
385
386    #[test]
387    fn cap_enforced() {
388        let mut c = Catalog::new();
389        for i in 0..MAX_INDEXES {
390            c.create(spec(&format!("i{i}"), "p:")).unwrap();
391        }
392        assert!(c.create(spec("over", "p:")).is_err());
393    }
394}
kevy_index/catalog.rs

kevy_index/
catalog.rs
