kevy_index/
catalog.rs

1//! [`Catalog`] — the index registry: declarations, states, and the
2//! compiled prefix matcher the write-path hook consults.
3
4use crate::value::IndexValue;
5use std::fmt::Write as _;
6
7/// Declared scalar type of an index (`TYPE i64|f64|str`).
8#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9pub enum ValType {
10    /// v2.8: f32 LE vector blob (ANN kinds parse the field
11    /// themselves — never coerced through IndexValue).
12    Vector,
13    /// Signed 64-bit integer.
14    I64,
15    /// Finite 64-bit float (NaN coerce-fails).
16    F64,
17    /// Raw bytes, memcmp order.
18    Str,
19}
20
21impl ValType {
22    /// Wire tag (catalog sidecar + IDX.LIST).
23    pub fn tag(self) -> &'static str {
24        match self {
25            ValType::I64 => "i64",
26            ValType::F64 => "f64",
27            ValType::Str => "str",
28            ValType::Vector => "vector",
29        }
30    }
31
32    /// Parse a wire tag.
33    pub fn parse(raw: &[u8]) -> Option<ValType> {
34        if raw.eq_ignore_ascii_case(b"i64") {
35            Some(ValType::I64)
36        } else if raw.eq_ignore_ascii_case(b"f64") {
37            Some(ValType::F64)
38        } else if raw.eq_ignore_ascii_case(b"str") {
39            Some(ValType::Str)
40        } else if raw.eq_ignore_ascii_case(b"vector") {
41            Some(ValType::Vector)
42        } else {
43            None
44        }
45    }
46}
47
48/// Index kind (`KIND range|unique`).
49#[derive(Debug, Clone, Copy, PartialEq, Eq)]
50pub enum IndexKind {
51    /// Ordered scan over `(value, key)` pairs.
52    Range,
53    /// Point lookup by value; duplicates recorded (declarative fence —
54    /// RFC D3: uniqueness is verified, not write-enforced).
55    Unique,
56    /// v2.7 full-text: the field tokenizes into an inverted segment
57    /// (kevy-text); queried with `MATCH`, BM25-ranked.
58    Text,
59    /// v2.8 ANN: the field holds an f32 LE vector indexed in an HNSW
60    /// graph (kevy-vector); queried with `KNN`, distance-ranked.
61    Ann,
62    /// v3.1 aggregate: per-group count/sum/min/max of the field,
63    /// grouped by `IndexSpec::group_by`; queried with `GROUP`/`GROUPS`.
64    Agg,
65}
66
67impl IndexKind {
68    /// Wire tag.
69    pub fn tag(self) -> &'static str {
70        match self {
71            IndexKind::Range => "range",
72            IndexKind::Unique => "unique",
73            IndexKind::Text => "text",
74            IndexKind::Ann => "ann",
75            IndexKind::Agg => "agg",
76        }
77    }
78
79    /// Parse a wire tag.
80    pub fn parse(raw: &[u8]) -> Option<IndexKind> {
81        if raw.eq_ignore_ascii_case(b"range") {
82            Some(IndexKind::Range)
83        } else if raw.eq_ignore_ascii_case(b"unique") {
84            Some(IndexKind::Unique)
85        } else if raw.eq_ignore_ascii_case(b"text") {
86            Some(IndexKind::Text)
87        } else if raw.eq_ignore_ascii_case(b"ann") {
88            Some(IndexKind::Ann)
89        } else if raw.eq_ignore_ascii_case(b"agg") {
90            Some(IndexKind::Agg)
91        } else {
92            None
93        }
94    }
95}
96
97/// Lifecycle state (RFC D5).
98#[derive(Debug, Clone, Copy, PartialEq, Eq)]
99pub enum IndexState {
100    /// Backfill in progress; queries answer `-INDEXBUILDING`.
101    Building,
102    /// Serving.
103    Ready,
104    /// Build aborted over budget (RFC D7); queries answer an error.
105    FailedOverBudget,
106}
107
108/// One declared index.
109#[derive(Debug, Clone, PartialEq, Eq)]
110pub struct IndexSpec {
111    /// Unique catalog name.
112    pub name: Vec<u8>,
113    /// Key-prefix domain (`ON PREFIX user:`).
114    pub prefix: Vec<u8>,
115    /// Hash field the value comes from.
116    pub field: Vec<u8>,
117    /// Declared scalar type.
118    pub ty: ValType,
119    /// Range or unique.
120    pub kind: IndexKind,
121    /// Optional per-index byte budget (`MAXMEM`); 0 = unlimited.
122    pub max_bytes: u64,
123    /// v2.8: ANN parameters (`Some` iff kind == Ann).
124    pub ann: Option<AnnSpec>,
125    /// v3.1: grouping field (`Some` iff kind == Agg).
126    pub group_by: Option<Vec<u8>>,
127}
128
129/// v2.8 — HNSW declaration (immutable once created; RFC D2).
130#[derive(Debug, Clone, Copy, PartialEq, Eq)]
131pub struct AnnSpec {
132    /// Vector dimensionality (field bytes must be dim×4 f32 LE).
133    pub dim: u32,
134    /// 0=cosine 1=l2 2=ip (kevy-vector's Distance tags).
135    pub distance: u8,
136    /// Max links per node per layer.
137    pub m: u16,
138    /// Construction beam width.
139    pub ef: u16,
140}
141
142/// Hard cap on declared indexes (RFC D1).
143pub const MAX_INDEXES: usize = 64;
144
145/// The registry. The runtime holds one per process behind an RCU-style
146/// swap; shards read their clone lock-free.
147#[derive(Debug, Clone, Default)]
148pub struct Catalog {
149    specs: Vec<(IndexSpec, IndexState)>,
150}
151
152impl Catalog {
153    /// Empty catalog.
154    pub fn new() -> Self {
155        Self::default()
156    }
157
158    /// Register a new index. Errors on duplicate name / cap.
159    pub fn create(&mut self, spec: IndexSpec) -> Result<(), &'static str> {
160        if self.specs.len() >= MAX_INDEXES {
161            return Err("ERR index limit reached (64)");
162        }
163        if self.specs.iter().any(|(s, _)| s.name == spec.name) {
164            return Err("ERR index already exists");
165        }
166        self.specs.push((spec, IndexState::Building));
167        Ok(())
168    }
169
170    /// Drop by name; `false` if absent.
171    pub fn drop_index(&mut self, name: &[u8]) -> bool {
172        let before = self.specs.len();
173        self.specs.retain(|(s, _)| s.name != name);
174        self.specs.len() != before
175    }
176
177    /// Set an index's lifecycle state; `false` if absent.
178    pub fn set_state(&mut self, name: &[u8], state: IndexState) -> bool {
179        for (s, st) in &mut self.specs {
180            if s.name == name {
181                *st = state;
182                return true;
183            }
184        }
185        false
186    }
187
188    /// Look up by name.
189    pub fn get(&self, name: &[u8]) -> Option<(&IndexSpec, IndexState)> {
190        self.specs.iter().find(|(s, _)| s.name == name).map(|(s, st)| (s, *st))
191    }
192
193    /// All specs with states, declaration order.
194    pub fn iter(&self) -> impl Iterator<Item = (&IndexSpec, IndexState)> {
195        self.specs.iter().map(|(s, st)| (s, *st))
196    }
197
198    /// Number of declared indexes.
199    pub fn len(&self) -> usize {
200        self.specs.len()
201    }
202
203    /// Whether no indexes are declared (the write hook's fast path).
204    pub fn is_empty(&self) -> bool {
205        self.specs.is_empty()
206    }
207
208    /// The write-path matcher: indexes whose prefix domain contains
209    /// `key`. Linear over ≤64 specs with a memcmp each — the compiled
210    /// trie of the RFC becomes worthwhile only past this cap, so the
211    /// simple form IS the fast form at our scale.
212    pub fn matching<'a>(&'a self, key: &'a [u8]) -> impl Iterator<Item = (&'a IndexSpec, IndexState)> {
213        self.specs
214            .iter()
215            .filter(move |(s, _)| key.starts_with(&s.prefix))
216            .map(|(s, st)| (s, *st))
217    }
218
219    /// Coerce a raw field value for `spec` (convenience passthrough).
220    pub fn coerce(spec: &IndexSpec, raw: &[u8]) -> Option<IndexValue> {
221        IndexValue::coerce(spec.ty, raw)
222    }
223
224    /// Serialize to the sidecar text form (one line per index:
225    /// `name<TAB>prefix<TAB>field<TAB>ty<TAB>kind<TAB>max_bytes[<TAB>ann]`,
226    /// fields hex-escaped for tabs/newlines via `%XX`; the 7th column
227    /// is `dim,distance,m,ef` for ANN kinds).
228    pub fn to_sidecar(&self) -> String {
229        let mut out = String::from("kevy-index-catalog v1\n");
230        for (s, _) in &self.specs {
231            let _ = write!(
232                out,
233                "{}\t{}\t{}\t{}\t{}\t{}",
234                esc(&s.name),
235                esc(&s.prefix),
236                esc(&s.field),
237                s.ty.tag(),
238                s.kind.tag(),
239                s.max_bytes
240            );
241            // 7th column is kind-interpreted: ann params for Ann,
242            // escaped group field for Agg.
243            if let Some(a) = &s.ann {
244                let _ = write!(out, "\t{},{},{},{}", a.dim, a.distance, a.m, a.ef);
245            } else if let Some(g) = &s.group_by {
246                let _ = write!(out, "\t{}", esc(g));
247            }
248            out.push('\n');
249        }
250        out
251    }
252
253    /// Parse the sidecar text form; all indexes load as `Building`
254    /// (boot rebuild, RFC D5). `None` on malformed input.
255    pub fn from_sidecar(text: &str) -> Option<Catalog> {
256        let mut lines = text.lines();
257        if lines.next()? != "kevy-index-catalog v1" {
258            return None;
259        }
260        let mut c = Catalog::new();
261        for line in lines {
262            if line.is_empty() {
263                continue;
264            }
265            c.create(spec_from_line(line)?).ok()?;
266        }
267        Some(c)
268    }
269}
270
271/// Parse one sidecar line back into an [`IndexSpec`] — the inverse of
272/// one `to_sidecar` line. `None` on malformed input.
273fn spec_from_line(line: &str) -> Option<IndexSpec> {
274    let parts: Vec<&str> = line.split('\t').collect();
275    if !(parts.len() == 6 || parts.len() == 7) {
276        return None;
277    }
278    let kind = IndexKind::parse(parts[4].as_bytes())?;
279    let (ann, group_by) = if parts.len() == 7 {
280        match kind {
281            IndexKind::Ann => {
282                let nums: Vec<&str> = parts[6].split(',').collect();
283                if nums.len() != 4 {
284                    return None;
285                }
286                (
287                    Some(AnnSpec {
288                        dim: nums[0].parse().ok()?,
289                        distance: nums[1].parse().ok()?,
290                        m: nums[2].parse().ok()?,
291                        ef: nums[3].parse().ok()?,
292                    }),
293                    None,
294                )
295            }
296            IndexKind::Agg => (None, Some(unesc(parts[6])?)),
297            _ => return None,
298        }
299    } else {
300        (None, None)
301    };
302    Some(IndexSpec {
303        name: unesc(parts[0])?,
304        prefix: unesc(parts[1])?,
305        field: unesc(parts[2])?,
306        ty: ValType::parse(parts[3].as_bytes())?,
307        kind,
308        max_bytes: parts[5].parse().ok()?,
309        ann,
310        group_by,
311    })
312}
313
314fn esc(b: &[u8]) -> String {
315    let mut out = String::with_capacity(b.len());
316    for &c in b {
317        if c == b'\t' || c == b'\n' || c == b'%' || !(32..127).contains(&c) {
318            let _ = write!(out, "%{c:02X}");
319        } else {
320            out.push(c as char);
321        }
322    }
323    out
324}
325
326fn unesc(s: &str) -> Option<Vec<u8>> {
327    let mut out = Vec::with_capacity(s.len());
328    let bytes = s.as_bytes();
329    let mut i = 0;
330    while i < bytes.len() {
331        if bytes[i] == b'%' {
332            let hex = s.get(i + 1..i + 3)?;
333            out.push(u8::from_str_radix(hex, 16).ok()?);
334            i += 3;
335        } else {
336            out.push(bytes[i]);
337            i += 1;
338        }
339    }
340    Some(out)
341}
342
343#[cfg(test)]
344mod tests {
345    use super::*;
346
347    fn spec(name: &str, prefix: &str) -> IndexSpec {
348        IndexSpec {
349            name: name.into(),
350            prefix: prefix.into(),
351            field: b"age".to_vec(),
352            ty: ValType::I64,
353            kind: IndexKind::Range,
354            ann: None,
355            max_bytes: 0,
356            group_by: None,
357        }
358    }
359
360    #[test]
361    fn create_drop_match_lifecycle() {
362        let mut c = Catalog::new();
363        c.create(spec("a", "user:")).unwrap();
364        c.create(spec("b", "sess:")).unwrap();
365        assert!(c.create(spec("a", "x:")).is_err(), "dup name");
366        assert_eq!(c.matching(b"user:42").count(), 1);
367        assert_eq!(c.matching(b"other:1").count(), 0);
368        assert_eq!(c.get(b"a").unwrap().1, IndexState::Building);
369        assert!(c.set_state(b"a", IndexState::Ready));
370        assert_eq!(c.get(b"a").unwrap().1, IndexState::Ready);
371        assert!(c.drop_index(b"b"));
372        assert!(!c.drop_index(b"b"));
373        assert_eq!(c.len(), 1);
374    }
375
376    #[test]
377    fn sidecar_roundtrip_with_escapes() {
378        let mut c = Catalog::new();
379        let mut s = spec("weird", "pre\tfix:");
380        s.field = b"f%\n".to_vec();
381        s.max_bytes = 1024;
382        c.create(s).unwrap();
383        let text = c.to_sidecar();
384        let c2 = Catalog::from_sidecar(&text).unwrap();
385        let (got, st) = c2.get(b"weird").unwrap();
386        assert_eq!(got.prefix, b"pre\tfix:".to_vec());
387        assert_eq!(got.field, b"f%\n".to_vec());
388        assert_eq!(got.max_bytes, 1024);
389        assert_eq!(st, IndexState::Building, "boot loads as Building");
390        assert!(Catalog::from_sidecar("bogus").is_none());
391    }
392
393    #[test]
394    fn cap_enforced() {
395        let mut c = Catalog::new();
396        for i in 0..MAX_INDEXES {
397            c.create(spec(&format!("i{i}"), "p:")).unwrap();
398        }
399        assert!(c.create(spec("over", "p:")).is_err());
400    }
401}
kevy_index/catalog.rs

kevy_index/
catalog.rs
