ingraph 0.1.0

A declaration-driven indexing framework: records and nodes are written as Rust types, and the storage rows, the dataflow between them and the GraphQL query surface are emitted from the declaration.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
//! The SET road, END TO END against the live store: an entity holding several embeddings, asked
//! with several, folded, and hydrated out of a real SQLite corpus.
//!
//! The vector mirror beside this file proves the pipeline is flavour-blind. What it cannot prove is
//! the claim this round makes, because that claim is about a grain it does not have: the engine here
//! answers at the ELEMENT grain — one hit per stored set member — and several of its hits name one
//! entity. Everything downstream has to be right about that:
//!
//!   * [`owner_best`](crate::retrieve::owner_best) collapses each query vector's element stream to
//!     one best per OWNER, which is the dedup stage read one grain up;
//!   * [`set_scored`](crate::retrieve::set_scored) folds the `m` per-owner bests into ONE score per
//!     entity, under the word the caller passed;
//!   * `limit` then counts ENTITIES, and the same live store hydrates them.
//!
//! # The physical form here is the UNIVERSAL one, and that is the delegation law's own default
//!
//! One row per member — owner key, ordinal, element vector — because that is what an engine without
//! native multi-vector scoring gets, and there is no engine with native multi-vector scoring in this
//! tree to take the other road. The two roads produce the same primitive by construction, so the
//! fold below is the half both share; the native half is an engine's own and has no home here yet.

use core::num::NonZeroUsize;
use std::sync::Arc;

use sqlx::Sqlite;

use super::corpus::{Segment, SegmentKey, SegmentRow, forget, shorter_than, store};
use crate::{
  AsyncVectorRead, AsyncVectorWrite, Connected, Embedding, Field, HashBests, HashSeen, Hits,
  Ranked, Scored, SlotHandle, VectorColumn, VectorEngine, VectorRecord,
  retrieve::{
    AsyncGather, FoldSemantics, Folded, FoldedQuery, Max, SqlGather, Sum, VectorQuery, owner_best,
    ranked_keys, set_scored, vector_keyed,
  },
  store::column::testkit::{Col, Fallback, FallbackRead, Refusal},
};

const DIM: usize = 4;

/// The one column this declaration elects, named exactly as the emission mints it: off the elected
/// field's own ident. Its TYPE is what says the column holds a SET.
const FRAME_EMBEDDINGS: VectorColumn = VectorColumn::new("frame_embeddings");

/// The DERIVED INDEX, written as its expansion writes it — the set form of the mirror beside this
/// file.
///
/// ```ignore
/// #[ingraph::vector(
///   upstream(node = "Segment", frame_embeddings = "frame_embeddings"),
///   embedding(frame_embeddings),
///   dedup(hash),
/// )]
/// pub struct Keyframes { frame_embeddings: Vec<Embedding<f32, 4>> }
/// ```
struct Keyframes {
  key: SegmentKey,
  frame_embeddings: Vec<Embedding<f32, DIM>>,
}

impl Keyframes {
  fn of(media: &str, id: i64, frame_embeddings: Vec<[f32; DIM]>) -> Self {
    Self {
      key: SegmentKey {
        media: media.to_owned(),
        id,
      },
      frame_embeddings: frame_embeddings
        .into_iter()
        .map(|coordinates| {
          Embedding::new(Arc::from(coordinates.to_vec())).expect("the fixture declares DIM")
        })
        .collect(),
    }
  }
}

/// The emitted `VectorRecord`, as the expansion writes it for a SET election: `vectors` reports
/// nothing — this declaration elects no single embedding — and `vector_sets` reports every member
/// under its ordinal, in field order.
impl VectorRecord for Keyframes {
  type Element = f32;
  type Filter = ();

  fn vectors(&self) -> impl Iterator<Item = (VectorColumn, &[f32])> {
    core::iter::empty()
  }

  fn vector_sets(&self) -> impl Iterator<Item = (VectorColumn, u32, &[f32])> {
    (0u32..)
      .zip(&self.frame_embeddings)
      .map(|(at, member)| (FRAME_EMBEDDINGS, at, member.coordinates()))
  }
}

/// A keyed table on the MULTI-ROW form: one row per set member, holding the owner key, the ordinal
/// and the coordinates, and answering a `knn` with a real cosine, DESCENDING.
///
/// The ordinal is a column of the ROW and not of the ADDRESS. That split is the whole reason the
/// existing row family ([`Field`](crate::Field)) needs no change for this round: a hit points at the
/// ENTITY, and which of its members produced the hit is a fact the engine used and the framework
/// never asks for.
struct Frames(Vec<Row>);

struct Row {
  owner: SegmentKey,
  ordinal: u32,
  coordinates: Vec<f32>,
}

/// `key`, sealed into this engine's address — the write half of the lifecycle, spelled here as the
/// framework's own walk spells it.
fn addressed(key: &SegmentKey) -> Vec<Col> {
  let mut buf = Frames::buffer();
  key.write(&mut buf);
  Frames::finalize(buf)
}

impl VectorEngine for Frames {
  type Options = ();
  type SearchOptions = ();
  type Error = Refusal;
  type Score = f32;
  type Element = f32;
  type Address = Vec<Col>;
  type Connection = ();
  type WriteBuffer = Fallback;
  type ReadBuffer = FallbackRead;

  fn buffer() -> Self::WriteBuffer {
    Fallback::default()
  }

  fn finalize(buf: Self::WriteBuffer) -> Self::Address {
    buf.0
  }

  fn open(address: Self::Address) -> Self::ReadBuffer {
    Fallback(address).read()
  }
}

impl AsyncVectorWrite for Frames {
  type Doc = Entry;
  type WriteContext<'a> = ();

  fn upsert(
    &self,
    _ctx: Connected<'_, (), Self::WriteContext<'_>>,
    _doc: Self::Doc,
  ) -> impl Future<Output = Result<(), Self::Error>> + Send {
    core::future::ready(Ok(()))
  }
}

impl AsyncVectorRead for Frames {
  type ReadContext<'a> = ();

  fn knn(
    &self,
    _ctx: Connected<'_, (), Self::ReadContext<'_>>,
    column: VectorColumn,
    query: &[Self::Element],
    _k: ::core::num::NonZeroUsize,
    _options: ::core::option::Option<&Self::SearchOptions>,
  ) -> impl Future<Output = Result<Hits<Self::Address, Self::Score>, Self::Error>> + Send {
    let mut hits: Vec<Scored<Vec<Col>, f32>> = self
      .0
      .iter()
      .filter(|_| column == FRAME_EMBEDDINGS)
      .map(|row| Scored::new(addressed(&row.owner), cosine(&row.coordinates, query)))
      .collect();
    hits.sort_by(|a, b| b.score().total_cmp(&a.score()));
    core::future::ready(Ok(hits))
  }

  /// The BY-KEY READ, answered off the same rows `knn` scores — so this walk's entity holds a SET
  /// here too, and the door that reads it says so. Every member of the owner's set comes back, in
  /// the ordinal order the rows were written in, which is what a set election's own read means.
  fn get(
    &self,
    _ctx: Connected<'_, (), Self::ReadContext<'_>>,
    column: VectorColumn,
    address: &Self::Address,
  ) -> impl Future<Output = Result<Vec<Vec<Self::Element>>, Self::Error>> + Send {
    let mut held: Vec<&Row> = self
      .0
      .iter()
      .filter(|row| column == FRAME_EMBEDDINGS && *address == addressed(&row.owner))
      .collect();
    held.sort_by_key(|row| row.ordinal);
    core::future::ready(Ok(
      held
        .into_iter()
        .map(|row| row.coordinates.clone())
        .collect(),
    ))
  }
}

/// The engine's own entry — the owner's address and every member of its set, each under its ordinal.
struct Entry {
  owner: SegmentKey,
  members: Vec<(VectorColumn, u32, Vec<f32>)>,
}

impl Frames {
  /// The KEYED REPLACE, at the identity the entry carries — and the identity is the PAIR.
  ///
  /// This is the one line the multi-row form adds to a writer, and getting it wrong is not a
  /// degradation: a replace matching on the OWNER alone collapses a whole set to whichever member
  /// was written last, silently, and the entity then answers every query with one vector. The
  /// ordinal is what makes the row unique, which is why it is stored beside the coordinates rather
  /// than derived from a position in a batch.
  fn upsert_entry(&mut self, entry: Entry) {
    for (column, ordinal, coordinates) in entry.members {
      assert_eq!(
        column, FRAME_EMBEDDINGS,
        "the declaration elects one column"
      );
      self
        .0
        .retain(|row| !(row.owner == entry.owner && row.ordinal == ordinal));
      self.0.push(Row {
        owner: entry.owner.clone(),
        ordinal,
        coordinates,
      });
    }
  }
}

fn cosine(a: &[f32], b: &[f32]) -> f32 {
  let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum();
  let norm = |v: &[f32]| v.iter().map(|x| x * x).sum::<f32>().sqrt();
  let scale = norm(a) * norm(b);
  if scale == 0.0 { 0.0 } else { dot / scale }
}

/// The index over the seeded corpus, written THROUGH the record face — so the rows this engine holds
/// are exactly what an emitted `vector_sets` handed a writer, ordinals included.
///
/// Segment 1's frames lean toward both query directions; segment 2 has one frame almost exactly on
/// the first direction and one on neither; segments 3–6 lean progressively further away.
fn corpus() -> [Keyframes; 6] {
  [
    Keyframes::of("m1", 1, vec![[1.0, 0.6, 0.0, 0.0], [0.6, 1.0, 0.0, 0.0]]),
    Keyframes::of("m1", 2, vec![[1.0, 0.02, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]]),
    Keyframes::of("m2", 3, vec![[1.0, 0.45, 0.0, 0.0]]),
    Keyframes::of("m2", 4, vec![[1.0, 0.80, 0.0, 0.0]]),
    Keyframes::of("m3", 5, vec![[1.0, 1.40, 0.0, 0.0]]),
    Keyframes::of("m3", 6, vec![]),
  ]
}

/// THE WRITE HALF, performed as a writer performs it: the entry built off the emitted `vector_sets`,
/// and the keyed replace at the pair that entry carries.
fn write(engine: &mut Frames, record: &Keyframes) {
  engine.upsert_entry(Entry {
    owner: record.key.clone(),
    members: record
      .vector_sets()
      .map(|(column, ordinal, coordinates)| (column, ordinal, coordinates.to_vec()))
      .collect(),
  });
}

fn index() -> SlotHandle<Frames, Keyframes> {
  let mut engine = Frames(Vec::new());
  for record in &corpus() {
    write(&mut engine, record);
  }
  // MINTED by hand, as both mirrors do — the fixture standing in for the launch walk.
  SlotHandle::mint(engine)
}

fn set_query(directions: Vec<[f32; DIM]>, floor: f32) -> FoldedQuery<f32, f32, DIM> {
  FoldedQuery::new(
    directions
      .into_iter()
      .map(|coordinates| {
        Embedding::new(Arc::from(coordinates.to_vec())).expect("the fixture declares DIM")
      })
      .collect(),
    floor,
  )
  .expect("the fixture asks with at least one vector")
}

/// TIER A, SET — exactly as the emitted `retrieve_keys_by_frame_embeddings_with` spells it: one engine call
/// per query vector, each collapsed to its per-owner bests, and the fold over all of them.
async fn retrieve_keys_with<F: FoldSemantics<f32>>(
  slot: &mut SlotHandle<Frames, Keyframes>,
  query: &FoldedQuery<f32, f32, DIM>,
  fold: &F,
  limit: usize,
) -> Vec<Folded<SegmentKey, f32>> {
  // THE ZERO FORK, spelled as the emission spells it: above the engine, once, before the loop.
  let Some(k) = NonZeroUsize::new(limit) else {
    return Vec::new();
  };
  let mut bests = Vec::with_capacity(query.inputs().len());
  for coordinates in query.coordinates() {
    let hits = slot
      .knn(
        Connected::new(&(), ()),
        FRAME_EMBEDDINGS,
        coordinates,
        k,
        None,
      )
      .await
      .expect("the toy index cannot fail");
    let hits = vector_keyed::<Frames, SegmentKey>(hits).expect("the addresses read back");
    bests.push(owner_best::<_, _, HashSeen<_>>(hits, query.floor()));
  }
  set_scored::<_, _, HashBests<_, _>, _>(bests, fold, limit)
}

/// The single-vector door beside it, spelled as the emission spells it — unchanged by this round,
/// and the thing the `m = 1` coincidence is measured against.
async fn retrieve_keys(
  slot: &mut SlotHandle<Frames, Keyframes>,
  query: &VectorQuery<f32, f32, DIM>,
  limit: usize,
) -> Vec<Ranked<SegmentKey, f32>> {
  let Some(k) = NonZeroUsize::new(limit) else {
    return Vec::new();
  };
  let hits = slot
    .knn(
      Connected::new(&(), ()),
      FRAME_EMBEDDINGS,
      query.coordinates(),
      k,
      None,
    )
    .await
    .expect("the toy index cannot fail");
  let hits = vector_keyed::<Frames, SegmentKey>(hits).expect("the addresses read back");
  ranked_keys::<_, _, HashSeen<_>>(hits, query.floor(), limit)
}

fn ids<S>(page: &[Folded<SegmentKey, S>]) -> Vec<i64> {
  page.iter().map(|hit| hit.node_ref().id).collect()
}

/// **The element grain never reaches the answer.** Segment 1 owns two frames above the floor and
/// segment 2 owns one; each is ONE result, which is what *`k` counts entities* means when the engine
/// answers per member.
#[tokio::test]
async fn an_entity_with_several_members_is_one_result() {
  let mut engine = index();

  let page = retrieve_keys_with(
    &mut engine,
    &set_query(vec![[1.0, 0.0, 0.0, 0.0]], 0.5),
    &Sum::new(),
    usize::MAX,
  )
  .await;

  let mut owners = ids(&page);
  owners.sort_unstable();
  owners.dedup();
  assert_eq!(owners.len(), ids(&page).len(), "{page:?}");
}

/// **THE KEYED REPLACE IS AT THE PAIR, and a re-write proves it.** Writing an entity's whole set a
/// second time replaces its members one for one; a replace matching on the OWNER alone would leave
/// the set holding a single member — the last one written — and every subsequent answer about that
/// entity would be quietly derived from one vector.
///
/// This is the multi-row form's own version of a law the engine audit already found on the native
/// side: a keyed store that deduplicates by the wrong key does not fail, it answers.
#[tokio::test]
async fn re_writing_a_set_replaces_its_members_rather_than_collapsing_it() {
  let mut engine = Frames(Vec::new());
  let record = Keyframes::of("m1", 1, vec![[1.0, 0.6, 0.0, 0.0], [0.6, 1.0, 0.0, 0.0]]);
  write(&mut engine, &record);
  write(&mut engine, &record);

  assert_eq!(engine.0.len(), 2, "one row per member, written twice");
  let mut ordinals: Vec<u32> = engine.0.iter().map(|row| row.ordinal).collect();
  ordinals.sort_unstable();
  assert_eq!(ordinals, vec![0, 1]);
}

/// **`Sum` is late interaction**: segment 1 answers BOTH directions moderately and outranks segment
/// 2, whose best frame is nearly perfect on one direction and absent from the other.
#[tokio::test]
async fn sum_ranks_the_entity_that_answered_both_directions_first() {
  let mut engine = index();

  let page = retrieve_keys_with(
    &mut engine,
    &set_query(vec![[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]], 0.5),
    &Sum::new(),
    10,
  )
  .await;

  assert_eq!(ids(&page)[0], 1, "{page:?}");
  assert!(ids(&page).contains(&2), "{page:?}");
}

/// **`Max` reverses it**: the single best pair wins, and segment 2's near-exact frame is it.
#[tokio::test]
async fn max_ranks_the_single_best_pair_first() {
  let mut engine = index();

  let page = retrieve_keys_with(
    &mut engine,
    &set_query(vec![[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]], 0.5),
    &Max::new(),
    10,
  )
  .await;

  assert_eq!(ids(&page)[0], 2, "{page:?}");
}

/// **An EMPTY set is never similarity-matched.** Segment 6 declares no frame_embeddings, so it contributes no
/// rows and no floor could admit it — the legality of the empty collection and its absence from
/// every answer are the same fact.
#[tokio::test]
async fn an_entity_with_an_empty_set_is_never_matched() {
  let mut engine = index();

  let page = retrieve_keys_with(
    &mut engine,
    &set_query(vec![[1.0, 0.0, 0.0, 0.0]], 0.0),
    &Sum::new(),
    usize::MAX,
  )
  .await;

  assert!(!ids(&page).contains(&6), "{page:?}");
}

/// **The `m = 1` coincidence, against the door beside it — and now FIELD-EXACT.** One query vector
/// makes every fold agree, and the agreement is with the single-vector road's own answer including
/// its engine ordinal: 单门.rank == 集合门.ranks[0].unwrap(), because the set road attributes each
/// stream's ordinal rather than dropping it.
#[tokio::test]
async fn one_query_vector_agrees_with_the_single_vector_road() {
  let mut engine = index();
  let direction = [1.0, 0.0, 0.0, 0.0];
  let floor = 0.5;

  let single = retrieve_keys(
    &mut engine,
    &VectorQuery::new(
      Embedding::new(Arc::from(direction.to_vec())).expect("the fixture declares DIM"),
      floor,
    ),
    usize::MAX,
  )
  .await;

  /// The comparison itself, written once and instantiated per fold — a `&dyn` list would have
  /// needed a reference to fold, which is a widening this round has no call for.
  async fn agrees<F: FoldSemantics<f32>>(
    slot: &mut SlotHandle<Frames, Keyframes>,
    query: &FoldedQuery<f32, f32, DIM>,
    fold: &F,
    single: &[Ranked<SegmentKey, f32>],
  ) {
    let set = retrieve_keys_with(slot, query, fold, usize::MAX).await;

    assert_eq!(set.len(), single.len());
    for (folded, ranked) in set.iter().zip(single) {
      assert_eq!(folded.node_ref(), ranked.node_ref());
      assert!((folded.score() - ranked.score()).abs() < f32::EPSILON);
      let [Some(only)] = folded.streams() else {
        panic!("one stream, and its own slot: {:?}", folded.streams())
      };
      assert_eq!(
        only.ordinal(),
        ranked.rank(),
        "one stream, and its own ordinal carried whole"
      );
      assert!(
        (only.score() - ranked.score()).abs() < f32::EPSILON,
        "the pre-fold score the stream measured is the single-vector road's own",
      );
    }
  }

  let query = set_query(vec![direction], floor);
  agrees(&mut engine, &query, &Sum::new(), &single).await;
  agrees(&mut engine, &query, &Max::new(), &single).await;
}

/// **THE LIVE HOP**: the folded keys hydrate out of the real SQLite corpus, through the same gather
/// the single-vector road's tier B uses.
///
/// There is no emitted tier-B door on this road — the ruling names two, both at tier A — so this is
/// what a caller writes, and it is one call: the whole set query's survivors, gathered positionally,
/// carrying the set scores the fold gave them.
#[tokio::test]
async fn the_folded_keys_hydrate_from_the_live_store() {
  let mut engine = index();
  let mut db = store().await;

  let page = retrieve_keys_with(
    &mut engine,
    &set_query(vec![[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]], 0.5),
    &Sum::new(),
    10,
  )
  .await;

  let keys: Vec<SegmentKey> = page.iter().map(|hit| hit.node_ref().clone()).collect();
  let mut gather = SqlGather::<SegmentRow, Sqlite>::new(&mut db);
  let rows: Vec<Option<Segment>> = AsyncGather::gather(&mut gather, &keys, None)
    .await
    .expect("the live store answers");

  assert_eq!(rows.len(), keys.len());
  let first = rows[0].as_ref().expect("the store holds segment 1");
  assert_eq!(first.id, 1);
  assert_eq!(first.transcript, "a long sunset over the harbour");
}

/// **The filter and the missing row behave as they do on every other road** — the set score does not
/// change what a gather is, so a pruned row and a forgotten one leave the same hole.
#[tokio::test]
async fn the_gather_prunes_and_forgets_the_same_way_under_a_set_score() {
  let mut engine = index();
  let mut db = store().await;
  forget(&mut db, 2).await;

  let page = retrieve_keys_with(
    &mut engine,
    &set_query(vec![[1.0, 0.0, 0.0, 0.0]], 0.5),
    &Sum::new(),
    usize::MAX,
  )
  .await;
  let keys: Vec<SegmentKey> = page.iter().map(|hit| hit.node_ref().clone()).collect();
  let short = shorter_than(15);
  let mut gather = SqlGather::<SegmentRow, Sqlite>::new(&mut db);
  let rows: Vec<Option<Segment>> = AsyncGather::gather(&mut gather, &keys, Some(&short))
    .await
    .expect("the live store answers");

  assert_eq!(rows.len(), keys.len());
  // Segment 2 was forgotten and every long segment is pruned, so the survivors are exactly the short
  // ones the store still holds.
  for (key, row) in keys.iter().zip(&rows) {
    if key.id == 2 {
      assert!(row.is_none(), "a forgotten row leaves a hole");
    }
  }
  assert!(rows.iter().any(Option::is_some), "{rows:?}");
}