ingraph 0.1.0

A declaration-driven indexing framework: records and nodes are written as Rust types, and the storage rows, the dataflow between them and the GraphQL query surface are emitted from the declaration.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
//! The vector MIRROR: the same pipeline, the same store, the same assertions — asked of a vector
//! face instead of a full-text one.
//!
//! The engine here is in-tree and deliberately small: a keyed embedding table that answers a `knn`
//! with cosine-shaped scores it computed itself. What it proves is not a backend — it is that
//! NOTHING about the flavour reaches the pipeline. The floor, the dedup, the ranks, the composed
//! gather and the fill loop are the same code as the full-text walk's, reached through
//! [`VectorEngine`](crate::VectorEngine) rather than through
//! [`FulltextEngine`](crate::FulltextEngine).

use core::num::NonZeroUsize;
use std::sync::Arc;

use sqlx::Sqlite;

use super::corpus::{Segment, SegmentFilter, SegmentKey, SegmentRow, forget, shorter_than, store};
use crate::{
  AsyncVectorRead, AsyncVectorWrite, Connected, Embedding, Field, Hits, Identifiable, Ranked,
  Scored, SlotHandle, Transformer, VectorColumn, VectorEngine, VectorRecord,
  retrieve::{
    EqSeen, HashSeen, RetrieveError, Seen, SqlGather, VectorQuery, filled, ranked_keys,
    vector_keyed,
  },
  store::column::testkit::{Col, Fallback, FallbackRead, Refusal},
};

/// The width every fixture here declares — spelled ONCE, so the record type, the query and the
/// assertions cannot drift apart by hand.
const DIM: usize = 4;

/// The one column this declaration elects, named exactly as the emission mints it: off the elected
/// field's own ident.
const EMBEDDING: VectorColumn = VectorColumn::new("embedding");

/// The DERIVED INDEX, written as its expansion writes it: the upstream's key carried whole, and the
/// embedding read out of the upstream.
///
/// ```ignore
/// #[ingraph::vector(upstream(node = "Segment", embedding = "embedding"), dedup(hash))]
/// pub struct Voiceprint {
///   #[ingraph(knn)]
///   embedding: Embedding<f32, 4>,
/// }
/// ```
pub(super) struct Voiceprint {
  key: SegmentKey,
  embedding: Embedding<f32, DIM>,
}

impl Voiceprint {
  pub(super) fn of(media: &str, id: i64, coordinates: [f32; DIM]) -> Self {
    Self {
      key: SegmentKey {
        media: media.to_owned(),
        id,
      },
      embedding: Embedding::new(Arc::from(coordinates.to_vec())).expect("the fixture declares DIM"),
    }
  }
}

impl Identifiable for Voiceprint {
  type Id = <SegmentKey as Identifiable>::Id;

  fn id_ref(&self) -> &i64 {
    Identifiable::id_ref(&self.key)
  }
}

impl Transformer for Voiceprint {
  type Key = SegmentKey;
  type Lineage = <Segment as Transformer>::Lineage;
  type Record = ();
  type Graphql = ();

  fn key(&self) -> SegmentKey {
    self.key.clone()
  }
}

impl VectorRecord for Voiceprint {
  type Element = f32;
  type Filter = ();

  fn vectors(&self) -> impl Iterator<Item = (VectorColumn, &[f32])> {
    core::iter::once((EMBEDDING, self.embedding.coordinates()))
  }
}

impl Voiceprint {
  /// This declaration's one elected column, as the engine below reads it back.
  fn coordinates(&self) -> &[f32] {
    self
      .vectors()
      .find(|(column, _)| *column == EMBEDDING)
      .expect("the declaration elects it")
      .1
  }
}

/// A keyed embedding table that answers with a real similarity, DESCENDING — the precondition the
/// whole pipeline reads.
///
/// It holds its entries as written, duplicates included, which is the honest shape: the store faces
/// are open traits, and a backend owes the framework no uniqueness across the sources sharing a
/// slot. That is what the dedup stage exists for, and this is where it is asked for it.
pub(super) struct Voiceprints(Vec<Voiceprint>);

/// `key`, sealed into this engine's address — the write half of the lifecycle, spelled here as the
/// framework's own walk spells it.
fn addressed(key: &SegmentKey) -> Vec<Col> {
  let mut buf = Voiceprints::buffer();
  key.write(&mut buf);
  Voiceprints::finalize(buf)
}

impl VectorEngine for Voiceprints {
  type Options = ();
  type SearchOptions = ();
  type Error = Refusal;
  type Score = f32;
  type Element = f32;

  /// The address it hands a hit back under — its OWN vocabulary, a column list, exactly as the
  /// full-text toy's is. It is not `SegmentKey`: an engine that handed keys back would let this file
  /// skip the address read the emitted method performs, and the point of the file is that it spells
  /// what the emission spells.
  type Address = Vec<Col>;

  type Connection = ();
  type WriteBuffer = Fallback;
  type ReadBuffer = FallbackRead;

  fn buffer() -> Self::WriteBuffer {
    Fallback::default()
  }

  fn finalize(buf: Self::WriteBuffer) -> Self::Address {
    buf.0
  }

  fn open(address: Self::Address) -> Self::ReadBuffer {
    Fallback(address).read()
  }
}

impl AsyncVectorWrite for Voiceprints {
  /// The entry an emission builds: this table stores whole voiceprints, so its own entry IS one.
  type Doc = Voiceprint;

  type WriteContext<'a> = ();

  /// A READY future rather than an `async fn`, which is the shape every synchronous face in this
  /// crate takes: the work is done inline and the outcome is wrapped, so the `+ Send` the seam
  /// declares is provable without asking anything of anybody.
  fn upsert(
    &self,
    _ctx: Connected<'_, (), Self::WriteContext<'_>>,
    _doc: Self::Doc,
  ) -> impl Future<Output = Result<(), Self::Error>> + Send {
    core::future::ready(Ok(()))
  }
}

impl AsyncVectorRead for Voiceprints {
  type ReadContext<'a> = ();

  /// One column, and it is READ rather than assumed: a slot may hold several, so an engine that
  /// ignored the selector would answer the wrong one the day a second election arrived. This table
  /// holds the one its declaration elects and refuses nothing else by refusing to score anything.
  fn knn(
    &self,
    _ctx: Connected<'_, (), Self::ReadContext<'_>>,
    column: VectorColumn,
    query: &[Self::Element],
    _k: ::core::num::NonZeroUsize,
    _options: ::core::option::Option<&Self::SearchOptions>,
  ) -> impl Future<Output = Result<Hits<Self::Address, Self::Score>, Self::Error>> + Send {
    let mut hits: Vec<Scored<Vec<Col>, f32>> = self
      .0
      .iter()
      .filter(|_| column == EMBEDDING)
      .map(|held| Scored::new(addressed(&held.key), cosine(held.coordinates(), query)))
      .collect();
    hits.sort_by(|a, b| b.score().total_cmp(&a.score()));
    core::future::ready(Ok(hits))
  }

  /// The BY-KEY READ, answered off the same table `knn` scores — so an address this face handed a
  /// hit back under is an address it can be asked about, which is the correspondence the seam names.
  ///
  /// It answers with EVERY entry at that key, duplicates included, exactly as `knn` does: this table
  /// holds what it was written and owes no uniqueness, which is what lets the similar door's own
  /// arity refusal be reachable rather than theoretical.
  fn get(
    &self,
    _ctx: Connected<'_, (), Self::ReadContext<'_>>,
    column: VectorColumn,
    address: &Self::Address,
  ) -> impl Future<Output = Result<Vec<Vec<Self::Element>>, Self::Error>> + Send {
    core::future::ready(Ok(
      self
        .0
        .iter()
        .filter(|held| column == EMBEDDING && *address == addressed(&held.key))
        .map(|held| held.coordinates().to_vec())
        .collect(),
    ))
  }
}

/// Cosine similarity — higher is closer, which is the direction this face reports in and which the
/// floor is therefore read against. A face measuring a DISTANCE would report the other way, and the
/// framework would not reinterpret that either.
fn cosine(a: &[f32], b: &[f32]) -> f32 {
  let dot: f32 = a.iter().zip(b).map(|(x, y)| x * y).sum();
  let norm = |v: &[f32]| v.iter().map(|x| x * x).sum::<f32>().sqrt();
  let scale = norm(a) * norm(b);
  if scale == 0.0 { 0.0 } else { dot / scale }
}

/// The index over the seeded corpus: each segment's embedding leans a little further from the query
/// direction than the last, so the answer's order is known before the engine is asked.
pub(super) fn index(
  extra: impl IntoIterator<Item = Voiceprint>,
) -> SlotHandle<Voiceprints, Voiceprint> {
  let mut held = vec![
    Voiceprint::of("m1", 1, [1.0, 0.00, 0.0, 0.0]),
    Voiceprint::of("m1", 2, [1.0, 0.20, 0.0, 0.0]),
    Voiceprint::of("m2", 3, [1.0, 0.45, 0.0, 0.0]),
    Voiceprint::of("m2", 4, [1.0, 0.80, 0.0, 0.0]),
    Voiceprint::of("m3", 5, [1.0, 1.40, 0.0, 0.0]),
    Voiceprint::of("m3", 6, [1.0, 2.60, 0.0, 0.0]),
  ];
  held.extend(extra);
  // MINTED by hand, as the full-text mirror does — the fixture standing in for the launch walk.
  SlotHandle::mint(Voiceprints(held))
}

pub(super) fn query(floor: f32) -> VectorQuery<f32, f32, DIM> {
  VectorQuery::new(
    Embedding::new(Arc::from(vec![1.0, 0.0, 0.0, 0.0])).expect("the fixture declares DIM"),
    floor,
  )
}

/// TIER A, exactly as the emitted `retrieve_keys_by_embedding` spells it — the election's own column
/// minted at the call, not taken from one.
pub(super) async fn retrieve_keys<D>(
  slot: &mut SlotHandle<Voiceprints, Voiceprint>,
  query: &VectorQuery<f32, f32, DIM>,
  limit: usize,
) -> Vec<Ranked<SegmentKey, f32>>
where
  D: Seen<SegmentKey>,
{
  let Some(k) = NonZeroUsize::new(limit) else {
    return Vec::new();
  };
  let hits = slot
    .knn(
      Connected::new(&(), ()),
      EMBEDDING,
      query.coordinates(),
      k,
      None,
    )
    .await
    .expect("the toy index cannot fail");
  let hits = vector_keyed::<Voiceprints, SegmentKey>(hits).expect("the addresses read back");
  ranked_keys::<_, _, D>(hits, query.floor(), limit)
}

/// TIER B, exactly as the emitted inherent method spells it.
async fn retrieve(
  slot: &mut SlotHandle<Voiceprints, Voiceprint>,
  db: &mut sqlx::sqlite::SqliteConnection,
  query: &VectorQuery<f32, f32, DIM>,
  filter: Option<&SegmentFilter<Sqlite>>,
  limit: usize,
) -> Result<
  Vec<Ranked<Segment, f32>>,
  RetrieveError<core::convert::Infallible, crate::query::sql::SqlQueryError>,
> {
  let ranked = retrieve_keys::<HashSeen<_>>(slot, query, usize::MAX).await;
  let mut gather = SqlGather::<SegmentRow, Sqlite>::new(db);
  filled(ranked, &mut gather, filter, limit)
    .await
    .map_err(RetrieveError::Gather)
}

fn ids(page: &[Ranked<Segment, f32>]) -> Vec<i64> {
  page.iter().map(|hit| hit.node_ref().id).collect()
}

/// **Tier A: the floor makes `k` a ceiling rather than a promise.** An ANN answers with every
/// neighbour it holds however poor; the floor is what turns that into *at most this many, none of
/// them noise*.
#[tokio::test]
async fn the_floor_turns_k_into_a_ceiling() {
  let mut engine = index([]);

  let unfloored = retrieve_keys::<HashSeen<_>>(&mut engine, &query(0.0), usize::MAX).await;
  assert_eq!(unfloored.len(), 6, "knn answers with everything it holds");

  let floored = retrieve_keys::<HashSeen<_>>(&mut engine, &query(0.9), usize::MAX).await;

  assert_eq!(
    floored
      .iter()
      .map(|hit| hit.node_ref().id)
      .collect::<Vec<_>>(),
    vec![1, 2, 3]
  );
  assert_eq!(
    floored.iter().map(Ranked::rank).collect::<Vec<_>>(),
    vec![1, 2, 3]
  );
}

/// **Tier A: a duplicate key collapses to its best sighting, and the rank hole stays.**
#[tokio::test]
async fn a_duplicate_key_collapses_and_leaves_its_rank_behind() {
  // The same key as the third entry, further from the query — so it lands BELOW its twin and the
  // survivor must be the first sighting.
  let mut engine = index([Voiceprint::of("m2", 3, [1.0, 1.90, 0.0, 0.0])]);

  let ranked = retrieve_keys::<HashSeen<_>>(&mut engine, &query(0.0), usize::MAX).await;

  assert_eq!(ranked.len(), 6, "seven hits, one duplicate key");
  assert_eq!(
    ranked
      .iter()
      .map(|hit| hit.node_ref().id)
      .collect::<Vec<_>>(),
    vec![1, 2, 3, 4, 5, 6]
  );
  // The duplicate stood at position 6 of the raw stream and no survivor wears it.
  assert_eq!(
    ranked.iter().map(Ranked::rank).collect::<Vec<_>>(),
    vec![1, 2, 3, 4, 5, 7]
  );
  // And the kept sighting is the BEST one: segment 3's score is its close embedding's, not the far
  // twin's.
  let kept = ranked
    .iter()
    .find(|hit| hit.node_ref().id == 3)
    .expect("the survivor");
  assert!(kept.score() > 0.9, "{kept:?}");
}

/// **Tier B: whole entities, engine order, engine ranks — the mirror of the full-text walk.**
#[tokio::test]
async fn tier_b_answers_whole_entities_in_engine_order() {
  let mut engine = index([]);
  let mut db = store().await;

  let page = retrieve(&mut engine, &mut db, &query(0.0), None, 10)
    .await
    .expect("the retrieve runs");

  assert_eq!(ids(&page), vec![1, 2, 3, 4, 5, 6]);
  assert_eq!(
    page.iter().map(Ranked::rank).collect::<Vec<_>>(),
    vec![1, 2, 3, 4, 5, 6]
  );
  assert_eq!(
    page[0].node_ref().transcript,
    "a long sunset over the harbour"
  );
}

/// **Tier B: the filter prunes mid-page and the loop goes back for more keys.**
#[tokio::test]
async fn a_filter_forces_a_second_key_batch() {
  let mut engine = index([]);
  let mut db = store().await;
  let short = shorter_than(15);

  let page = retrieve(&mut engine, &mut db, &query(0.0), Some(&short), 2)
    .await
    .expect("the retrieve runs");

  // Segments 2 (12s) and 4 (8s) are the only ones under the bound; 2 stands at rank 2 and 4 at rank
  // 4, so the page's numbers carry the pruned rows' holes.
  assert_eq!(ids(&page), vec![2, 4]);
  assert_eq!(
    page.iter().map(Ranked::rank).collect::<Vec<_>>(),
    vec![2, 4]
  );
}

/// **Tier B: a store-missing row is skipped exactly as a prune is.**
#[tokio::test]
async fn a_store_missing_row_is_skipped_and_the_page_fills() {
  let mut engine = index([]);
  let mut db = store().await;
  forget(&mut db, 2).await;

  let page = retrieve(&mut engine, &mut db, &query(0.0), None, 10)
    .await
    .expect("the retrieve runs");

  assert_eq!(ids(&page), vec![1, 3, 4, 5, 6]);
  assert_eq!(
    page.iter().map(Ranked::rank).collect::<Vec<_>>(),
    vec![1, 3, 4, 5, 6]
  );
}

/// **The short final page**, on the flavour the ruling named it for: `knn` always answers with `k`,
/// and the floor is what makes a shorter answer the honest one.
#[tokio::test]
async fn a_floored_retrieve_answers_short() {
  let mut engine = index([]);
  let mut db = store().await;

  // Two of the six embeddings sit above this similarity; the other four are the tail `knn` would
  // have padded the answer with.
  let page = retrieve(&mut engine, &mut db, &query(0.97), None, 10)
    .await
    .expect("the retrieve runs");

  assert_eq!(ids(&page), vec![1, 2]);
}

/// The mechanism is a bounds choice and not a behaviour, on this flavour too.
#[tokio::test]
async fn the_equality_mechanism_answers_the_same_stream() {
  let mut engine = index([]);

  let hashed = retrieve_keys::<HashSeen<_>>(&mut engine, &query(0.0), usize::MAX).await;
  let listed = retrieve_keys::<EqSeen<_>>(&mut engine, &query(0.0), usize::MAX).await;

  assert_eq!(hashed, listed);
}