ingraph 0.1.0

A declaration-driven indexing framework: records and nodes are written as Rust types, and the storage rows, the dataflow between them and the GraphQL query surface are emitted from the declaration.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
use core::{
  convert::Infallible,
  future::{Future, ready},
  num::NonZeroUsize,
};

use std::sync::{
  Arc,
  atomic::{AtomicUsize, Ordering},
};

use super::{AsyncServes, FulltextAsk, Responding, Serve, Serves, SlotLane, VectorAsk};
use crate::{
  AsyncFulltextRead, AsyncFulltextWrite, AsyncVectorRead, AsyncVectorWrite, ColumnReader,
  ColumnWriter, Connected, Elected, Embedding, Field, FulltextEngine, FulltextSubset, Hits, Scored,
  SlotHandle, VectorColumn, VectorEngine,
  drive::{Answers, AsyncAnswers, QueryArm, door, lane::block_on},
  store::column::testkit::{Col, Fallback, FallbackRead, Refusal},
};

/// The `k` every ask below carries — wider than any fixture here, so what an assertion counts is
/// the LANE's doing rather than the ceiling's.
const K: NonZeroUsize = NonZeroUsize::new(10).expect("10 is not zero");

/// The declaration the fixtures' slots are branded for. NAMED and never built — a brand is carried
/// in the type and has no value.
struct Transcript;

/// A toy full-text engine: a shelf of `(text, key)` rows that answers a `search` with every row
/// whose text contains the query.
///
/// Read-only by construction, which is the honest shape for a READ lane's fixture: `upsert` takes
/// `&self` and this engine keeps its rows as it was built with them.
struct Shelf(Vec<(&'static str, u64)>);

impl FulltextEngine for Shelf {
  type Options = ();
  type SearchOptions = ();
  type Error = Refusal;
  type Score = f32;
  type Address = Vec<Col>;
  type Connection = ();
  type WriteBuffer = Fallback;
  type ReadBuffer = FallbackRead;

  fn buffer() -> Self::WriteBuffer {
    Fallback::default()
  }

  fn finalize(buf: Self::WriteBuffer) -> Self::Address {
    buf.0
  }

  fn open(address: Self::Address) -> Self::ReadBuffer {
    Fallback(address).read()
  }
}

impl AsyncFulltextWrite for Shelf {
  type Doc = (&'static str, u64);
  type WriteContext<'a> = ();

  fn upsert(
    &self,
    _ctx: Connected<'_, (), Self::WriteContext<'_>>,
    _doc: Self::Doc,
  ) -> impl Future<Output = Result<(), Self::Error>> + Send {
    ready(Ok(()))
  }
}

impl AsyncFulltextRead for Shelf {
  type ReadContext<'a> = ();

  fn search(
    &self,
    _ctx: Connected<'_, (), Self::ReadContext<'_>>,
    _subset: FulltextSubset<'_>,
    query: &str,
    _k: NonZeroUsize,
    _options: ::core::option::Option<&Self::SearchOptions>,
  ) -> impl Future<Output = Result<Hits<Self::Address, Self::Score>, Self::Error>> + Send {
    let hits = self
      .0
      .iter()
      .filter(|(text, _)| text.contains(query))
      .map(|(_, key)| Scored::new(shelved(*key), 1.0))
      .collect();
    ready(Ok(hits))
  }
}

/// `key`, sealed into the shelf's own address — the full-text twin of [`addressed`].
fn shelved(key: u64) -> Vec<Col> {
  let mut buf = Shelf::buffer();
  Held(key).write(&mut buf);
  Shelf::finalize(buf)
}

/// THE TWO ELECTIONS this fixture's declaration makes, spelled as the emission spells them: a
/// zero-sized marker per column, implementing [`Elected<Transcript>`](Elected).
///
/// Hand-written here because `ingraph` cannot invoke its own derive; `ingraph-derive`'s
/// `multi_election` corpus is where the EMITTED pair is proved, and these two are the same shape
/// that emission mints.
struct ByText;
struct ByImage;

impl Elected<Transcript> for ByText {
  const COLUMN: VectorColumn = TEXT;
}

impl Elected<Transcript> for ByImage {
  const COLUMN: VectorColumn = IMAGE;
}

const TEXT: VectorColumn = VectorColumn::new("text_embedding");
const IMAGE: VectorColumn = VectorColumn::new("image_embedding");

/// The vector twin, so the second [`Serves`] impl is exercised rather than merely compiled: a keyed
/// coordinate table answering a `knn` with a similarity it computed itself.
///
/// MULTI-COLUMN, which is what makes the election checkable at all: a table holding one list answers
/// identically whatever column it is asked for, so a lane that ignored `C::COLUMN` would pass over
/// it. Rows are `(column, coordinates, key)` and only the asked column's are scored.
struct Coordinates(Vec<(VectorColumn, Embedding<f32, 2>, u64)>);

/// This table's key is one `u64`, so its whole ROW is one column — the degenerate end of the
/// vocabulary, and still a real crossing: what a hit is handed back under is what this wrote.
struct Held(u64);

impl Field for Held {
  fn write(&self, w: &mut impl ColumnWriter) {
    self.0.write(w);
  }

  fn read<R: ColumnReader>(r: &mut R) -> Result<Self, R::Error> {
    u64::read(r).map(Self)
  }
}

/// `key`, sealed into this engine's address.
fn addressed(key: u64) -> Vec<Col> {
  let mut buf = Coordinates::buffer();
  Held(key).write(&mut buf);
  Coordinates::finalize(buf)
}

impl VectorEngine for Coordinates {
  type Options = ();
  type SearchOptions = ();
  type Error = Refusal;
  type Score = f32;
  type Element = f32;
  type Address = Vec<Col>;
  type Connection = ();
  type WriteBuffer = Fallback;
  type ReadBuffer = FallbackRead;

  fn buffer() -> Self::WriteBuffer {
    Fallback::default()
  }

  fn finalize(buf: Self::WriteBuffer) -> Self::Address {
    buf.0
  }

  fn open(address: Self::Address) -> Self::ReadBuffer {
    Fallback(address).read()
  }
}

impl AsyncVectorWrite for Coordinates {
  type Doc = (Embedding<f32, 2>, u64);
  type WriteContext<'a> = ();

  fn upsert(
    &self,
    _ctx: Connected<'_, (), Self::WriteContext<'_>>,
    _doc: Self::Doc,
  ) -> impl Future<Output = Result<(), Self::Error>> + Send {
    ready(Ok(()))
  }
}

impl AsyncVectorRead for Coordinates {
  type ReadContext<'a> = ();

  fn knn(
    &self,
    _ctx: Connected<'_, (), Self::ReadContext<'_>>,
    column: VectorColumn,
    query: &[Self::Element],
    _k: NonZeroUsize,
    _options: ::core::option::Option<&Self::SearchOptions>,
  ) -> impl Future<Output = Result<Hits<Self::Address, Self::Score>, Self::Error>> + Send {
    let mut hits: Vec<Scored<Vec<Col>, f32>> = self
      .0
      .iter()
      .filter(|(held, _, _)| *held == column)
      .map(|(_, held, key)| {
        let distance: f32 = held
          .coordinates()
          .iter()
          .zip(query)
          .map(|(a, b)| (a - b) * (a - b))
          .sum();
        Scored::new(addressed(*key), -distance)
      })
      .collect();
    hits.sort_by(|a, b| b.score().total_cmp(&a.score()));
    ready(Ok(hits))
  }

  /// The by-key read, answered for real rather than stubbed: this table already holds
  /// `(column, coordinates, key)` rows, so *what does this key hold in this column* is a lookup it
  /// can serve — and a face answering an empty set here would be one whose `get` and whose `knn`
  /// disagree about what it stores.
  ///
  /// The address is the same one-key list `knn` hands a hit back under, which is exactly the
  /// correspondence the seam names.
  fn get(
    &self,
    _ctx: Connected<'_, (), Self::ReadContext<'_>>,
    column: VectorColumn,
    address: &Self::Address,
  ) -> impl Future<Output = Result<Vec<Vec<Self::Element>>, Self::Error>> + Send {
    ready(Ok(
      self
        .0
        .iter()
        .filter(|(held, _, key)| *held == column && *address == addressed(*key))
        .map(|(_, held, _)| held.coordinates().to_vec())
        .collect(),
    ))
  }
}

/// THE ONE WRAPPER the seam's proof needs — a telemetry lane, and the shape every other one takes.
///
/// It implements [`Serves`] by DELEGATING, which is the whole composability claim: it names no
/// engine, no slot and no question, so it wraps the shipped lane and a hand-written responder
/// alike. The count is taken BEFORE the inner future is built rather than inside an `async` body,
/// which is what the `fn`-returning-a-future spelling buys — an observation that happens on the
/// call rather than on the first poll.
struct Counting<S> {
  inner: S,
  served: Arc<AtomicUsize>,
}

impl<S> Counting<S> {
  const fn new(inner: S, served: Arc<AtomicUsize>) -> Self {
    Self { inner, served }
  }
}

/// A wrapper that changes what answering COSTS and not what it is configured with, so it projects
/// the seat it wraps: whatever knobs the inner server takes, this one takes, and a caller reaching
/// through the stack spells the innermost server's own options type.
///
/// ONE row for the wrapper, under BOTH the face rows below — which is the base's whole arithmetic:
/// a delegating lane that carried the seat per face could wrap a two-faced server and disagree with
/// itself about what configures it.
impl<S: Serve> Serve for Counting<S> {
  type Options = S::Options;
}

impl<S, Q, A> Serves<Q, A> for Counting<S>
where
  S: Serves<Q, A>,
{
  type Error = S::Error;

  fn serve(&mut self, q: Q, options: Option<&Self::Options>) -> Result<A, Self::Error> {
    self.served.fetch_add(1, Ordering::Relaxed);
    self.inner.serve(q, options)
  }
}

/// The SAME wrapper on the other face — and that a delegating lane needs BOTH rows is the honest
/// count rather than a duplication.
///
/// A wrapper's whole content is *whatever is under me, plus this*, so it has to be able to sit over
/// either face; and each row demands the face it delegates to, so wrapping a blocking lane yields a
/// blocking lane and wrapping an awaited one yields an awaited one. A blanket bridge would have
/// collapsed that into *everything is awaited*, which is the mechanism the law refuses.
impl<S, Q, A> AsyncServes<Q, A> for Counting<S>
where
  S: AsyncServes<Q, A>,
{
  type Error = S::Error;

  fn serve(
    &mut self,
    q: Q,
    options: Option<&Self::Options>,
  ) -> impl Future<Output = Result<A, Self::Error>> + Send {
    self.served.fetch_add(1, Ordering::Relaxed);
    self.inner.serve(q, options)
  }
}

/// A responder written against the ROLE, not against the seam — the shape every query node's author
/// writes, and what the bridge has to carry.
struct Words;

impl Answers<String> for Words {
  type Output = usize;
  type Error = Infallible;

  fn on_invoke(&mut self, input: String) -> Result<usize, Self::Error> {
    Ok(input.split_whitespace().count())
  }
}

/// [`Words`]' AWAITED sibling: the same answer, on the other face.
///
/// A separate type rather than a second row on [`Words`], and the reason is what each proof claims.
/// [`a_responder_reaches_the_seam_through_the_bridge_without_naming_it`] rests on `Words` wearing
/// exactly one face — that is how it shows the bridge carries a responder that named no seam — so a
/// second row there would make its own `serve` call `E0034` and cost the test its subject. The
/// async cell needs a responder that genuinely awaits, and this is it.
struct AwaitedWords;

impl AsyncAnswers<String> for AwaitedWords {
  type Output = usize;
  type Error = Infallible;

  async fn on_invoke(&mut self, input: String) -> Result<usize, Self::Error> {
    Ok(input.split_whitespace().count())
  }
}

fn shelf() -> SlotLane<Shelf, Transcript> {
  SlotLane::new(SlotHandle::mint(Shelf(vec![
    ("a red sunset over the bay", 1),
    ("a blue car", 2),
    ("sunset boulevard", 3),
  ])))
}

/// The seeded table, and the corpus is what makes the two elections DISTINGUISHABLE: the same two
/// keys sit at opposite ends of the two columns, so one query direction answers `1` in `text_embedding`
/// and `9` in `image_embedding`.
fn table() -> Coordinates {
  let at = |column: VectorColumn, x: f32, key: u64| {
    (
      column,
      Embedding::new(Arc::from(vec![x, 0.0])).expect("the fixture declares the width"),
      key,
    )
  };
  Coordinates(vec![
    at(TEXT, 1.0, 1),
    at(TEXT, 9.0, 9),
    at(IMAGE, 9.0, 1),
    at(IMAGE, 1.0, 9),
  ])
}

/// One lane PER ELECTION, over one slot — the wiring the ruling asks for, written out.
fn by_text() -> SlotLane<Coordinates, Transcript, ByText> {
  SlotLane::new(SlotHandle::mint(table()))
}

fn by_image() -> SlotLane<Coordinates, Transcript, ByImage> {
  SlotLane::new(SlotHandle::mint(table()))
}

#[test]
fn the_shipped_lane_serves_the_branded_slot_through_the_engines_own_verb() {
  let mut lane = shelf();
  let hits = block_on(lane.serve(
    FulltextAsk::new(Connected::new(&(), ()), FulltextSubset::All, "sunset", K),
    None,
  ))
  .expect("this engine cannot refuse");

  // The engine's OWN answer, unaltered: addresses and scores, no floor, no dedup, no rank. Those
  // are the emitted method's, which is where the declaration is concrete.
  assert_eq!(
    hits.iter().map(Scored::data_ref).collect::<Vec<_>>(),
    vec![&shelved(1), &shelved(3)]
  );
  assert_eq!(lane.slot().engine().0.len(), 3);
}

#[test]
fn one_lane_serves_both_engine_kinds_and_what_separates_them_is_the_question() {
  let mut lane = by_image();
  let hits = block_on(lane.serve(
    VectorAsk::new(Connected::new(&(), ()), &[8.6, 0.0], K),
    None,
  ))
  .expect("this engine cannot refuse");

  assert_eq!(hits.first().expect("a nearest").data_ref(), &addressed(1));
}

/// **TWO DOORS, TWO ELECTIONS, TWO ANSWERS — and the ask is the same value.**
///
/// One slot, one seeded table, one query direction, one `VectorAsk`: what differs between the two
/// results is the LANE's third parameter and nothing else. The doors are typed per election, which
/// is the wiring the compile-fail beside this pins the other half of.
///
/// MUTATION-CHECKED, twice. Serve through `self.slot.knn(scope, TEXT, coordinates)` — a column
/// ignored, one election answering both — and the two answers coincide, `assert_ne!` reds. Give both
/// markers one column — the emission collapsing to a single marker — and they coincide again.
#[test]
fn a_door_typed_for_an_election_answers_from_that_elections_column() {
  fn text_door(lane: &mut SlotLane<Coordinates, Transcript, ByText>) -> Vec<u64> {
    nearest(lane)
  }

  fn image_door(lane: &mut SlotLane<Coordinates, Transcript, ByImage>) -> Vec<u64> {
    nearest(lane)
  }

  fn nearest<C: Elected<Transcript>>(lane: &mut SlotLane<Coordinates, Transcript, C>) -> Vec<u64> {
    block_on(lane.serve(
      VectorAsk::new(Connected::new(&(), ()), &[8.6, 0.0], K),
      None,
    ))
    .expect("this engine cannot refuse")
    .into_iter()
    .map(|hit| {
      let mut phase = Coordinates::open(hit.into_data());
      let Held(key) = Held::read(&mut phase).expect("the toy sealed this key");
      phase.finish().expect("and nothing beyond it");
      key
    })
    .collect()
  }

  let by_text = text_door(&mut by_text());
  let by_image = image_door(&mut by_image());

  assert_eq!(by_text, vec![9, 1], "the text column's neighbours");
  assert_eq!(by_image, vec![1, 9], "the image column's neighbours");
  assert_ne!(
    by_text, by_image,
    "two elections, two columns, two answers — over ONE slot",
  );
}

#[test]
fn a_wrapping_lane_composes_over_the_shipped_one_without_naming_its_engine() {
  let served = Arc::new(AtomicUsize::new(0));
  let mut lane = Counting::new(shelf(), Arc::clone(&served));

  let hits = block_on(lane.serve(
    FulltextAsk::new(Connected::new(&(), ()), FulltextSubset::All, "blue", K),
    None,
  ))
  .expect("this engine cannot refuse");

  assert_eq!(hits.len(), 1);
  assert_eq!(served.load(Ordering::Relaxed), 1);
}

#[test]
fn a_responder_reaches_the_seam_through_the_bridge_without_naming_it() {
  // `Words` implements `Answers` and nothing else. That it can be `serve`d once seated is the
  // bridge, and it is what keeps every node written against the responder role working through the
  // arm's rerouted call.
  let mut words = Responding::new(Words);
  // NO `block_on`, and its absence is the face pair's own dividend: the seating carries the
  // responder's BLOCKING face onto the blocking server face, so the call is the answer.
  let count = words
    .serve("a red car".to_owned(), None)
    .expect("infallible");
  assert_eq!(count, 3);
  assert_eq!(
    core::mem::size_of::<Responding<Words>>(),
    core::mem::size_of::<Words>()
  );
}

#[test]
fn a_wrapping_lane_counts_a_live_round_trip_the_arm_cannot_see() {
  // THE SEAM'S PROOF, end to end: the arm holds a wrapper, the wrapper holds a seated responder,
  // and the caller gets the same answer it would have got from the responder alone — while the
  // wrapper counted. Nothing on the arm and nothing in `Words` mentions the wrapper.
  //
  // MUTATION-CHECKED: route the arm back through `Answers::on_invoke` — the bypass the seam
  // replaced — and this stops COMPILING, `Counting<Responding<Words>>` implementing `Serves` and
  // not `Answers`. That is the pin on *the arm's answering half goes through the seam*.
  //
  // It is also the BLOCKING cell: `run` bounds `Serves`, so nothing here mints a future and no
  // runtime is built. The test below is the same wrapper over the same responder on the other.
  let served = Arc::new(AtomicUsize::new(0));
  let (front, invokes) = door::<String, usize>(4);
  let mut arm = QueryArm::serving(
    "words",
    Counting::new(Responding::new(Words), Arc::clone(&served)),
    invokes,
  );

  let awaiting = front.invoke("a red car".to_owned()).expect("listening");
  drop(front);
  let report = arm.run();

  assert_eq!(awaiting.take().expect("answered"), 3);
  assert_eq!(report.invokes(), 1);
  assert_eq!(report.passed(), 1);
  assert_eq!(served.load(Ordering::Relaxed), 1);
}

/// THE OTHER CELL: the same wrapper, over an AWAITED responder, driven by `run_async`.
///
/// What is held constant is the thing under test — `Counting` is the identical wrapper type, and its
/// count is taken on the call in both cells. What changes is the face beneath it, and the arm's verb
/// that selects one: `run` bounds `Serves` and `run_async` bounds `AsyncServes`, so a wrapper that
/// composes over both is reached through both without either bound being weakened.
#[test]
fn the_async_engine_serves_through_the_same_wrapper() {
  let served = Arc::new(AtomicUsize::new(0));
  let (front, invokes) = door::<String, usize>(4);
  let mut arm = QueryArm::serving(
    "words",
    Counting::new(Responding::new(AwaitedWords), Arc::clone(&served)),
    invokes,
  );

  let awaiting = front.invoke("a red car".to_owned()).expect("listening");
  drop(front);
  let report = tokio::runtime::Builder::new_current_thread()
    .build()
    .expect("a current-thread runtime")
    .block_on(arm.run_async());

  assert_eq!(awaiting.take().expect("answered"), 3);
  assert_eq!(report.passed(), 1);
  assert_eq!(served.load(Ordering::Relaxed), 1);
}

/// A SPACE, in the sealed form the space axis rules.
struct Clip(());

/// A TOWER: it takes a question and answers coordinates in a named space — the last hop of an `ask`
/// chain, whose product is the very type an election's field declares.
struct ClipTower;

impl Answers<String> for ClipTower {
  type Output = Embedding<f32, 4, Clip>;
  type Error = Infallible;

  fn on_invoke(&mut self, input: String) -> Result<Self::Output, Self::Error> {
    let weight = input.len() as f32;
    Ok(
      Embedding::new(Arc::from([weight, 0.0, 0.0, 0.0].as_slice()))
        .expect("four coordinates of four"),
    )
  }
}

/// A door is a way of ASKING: the emitted `ask` road takes a hop that runs as its own resident
/// through the very parameter it takes an in-process one through, because both wear the server
/// seam. The road is on the AWAITED face — which the door answers too — while the arm behind it is
/// driven by the blocking one, so this is also the matrix's mixed cell: the asking side awaits and
/// the answering side never mints a future at all.
#[test]
fn a_door_serves_the_arm_behind_it() {
  let (mut front, invokes) = door::<String, Embedding<f32, 4, Clip>>(4);
  let mut arm = QueryArm::new("clip", ClipTower, invokes);
  let asking = std::thread::spawn(move || {
    block_on(async move { AsyncServes::serve(&mut front, "a cat".to_owned(), None).await })
  });
  let served = arm.run::<Embedding<f32, 4, Clip>>();
  let answer = asking.join().expect("the asking thread finishes");
  assert_eq!(served.invokes(), 1);
  assert_eq!(
    answer.expect("the door answered").coordinates(),
    &[5.0, 0.0, 0.0, 0.0]
  );
}

/// The other half of the same sentence: a door whose arm has gone reports a refusal rather than
/// waiting, which is what makes the ask road's `Asked` arm reachable.
#[test]
fn a_door_with_no_arm_refuses() {
  let (mut front, invokes) = door::<String, Embedding<f32, 4, Clip>>(4);
  drop(invokes);
  let answer =
    block_on(async move { AsyncServes::serve(&mut front, "a cat".to_owned(), None).await });
  assert!(answer.is_err());
}