ingraph 0.1.0

A declaration-driven indexing framework: records and nodes are written as Rust types, and the storage rows, the dataflow between them and the GraphQL query surface are emitted from the declaration.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
//! THE WALK, run over the whole census on three engines — and over the rows written to break it.
//!
//! The census is `store::field`'s nineteen rows, one representative value each, driven through
//! [`round_trip_fulltext`] at the SHIPPED engine (under the `tantivy` feature), through
//! [`round_trip_vector`] at a toy one, and through [`round_trip_fulltext`] again at a second toy
//! whose sink leaves every non-required kind on its DEFAULT. Both toys are built out of
//! `store::column`'s shared testkit, and the two sinks are the two shapes a backend has: a column
//! per width, or the five required kinds and a fallback chain.
//!
//! Three engines, ONE list, one `census!` — because a row that conforms at one backend and not
//! another is exactly what the walk exists to find, and a list per engine is three lists free to
//! drift.
//!
//! The broken rows below are not a demonstration. Each is a shape the language ADMITS and the walk
//! has to refuse, and the kind-split pair is the one no engine can refuse for itself: it round-trips
//! correctly through tantivy's own three phases, which
//! [`the_engine_leg_alone_passes_the_row_the_kind_leg_refuses`] proves on the spot.

// The three pointer rows live in `alloc`, and the census exercises all three.
extern crate alloc;

use core::{
  fmt::Debug,
  net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6},
  num::{
    NonZeroI8, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI128, NonZeroU8, NonZeroU16, NonZeroU32,
    NonZeroU64, NonZeroU128,
  },
};

use smol_bytes::Utf8Bytes;
use uuid::Uuid;

use super::{ColumnDoor, ConformanceError, round_trip_fulltext, round_trip_vector};
use crate::store::{
  ColumnReader, ColumnWriter, Field, FulltextEngine, NoColumns, VectorEngine,
  column::testkit::{Col, Fallback, FallbackRead, Native, NativeRead, Refusal},
};

/// The entity a typed id addresses — it implements nothing, because `E` reaches no column.
struct Scene;

/// A TOY VECTOR ENGINE, over the shared testkit's own column store.
///
/// It exists because the vector family ships no backend, and the walk is not a claim about one
/// family: the same census has to cross a `VectorEngine`, so one is built out of the sink
/// `store::field`'s rows are already read back off. Reusing `Native` rather than writing a third
/// column store is the same argument the testkit was extracted for — a toy copied per file is two
/// engines free to drift.
///
/// `Native` lands a column per integer WIDTH and leaves [`fixed`](ColumnWriter::fixed) on its
/// fallback, which is a real engine shape: narrow integers are native here and fixed-width byte
/// columns are not.
struct ToyVector;

impl VectorEngine for ToyVector {
  type Options = ();
  type SearchOptions = ();
  type Error = Refusal;
  type Score = f32;
  type Element = f32;
  type Address = Vec<Col>;
  type Connection = ();
  type WriteBuffer = Native;
  type ReadBuffer = NativeRead;

  fn buffer() -> Self::WriteBuffer {
    Native::default()
  }

  fn finalize(buf: Self::WriteBuffer) -> Self::Address {
    buf.0
  }

  fn open(address: Self::Address) -> Self::ReadBuffer {
    Native(address).read()
  }
}

/// A FULL-TEXT ENGINE OVER THE REQUIRED-KINDS-ONLY SINK — the other real engine shape.
///
/// [`ToyVector`] is built on `Native`, which lands a column per integer width. This one is built on
/// `Fallback`, which overrides the five REQUIRED kinds and nothing else — so every defaulted body in
/// the column language runs on the way through: an `i8` lands through
/// [`fixed`](ColumnWriter::fixed) and `fixed` in turn through [`bytes`](ColumnWriter::bytes), on
/// both halves. Those are the two shapes a backend can have, and a row can conform at one and not
/// the other; before this, the census crossed only the first.
///
/// It buys a second thing the vector toy cannot: the FULL-TEXT entry gets a census run that no
/// feature gates. `the_whole_census_conforms_at_real_tantivy` is the only other one, and it is
/// `#[cfg(feature = "tantivy")]` — so in a build without that feature
/// [`round_trip_fulltext`] had no row of the census through it at all.
struct FallbackFulltext;

impl FulltextEngine for FallbackFulltext {
  type Options = ();
  type SearchOptions = ();
  type Error = Refusal;
  type Score = f32;
  type Address = Vec<Col>;
  type Connection = ();
  type WriteBuffer = Fallback;
  type ReadBuffer = FallbackRead;

  fn buffer() -> Self::WriteBuffer {
    Fallback::default()
  }

  fn finalize(buf: Self::WriteBuffer) -> Self::Address {
    buf.0
  }

  fn open(address: Self::Address) -> Self::ReadBuffer {
    Fallback(address).read()
  }
}

/// One row through the VECTOR entry at the toy engine, naming the row if it does not conform.
#[track_caller]
fn on_toy<K: Field + PartialEq + Debug>(key: K) {
  if let Err(refusal) = round_trip_vector::<ToyVector, K>(key) {
    panic!(
      "{} does not conform at the toy vector engine: {refusal:?}",
      core::any::type_name::<K>()
    );
  }
}

/// One row through the FULL-TEXT entry at the sink where every defaulted body runs.
#[track_caller]
fn on_fallback<K: Field + PartialEq + Debug>(key: K) {
  if let Err(refusal) = round_trip_fulltext::<FallbackFulltext, K>(key) {
    panic!(
      "{} does not conform at the fallback-only engine: {refusal:?}",
      core::any::type_name::<K>()
    );
  }
}

/// One row through the FULL-TEXT entry at the SHIPPED engine.
#[cfg(feature = "tantivy")]
#[track_caller]
fn on_tantivy<K: Field + PartialEq + Debug>(key: K) {
  if let Err(refusal) = round_trip_fulltext::<crate::store::TantivyFulltext, K>(key) {
    panic!(
      "{} does not conform at the shipped engine: {refusal:?}",
      core::any::type_name::<K>()
    );
  }
}

/// THE NINETEEN CENSUS ROWS, one representative value each — `store::field`'s whole vocabulary, in
/// the order that module declares it.
///
/// The gated rows carry their gate here, so the census is complete over the member set of the build
/// it is compiled in rather than over `--all-features`'s.
macro_rules! census {
  ($run:ident) => {{
    // 1–2: the ten integer widths, each at its own.
    $run(i8::MIN);
    $run(i16::MIN);
    $run(i32::MIN);
    $run(i64::MIN);
    $run(i128::MIN);
    $run(u8::MAX);
    $run(u16::MAX);
    $run(u32::MAX);
    $run(u64::MAX);
    $run(u128::MAX);
    // 3–4: the ten `NonZero*`, transparent to the integer each holds — and the walk restoring one is
    // the transparency law read back rather than asserted.
    $run(NonZeroI8::new(-1).expect("-1 is not zero"));
    $run(NonZeroI16::new(-1).expect("-1 is not zero"));
    $run(NonZeroI32::new(-1).expect("-1 is not zero"));
    $run(NonZeroI64::new(-1).expect("-1 is not zero"));
    $run(NonZeroI128::new(-1).expect("-1 is not zero"));
    $run(NonZeroU8::new(u8::MAX).expect("the maximum is not zero"));
    $run(NonZeroU16::new(u16::MAX).expect("the maximum is not zero"));
    $run(NonZeroU32::new(u32::MAX).expect("the maximum is not zero"));
    $run(NonZeroU64::new(u64::MAX).expect("the maximum is not zero"));
    $run(NonZeroU128::new(u128::MAX).expect("the maximum is not zero"));
    // 5: the uuid member.
    $run(identity());
    // 6–8: the text members, including the empty string and a value whose byte width is not its
    // character count.
    $run("a caption".to_owned());
    $run(String::new());
    $run(Utf8Bytes::from("médià".to_owned()));
    #[cfg(feature = "smol_str")]
    $run(smol_str::SmolStr::from("médià"));
    // 9–14: the network family, both arms of both untagged enums.
    $run(Ipv4Addr::new(10, 0, 0, 1));
    $run(host());
    $run(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)));
    $run(IpAddr::V6(host()));
    $run(SocketAddrV4::new(Ipv4Addr::new(10, 0, 0, 1), 8080));
    $run(SocketAddrV6::new(host(), 8080, 3, 5));
    $run(SocketAddr::V4(SocketAddrV4::new(
      Ipv4Addr::new(10, 0, 0, 1),
      8080,
    )));
    $run(SocketAddr::V6(SocketAddrV6::new(host(), 8080, 3, 5)));
    // 15: the fixed-width byte member, at two widths — one of them the bson id's.
    $run([1_u8, 2, 3, 4]);
    $run([7_u8; 12]);
    // 16: the three pointer members, and one nested.
    $run(alloc::sync::Arc::new(7_u64));
    $run(alloc::rc::Rc::new(7_u64));
    $run(alloc::boxed::Box::new(7_u64));
    $run(alloc::sync::Arc::new(alloc::boxed::Box::new(7_u64)));
    // 17: the typed id, whose entity marker reaches no column.
    $run(crate::TypedId::<Uuid, Scene>::new(identity()));
    // 18–19: the bson pair, under the feature that supplies the types.
    #[cfg(feature = "mongo")]
    $run(bson::Uuid::from_bytes(*identity().as_bytes()));
    #[cfg(feature = "mongo")]
    $run(bson::oid::ObjectId::from_bytes([7; 12]));
  }};
}

/// The uuid every row carries, so a failure names a value rather than a fresh one.
fn identity() -> Uuid {
  Uuid::from_bytes([
    0x9f, 0x1c, 0x2d, 0x3e, 0x4a, 0x5b, 0x6c, 0x7d, 0x8e, 0x9f, 0xa0, 0xb1, 0xc2, 0xd3, 0xe4, 0xf5,
  ])
}

/// The v6 host the network rows share.
fn host() -> Ipv6Addr {
  Ipv6Addr::new(0x2001, 0x0db8, 0, 0, 0, 0, 0, 1)
}

/// A KEY WITH NO COLUMNS — the one shape the unit engine can hold, and the other half of that
/// engine's answer.
#[derive(Debug, PartialEq)]
struct Anonymous;

impl Field for Anonymous {
  fn write(&self, _w: &mut impl ColumnWriter) {}

  fn read<R: ColumnReader>(_r: &mut R) -> Result<Self, R::Error> {
    Ok(Self)
  }
}

/// A ROW SPLIT ACROSS TWO DOORS — written through [`fixed`](ColumnWriter::fixed) and read through
/// [`bytes`](ColumnWriter::bytes).
///
/// This is the shape the shipped [`SocketAddr`](core::net::SocketAddr) row had before T2 repaired
/// it, reduced to one column. It is correct on every engine whose `fixed` is the defaulted one and
/// wrong on the first engine that gives fixed-width columns a home of their own.
#[derive(Debug, PartialEq)]
struct Split(u32);

impl Field for Split {
  fn write(&self, w: &mut impl ColumnWriter) {
    w.fixed(&self.0.to_be_bytes());
  }

  fn read<R: ColumnReader>(r: &mut R) -> Result<Self, R::Error> {
    let value = r.bytes()?;
    Ok(Self(u32::from_be_bytes(
      <[u8; 4]>::try_from(value).expect("a column this row measured at four bytes"),
    )))
  }
}

/// THE SAME SPLIT, one column IN — so the refusal has an index to be wrong about.
#[derive(Debug, PartialEq)]
struct SplitSecond(u64, u32);

impl Field for SplitSecond {
  fn write(&self, w: &mut impl ColumnWriter) {
    w.u64(self.0);
    w.fixed(&self.1.to_be_bytes());
  }

  fn read<R: ColumnReader>(r: &mut R) -> Result<Self, R::Error> {
    let first = r.u64()?;
    let second = r.bytes()?;
    Ok(Self(
      first,
      u32::from_be_bytes(<[u8; 4]>::try_from(second).expect("four bytes")),
    ))
  }
}

/// A ROW WHOSE NARROW INTEGER IS READ AS A FIXED COLUMN — the split that only exists because both
/// halves of the kind engine override every DEFAULTED method.
///
/// `u32` lands through its own door and `fixed::<4>` is what the default chain would have turned
/// that door into. A log that inherited either default would record one kind for both and this row
/// would pass.
#[derive(Debug, PartialEq)]
struct SplitDefaulted(u32);

impl Field for SplitDefaulted {
  fn write(&self, w: &mut impl ColumnWriter) {
    w.u32(self.0);
  }

  fn read<R: ColumnReader>(r: &mut R) -> Result<Self, R::Error> {
    r.fixed::<4>().map(|value| Self(u32::from_be_bytes(value)))
  }
}

/// A ROW THAT READS PAST ITS OWN WRITE.
#[derive(Debug, PartialEq)]
struct Greedy(u64);

impl Field for Greedy {
  fn write(&self, w: &mut impl ColumnWriter) {
    w.u64(self.0);
  }

  fn read<R: ColumnReader>(r: &mut R) -> Result<Self, R::Error> {
    let value = r.u64()?;
    r.u64()?;
    Ok(Self(value))
  }
}

/// A ROW THAT LANDS A COLUMN ITS READ NEVER TAKES.
#[derive(Debug, PartialEq)]
struct Chatty(u64);

impl Field for Chatty {
  fn write(&self, w: &mut impl ColumnWriter) {
    w.u64(self.0);
    w.u64(0);
  }

  fn read<R: ColumnReader>(r: &mut R) -> Result<Self, R::Error> {
    r.u64().map(Self)
  }
}

/// A ROW THAT COMES BACK AS ANOTHER VALUE — the round trip completing without restoring.
#[derive(Debug, PartialEq)]
struct Lossy(u64);

impl Field for Lossy {
  fn write(&self, w: &mut impl ColumnWriter) {
    w.u64(self.0 | 1);
  }

  fn read<R: ColumnReader>(r: &mut R) -> Result<Self, R::Error> {
    r.u64().map(Self)
  }
}

#[test]
fn the_whole_census_conforms_at_a_toy_vector_engine() {
  census!(on_toy);
}

/// The same list at the sink where every DEFAULTED body runs, through the full-text entry.
#[test]
fn the_whole_census_conforms_at_a_fallback_only_engine() {
  census!(on_fallback);
}

/// The same list at the SHIPPED backend — the acceptance the walk was written for.
#[cfg(feature = "tantivy")]
#[test]
fn the_whole_census_conforms_at_real_tantivy() {
  census!(on_tantivy);
}

/// THE T2 LAW, ENFORCED GENERICALLY — and it holds at both families, because the leg that raises it
/// has no engine in it.
#[test]
fn a_row_split_across_two_doors_is_refused_with_the_column_named() {
  assert!(matches!(
    round_trip_vector::<ToyVector, Split>(Split(7)),
    Err(ConformanceError::KindSplit {
      at: 0,
      wrote: ColumnDoor::Fixed(4),
      read: ColumnDoor::Bytes,
    })
  ));

  #[cfg(feature = "tantivy")]
  assert!(matches!(
    round_trip_fulltext::<crate::store::TantivyFulltext, Split>(Split(7)),
    Err(ConformanceError::KindSplit {
      at: 0,
      wrote: ColumnDoor::Fixed(4),
      read: ColumnDoor::Bytes,
    })
  ));
}

/// THE ENGINE CANNOT SEE IT, which is the whole reason the first leg exists.
///
/// The split row goes through the shipped engine's own three phases and comes back as itself,
/// because `TantivyTerms` leaves [`fixed`](ColumnWriter::fixed) on its fallback and the two doors
/// land identical bytes there. So this passes and the test above fails, on the same row, and the
/// difference between them is the entire value of the kind leg.
#[cfg(feature = "tantivy")]
#[test]
fn the_engine_leg_alone_passes_the_row_the_kind_leg_refuses() {
  use crate::{FulltextEngine, store::TantivyFulltext};

  let mut buf = TantivyFulltext::buffer();
  Split(7).write(&mut buf);
  let mut phase = TantivyFulltext::open(TantivyFulltext::finalize(buf));
  assert_eq!(
    Split::read(&mut phase).expect("the address reads back"),
    Split(7)
  );
  phase.finish().expect("the key took every column");
}

#[test]
fn the_split_column_is_named_by_its_index() {
  assert!(matches!(
    round_trip_vector::<ToyVector, SplitSecond>(SplitSecond(7, 9)),
    Err(ConformanceError::KindSplit {
      at: 1,
      wrote: ColumnDoor::Fixed(4),
      read: ColumnDoor::Bytes,
    })
  ));
}

/// A NARROW INTEGER IS ITS OWN KIND, and the defaulted door it falls through is not.
///
/// MUTATION-CHECKED, and what it pins is the write log's OVERRIDES: dropping `u32 => U32` from
/// `log_integer!` — exactly what inheriting [`ColumnWriter::u32`]'s default does — turns this RED.
/// The log then records `fixed<4>` where the row said `u32`, the two halves agree, and the split
/// this asserts stops being reported at all. Measured on the branch, restored, byte-compared.
#[test]
fn a_narrow_integer_is_not_the_fixed_column_its_default_lands_in() {
  assert!(matches!(
    round_trip_vector::<ToyVector, SplitDefaulted>(SplitDefaulted(7)),
    Err(ConformanceError::KindSplit {
      at: 0,
      wrote: ColumnDoor::U32,
      read: ColumnDoor::Fixed(4),
    })
  ));
}

#[test]
fn a_row_that_reads_past_its_own_write_is_short() {
  assert!(matches!(
    round_trip_vector::<ToyVector, Greedy>(Greedy(7)),
    Err(ConformanceError::Short {
      at: 1,
      wanted: ColumnDoor::U64,
    })
  ));
}

#[test]
fn a_row_that_writes_more_than_it_reads_leaves_a_surplus() {
  assert!(matches!(
    round_trip_vector::<ToyVector, Chatty>(Chatty(7)),
    Err(ConformanceError::Surplus { unread: 1 })
  ));
}

#[test]
fn a_row_that_does_not_restore_is_a_mismatch() {
  let Err(ConformanceError::Mismatch { wrote, read }) =
    round_trip_vector::<ToyVector, Lossy>(Lossy(2))
  else {
    panic!("a row that renders another value has to be caught");
  };
  assert_eq!(wrote, "Lossy(2)");
  assert_eq!(read, "Lossy(3)");
}

/// THE UNIT ENGINE refuses a key it could never have held, in its OWN word for why.
///
/// Both families, because the answer is the column language's rather than either family's.
#[test]
fn the_unit_engine_refuses_a_key_it_never_held() {
  assert!(matches!(
    round_trip_vector::<(), u64>(7),
    Err(ConformanceError::Read(NoColumns))
  ));
  assert!(matches!(
    round_trip_fulltext::<(), u64>(7),
    Err(ConformanceError::Read(NoColumns))
  ));
}

/// And it PASSES a key with no columns — the consistent other half of the same reading: a phase
/// holding nothing has nothing left over, so a key that read nothing off it is whole.
#[test]
fn the_unit_engine_passes_a_key_with_no_columns() {
  assert!(round_trip_vector::<(), Anonymous>(Anonymous).is_ok());
  assert!(round_trip_fulltext::<(), Anonymous>(Anonymous).is_ok());
}

/// A REFUSAL PRINTS what it found, which is what a third-party engine author reads.
#[test]
fn a_refusal_names_the_column_and_both_kinds() {
  let refusal = ConformanceError::<NoColumns>::KindSplit {
    at: 3,
    wrote: ColumnDoor::Fixed(6),
    read: ColumnDoor::Bytes,
  };
  assert_eq!(
    refusal.to_string(),
    "column 3 was written as fixed<6> and read as bytes"
  );
}