ingraph 0.1.0

A declaration-driven indexing framework: records and nodes are written as Rust types, and the storage rows, the dataflow between them and the GraphQL query surface are emitted from the declaration.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
use core::{num::NonZeroUsize, time::Duration};

use std::{
  collections::BTreeMap,
  panic::AssertUnwindSafe,
  sync::{
    Arc,
    atomic::{AtomicBool, AtomicUsize, Ordering},
  },
  thread::{self, ThreadId},
  time::Instant,
};

use super::{Bench, Crew, CrewOptions, DEFAULT_LABEL, HashRouter, Router};
use crate::conduit::{self, park_on};

/// How long a test waits for a running crew to have finished — a DEADLINE and not a sleep, so a
/// broken device reddens a run rather than hanging it.
const PATIENCE: Duration = Duration::from_secs(10);

/// Long enough that a call which genuinely waited cannot be mistaken for one that did not, short
/// enough that the suite does not notice.
const HELD: Duration = Duration::from_millis(150);

/// A crew of `workers`, with `queue` per in-queue and `returns` on the line.
fn seats(workers: usize, queue: usize, returns: usize) -> CrewOptions {
  CrewOptions::new()
    .with_workers(NonZeroUsize::new(workers).expect("not zero"))
    .with_queue_depth(NonZeroUsize::new(queue).expect("not zero"))
    .with_return_depth(NonZeroUsize::new(returns).expect("not zero"))
}

/// EVERY KEY TO WORKER ZERO — what a test spends when it needs to know which bench a task reached
/// rather than to exercise the hashing.
struct ToZero;

impl<K: ?Sized> Router<K> for ToZero {
  fn route(&self, _: &K) -> u64 {
    0
  }
}

// ---------------------------------------------------------------------------------------------
// AFFINITY: one key, one bench, and the order it was given in.
// ---------------------------------------------------------------------------------------------

/// A bench that says WHICH WORKER it is on every result, which is how the observable trace below
/// can name the bench a key reached without reaching inside the device.
struct Ledger {
  index: usize,
}

impl Bench for Ledger {
  type Task = (u64, u64);
  type Yield = (usize, u64, u64);

  fn work(&mut self, (key, seq): (u64, u64)) -> impl IntoIterator<Item = (usize, u64, u64)> {
    [(self.index, key, seq)]
  }
}

/// Route four keys six times each, INTERLEAVED, and answer what came back in the order it came.
fn interleaved(options: CrewOptions) -> Vec<(usize, u64, u64)> {
  let crew =
    Crew::spawn(options, HashRouter::new(), |index| Ledger { index }).expect("the crew spawns");

  let mut harvested = Vec::new();
  for seq in 0..6 {
    for key in 0..4 {
      crew.route(&key, (key, seq)).expect("the crew is alive");
      harvested.extend(crew.harvest());
    }
  }
  harvested.extend(crew.close());
  harvested
}

/// Group a walk of results into one sequence per key, in the order they came out.
fn per_key(results: &[(usize, u64, u64)]) -> BTreeMap<u64, Vec<u64>> {
  let mut grouped: BTreeMap<u64, Vec<u64>> = BTreeMap::new();
  for &(_, key, seq) in results {
    grouped.entry(key).or_default().push(seq);
  }
  grouped
}

#[test]
fn one_key_reaches_one_bench_and_keeps_its_order() {
  let results = interleaved(seats(4, 4, 64));
  assert_eq!(results.len(), 24, "nothing was lost");

  // AFFINITY: every result for a key names the same worker.
  let mut bench_of: BTreeMap<u64, usize> = BTreeMap::new();
  for &(worker, key, _) in &results {
    let claimed = *bench_of.entry(key).or_insert(worker);
    assert_eq!(claimed, worker, "key {key} reached two benches");
  }

  // ORDER: each key's own sequence came back in the order it was routed, however the four keys
  // interleaved on the way out.
  assert_eq!(
    per_key(&results),
    BTreeMap::from([
      (0, vec![0, 1, 2, 3, 4, 5]),
      (1, vec![0, 1, 2, 3, 4, 5]),
      (2, vec![0, 1, 2, 3, 4, 5]),
      (3, vec![0, 1, 2, 3, 4, 5]),
    ])
  );
}

#[test]
fn the_two_forms_answer_the_same_per_key_sequences() {
  let inline = interleaved(seats(1, 4, 64));
  let pool = interleaved(seats(4, 4, 64));

  assert_eq!(inline.len(), pool.len());
  assert_eq!(
    per_key(&inline),
    per_key(&pool),
    "the width changes who does the work and not what comes out of it"
  );
}

// ---------------------------------------------------------------------------------------------
// THE INLINE FORM: no thread at all, and the arm that never fires.
// ---------------------------------------------------------------------------------------------

/// A bench that reports WHICH THREAD it ran on, which is what turns *no thread was spawned* into an
/// assertion rather than an assumption.
struct Whose;

impl Bench for Whose {
  type Task = ();
  type Yield = (ThreadId, Option<String>);

  fn work(&mut self, (): ()) -> impl IntoIterator<Item = (ThreadId, Option<String>)> {
    let here = thread::current();
    [(here.id(), here.name().map(str::to_owned))]
  }
}

#[test]
fn the_inline_form_runs_on_the_calling_thread() {
  let crew = Crew::spawn(seats(1, 4, 8), ToZero, |_| Whose).expect("the crew spawns");
  assert!(crew.is_inline());
  assert_eq!(crew.workers().get(), 1);

  crew.route(&(), ()).expect("the crew is alive");
  let ran: Vec<_> = crew.close().collect();

  assert_eq!(ran.len(), 1);
  assert_eq!(
    ran[0].0,
    thread::current().id(),
    "the inline form spawned a thread"
  );
  assert_ne!(
    ran[0].1.as_deref(),
    Some(DEFAULT_LABEL),
    "no worker thread was named"
  );
}

#[test]
fn the_pool_form_runs_on_threads_of_its_own() {
  let crew = Crew::spawn(seats(3, 4, 8), HashRouter::new(), |_| Whose).expect("the crew spawns");
  assert!(!crew.is_inline());

  for key in 0..12u64 {
    crew.route(&key, ()).expect("the crew is alive");
  }
  let ran: Vec<_> = crew.close().collect();

  assert_eq!(ran.len(), 12);
  let here = thread::current().id();
  for (id, name) in &ran {
    assert_ne!(*id, here, "a worker ran on the calling thread");
    assert!(
      name
        .as_deref()
        .is_some_and(|name| name.starts_with(DEFAULT_LABEL)),
      "a worker thread went unnamed: {name:?}"
    );
  }
}

#[test]
fn the_inline_form_s_arm_parks_rather_than_closing() {
  let crew = Crew::spawn(seats(1, 4, 8), ToZero, |_| Whose).expect("the crew spawns");
  crew.route(&(), ()).expect("the crew is alive");

  // The result is HERE, staged — and the line stayed silent, so an arm offered over it neither
  // fires nor reports an end.
  assert_eq!(crew.pending(), 1);
  assert!(
    park_on(crew.returns().arm(true, || ()), Duration::from_millis(100)).is_none(),
    "the inline form's arm answered on a line nothing speaks on"
  );
  assert_eq!(crew.harvest().count(), 1);
}

#[test]
fn the_pool_form_s_arm_fires_and_leaves_the_order_alone() {
  let crew = Crew::spawn(seats(2, 4, 8), ToZero, |_| Whose).expect("the crew spawns");
  crew.route(&(), ()).expect("the crew is alive");

  assert!(
    park_on(crew.returns().arm(true, || "spoke"), PATIENCE)
      .is_some_and(|heard| heard == Some("spoke")),
    "the crew never spoke"
  );
  // The arm answered a MARKER and took nothing: the result is still where the one door hands it out.
  assert_eq!(crew.harvest().count(), 1, "the arm swallowed the result");
}

// ---------------------------------------------------------------------------------------------
// BACKPRESSURE, both ways.
// ---------------------------------------------------------------------------------------------

/// A bench that will not finish a task until it is let go.
struct Gated {
  gate: conduit::Receiver<()>,
}

impl Bench for Gated {
  type Task = u64;
  type Yield = u64;

  fn work(&mut self, task: u64) -> impl IntoIterator<Item = u64> {
    let _ = self.gate.recv();
    [task]
  }
}

#[test]
fn a_full_in_queue_makes_route_wait() {
  let (open, gate) = conduit::bounded::<()>(16);
  let crew = Crew::spawn(seats(2, 1, 16), ToZero, |_| Gated { gate: gate.clone() })
    .expect("the crew spawns");

  // Nobody is let go until the sleep is over, so the third route cannot land before it.
  let releasing = thread::spawn(move || {
    thread::sleep(HELD);
    for _ in 0..8 {
      let _ = open.send(());
    }
  });

  let began = Instant::now();
  for task in 0..3 {
    crew.route(&(), task).expect("the crew is alive");
  }
  let waited = began.elapsed();
  releasing.join().expect("the releasing thread");
  drop(crew.close());

  assert!(
    waited >= HELD / 2,
    "routing three tasks into a queue of one never waited: {waited:?}"
  );
}

#[test]
fn a_deep_in_queue_does_not_make_route_wait() {
  let (open, gate) = conduit::bounded::<()>(16);
  let crew = Crew::spawn(seats(2, 8, 16), ToZero, |_| Gated { gate: gate.clone() })
    .expect("the crew spawns");

  // THE CONTROL for the test above: the same gated benches, the same three tasks, a queue with room
  // — and the routes return at once, which is what says the wait there was the queue's.
  let began = Instant::now();
  for task in 0..3 {
    crew.route(&(), task).expect("the crew is alive");
  }
  let waited = began.elapsed();

  for _ in 0..8 {
    let _ = open.send(());
  }
  drop(crew.close());
  assert!(
    waited < HELD / 2,
    "a queue with room still waited: {waited:?}"
  );
}

/// A bench whose results are made ONE AT A TIME and counted as they are made, so a worker parked
/// half-way through a task is visible from outside it.
struct Spilling {
  made: Arc<AtomicUsize>,
  each: usize,
}

/// The walk `Spilling::work` answers — lazy on purpose, so the count is of results that exist rather
/// than of results that were planned.
struct Spill {
  made: Arc<AtomicUsize>,
  left: usize,
}

impl Iterator for Spill {
  type Item = usize;

  fn next(&mut self) -> Option<usize> {
    if self.left == 0 {
      return None;
    }
    self.left -= 1;
    Some(self.made.fetch_add(1, Ordering::SeqCst))
  }
}

impl Bench for Spilling {
  type Task = ();
  type Yield = usize;

  fn work(&mut self, (): ()) -> impl IntoIterator<Item = usize> {
    Spill {
      made: Arc::clone(&self.made),
      left: self.each,
    }
  }
}

#[test]
fn a_full_return_line_makes_a_worker_wait() {
  let made = Arc::new(AtomicUsize::new(0));
  let crew = Crew::spawn(seats(2, 4, 2), ToZero, |_| Spilling {
    made: Arc::clone(&made),
    each: 10,
  })
  .expect("the crew spawns");

  crew.route(&(), ()).expect("the crew is alive");

  // Two on the line and one in the worker's hand: past that it is parked, and a bench that could
  // run ahead would have made all ten by now.
  thread::sleep(HELD);
  assert!(
    made.load(Ordering::SeqCst) <= 3,
    "the worker ran past a full return line: {} made",
    made.load(Ordering::SeqCst)
  );
  assert_eq!(crew.pending(), 2, "the line is not full");

  // Harvesting is what frees it, and the rest follows.
  let mut taken = crew.harvest().count();
  let deadline = Instant::now() + PATIENCE;
  while taken < 10 {
    assert!(Instant::now() < deadline, "the worker never resumed");
    taken += crew.harvest().count();
  }
  assert_eq!(made.load(Ordering::SeqCst), 10);
  assert_eq!(crew.close().count(), 0, "everything was already harvested");
}

#[test]
fn a_full_return_line_does_not_wedge_a_full_in_queue() {
  // THE TWO BOUNDS MEETING: benches that spill more than the line holds, an in-queue small enough to
  // fill, and a body that routes a burst without harvesting between the routes. Each side is waiting
  // on the other unless `route` takes what the line is holding, which is what this pins.
  let made = Arc::new(AtomicUsize::new(0));
  let crew = Crew::spawn(seats(2, 1, 2), ToZero, |_| Spilling {
    made: Arc::clone(&made),
    each: 8,
  })
  .expect("the crew spawns");

  let routed = Arc::new(AtomicBool::new(false));
  let hand = thread::spawn({
    let routed = Arc::clone(&routed);
    move || {
      for _ in 0..6 {
        crew.route(&(), ()).expect("the crew is alive");
      }
      routed.store(true, Ordering::SeqCst);
      crew.close().count()
    }
  });

  let deadline = Instant::now() + PATIENCE;
  while !routed.load(Ordering::SeqCst) {
    assert!(
      Instant::now() < deadline,
      "a full return line wedged a full in-queue"
    );
    thread::sleep(Duration::from_millis(5));
  }
  let tail = hand.join().expect("the routing thread");
  assert_eq!(
    made.load(Ordering::SeqCst),
    48,
    "six tasks of eight results"
  );
  assert!(tail <= 48);
}

// ---------------------------------------------------------------------------------------------
// THE EPILOGUE and its backstop.
// ---------------------------------------------------------------------------------------------

/// A bench that takes a moment over each task and says when it is dropped — which only happens once
/// its worker has returned and been joined.
struct Slow {
  gone: Arc<AtomicUsize>,
}

impl Bench for Slow {
  type Task = u64;
  type Yield = u64;

  fn work(&mut self, task: u64) -> impl IntoIterator<Item = u64> {
    thread::sleep(Duration::from_millis(5));
    [task]
  }
}

impl Drop for Slow {
  fn drop(&mut self) {
    self.gone.fetch_add(1, Ordering::SeqCst);
  }
}

#[test]
fn close_drains_the_tails_then_joins() {
  let gone = Arc::new(AtomicUsize::new(0));
  let crew = Crew::spawn(seats(2, 8, 64), HashRouter::new(), |_| Slow {
    gone: Arc::clone(&gone),
  })
  .expect("the crew spawns");

  for task in 0..8 {
    crew.route(&task, task).expect("the crew is alive");
  }
  // Closed with every task still in flight: what comes back is the whole of it, because a task
  // already in a queue is worked before that queue reports its end.
  let mut tail: Vec<u64> = crew.close().collect();
  tail.sort_unstable();

  assert_eq!(tail, (0..8).collect::<Vec<_>>());
  assert_eq!(gone.load(Ordering::SeqCst), 2, "a worker was not joined");
}

#[test]
fn dropping_without_closing_winds_the_crew_down() {
  let gone = Arc::new(AtomicUsize::new(0));
  let crew = Crew::spawn(seats(3, 8, 64), HashRouter::new(), |_| Slow {
    gone: Arc::clone(&gone),
  })
  .expect("the crew spawns");
  for task in 0..6 {
    crew.route(&task, task).expect("the crew is alive");
  }

  drop(crew);
  assert_eq!(
    gone.load(Ordering::SeqCst),
    3,
    "the backstop returned before every worker was joined"
  );
}

#[test]
fn a_drop_during_an_unwind_neither_deadlocks_nor_leaks_a_worker() {
  let gone = Arc::new(AtomicUsize::new(0));
  let made = Arc::new(AtomicUsize::new(0));
  let finished = Arc::new(AtomicBool::new(false));

  // The hard case on purpose: benches that spill more than the line holds, so the workers are parked
  // on a FULL return line at the moment the body dies. A drop that joined before it drained would
  // wait on a thread that is waiting on it.
  let dying = thread::spawn({
    let (gone, made, finished) = (Arc::clone(&gone), Arc::clone(&made), Arc::clone(&finished));
    move || {
      let outcome = std::panic::catch_unwind(AssertUnwindSafe(|| {
        let crew = Crew::spawn(seats(2, 4, 2), ToZero, |_| Unwound {
          gone: Arc::clone(&gone),
          made: Arc::clone(&made),
        })
        .expect("the crew spawns");
        crew.route(&(), ()).expect("the crew is alive");
        crew.route(&(), ()).expect("the crew is alive");
        panic!("the body dies inside its own loop");
      }));
      assert!(outcome.is_err(), "the panic was swallowed");
      finished.store(true, Ordering::SeqCst);
    }
  });

  let deadline = Instant::now() + PATIENCE;
  while !finished.load(Ordering::SeqCst) {
    assert!(
      Instant::now() < deadline,
      "the drop backstop deadlocked on the unwind path"
    );
    thread::sleep(Duration::from_millis(5));
  }
  dying.join().expect("the dying thread");
  assert_eq!(
    gone.load(Ordering::SeqCst),
    2,
    "a worker was left behind by the unwind"
  );
}

/// [`Spilling`] with [`Slow`]'s drop on it — what the unwind test needs to say both things at once.
struct Unwound {
  gone: Arc<AtomicUsize>,
  made: Arc<AtomicUsize>,
}

impl Bench for Unwound {
  type Task = ();
  type Yield = usize;

  fn work(&mut self, (): ()) -> impl IntoIterator<Item = usize> {
    Spill {
      made: Arc::clone(&self.made),
      left: 32,
    }
  }
}

impl Drop for Unwound {
  fn drop(&mut self) {
    self.gone.fetch_add(1, Ordering::SeqCst);
  }
}

// ---------------------------------------------------------------------------------------------
// THE ROUTER and THE SEATS.
// ---------------------------------------------------------------------------------------------

#[test]
fn a_hash_router_answers_the_same_key_the_same_way() {
  let router = HashRouter::new();
  let first = router.route("track/7");
  assert_eq!(first, router.route("track/7"));
  assert_eq!(first, router.route("track/7"), "and again");

  // A router over a FIXED hasher agrees with a second one of its own kind, which is what a caller
  // reaches `with_hasher` for.
  #[derive(Clone, Default)]
  struct Zeroed;

  impl core::hash::BuildHasher for Zeroed {
    type Hasher = std::collections::hash_map::DefaultHasher;

    fn build_hasher(&self) -> Self::Hasher {
      Self::Hasher::default()
    }
  }

  assert_eq!(
    HashRouter::with_hasher(Zeroed).route("track/7"),
    HashRouter::with_hasher(Zeroed).route("track/7"),
    "a fixed hasher is what makes two routers agree"
  );
}

#[test]
fn the_seats_ship_their_defaults() {
  let seats = CrewOptions::new();
  assert_eq!(seats, CrewOptions::default());
  assert_eq!(seats.workers(), super::DEFAULT_WORKERS);
  assert_eq!(seats.queue_depth(), super::DEFAULT_QUEUE_DEPTH);
  assert_eq!(seats.return_depth(), super::DEFAULT_RETURN_DEPTH);
  assert_eq!(seats.label(), DEFAULT_LABEL);

  // ONE worker by default, which is the inline form: a device's default may not read the host.
  assert_eq!(seats.workers().get(), 1);

  let tuned = seats
    .with_workers(NonZeroUsize::new(4).expect("not zero"))
    .with_queue_depth(NonZeroUsize::new(2).expect("not zero"))
    .with_return_depth(NonZeroUsize::new(8).expect("not zero"))
    .with_label("resample");
  assert_eq!(tuned.workers().get(), 4);
  assert_eq!(tuned.queue_depth().get(), 2);
  assert_eq!(tuned.return_depth().get(), 8);
  assert_eq!(tuned.label(), "resample");

  let mut in_place = CrewOptions::new();
  in_place
    .set_workers(NonZeroUsize::new(4).expect("not zero"))
    .set_queue_depth(NonZeroUsize::new(2).expect("not zero"))
    .set_return_depth(NonZeroUsize::new(8).expect("not zero"))
    .set_label("resample");
  assert_eq!(in_place, tuned, "the two spellings say the same thing");
}

// ---------------------------------------------------------------------------------------------
// THE ORDER, through the arm and through backpressure at once — the two waits that used to be two
// readers of one line.
// ---------------------------------------------------------------------------------------------

/// A bench that TAKES ITS TIME and then speaks TWICE, which is the shape the defect needed: a body
/// waiting on the crew while it routes, and a run of results arriving in one breath.
struct Twinning {
  taking: Duration,
  started: conduit::Sender<()>,
}

impl Bench for Twinning {
  type Task = u64;
  type Yield = u64;

  fn work(&mut self, task: u64) -> impl IntoIterator<Item = u64> {
    // Said BEFORE the pause, so the script below knows the in-queue has been read.
    drop(self.started.try_send(()));
    thread::sleep(self.taking);
    [task * 2, task * 2 + 1]
  }
}

/// THE DEFECT'S OWN INTERLEAVE, scripted rather than raced (ingraph#409).
///
/// The order broke when a body was doing BOTH of the things a body does at once: an arm pinned over
/// the crew's return line, and a [`route`](Crew::route) waiting for a full in-queue underneath it.
/// Both waits used to REGISTER on the line, and a bounded channel hands an item straight to a
/// waiting reader rather than queueing it — so the first result of a task went to the arm's wait and
/// into the queue, the second went STRAIGHT to the route's wait, and the route's was read first. One
/// bench's two results came out swapped.
///
/// The script forces exactly that: the arm is parked first, the in-queue is filled so the third
/// route waits underneath it, and the bench then speaks twice while both waits are outstanding.
#[test]
fn a_pinned_arm_over_a_waiting_route_still_hands_one_bench_its_own_order() {
  use core::pin::pin;

  // SIX TIMES, because what used to fail was a window and not a certainty: the route's wait is
  // re-offered every millisecond, so one pass is a very likely miss and six is an assurance.
  for attempt in 0..6 {
    let (started, starts) = conduit::bounded::<()>(8);
    let crew = Crew::spawn(seats(2, 1, 64), ToZero, |_| Twinning {
      taking: Duration::from_millis(25),
      started: started.clone(),
    })
    .expect("the crew spawns");
    drop(started);

    {
      // ① THE ARM, pinned and parked — a body's own `select` over the crew, mid-flight.
      let mut worked = pin!(crew.returns().arm(true, || ()));
      assert!(
        park_on(worked.as_mut(), Duration::from_millis(5)).is_none(),
        "attempt {attempt}: the crew spoke before it had been given anything"
      );

      // ② THE FIRST TASK reaches the bench, which says so and then takes its time.
      crew.route(&(), 0).expect("the crew is alive");
      assert!(
        park_on(starts.recv_async(), PATIENCE).is_some(),
        "attempt {attempt}: the bench never started"
      );
      // ③ THE IN-QUEUE FILLS — one seat, and this takes it.
      crew.route(&(), 1).expect("the crew is alive");
      // ④ AND THIS ONE WAITS UNDERNEATH THE ARM, which is the interleave: the bench's two results
      //    land while both the arm's wait and this call's wait are outstanding.
      crew.route(&(), 2).expect("the crew is alive");
    }

    let mut got: Vec<u64> = crew.harvest().collect();
    got.extend(crew.close());
    assert_eq!(
      got,
      vec![0, 1, 2, 3, 4, 5],
      "attempt {attempt}: one bench's results came back out of the order it made them in"
    );
  }
}

/// The same promise under CPU CONTENTION, which is the shape the defect was first measured in: a
/// loaded machine, an armed loop, and per-key sequences that must not interleave with themselves.
#[test]
fn one_key_keeps_its_order_through_an_armed_loop_under_contention() {
  use core::pin::pin;

  use crate::conduit::{Arm, select};

  /// What the loop below waits for — a task to route, or the crew.
  enum Turn {
    Fed((u64, u64)),
    Worked,
  }

  /// A bench that answers a task with a RUN of results, so a worker can be caught mid-task.
  struct Fanning;

  impl Bench for Fanning {
    type Task = (u64, u64);
    type Yield = (u64, u64);

    fn work(&mut self, (key, seq): (u64, u64)) -> impl IntoIterator<Item = (u64, u64)> {
      (0..3).map(move |step| (key, seq * 3 + step))
    }
  }

  /// The SHIPPED loop shape: one select over a feed and the crew's line, routing what it hears and
  /// harvesting every pass.
  fn drive() -> BTreeMap<u64, Vec<u64>> {
    let crew =
      Crew::spawn(seats(4, 2, 4), HashRouter::new(), |_| Fanning).expect("the crew spawns");
    let (feed, from_feed) = conduit::bounded::<(u64, u64)>(256);
    for seq in 0..8 {
      for key in 0..4 {
        feed.send((key, seq)).expect("the feed holds it");
      }
    }
    drop(feed);

    let mut out: Vec<(u64, u64)> = Vec::new();
    let mut turn = 0usize;
    let mut feeding = true;
    loop {
      {
        let mut fed = pin!(conduit::arm(feeding, &from_feed, Turn::Fed));
        let mut worked = pin!(crew.returns().arm(true, || Turn::Worked));
        let mut arms: [Arm<'_, Option<Turn>>; 2] = [fed.as_mut(), worked.as_mut()];

        match park_on(select(turn, &mut arms), Duration::from_millis(20)) {
          None | Some(None) => {}
          Some(Some((index, heard))) => {
            turn = index + 1;
            match heard {
              None => feeding = false,
              Some(Turn::Fed((key, seq))) => {
                crew.route(&key, (key, seq)).expect("the crew is alive");
              }
              Some(Turn::Worked) => {}
            }
          }
        }
      }
      out.extend(crew.harvest());
      if !feeding {
        break;
      }
    }
    out.extend(crew.close());

    let mut grouped: BTreeMap<u64, Vec<u64>> = BTreeMap::new();
    for (key, seq) in out {
      grouped.entry(key).or_default().push(seq);
    }
    grouped
  }

  // The load is what the defect needed: it was never reproducible on a quiet machine and always
  // reproducible on a busy one.
  let stop = Arc::new(AtomicBool::new(false));
  let burners: Vec<_> = (0..8)
    .map(|_| {
      let stop = Arc::clone(&stop);
      thread::spawn(move || {
        let mut spun = 0u64;
        while !stop.load(Ordering::Relaxed) {
          spun = spun.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1);
          std::hint::black_box(spun);
        }
      })
    })
    .collect();

  let want: Vec<u64> = (0..24).collect();
  let mut broke = 0usize;
  // The first few, named. A defect here fails in nearly every round, and a report that printed all
  // of them would bury the one line a reader needs.
  let mut first: Vec<(usize, u64, Vec<u64>)> = Vec::new();
  for round in 0..120 {
    for (key, seq) in drive() {
      if seq != want {
        broke += 1;
        if first.len() < 3 {
          first.push((round, key, seq));
        }
      }
    }
  }
  stop.store(true, Ordering::Relaxed);
  for burner in burners {
    burner.join().expect("the burner ends");
  }
  assert_eq!(
    broke, 0,
    "a key's own sequence interleaved with itself, {broke} times — first: {first:?}"
  );
}