ingraph 0.1.0

A declaration-driven indexing framework: records and nodes are written as Rust types, and the storage rows, the dataflow between them and the GraphQL query surface are emitted from the declaration.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
//! THE GATES: the four seats where a policy is actually consulted, with 叉 A written once.
//!
//! A policy is a decision and a gate is the thing that ASKS for one. Everything the hook law says
//! about the ask lives here rather than at four call sites: the hook is consulted, an `Err` becomes
//! a tap record and the shipped default, and the framework performs whatever the decision was —
//! which is the sleep, the drop, the extra ticks. A caller holds a gate and never a policy, so the
//! fallback cannot be forgotten at one of the four and remembered at the other three.
//!
//! Each gate carries the node it speaks for and the taps it reports through, because a report has
//! to name a node and a policy does not know one: one policy serves the whole arrangement, and the
//! gate is where it meets a particular node.

use core::{any::Any, hash::Hash, time::Duration};

use std::{collections::HashSet, sync::Arc};

use jiff::SignedDuration;
use smol_bytes::Utf8Bytes;

use crate::{
  BoxedError,
  drive::{
    Awaiting, Delivery, Evictor, Failure, Fate, LifecycleTap, NodeName, Observation, Outbox,
    QueryError, Station, Taps,
  },
};

use super::{
  Concern, DeadlinePolicy, Missed, Occupancy, OverflowDecision, OverflowPolicy, RetryDecision,
  RetryPolicy, ShedKey, TickDecision, TickPolicy,
};

#[cfg(test)]
mod tests;

/// 叉 A, WRITTEN ONCE: take the policy's answer, or record why there is none and take the shipped
/// default.
///
/// It is a free function rather than four copies because the sentence is one sentence. A mad
/// advisor must not stop the factory: the data path continues at the framework's own decision, and
/// the ONLY trace of the sickness is the record — which is exactly why the record keeps the error's
/// type instead of a rendering.
fn advise<D>(
  taps: &Taps,
  node: &Utf8Bytes,
  concern: Concern,
  asked: Result<D, BoxedError>,
  shipped: D,
) -> D {
  match asked {
    Ok(decision) => decision,
    Err(error) => {
      taps.observe(
        NodeName::new(node),
        &Observation::PolicyFailed { concern, error },
      );
      shipped
    }
  }
}

/// THE TICK SEAT: what one node's clock does when it ran late.
///
/// Held by a resident and asked only where the fork has actually arisen — a tick that fires on time
/// missed nothing, so no policy is consulted and a deployment at the shipped default does, to the
/// instruction, what it did before this seam existed.
#[derive(Clone)]
pub struct Ticks {
  policy: Arc<dyn TickPolicy>,
  node: Utf8Bytes,
  taps: Taps,
}

impl Ticks {
  /// The gate over `policy`, for the node called `node`, reporting through `taps`.
  #[inline]
  pub(super) const fn new(policy: Arc<dyn TickPolicy>, node: Utf8Bytes, taps: Taps) -> Self {
    Self { policy, node, taps }
  }

  /// What to do about `missed` — the policy's answer, or [`TickDecision::Coalesce`] where it had
  /// none.
  #[must_use]
  pub fn on_missed(&self, missed: Missed) -> TickDecision {
    advise(
      &self.taps,
      &self.node,
      Concern::Tick,
      self.policy.on_missed(missed),
      TickDecision::Coalesce,
    )
  }
}

impl core::fmt::Debug for Ticks {
  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
    f.debug_struct("Ticks")
      .field("node", &self.node.as_str())
      .finish_non_exhaustive()
  }
}

/// THE RETRY SEAT: attached where [`Station::is_settled`] already answers whether an item is done.
///
/// A settled item never asks — there is nothing to retry about work that finished — so the policy
/// is reached by exactly the station that holds the window, which is what makes the seat the one
/// the R-line named rather than a new place to check.
#[derive(Clone)]
pub struct Retries {
  policy: Arc<dyn RetryPolicy>,
  node: Utf8Bytes,
  taps: Taps,
}

impl Retries {
  /// The gate over `policy`, for the node called `node`, reporting through `taps`.
  #[inline]
  pub(super) const fn new(policy: Arc<dyn RetryPolicy>, node: Utf8Bytes, taps: Taps) -> Self {
    Self { policy, node, taps }
  }

  /// What an item at `station` earns on attempt `attempt`.
  ///
  /// A station that [`is_settled`](Station::is_settled) answers [`GiveUp`](RetryDecision::GiveUp)
  /// without asking anybody: persisted, processed and skipped are outcomes, and an outcome is not a
  /// thing to try again. `failure` is what the station carried, and it is `None` for every settled
  /// one.
  ///
  /// That early answer records NOTHING, and the distinction is the vocabulary's: a settled item did
  /// not have the framework give up on it, it finished. Only a failure whose retries ran out is a
  /// [`Fate::GivenUp`].
  #[must_use]
  pub fn settling(
    &self,
    station: Station,
    attempt: u32,
    failure: Option<&Failure>,
  ) -> RetryDecision {
    if station.is_settled() {
      return RetryDecision::GiveUp;
    }
    let decision = match failure {
      Some(failure) => self.ask(attempt, failure.error()),
      // A failed station with no failure to show is a caller that erased it. The item still failed,
      // so the policy is still the party that decides — it is shown the station's own error type,
      // which is this one.
      None => self.ask(attempt, &UnstatedFailure),
    };
    // The failure travels INTO the record here and nowhere else: this is the one seat that was
    // handed the whole `Failure` rather than a borrowed `&dyn Error`, so it is the one that can put
    // the error's own type into a give-up's obituary.
    self.record(attempt, decision, failure.cloned());
    decision
  }

  /// Ask outright, and RECORD what came back.
  ///
  /// The record is made here rather than by whoever performs the wait, because this is the moment
  /// the decision exists: a tap watching a deployment's retries is watching what its policy
  /// ANSWERED, and a report written after the sleep would be a report of what survived it. The same
  /// is true of the ending: a give-up is decided here and performed elsewhere.
  #[must_use]
  pub fn on_failure(
    &self,
    attempt: u32,
    error: &(dyn core::error::Error + 'static),
  ) -> RetryDecision {
    let decision = self.ask(attempt, error);
    // No failure, and it is a limitation of this door rather than a choice: `&dyn Error` is neither
    // owned nor `Send + Sync`, so nothing here can build the `Failure` an obituary would carry.
    // [`settling`](Self::settling) is the door that has one.
    self.record(attempt, decision, None);
    decision
  }

  /// Consult the policy, with 叉 A's fallback and no record.
  fn ask(&self, attempt: u32, error: &(dyn core::error::Error + 'static)) -> RetryDecision {
    advise(
      &self.taps,
      &self.node,
      Concern::Retry,
      self.policy.on_failure(attempt, error),
      RetryDecision::GiveUp,
    )
  }

  /// Say what was decided — a scheduled retry, or the ending.
  ///
  /// BOTH arms, which is what C-2's one law asks for: a retry that will happen has been announced
  /// since the policy seam landed, and a retry that will not was the same decision with nothing said
  /// about it.
  fn record(&self, attempt: u32, decision: RetryDecision, failure: Option<Failure>) {
    let observation = match decision {
      RetryDecision::Retry { after } => Observation::Retrying { attempt, after },
      RetryDecision::GiveUp => {
        Fate::GivenUp {
          attempt,
          failure,
          message: (),
        }
        .erase()
        .0
      }
    };
    self.taps.observe(NodeName::new(&self.node), &observation);
  }
}

impl core::fmt::Debug for Retries {
  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
    f.debug_struct("Retries")
      .field("node", &self.node.as_str())
      .finish_non_exhaustive()
  }
}

/// A failed station whose error did not travel — see [`Retries::settling`].
#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)]
#[display("the item failed and its error was not carried")]
struct UnstatedFailure;

impl core::error::Error for UnstatedFailure {}

/// WHAT ONE POLICY-GOVERNED HAND-OVER DID.
///
/// Four outcomes and not a `Result`, because two of them are successes that lost something and one
/// is a failure that lost nothing. What the caller HOLDS differs by arm — a shed delivery comes back
/// spent, a refused one comes back whole and still owing its answer — so the arms are named rather
/// than folded into an `Err`.
#[derive(Debug)]
pub enum Handover<T, K> {
  /// It went in, and nothing was shed.
  Sent,
  /// It went in, and this came out to make room — [`OverflowDecision::DropOldest`]. The obituary has
  /// been recorded and the token ANSWERED, so what comes back carries none — see
  /// [`Sheds::deliver`].
  Evicted(Delivery<T, K>),
  /// It did not go in, and nothing came out — [`OverflowDecision::DropNewest`]. Recorded and
  /// answered exactly as [`Evicted`](Self::Evicted) is.
  Dropped(Delivery<T, K>),
  /// The consumer is gone. Nothing was shed, nothing was answered, and the delivery comes back
  /// WHOLE — token and all — exactly as [`Outbox::deliver`]'s own `Err` hands it back. A consumer
  /// that died is not an ending a policy chose, so it is the sender's to state and not this seat's.
  Gone(Delivery<T, K>),
}

impl<T, K> Handover<T, K> {
  /// Whether the delivery reached the edge.
  #[inline]
  #[must_use]
  pub const fn is_sent(&self) -> bool {
    matches!(self, Self::Sent | Self::Evicted(_))
  }

  /// Whether an item was lost to a policy.
  #[inline]
  #[must_use]
  pub const fn is_shed(&self) -> bool {
    matches!(self, Self::Evicted(_) | Self::Dropped(_))
  }
}

/// THE OVERFLOW SEAT: a full bounded edge, and what the deployment said to do about one.
///
/// # It performs the decision
///
/// The hook answers `Block`, `DropOldest` or `DropNewest` and this is what carries it out —
/// waiting, evicting, refusing — and what records every obituary. That division is the hook law's:
/// a policy that could do the dropping itself would be a policy that could touch the transport.
#[derive(Clone)]
pub struct Sheds {
  policy: Arc<dyn OverflowPolicy>,
  node: Utf8Bytes,
  taps: Taps,
}

impl Sheds {
  /// The gate over `policy`, for the node called `node`, reporting through `taps`.
  #[inline]
  pub(super) const fn new(policy: Arc<dyn OverflowPolicy>, node: Utf8Bytes, taps: Taps) -> Self {
    Self { policy, node, taps }
  }

  /// What to do about an edge full at `occupancy` — the policy's answer, or
  /// [`OverflowDecision::Block`] where it had none.
  #[must_use]
  pub fn on_full(&self, occupancy: Occupancy) -> OverflowDecision {
    advise(
      &self.taps,
      &self.node,
      Concern::Overflow,
      self.policy.on_full(occupancy),
      OverflowDecision::Block,
    )
  }

  /// Record ONE item's obituary: it was lost at `occupancy`, by `decision`.
  pub fn shed<K>(&self, key: K, decision: OverflowDecision, occupancy: Occupancy)
  where
    K: Any + core::fmt::Display + Send + Sync,
  {
    self.taps.observe(
      NodeName::new(&self.node),
      &Observation::Shed {
        key: ShedKey::new(key),
        decision,
        occupancy,
      },
    );
  }

  /// Hand `delivery` to `outbox`, letting the policy answer a full edge.
  ///
  /// # The eviction hand is OPTIONAL, and what changes without one
  ///
  /// [`DropOldest`](OverflowDecision::DropOldest) names the FAR end of the queue, and only a
  /// receiving half can reach it — so a sender that was given no [`Evictor`] has exactly one item it
  /// can lose, which is the one in its hand. It loses that one, and the obituary says
  /// [`DropNewest`](OverflowDecision::DropNewest) because that is what happened: the record is of
  /// the shed, not of the wish.
  ///
  /// That substitution is the honest one of the three available. Blocking would ignore a
  /// deployment's explicit *shed under pressure*; refusing at launch would make the hand mandatory
  /// on every edge, and an evictor holds a receiver — an edge with one outstanding never reports a
  /// disconnect, so a mandatory hand would put a shutdown-blocking handle on every producer in the
  /// arrangement to serve the one policy that needs it.
  ///
  /// # What a shed costs, and what it does not
  ///
  /// Every item lost leaves an [`Observation::Shed`] carrying its own key, and the delivery's OWN
  /// TOKEN IS ANSWERED with the shed [`Fate`] before it comes back in the [`Handover`]. The second
  /// half is C-1's ruling and it replaces the older division, in which answering belonged to
  /// whoever held the delivery next: nobody holds a shed delivery next, so the token was dropped, and
  /// a dropped token reaches its sender as a disconnected oneshot — indistinguishable from a
  /// consumer that died, and holding the sender's window forever on an item that no longer exists.
  /// The fate says which ending it was and credits the window in the same breath.
  ///
  /// The delivery handed back therefore carries no token — [`Delivery::in_custody`]'s shape — and
  /// its holder owes nothing.
  ///
  /// A racing producer that refills the edge between the eviction and the retry is not shed twice —
  /// the retry WAITS, the policy having asked to lose the oldest and already got one.
  pub fn deliver<K, T>(
    &self,
    outbox: &Outbox<K, T>,
    evictor: Option<&Evictor<K, T>>,
    delivery: Delivery<T, K>,
  ) -> Handover<T, K>
  where
    K: Any + Clone + core::fmt::Display + Eq + Hash + Send + Sync,
  {
    let Err(delivery) = outbox.try_deliver(delivery) else {
      return Handover::Sent;
    };
    if outbox.is_disconnected() {
      return Handover::Gone(delivery);
    }
    let occupancy = occupancy(outbox);
    match self.on_full(occupancy) {
      OverflowDecision::Block => self.block(outbox, delivery),
      OverflowDecision::DropNewest => self.drop_newest(delivery, occupancy),
      // Nothing to evict is the same story as no hand to evict WITH: the consumer drained the edge
      // between the refusal and the look, and either way the only item left to lose is the one in
      // hand.
      OverflowDecision::DropOldest => match evictor.and_then(Evictor::evict) {
        None => self.drop_newest(delivery, occupancy),
        Some(oldest) => {
          let oldest = self.shed_whole(oldest, OverflowDecision::DropOldest, occupancy);
          match outbox.try_deliver(delivery) {
            Ok(()) => Handover::Evicted(oldest),
            Err(delivery) => match self.block(outbox, delivery) {
              Handover::Sent => Handover::Evicted(oldest),
              other => other,
            },
          }
        }
      },
    }
  }

  /// Lose the delivery in hand, and say so.
  fn drop_newest<K, T>(&self, delivery: Delivery<T, K>, occupancy: Occupancy) -> Handover<T, K>
  where
    K: Any + Clone + core::fmt::Display + Eq + Hash + Send + Sync,
  {
    Handover::Dropped(self.shed_whole(delivery, OverflowDecision::DropNewest, occupancy))
  }

  /// SHED THIS DELIVERY WHOLE: mourn every item it carried, answer its token with the shed fate,
  /// and hand back what is left of it.
  ///
  /// # One fate for a delivery, one obituary per item
  ///
  /// The two records answer different questions and the asymmetry is the transport's: the eviction
  /// hand takes a MESSAGE out of a queue and never an item out of a message, so the ending is one
  /// ending, while what a tap counts and re-derives by is a key. So the fate names the delivery's
  /// first key and CREDITS every one of them, and the per-item obituaries beside it name the rest.
  ///
  /// A delivery carrying no items at all — an empty batch — has no key to name and is answered with
  /// nothing, which loses nothing: its window entry holds no item to stall on.
  fn shed_whole<K, T>(
    &self,
    delivery: Delivery<T, K>,
    decision: OverflowDecision,
    occupancy: Occupancy,
  ) -> Delivery<T, K>
  where
    K: Any + Clone + core::fmt::Display + Eq + Hash + Send + Sync,
  {
    self.mourn(&delivery, decision, occupancy);
    let (payload, custody) = delivery.into_parts();
    if let Some(custody) = custody
      && let Some(first) = payload.keys().next().cloned()
    {
      let credited: HashSet<K> = payload.keys().cloned().collect();
      custody.fate(
        credited,
        // THIS node's own name, which is the one case where that is right: no consumer took the
        // delivery, so there is no worker to name. It is the framework speaking for the sender about
        // a hand-over that reached nowhere — `Resident::hung_up`'s attribution, for its reason.
        self.node.clone(),
        Fate::Shed {
          key: ShedKey::new(first),
          decision,
          occupancy,
          message: (),
        },
      );
    }
    Delivery::in_custody(payload)
  }

  /// Wait for room, which is what [`OverflowDecision::Block`] IS.
  fn block<K, T>(&self, outbox: &Outbox<K, T>, delivery: Delivery<T, K>) -> Handover<T, K> {
    match outbox.deliver(delivery) {
      Ok(()) => Handover::Sent,
      Err(delivery) => Handover::Gone(delivery),
    }
  }

  /// One obituary per item the delivery carried.
  fn mourn<K, T>(&self, delivery: &Delivery<T, K>, decision: OverflowDecision, occupancy: Occupancy)
  where
    K: Any + Clone + core::fmt::Display + Send + Sync,
  {
    for key in delivery.payload().keys() {
      self.shed(key.clone(), decision, occupancy);
    }
  }
}

impl core::fmt::Debug for Sheds {
  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
    f.debug_struct("Sheds")
      .field("node", &self.node.as_str())
      .finish_non_exhaustive()
  }
}

/// How full `outbox` is, in the vocabulary a policy is asked in.
fn occupancy<K, T>(outbox: &Outbox<K, T>) -> Occupancy {
  match outbox.capacity() {
    Some(capacity) => Occupancy::bounded(outbox.len(), capacity),
    None => Occupancy::unbounded(outbox.len()),
  }
}

/// THE DEADLINE SEAT: how long a silent caller waits at a door.
///
/// # EXPLICIT BEATS AMBIENT
///
/// [`resolve`](Self::resolve) is that ruling and the whole of it: a call site that named a budget
/// keeps it, and the policy is asked only where none was named. Nothing here can shorten a caller's
/// own patience, which is what keeps an ambient default from becoming a thing that has to be
/// fought.
#[derive(Clone)]
pub struct Deadlines {
  policy: Arc<dyn DeadlinePolicy>,
  node: Utf8Bytes,
  taps: Taps,
}

impl Deadlines {
  /// The gate over `policy`, for the node called `node`, reporting through `taps`.
  #[inline]
  pub(super) const fn new(policy: Arc<dyn DeadlinePolicy>, node: Utf8Bytes, taps: Taps) -> Self {
    Self { policy, node, taps }
  }

  /// The budget in force: `explicit` where the call site named one, the policy's where it did not.
  ///
  /// A NEGATIVE ambient budget is a deadline already past, which reads as *do not wait* and is
  /// answered as [`Duration::ZERO`] rather than refused: the policy is advisory, and an advisor's
  /// arithmetic mistake must not become a launch failure.
  #[must_use]
  pub fn resolve(&self, explicit: Option<Duration>) -> Option<Duration> {
    if explicit.is_some() {
      return explicit;
    }
    advise(
      &self.taps,
      &self.node,
      Concern::Deadline,
      self.policy.deadline(),
      None,
    )
    .map(unsigned)
  }

  /// Wait for `awaiting`'s answer under the resolved budget.
  ///
  /// [`Awaiting::take`] where nothing named one, [`Awaiting::take_within`] where something did — so
  /// a door with no policy and a caller with no word behave exactly as they always have.
  ///
  /// # An expiry is SAID, and the caller's answer does not change
  ///
  /// Where the budget runs out, a [`Fate::Expired`] reaches every tap under this node's name and the
  /// caller still gets [`QueryError::TimedOut`]. Two views of one truth, and the pair is C-2's
  /// ruling: the asker learns its own question went unanswered and can say nothing about which node
  /// ran out of time, while the arrangement learns exactly that and can say nothing about who was
  /// asking.
  ///
  /// # Errors
  ///
  /// [`QueryError::Dropped`] where the chain dropped the reply, and [`QueryError::TimedOut`] where
  /// the resolved budget elapsed first.
  pub fn take<R>(
    &self,
    awaiting: Awaiting<R>,
    explicit: Option<Duration>,
  ) -> Result<R, QueryError> {
    let Some(budget) = self.resolve(explicit) else {
      return awaiting.take();
    };
    let taken = awaiting.take_within(budget);
    if matches!(taken, Err(QueryError::TimedOut)) {
      self.expired(budget);
    }
    taken
  }

  /// Say that `budget` ran out.
  fn expired(&self, budget: Duration) {
    let observation = Fate::Expired {
      // A budget past the signed range is 292 years of patience, which no caller has and no policy
      // means; saturating says *the longest wait this vocabulary can name* rather than refusing to
      // record the expiry at all.
      budget: SignedDuration::try_from(budget).unwrap_or(SignedDuration::MAX),
      message: (),
    }
    .erase()
    .0;
    self.taps.observe(NodeName::new(&self.node), &observation);
  }
}

impl core::fmt::Debug for Deadlines {
  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
    f.debug_struct("Deadlines")
      .field("node", &self.node.as_str())
      .finish_non_exhaustive()
  }
}

/// A signed budget as a wait: the past is no wait at all.
#[inline]
fn unsigned(budget: SignedDuration) -> Duration {
  if budget.is_negative() {
    return Duration::ZERO;
  }
  budget.unsigned_abs()
}