axond 0.3.25

Axond — a stateless, single-binary, self-hosted AI gateway: one place for provider keys, model routing, usage, and telemetry.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
//! The journal contract, asserted against the in-memory oracle.
//!
//! These are the properties a durable implementation has to reproduce, written
//! as the failures they exist to prevent: an event lost because the process died
//! between settlement and delivery, an event delivered twice and billed twice, a
//! caller's events reordered, one bad event stalling every later one, and a
//! journal that fills up without saying so.

use std::time::{Duration, SystemTime};

use super::super::ObservedRecord;
use super::super::identity::next_request_id;
use super::super::tests::sample_record;
use super::oracle::InMemoryUsageJournal;
use super::*;

/// A settled record for `subject`, with a fresh event identity.
fn event_for(subject: &str) -> UsageEvent {
    let mut record = sample_record();
    record.request_id = next_request_id().to_string();
    record.subject = subject.to_owned();
    UsageEvent::new(ObservedRecord::now(record)).expect("a minted id is an event identity")
}

fn event() -> UsageEvent {
    event_for("GW_INBOUND_ACME_KEY")
}

fn consumer(name: &str) -> ConsumerId {
    ConsumerId::parse(name).expect("a valid consumer name")
}

fn claim_of(max_events: usize, now: SystemTime) -> Claim {
    Claim {
        max_events,
        lease: Duration::from_secs(30),
        now,
    }
}

fn claim(max_events: usize) -> Claim {
    claim_of(max_events, SystemTime::now())
}

#[test]
fn a_record_that_lost_its_identity_is_not_an_event() {
    let mut record = sample_record();
    // The shape a counter-era writer produced. Journaling it would mean minting a
    // new identity for an event a consumer may already hold under another one.
    record.request_id = "req_0000000000000001".to_owned();
    let error = UsageEvent::new(ObservedRecord::now(record)).expect_err("must be refused");
    assert!(matches!(error, InvalidEvent::Identity { .. }), "{error:?}");
}

#[test]
fn an_event_carries_the_key_a_consumer_deduplicates_on() {
    let event = event();
    assert_eq!(event.idempotency_key().as_str(), event.id().to_string());
    assert_eq!(
        event.idempotency_key().as_str(),
        event.record().request_id,
        "the key and the stored column must be the same string"
    );
    assert_eq!(
        event.ordering_key(),
        &OrderingKey {
            namespace: event.record().namespace.clone(),
            subject: event.record().subject.clone(),
        }
    );
}

#[test]
fn the_default_delivery_mode_promises_nothing_durable() {
    assert_eq!(DeliveryMode::default(), DeliveryMode::TelemetryGrade);
    assert!(!DeliveryMode::default().is_durable());
    assert!(DeliveryMode::BillingGrade.is_durable());
    // The oracle implements every operation and still refuses to claim the
    // guarantee, because an in-memory log cannot keep it.
    assert_eq!(
        InMemoryUsageJournal::new().mode(),
        DeliveryMode::TelemetryGrade
    );
}

#[test]
fn a_billing_grade_journal_refuses_rather_than_loses_by_default() {
    assert_eq!(Capacity::BILLING_GRADE.policy, CapacityPolicy::Refuse);
    assert!(!Capacity::BILLING_GRADE.policy.can_lose_events());
    assert!(CapacityPolicy::DropOldest.can_lose_events());
}

#[tokio::test]
async fn appending_the_same_event_twice_journals_it_once() {
    let journal = InMemoryUsageJournal::new();
    let event = event();
    let first = journal.append(&event).await.expect("append");
    let second = journal.append(&event).await.expect("re-append");
    assert!(first.is_new());
    assert!(!second.is_new(), "{second:?}");
    assert_eq!(first.position(), second.position());
    assert_eq!(journal.stored_events(), 1);
    assert_eq!(
        journal
            .stats(&consumer("billing"))
            .await
            .expect("stats")
            .pending,
        1
    );
}

/// A caller that never learned its append's outcome retries it, and after a
/// restart it cannot recover the original `observed_at` — so it rebuilds the
/// envelope with a new one. That is the ordinary retry, not a conflict, and the
/// first observation is the one kept: a consumer may already have written it.
#[tokio::test]
async fn a_retry_that_re_observed_the_record_is_still_the_same_event() {
    let journal = InMemoryUsageJournal::new();
    let event = event();
    let first = journal.append(&event).await.expect("append");

    let re_observed = UsageEvent::new(ObservedRecord {
        record: event.record().clone(),
        observed_at: event.observed_at() + Duration::from_secs(60),
    })
    .expect("identity is unchanged");
    let second = journal.append(&re_observed).await.expect("re-append");

    assert!(!second.is_new(), "{second:?}");
    assert_eq!(first.position(), second.position());
    assert_eq!(journal.stored_events(), 1);
    let delivered = journal
        .claim(&consumer("billing"), claim_of(1, SystemTime::now()))
        .await
        .expect("claim");
    assert_eq!(delivered[0].event.observed_at(), event.observed_at());
}

#[tokio::test]
async fn the_same_identity_with_different_content_is_a_conflict() {
    let journal = InMemoryUsageJournal::new();
    let event = event();
    journal.append(&event).await.expect("append");

    let mut mutated = event.record().clone();
    mutated.cost_microdollars += 1;
    let mutated = UsageEvent::new(ObservedRecord {
        record: mutated,
        observed_at: event.observed_at(),
    })
    .expect("identity is unchanged");

    let error = journal
        .append(&mutated)
        .await
        .expect_err("a reused identity must not overwrite a journaled fact");
    assert!(
        matches!(&error, JournalError::Conflict { key } if key == event.idempotency_key()),
        "{error:?}"
    );
    assert_eq!(journal.stored_events(), 1);
}

/// The failure #155 exists for: the request settled, the process died before any
/// sink acknowledged the event, and the event has to still be there.
#[tokio::test]
async fn a_crash_after_settlement_keeps_the_event_deliverable() {
    let journal = InMemoryUsageJournal::new();
    let event = event();
    journal.append(&event).await.expect("append");

    let restarted = journal.restart();
    let claimed = restarted
        .claim(&consumer("billing"), claim(10))
        .await
        .expect("claim");
    assert_eq!(claimed.len(), 1);
    assert_eq!(claimed[0].event, event);
    assert_eq!(claimed[0].id.attempt, 1);
    assert!(!claimed[0].id.is_redelivery());
}

#[tokio::test]
async fn a_restart_resumes_delivery_state_instead_of_redelivering_everything() {
    let journal = InMemoryUsageJournal::new();
    let billing = consumer("billing");
    let first = event_for("acme-one");
    let second = event_for("acme-two");
    journal.append(&first).await.expect("append");
    journal.append(&second).await.expect("append");

    let claimed = journal.claim(&billing, claim(1)).await.expect("claim");
    assert_eq!(claimed[0].event, first);
    journal.ack(&claimed[0].id).await.expect("ack");

    let restarted = journal.restart();
    // Well past the lease, so nothing is held back by an in-flight claim.
    let later = SystemTime::now() + Duration::from_secs(300);
    let replayed = restarted
        .claim(&billing, claim_of(10, later))
        .await
        .expect("claim");
    assert_eq!(
        replayed.iter().map(|d| d.event.id()).collect::<Vec<_>>(),
        vec![second.id()],
        "an acknowledged event must not be delivered again"
    );
}

#[tokio::test]
async fn an_expired_lease_redelivers_the_same_event_as_a_new_attempt() {
    let journal = InMemoryUsageJournal::new();
    let billing = consumer("billing");
    let event = event();
    journal.append(&event).await.expect("append");

    let now = SystemTime::now();
    let first = journal
        .claim(&billing, claim_of(10, now))
        .await
        .expect("claim");
    assert_eq!(first.len(), 1);

    // While the lease holds, the event is invisible: two workers must not
    // deliver it at once.
    assert!(
        journal
            .claim(&billing, claim_of(10, now + Duration::from_secs(1)))
            .await
            .expect("claim")
            .is_empty()
    );

    let second = journal
        .claim(&billing, claim_of(10, now + Duration::from_secs(31)))
        .await
        .expect("claim");
    assert_eq!(second.len(), 1);
    let (first, second) = (&first[0], &second[0]);
    assert_eq!(
        first.event.idempotency_key(),
        second.event.idempotency_key(),
        "a replay is the same billable fact"
    );
    assert_ne!(first.id, second.id, "and a distinguishable attempt");
    assert_eq!(second.id.attempt, 2);
    assert!(second.id.is_redelivery());
    assert_eq!(
        second.id.to_string(),
        format!("billing:{}#2", event.id()),
        "a delivery identity names the consumer, the event, and the attempt"
    );
}

#[tokio::test]
async fn acknowledging_twice_acknowledges_once() {
    let journal = InMemoryUsageJournal::new();
    let billing = consumer("billing");
    journal.append(&event()).await.expect("append");
    let claimed = journal.claim(&billing, claim(10)).await.expect("claim");

    journal.ack(&claimed[0].id).await.expect("ack");
    // The recovery case: the destination row was written, the acknowledgement's
    // outcome was unknown, so the worker repeats it.
    journal
        .ack(&claimed[0].id)
        .await
        .expect("a repeated ack must not fail");
    let stats = journal.stats(&billing).await.expect("stats");
    assert!(stats.is_drained());
    assert_eq!(stats.quarantined, 0);
}

#[tokio::test]
async fn a_delivery_that_was_never_claimed_cannot_be_acknowledged() {
    let journal = InMemoryUsageJournal::new();
    let event = event();
    journal.append(&event).await.expect("append");
    let delivery = DeliveryId {
        consumer: consumer("billing"),
        event: event.id(),
        attempt: 1,
    };
    let error = journal.ack(&delivery).await.expect_err("not outstanding");
    assert!(
        matches!(error, JournalError::NotOutstanding { .. }),
        "{error:?}"
    );

    let unknown = DeliveryId {
        consumer: consumer("billing"),
        event: next_request_id(),
        attempt: 1,
    };
    assert!(journal.ack(&unknown).await.is_err());
}

#[tokio::test]
async fn one_callers_events_are_delivered_in_append_order() {
    let journal = InMemoryUsageJournal::new();
    let billing = consumer("billing");
    let first = event_for("acme");
    let second = event_for("acme");
    let other = event_for("globex");
    for event in [&first, &second, &other] {
        journal.append(event).await.expect("append");
    }

    let claimed = journal.claim(&billing, claim(10)).await.expect("claim");
    assert_eq!(
        claimed.iter().map(|d| d.event.id()).collect::<Vec<_>>(),
        vec![first.id(), other.id()],
        "one event per ordering key is in flight; a second caller is not held up by the first"
    );

    // The next event for `acme` becomes claimable only once its predecessor is
    // settled, which is what "per-key ordering" means under a replaying consumer.
    journal.ack(&claimed[0].id).await.expect("ack");
    let next = journal.claim(&billing, claim(10)).await.expect("claim");
    assert_eq!(
        next.iter().map(|d| d.event.id()).collect::<Vec<_>>(),
        vec![second.id()]
    );
}

#[tokio::test]
async fn a_quarantined_event_stops_blocking_its_ordering_key() {
    let journal = InMemoryUsageJournal::new();
    let billing = consumer("billing");
    let poison = event_for("acme");
    let next = event_for("acme");
    journal.append(&poison).await.expect("append");
    journal.append(&next).await.expect("append");

    let claimed = journal.claim(&billing, claim(10)).await.expect("claim");
    assert_eq!(claimed[0].event.id(), poison.id());
    journal
        .quarantine(&claimed[0].id, PoisonReason::Malformed)
        .await
        .expect("quarantine");

    let after = journal.claim(&billing, claim(10)).await.expect("claim");
    assert_eq!(
        after.iter().map(|d| d.event.id()).collect::<Vec<_>>(),
        vec![next.id()],
        "the caller's later events must not be stuck behind a poison event"
    );
    let stats = journal.stats(&billing).await.expect("stats");
    assert_eq!(stats.quarantined, 1);
    assert_eq!(PoisonReason::Malformed.as_str(), "malformed");
}

/// A stale attempt number is not a reason to refuse an acknowledgement: the
/// worker that crashed after writing its destination row can only repeat the
/// delivery id it holds, and refusing it would redeliver an event that was already
/// delivered until its attempt budget quarantined it.
#[tokio::test]
async fn an_acknowledgement_from_a_crashed_worker_is_honoured_after_redelivery() {
    let journal = InMemoryUsageJournal::new();
    let billing = consumer("billing");
    let event = event();
    journal.append(&event).await.expect("append");

    let now = SystemTime::now();
    let first = journal
        .claim(&billing, claim_of(1, now))
        .await
        .expect("claim");
    // The worker wrote its row, then died before acknowledging. The lease expires
    // and the event comes back as a second attempt.
    let later = now + Duration::from_secs(31);
    let second = journal
        .claim(&billing, claim_of(1, later))
        .await
        .expect("claim");
    assert!(second[0].id.is_redelivery());

    journal
        .ack(&first[0].id)
        .await
        .expect("the recovered worker acknowledges the attempt it was handed");

    let stats = journal.stats(&billing).await.expect("stats");
    assert!(stats.is_drained(), "{stats:?}");
    assert!(
        journal
            .claim(&billing, claim_of(1, later + Duration::from_secs(60)))
            .await
            .expect("claim")
            .is_empty()
    );
}

/// Quarantine is terminal until an operator intervenes. An acknowledgement that
/// cleared it would take the event off the poison count and make it prunable,
/// losing the one copy of the record somebody was asked to look at.
#[tokio::test]
async fn an_acknowledgement_cannot_quietly_release_a_quarantined_event() {
    let journal = InMemoryUsageJournal::with_capacity(Capacity {
        retain_acknowledged: Duration::ZERO,
        ..Capacity::BILLING_GRADE
    });
    let billing = consumer("billing");
    let event = event();
    journal.append(&event).await.expect("append");
    let claimed = journal
        .claim(&billing, claim_of(1, SystemTime::now()))
        .await
        .expect("claim");
    journal
        .quarantine(&claimed[0].id, PoisonReason::Rejected)
        .await
        .expect("quarantine");

    let error = journal
        .ack(&claimed[0].id)
        .await
        .expect_err("a quarantined event is out of the delivery path");
    assert!(
        matches!(&error, JournalError::Quarantined { delivery } if delivery == &claimed[0].id),
        "{error:?}"
    );

    journal.append(&event_for("other")).await.expect("append");
    assert_eq!(
        journal.stored_events(),
        2,
        "a quarantined event is not prunable"
    );
    let stats = journal.stats(&billing).await.expect("stats");
    assert_eq!(stats.quarantined, 1, "the poison count stays visible");
}

/// Capacity bounds what is *waiting*; retention is what bounds a journal that is
/// keeping up. Without it a drained journal grows forever, which is the quiet way
/// a durable store runs out of disk.
#[tokio::test]
async fn an_event_every_consumer_finished_with_is_pruned_once_its_window_passes() {
    // Zero retention: an event is prunable as soon as every consumer is done with
    // it, which is the boundary case a duration-based window has to get right.
    let journal = InMemoryUsageJournal::with_capacity(Capacity {
        retain_acknowledged: Duration::ZERO,
        ..Capacity::BILLING_GRADE
    });
    let billing = consumer("billing");
    let delivered = event();
    let poison = event_for("acme");
    journal.append(&delivered).await.expect("append");
    journal.append(&poison).await.expect("append");

    let claimed = journal
        .claim(&billing, claim_of(10, SystemTime::now()))
        .await
        .expect("claim");
    for delivery in &claimed {
        if delivery.event.id() == poison.id() {
            journal
                .quarantine(&delivery.id, PoisonReason::Malformed)
                .await
                .expect("quarantine");
        } else {
            journal.ack(&delivery.id).await.expect("ack");
        }
    }

    // Pruning happens on the next append, as a store's would happen in its own
    // maintenance statement rather than on the delivery path.
    journal.append(&event_for("other")).await.expect("append");

    assert_eq!(
        journal.stored_events(),
        2,
        "the acknowledged event is gone; the quarantined one waits for an operator"
    );
    let stats = journal.stats(&billing).await.expect("stats");
    assert_eq!(stats.quarantined, 1);
    assert_eq!(stats.pending, 1);
}

/// A journal with a real retention window keeps what it delivered, so a consumer
/// that re-acknowledges after a restart finds the event rather than an absence.
#[tokio::test]
async fn a_retained_event_is_still_there_after_it_was_acknowledged() {
    let journal = InMemoryUsageJournal::new();
    let billing = consumer("billing");
    let event = event();
    journal.append(&event).await.expect("append");
    let claimed = journal
        .claim(&billing, claim_of(1, SystemTime::now()))
        .await
        .expect("claim");
    journal.ack(&claimed[0].id).await.expect("ack");

    journal.append(&event_for("other")).await.expect("append");

    assert_eq!(journal.stored_events(), 2);
    assert_eq!(
        Capacity::BILLING_GRADE.retain_acknowledged,
        Duration::from_secs(24 * 60 * 60)
    );
    journal
        .ack(&claimed[0].id)
        .await
        .expect("a re-acknowledgement inside the window finds its event");
}

/// Quarantine is gated like `ack`, and for the same reason: a verdict on an event
/// is a consumer's to give only once the event was handed to it, so a second
/// consumer cannot remove an event from a delivery path it never read. Repeating
/// the verdict is `Ok`, and the first reason stands.
#[tokio::test]
async fn only_a_consumer_that_was_handed_an_event_can_condemn_it() {
    let journal = InMemoryUsageJournal::new();
    let billing = consumer("billing");
    let event = event();
    journal.append(&event).await.expect("append");

    let unclaimed = DeliveryId {
        consumer: consumer("warehouse"),
        event: event.id(),
        attempt: 1,
    };
    let error = journal
        .quarantine(&unclaimed, PoisonReason::Malformed)
        .await
        .expect_err("a consumer that never claimed the event has no verdict to give");
    assert!(
        matches!(&error, JournalError::NotOutstanding { delivery } if delivery == &unclaimed),
        "{error:?}"
    );

    let claimed = journal
        .claim(&billing, claim_of(1, SystemTime::now()))
        .await
        .expect("claim");
    journal
        .quarantine(&claimed[0].id, PoisonReason::Malformed)
        .await
        .expect("quarantine");
    journal
        .quarantine(&claimed[0].id, PoisonReason::Rejected)
        .await
        .expect("a repeated verdict is not an error");

    let stats = journal.stats(&billing).await.expect("stats");
    assert_eq!(stats.quarantined, 1);
    assert_eq!(
        journal
            .stats(&consumer("warehouse"))
            .await
            .expect("stats")
            .quarantined,
        0,
        "one consumer's verdict is not another's"
    );
}

#[tokio::test]
async fn an_event_that_exhausts_its_attempts_is_quarantined_not_retried_forever() {
    let capacity = Capacity {
        max_delivery_attempts: 2,
        ..Capacity::BILLING_GRADE
    };
    let journal = InMemoryUsageJournal::with_capacity(capacity);
    let billing = consumer("billing");
    let event = event();
    journal.append(&event).await.expect("append");

    let mut now = SystemTime::now();
    for attempt in 1..=capacity.max_delivery_attempts {
        let claimed = journal
            .claim(&billing, claim_of(10, now))
            .await
            .expect("claim");
        assert_eq!(claimed[0].id.attempt, attempt);
        // The worker crashes every time, so every lease expires unacknowledged.
        now += Duration::from_secs(31);
    }

    let after = journal
        .claim(&billing, claim_of(10, now))
        .await
        .expect("claim");
    assert!(
        after.is_empty(),
        "an event past its attempt budget must leave the delivery path"
    );
    let stats = journal.stats(&billing).await.expect("stats");
    assert_eq!(stats.quarantined, 1);
    assert_eq!(stats.pending, 0);
    assert_eq!(stats.capacity.max_delivery_attempts, 2);
    assert_eq!(
        PoisonReason::AttemptsExhausted.as_str(),
        "attempts_exhausted"
    );
}

#[tokio::test]
async fn a_full_journal_refuses_the_append_and_says_what_it_is_bounded_by() {
    let capacity = Capacity {
        max_events: 2,
        ..Capacity::BILLING_GRADE
    };
    let journal = InMemoryUsageJournal::with_capacity(capacity);
    journal.append(&event()).await.expect("append");
    journal.append(&event()).await.expect("append");

    let error = journal
        .append(&event())
        .await
        .expect_err("a full journal must refuse");
    assert!(
        matches!(
            &error,
            JournalError::AtCapacity { pending, capacity } if *pending == 2 && capacity.max_events == 2
        ),
        "{error:?}"
    );
    assert!(
        error.to_string().contains("was not journaled"),
        "the caller has to be able to tell the event is not durable: {error}"
    );
    assert_eq!(journal.stored_events(), 2);
    let stats = journal.stats(&consumer("billing")).await.expect("stats");
    assert_eq!(stats.dropped, 0, "refusing is not losing");
    assert!(stats.oldest_pending_age.is_some());
}

/// Capacity bounds the footprint, so a delivered event inside its retention window
/// occupies it like any other — otherwise the limit an operator sizes storage
/// against would be false by a whole window's worth of rows. The room is taken from
/// the delivered tail rather than by refusing, because a redundant
/// re-acknowledgement is cheaper than an event nobody has delivered.
#[tokio::test]
async fn a_delivered_event_still_in_its_window_occupies_capacity_and_yields_it_first() {
    let journal = InMemoryUsageJournal::with_capacity(Capacity {
        max_events: 2,
        // Long enough that nothing is pruned by the window during the test.
        retain_acknowledged: Duration::from_secs(60 * 60),
        ..Capacity::BILLING_GRADE
    });
    let billing = consumer("billing");
    for subject in ["acme-one", "acme-two", "acme-three"] {
        let event = event_for(subject);
        journal
            .append(&event)
            .await
            .expect("the delivered tail makes room instead of refusing");
        let claimed = journal.claim(&billing, claim(1)).await.expect("claim");
        journal.ack(&claimed[0].id).await.expect("ack");
        assert!(
            journal.stored_events() <= 2,
            "capacity is the bound on everything stored, delivered events included"
        );
    }

    let stats = journal.stats(&billing).await.expect("stats");
    assert!(stats.is_drained(), "{stats:?}");
    assert_eq!(
        stats.dropped, 0,
        "giving up a retention courtesy is not losing an event"
    );
}

/// An undelivered backlog is never sacrificed to keep a delivered event's retention
/// window: the courtesy is what yields, and once it is gone the policy decides.
#[tokio::test]
async fn a_full_journal_refuses_rather_than_drop_what_it_has_not_delivered() {
    let journal = InMemoryUsageJournal::with_capacity(Capacity {
        max_events: 2,
        retain_acknowledged: Duration::from_secs(60 * 60),
        ..Capacity::BILLING_GRADE
    });
    let billing = consumer("billing");
    let delivered = event_for("acme-one");
    journal.append(&delivered).await.expect("append");
    let claimed = journal.claim(&billing, claim(1)).await.expect("claim");
    journal.ack(&claimed[0].id).await.expect("ack");
    journal
        .append(&event_for("acme-two"))
        .await
        .expect("append");

    journal
        .append(&event_for("acme-three"))
        .await
        .expect("the acknowledged event yields its window");
    assert_eq!(journal.stored_events(), 2);

    let error = journal
        .append(&event_for("acme-four"))
        .await
        .expect_err("nothing delivered is left to give up");
    assert!(
        matches!(&error, JournalError::AtCapacity { pending, .. } if *pending == 2),
        "{error:?}"
    );

    // Its idempotency key went with it, which is the guarantee retention does not
    // make: re-appending it is a new event competing for room, not `AlreadyPresent`.
    let error = journal.append(&delivered).await.expect_err("a new event");
    assert!(
        matches!(&error, JournalError::AtCapacity { .. }),
        "{error:?}"
    );
}

/// A verdict from a consumer that never read the journal must not register it.
/// Retention waits on *every* registered consumer, so one phantom registration
/// would freeze pruning permanently — the journal would then grow until capacity
/// refused appends, for a consumer that never existed.
#[tokio::test]
async fn a_stray_verdict_does_not_register_the_consumer_that_sent_it() {
    let journal = InMemoryUsageJournal::with_capacity(Capacity {
        retain_acknowledged: Duration::ZERO,
        ..Capacity::BILLING_GRADE
    });
    let billing = consumer("billing");
    journal.append(&event()).await.expect("append");
    let claimed = journal.claim(&billing, claim(1)).await.expect("claim");

    // A mistyped consumer name, or a worker pointed at the wrong journal.
    let stray = DeliveryId {
        consumer: consumer("typo"),
        ..claimed[0].id.clone()
    };
    for error in [
        journal.ack(&stray).await.expect_err("never claimed"),
        journal
            .quarantine(&stray, PoisonReason::Malformed)
            .await
            .expect_err("never claimed"),
    ] {
        assert!(
            matches!(&error, JournalError::NotOutstanding { delivery } if delivery == &stray),
            "{error:?}"
        );
    }

    journal.ack(&claimed[0].id).await.expect("ack");
    journal.append(&event_for("other")).await.expect("append");
    assert_eq!(
        journal.stored_events(),
        1,
        "retention still prunes: the stray verdict registered nothing to wait for"
    );
}

/// Quarantine is exempt from dropping and pruning, so it has to be inside the
/// capacity bound — otherwise a destination that rejects everything (a schema
/// mismatch, say) would quietly grow the store without limit, which is the failure
/// capacity exists to prevent.
#[tokio::test]
async fn a_journal_whose_backlog_is_all_poison_fills_up_instead_of_growing() {
    let journal = InMemoryUsageJournal::with_capacity(Capacity {
        max_events: 2,
        max_delivery_attempts: 1,
        ..Capacity::BILLING_GRADE
    });
    let billing = consumer("billing");
    for subject in ["acme-one", "acme-two"] {
        journal.append(&event_for(subject)).await.expect("append");
    }

    // Every event is condemned, by the only consumer there is.
    for delivery in journal.claim(&billing, claim(10)).await.expect("claim") {
        journal
            .quarantine(&delivery.id, PoisonReason::Rejected)
            .await
            .expect("quarantine");
    }

    let stats = journal.stats(&billing).await.expect("stats");
    assert!(stats.is_drained(), "nothing is waiting for delivery");
    assert_eq!(stats.quarantined, 2);

    let error = journal
        .append(&event_for("acme-three"))
        .await
        .expect_err("a journal full of poison is still full");
    assert!(
        matches!(&error, JournalError::AtCapacity { pending, .. } if *pending == 2),
        "{error:?}"
    );
    assert_eq!(journal.stored_events(), 2);
}

/// The two verdicts are exclusive in both directions: `ack` cannot release a
/// quarantine, and a late quarantine cannot retract an acknowledgement — it would
/// put a delivered event on the poison count and, being exempt from pruning, keep
/// it there forever.
#[tokio::test]
async fn an_acknowledged_event_cannot_be_condemned_afterwards() {
    let journal = InMemoryUsageJournal::new();
    let billing = consumer("billing");
    journal.append(&event()).await.expect("append");
    let claimed = journal.claim(&billing, claim(1)).await.expect("claim");
    journal.ack(&claimed[0].id).await.expect("ack");

    let error = journal
        .quarantine(&claimed[0].id, PoisonReason::Rejected)
        .await
        .expect_err("there is no delivery left to condemn");
    assert!(
        matches!(&error, JournalError::AlreadyAcknowledged { delivery } if delivery == &claimed[0].id),
        "{error:?}"
    );
    let stats = journal.stats(&billing).await.expect("stats");
    assert_eq!(stats.quarantined, 0);
    assert!(stats.is_drained(), "{stats:?}");
}

/// A lossy policy may drop a backlog; it may not drop the evidence an operator was
/// asked to look at. When the only room left to make is somebody's quarantine, the
/// journal refuses instead — the same answer `Refuse` gives, for the same reason.
#[tokio::test]
async fn drop_oldest_will_not_make_room_by_deleting_a_quarantined_event() {
    let capacity = Capacity {
        max_events: 1,
        policy: CapacityPolicy::DropOldest,
        ..Capacity::BILLING_GRADE
    };
    let journal = InMemoryUsageJournal::with_capacity(capacity);
    let billing = consumer("billing");
    let warehouse = consumer("warehouse");
    let poison = event_for("acme-one");
    journal.append(&poison).await.expect("append");

    // Two consumers, disagreeing about the event: billing has condemned it, the
    // warehouse has not finished with it, so it is still counted as a backlog.
    let claimed = journal.claim(&billing, claim(10)).await.expect("claim");
    journal
        .quarantine(&claimed[0].id, PoisonReason::Malformed)
        .await
        .expect("quarantine");
    journal.claim(&warehouse, claim(10)).await.expect("claim");

    let error = journal
        .append(&event_for("acme-two"))
        .await
        .expect_err("the only droppable event is somebody's evidence");
    assert!(
        matches!(&error, JournalError::AtCapacity { pending, .. } if *pending == 1),
        "{error:?}"
    );
    assert_eq!(journal.stored_events(), 1);
    assert_eq!(
        journal.stats(&billing).await.expect("stats").quarantined,
        1,
        "the poison count an operator is watching must not be decremented by a drop"
    );
    assert_eq!(
        journal.stats(&billing).await.expect("stats").dropped,
        0,
        "nothing was lost"
    );
}

#[tokio::test]
async fn drop_oldest_bounds_storage_and_counts_what_it_lost() {
    let capacity = Capacity {
        max_events: 2,
        policy: CapacityPolicy::DropOldest,
        ..Capacity::BILLING_GRADE
    };
    let journal = InMemoryUsageJournal::with_capacity(capacity);
    let oldest = event_for("acme-one");
    journal.append(&oldest).await.expect("append");
    journal
        .append(&event_for("acme-two"))
        .await
        .expect("append");
    journal
        .append(&event_for("acme-three"))
        .await
        .expect("append");

    assert_eq!(journal.stored_events(), 2, "storage stays bounded");
    let billing = consumer("billing");
    let claimed = journal.claim(&billing, claim(10)).await.expect("claim");
    assert!(
        !claimed.iter().any(|d| d.event.id() == oldest.id()),
        "the dropped event is gone, not merely deprioritised"
    );
    let stats = journal.stats(&billing).await.expect("stats");
    assert_eq!(stats.dropped, 1, "a lossy policy has to report its cost");
}

#[tokio::test]
async fn consumers_acknowledge_independently() {
    let journal = InMemoryUsageJournal::new();
    let (billing, warehouse) = (consumer("billing"), consumer("warehouse"));
    let event = event();
    journal.append(&event).await.expect("append");

    let for_billing = journal.claim(&billing, claim(10)).await.expect("claim");
    let for_warehouse = journal.claim(&warehouse, claim(10)).await.expect("claim");
    assert_eq!(for_billing[0].event.id(), event.id());
    assert_eq!(for_warehouse[0].event.id(), event.id());
    assert_ne!(for_billing[0].id, for_warehouse[0].id);

    journal.ack(&for_billing[0].id).await.expect("ack");
    assert!(journal.stats(&billing).await.expect("stats").is_drained());
    assert_eq!(
        journal.stats(&warehouse).await.expect("stats").in_flight,
        1,
        "one consumer's acknowledgement is not another's"
    );
}

#[tokio::test]
async fn a_claim_is_bounded_by_what_the_caller_asked_for() {
    let journal = InMemoryUsageJournal::new();
    for index in 0..5 {
        journal
            .append(&event_for(&format!("acme-{index}")))
            .await
            .expect("append");
    }
    let claimed = journal
        .claim(&consumer("billing"), claim(2))
        .await
        .expect("claim");
    assert_eq!(claimed.len(), 2);
    assert!(
        claimed
            .iter()
            .all(|delivery| delivery.lease_expires_at > SystemTime::now())
    );
}

#[test]
fn consumer_names_are_narrow_enough_to_be_storage_keys() {
    assert_eq!(consumer("billing-postgres").as_str(), "billing-postgres");
    for bad in [
        "",
        "Billing",
        "billing table",
        "billing;drop",
        &"x".repeat(64),
    ] {
        assert!(ConsumerId::parse(bad).is_err(), "accepted `{bad}`");
    }
}

#[test]
fn the_event_carries_what_a_sink_needs_unchanged() {
    let event = event();
    let observed = event.observed();
    assert_eq!(observed.record.request_id, event.record().request_id);
    assert_eq!(
        observed.observed_at,
        event.observed_at(),
        "a replay written later still says when the request happened"
    );
}