keysink 0.2.3

Local gateway that keeps real AI API keys out of agents (Claude Code, OpenAI, MCP): agents hold revocable virtual keys, real keys stay in an encrypted vault.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
use crate::error::{Error, Result};
use crate::util;
use arc_swap::ArcSwapOption;
use serde::Serialize;
use std::collections::HashMap;
use std::fs::File;
use std::io::{BufWriter, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender, TrySendError, sync_channel};
use std::thread::JoinHandle;
use std::time::{Duration, Instant};

const MAX_AUDIT_BYTES: u64 = 10 * 1024 * 1024;
/// Rotated generations kept on disk: `.1` (newest) through `.{N}` (oldest).
/// Rotation is the log's only retention mechanism, and rejected requests are
/// audited too — without generations, a flood of unauthenticated requests
/// evicts the forensic trail of real traffic (60MB of history at N=5, plus
/// the live file) in a few minutes at loopback speeds.
const AUDIT_GENERATIONS: usize = 5;
/// Bounded queue depth: under disk-write jitter, prefer dropping audit lines (with a counted warning) over blocking the request path.
const QUEUE_DEPTH: usize = 1024;
/// How long the writer idles before flushing buffered lines: under load,
/// lines batch into the BufWriter (flushed at its 8KB capacity), so one
/// syscall carries many lines; when the queue goes quiet, this bounds how
/// long a `tail -f` style consumer waits for the tail.
const FLUSH_IDLE: Duration = Duration::from_millis(50);
/// Per-fingerprint admission interval for rejection rows: a single probing
/// key leaves a steady, complete trail (one row per second) without being
/// able to rotate the log on its own.
const REJECT_FP_INTERVAL: Duration = Duration::from_secs(1);
/// Global rejection admission per window: bounds floods that randomize the
/// fingerprint (each fake token is a fresh bucket, so the per-fingerprint
/// gate alone cannot cap them).
const REJECT_GLOBAL_WINDOW: Duration = Duration::from_secs(1);
const REJECT_GLOBAL_MAX: usize = 100;
/// Per-fingerprint table bound: randomized-token floods would otherwise grow
/// it without limit; when full it resets (buckets re-arm, the global cap
/// still holds).
const REJECT_FP_TABLE_MAX: usize = 4096;
/// First wait after a failed audit write, doubling per consecutive failure and
/// capped: a full disk or a vanished directory would otherwise cost one failed
/// open + one warn line *per record*, turning an audit-on gateway into a
/// syscall spin and a log flood.
const WRITE_BACKOFF_START: Duration = Duration::from_millis(50);
const WRITE_BACKOFF_MAX: Duration = Duration::from_secs(5);
/// How long `close()` waits for the writer thread to drain before detaching
/// it: long enough for a normal flush (the queue is at most 1024 records), and
/// short enough that a writer stuck on a hung mount cannot hold shutdown past
/// the service manager's stop timeout.
const CLOSE_JOIN_TIMEOUT: Duration = Duration::from_secs(3);

/// Audit disk writes run on a dedicated thread + bounded channel:
/// - the request path only does try_send, never a synchronous open/write/chmod on a tokio worker;
/// - a single writer naturally eliminates the race where two concurrent requests trigger the rotation rename at the same time.
pub struct Audit {
    /// Startup-fixed sender; `close()` swaps it out. ArcSwap instead of the
    /// old Mutex<Option<Writer>> so the request path (`is_enabled` + `record`)
    /// is two lock-free loads with no lock and no per-record sender clone; an
    /// in-flight record keeps the channel alive past `close()` through its own
    /// Arc. SyncSender is Sync (T: Send), so the ArcSwap needs no wrapper.
    writer: ArcSwapOption<SyncSender<AuditEntry>>,
    /// Writer thread handle, parked for `close()`: the thread exits only when
    /// the channel disconnects, so the handle cannot live inside the swapped
    /// Arc next to the sender it must outlive.
    handle: Mutex<Option<JoinHandle<()>>>,
    /// Rejection-side admission state (see record_reject): bounded, checked
    /// on the calling thread before the queue.
    rejects: Mutex<RejectGate>,
    /// Liveness/counters shared with the writer thread, the request path and
    /// the health endpoint.
    state: Arc<AuditState>,
}

/// Writer-thread liveness and counters.
///
/// A dead writer thread used to be completely silent: every record hit
/// `Disconnected` and returned, `is_enabled()` still said yes, and nothing
/// anywhere said audit had stopped recording. `alive`/`closed` make the
/// difference between "closed during shutdown" (expected) and "the writer
/// died" (an alert) visible, and `dropped` gives the health endpoint the loss
/// count that used to live only in a static counter nobody could read.
struct AuditState {
    alive: AtomicBool,
    closed: AtomicBool,
    /// The last write/flush attempt failed and audit has not recovered yet.
    /// `alive` only says the writer thread exists; a writer that exists but
    /// cannot write (an unwritable path, a full disk) must still make
    /// `healthy()` false.
    write_failed: AtomicBool,
    dropped: AtomicU64,
    dead_warned: AtomicBool,
}

struct RejectGate {
    per_fp: HashMap<String, Instant>,
    window_start: Instant,
    window_count: usize,
    suppressed: u64,
}

impl Default for RejectGate {
    fn default() -> Self {
        Self {
            per_fp: HashMap::new(),
            window_start: Instant::now(),
            window_count: 0,
            suppressed: 0,
        }
    }
}

#[derive(Serialize)]
pub struct AuditEntry {
    pub ts: String,
    pub rid: String,
    pub route: String,
    pub key_fp: String,
    pub method: String,
    pub path: String,
    pub status: u16,
    pub ms: u64,
    pub bytes_in: usize,
    pub stream: bool,
    pub attempts: u8,
    /// Whether this is a rejection row (auth failure, disabled route, body
    /// limits, ...). Rejection rows are the one audit input an unauthenticated
    /// peer controls, so they are written to a separate rotation chain
    /// (`<audit-file>.rej`): a flood can only ever rotate its own chain away,
    /// never the main chain's real-traffic rows. Not serialized on the main
    /// chain — existing line formats stay byte-identical for consumers; on
    /// the reject chain the field is redundant but self-describing.
    #[serde(skip_serializing_if = "std::ops::Not::not")]
    pub rejected: bool,
}

/// The separate rotation chain rejection rows are written to: same directory,
/// `.rej` suffix, its own five generations. A flood of unauthenticated
/// requests — the one audit input an attacker controls without any
/// credential — can rotate this chain away, but never the main chain's
/// real-traffic rows.
fn reject_chain_path(path: &Path) -> PathBuf {
    PathBuf::from(format!("{}.rej", path.display()))
}

impl Audit {
    pub fn disabled() -> Self {
        Self {
            writer: ArcSwapOption::empty(),
            handle: Mutex::new(None),
            rejects: Mutex::new(RejectGate::default()),
            state: Arc::new(AuditState {
                alive: AtomicBool::new(false),
                closed: AtomicBool::new(true),
                write_failed: AtomicBool::new(false),
                dropped: AtomicU64::new(0),
                dead_warned: AtomicBool::new(false),
            }),
        }
    }

    /// Whether rows are being written. The writer is chosen once at startup
    /// (--audit-log or not), so the answer cannot change mid-request; callers
    /// skip `AuditEntry` construction entirely when this is false — its
    /// RFC3339 local-time format plus several String allocations are pure
    /// waste on the default audit-off path.
    #[inline]
    pub fn is_enabled(&self) -> bool {
        self.writer.load().is_some()
    }

    /// Whether audit is doing what it says: disabled counts as healthy, an
    /// enabled audit whose writer thread died or whose last write failed does
    /// not.
    pub fn healthy(&self) -> bool {
        !self.is_enabled()
            || (self.state.alive.load(Ordering::Relaxed)
                && !self.state.write_failed.load(Ordering::Relaxed))
    }

    /// Lines dropped so far (queue full or a failed write). Surfaced on the
    /// health endpoint; throttled rejection rows are counted separately in the
    /// log and never reach the writer at all.
    pub fn dropped(&self) -> u64 {
        self.state.dropped.load(Ordering::Relaxed)
    }

    /// Test hook for other modules' health tests: simulate a writer thread
    /// that died or a write that cannot land.
    #[cfg(test)]
    pub(crate) fn mark_unhealthy_for_test(&self) {
        self.state.write_failed.store(true, Ordering::Relaxed);
    }

    pub fn new(path: PathBuf) -> Self {
        // mode(0o600) only takes effect at creation; if an existing file has overly broad permissions, warn once at startup.
        for existing in [path.as_path(), reject_chain_path(&path).as_path()] {
            if existing.exists()
                && let Err(e) = util::ensure_private_file(existing, "audit log")
            {
                tracing::warn!(error = %e, "audit log permissions too wide; consider tightening to 0600");
            }
        }
        let reject_path = reject_chain_path(&path);
        let (tx, rx) = sync_channel(QUEUE_DEPTH);
        let state = Arc::new(AuditState {
            alive: AtomicBool::new(true),
            closed: AtomicBool::new(false),
            write_failed: AtomicBool::new(false),
            dropped: AtomicU64::new(0),
            dead_warned: AtomicBool::new(false),
        });
        let thread_state = state.clone();
        match std::thread::Builder::new()
            .name("keysink-audit".into())
            .spawn(move || writer_loop(path, reject_path, rx, thread_state))
        {
            Ok(handle) => Self {
                writer: ArcSwapOption::from_pointee(tx),
                handle: Mutex::new(Some(handle)),
                rejects: Mutex::new(RejectGate::default()),
                state,
            },
            Err(e) => {
                tracing::warn!(error = %e, "failed to start audit writer thread; audit disabled for this run");
                Self::disabled()
            }
        }
    }

    /// Records only route/fingerprint/status/latency/byte counts; never headers, bodies, or query.
    pub fn record(&self, entry: AuditEntry) {
        // One lock-free load; the Arc keeps the channel open until this record
        // is queued, even if close() swaps the writer out in between.
        let Some(tx) = self.writer.load_full() else {
            return;
        };
        match tx.try_send(entry) {
            Ok(()) => {}
            Err(TrySendError::Full(_)) => {
                let n = self.state.dropped.fetch_add(1, Ordering::Relaxed) + 1;
                if n % 100 == 1 {
                    tracing::warn!(dropped = n, "audit queue full; dropping audit lines");
                }
            }
            Err(TrySendError::Disconnected(_)) => {
                // The writer thread is gone. During a clean close that is
                // expected; otherwise audit silently stopped recording, which
                // is exactly what used to go unnoticed (health now shows
                // audit_ok=false, and this line says it once).
                if !self.state.closed.load(Ordering::Relaxed)
                    && !self.state.dead_warned.swap(true, Ordering::Relaxed)
                {
                    tracing::error!(
                        "audit writer thread exited; audit is no longer recording (see /__keysink/health audit_ok)"
                    );
                }
            }
        }
    }

    /// Audits a rejected request (auth failure, disabled route, body limits,
    /// route configuration errors). Rejection rows are the one audit input an
    /// unauthenticated peer controls, so they pass an admission gate before
    /// the queue: per-fingerprint (one row per `REJECT_FP_INTERVAL`) plus a
    /// global cap per window — and they land on the separate `.rej` rotation
    /// chain, so even a sustained fingerprint-randomizing flood (every fake
    /// token a fresh bucket, admitted up to the global cap) can only evict
    /// its own trail, never the main chain's real-traffic rows. Suppressed
    /// rows are counted, never silent.
    pub fn record_reject(&self, mut entry: AuditEntry) {
        // Cheapest possible off path: without this, a credential flood on an
        // audit-off gateway still takes the rejection-gate mutex and touches
        // its fingerprint map for every request.
        if !self.is_enabled() {
            return;
        }
        entry.rejected = true;
        let admit = {
            let mut gate = self.rejects.lock().unwrap_or_else(|e| e.into_inner());
            let now = Instant::now();
            if now.duration_since(gate.window_start) >= REJECT_GLOBAL_WINDOW {
                gate.window_start = now;
                gate.window_count = 0;
            }
            let admitted = if gate.window_count >= REJECT_GLOBAL_MAX {
                false
            } else {
                match gate.per_fp.get_mut(&entry.key_fp) {
                    Some(last) => {
                        if now.duration_since(*last) >= REJECT_FP_INTERVAL {
                            *last = now;
                            true
                        } else {
                            false
                        }
                    }
                    None => {
                        if gate.per_fp.len() >= REJECT_FP_TABLE_MAX {
                            gate.per_fp.clear();
                        }
                        gate.per_fp.insert(entry.key_fp.clone(), now);
                        true
                    }
                }
            };
            if admitted {
                gate.window_count += 1;
            } else {
                gate.suppressed += 1;
                let n = gate.suppressed;
                if n % 100 == 1 {
                    tracing::warn!(
                        suppressed = n,
                        global_per_sec = REJECT_GLOBAL_MAX,
                        "rejection audit rows throttled (1/s per fingerprint); a flood can no longer rotate the log away"
                    );
                }
            }
            admitted
        };
        if admit {
            self.record(entry);
        }
    }

    /// On shutdown, drain the queue and wait for the writer thread to exit so
    /// tail audit lines are not lost — but only for a bounded time.
    ///
    /// The writer does blocking file I/O. On a hung mount (NFS, a stalled
    /// disk) `join()` never returns, and shutdown would hang until the service
    /// manager SIGKILLs the process — losing the very flush this close exists
    /// for. Past the timeout the writer is detached: the process is exiting
    /// anyway, and the alternative (a stuck gateway) is worse than a truncated
    /// audit tail, which the health endpoint already reported.
    pub fn close(&self) {
        self.state.closed.store(true, Ordering::Relaxed);
        // Dropping the swapped-out sender — together with any in-flight
        // record's Arc — disconnects the channel; the writer thread then
        // drains the queue and exits, and the join below waits for exactly
        // that. Records that load the sender after the swap see None and
        // return silently.
        drop(self.writer.swap(None));
        let Some(handle) = self.handle.lock().unwrap_or_else(|e| e.into_inner()).take() else {
            return;
        };
        let deadline = Instant::now() + CLOSE_JOIN_TIMEOUT;
        while !handle.is_finished() && Instant::now() < deadline {
            std::thread::sleep(Duration::from_millis(5));
        }
        if handle.is_finished() {
            let _ = handle.join();
        } else {
            // Detach: the thread is stuck in a write. Say so once; there is
            // nothing else the exiting process can do about it.
            tracing::warn!(
                timeout_ms = CLOSE_JOIN_TIMEOUT.as_millis(),
                "audit writer did not finish flushing in time; detaching it (tail lines may be lost)"
            );
        }
    }
}

/// Verifies both rotation chains (the main path and its `.rej` sibling) can be
/// opened for append before the gateway starts.
///
/// `Audit::new` only spawns the writer; it used to accept an unwritable path
/// (a symlink, a directory, an unwritable parent) and the gateway started with
/// audit "enabled" while every record was dropped. `serve` calls this first so
/// a bad `--audit-log` is a startup error instead of a silent loss of the
/// forensic trail.
pub fn probe(path: &Path) -> Result<()> {
    let reject_path = reject_chain_path(path);
    for target in [path, reject_path.as_path()] {
        util::private_open_options(0o600)
            .create(true)
            .append(true)
            .open(target)
            .map_err(|e| {
                Error::msg(format!(
                    "audit log {} cannot be opened for append: {e}",
                    target.display()
                ))
            })?;
    }
    Ok(())
}

/// Marks the writer thread dead on every exit path, a panic included.
struct AliveGuard(Arc<AuditState>);

impl Drop for AliveGuard {
    fn drop(&mut self) {
        self.0.alive.store(false, Ordering::Relaxed);
    }
}

/// Both rotation chains the writer maintains: real-traffic rows on the main
/// path, rejection rows on the `.rej` path. Each chain rotates independently
/// at MAX_AUDIT_BYTES; a flood can only ever cycle the reject chain.
struct WriterLogs {
    main: OpenLog,
    reject: OpenLog,
}

fn writer_loop(
    path: PathBuf,
    reject_path: PathBuf,
    rx: Receiver<AuditEntry>,
    state: Arc<AuditState>,
) {
    let _alive = AliveGuard(state.clone());
    // The handles stay open, buffered: lines batch in a BufWriter instead of
    // costing one write syscall each; the buffer flushes at capacity, when
    // the queue goes idle (FLUSH_IDLE), and before rotation or shutdown. On
    // rotation or write failure the handle becomes None and the next record
    // reopens.
    let mut logs = WriterLogs {
        main: OpenLog {
            file: None,
            written: 0,
        },
        reject: OpenLog {
            file: None,
            written: 0,
        },
    };
    let mut failures = 0u32;
    loop {
        match rx.recv_timeout(FLUSH_IDLE) {
            Ok(entry) => {
                let (chain_path, chain) = if entry.rejected {
                    (&reject_path, &mut logs.reject)
                } else {
                    (&path, &mut logs.main)
                };
                match append(chain_path, chain, &entry) {
                    Ok(()) => {
                        failures = 0;
                        state.write_failed.store(false, Ordering::Relaxed);
                    }
                    Err(e) => {
                        // Dropping the BufWriter attempts a best-effort flush of
                        // whatever is still buffered; the next record reopens.
                        chain.file = None;
                        state.write_failed.store(true, Ordering::Relaxed);
                        failures += 1;
                        state.dropped.fetch_add(1, Ordering::Relaxed);
                        if failures == 1 || failures.is_multiple_of(100) {
                            tracing::warn!(
                                error = %e,
                                failures,
                                "failed to write audit log (backing off; audit lines are dropped until it recovers)"
                            );
                        }
                        // Back off instead of retrying the open/write per line: a
                        // full disk would otherwise cost a failed syscall and a
                        // warn line for every record.
                        if !wait_for_recovery(&rx, write_backoff(failures), &state) {
                            break;
                        }
                    }
                }
            }
            Err(RecvTimeoutError::Timeout) => {
                // Recovery probe first, flush second: a probe that lands is
                // immediately re-verified by the flush of any still-buffered
                // line, so a still-broken write path ends the tick unhealthy.
                // Without the probe, a write failure with no further traffic
                // stuck `write_failed` forever — nothing ever retried the
                // closed handle. The probe is the same open-for-append that a
                // new record (and startup's `probe`) would perform, and any
                // real write that still fails re-flags it.
                if state.write_failed.load(Ordering::Relaxed)
                    && reopen(&path, &mut logs.main).is_ok()
                    && reopen(&reject_path, &mut logs.reject).is_ok()
                {
                    state.write_failed.store(false, Ordering::Relaxed);
                }
                flush_dirty(&mut logs.main, &state);
                flush_dirty(&mut logs.reject, &state);
            }
            // Disconnected (after close): drain, flush, exit so tail audit lines are not lost.
            Err(RecvTimeoutError::Disconnected) => break,
        }
    }
    flush_dirty(&mut logs.main, &state);
    flush_dirty(&mut logs.reject, &state);
}

/// Wait after `failures` consecutive write failures: doubling, capped.
fn write_backoff(failures: u32) -> Duration {
    let shift = failures.saturating_sub(1).min(16);
    WRITE_BACKOFF_START
        .saturating_mul(1u32 << shift)
        .min(WRITE_BACKOFF_MAX)
}

/// Sleeps out a write backoff while draining (and counting) the queue, so a
/// shutdown (`close`) is noticed immediately instead of waiting for the full
/// backoff. Returns false when the channel closed, i.e. the writer must exit.
fn wait_for_recovery(rx: &Receiver<AuditEntry>, backoff: Duration, state: &AuditState) -> bool {
    let until = Instant::now() + backoff;
    loop {
        let left = until.saturating_duration_since(Instant::now());
        if left.is_zero() {
            return true;
        }
        match rx.recv_timeout(left.min(FLUSH_IDLE)) {
            Ok(_) => {
                state.dropped.fetch_add(1, Ordering::Relaxed);
            }
            Err(RecvTimeoutError::Timeout) => {}
            Err(RecvTimeoutError::Disconnected) => return false,
        }
    }
}

fn flush_dirty(log: &mut OpenLog, state: &AuditState) {
    let Some(writer) = log.file.as_mut() else {
        return;
    };
    if writer.buffer().is_empty() {
        return;
    }
    if let Err(e) = writer.flush() {
        tracing::warn!(error = %e, "failed to flush audit log");
        log.file = None;
        state.write_failed.store(true, Ordering::Relaxed);
    }
}

struct OpenLog {
    file: Option<BufWriter<File>>,
    written: u64,
}

/// (Re)opens a chain for append, rotating first if the previous generation
/// filled up. Shared by the record path and the idle recovery probe, so a
/// reopened chain gets the same rotation bookkeeping a new record would.
/// A chain that is already open is left alone (its buffer is preserved).
fn reopen(path: &Path, log: &mut OpenLog) -> std::io::Result<()> {
    if log.file.is_some() {
        return Ok(());
    }
    rotate_if_needed(path, log)?;
    let file = util::private_open_options(0o600)
        .create(true)
        .append(true)
        .open(path)?;
    log.written = file.metadata()?.len();
    log.file = Some(BufWriter::new(file));
    Ok(())
}

fn append(path: &Path, log: &mut OpenLog, entry: &AuditEntry) -> std::io::Result<()> {
    let mut line = serde_json::to_vec(entry).map_err(std::io::Error::other)?;
    line.push(b'\n');
    if log.file.is_none() {
        reopen(path, log)?;
    }
    // Unreachable in practice (the block above just opened it), but the
    // invariant is enforced here instead of asserted with an unwrap.
    let Some(writer) = log.file.as_mut() else {
        return Ok(());
    };
    writer.write_all(&line)?;
    log.written += line.len() as u64;
    if log.written >= MAX_AUDIT_BYTES {
        // Flush before the handle is dropped: the rotation rename happens on
        // the next reopen, and buffered-but-unwritten tail lines must not be
        // left behind in a file that is about to be renamed. (Windows also
        // refuses to rename an open file.)
        writer.flush()?;
        log.file = None;
    }
    Ok(())
}

fn rotate_if_needed(path: &Path, log: &mut OpenLog) -> std::io::Result<()> {
    if log.written < MAX_AUDIT_BYTES {
        return Ok(());
    }
    // Shift the chain one generation older (.N-1 → .N … current → .1),
    // dropping the previous .N. remove-before-rename keeps Windows happy
    // (rename refuses to replace an existing file there); the live file's
    // replacement target (.1) was just vacated by the shift.
    for generation in (2..=AUDIT_GENERATIONS).rev() {
        let from = PathBuf::from(format!("{}.{}", path.display(), generation - 1));
        let to = PathBuf::from(format!("{}.{}", path.display(), generation));
        if from.exists() {
            let _ = std::fs::remove_file(&to);
            std::fs::rename(&from, &to)?;
        }
    }
    let rotated = PathBuf::from(format!("{}.1", path.display()));
    std::fs::rename(path, rotated)?;
    log.written = 0;
    Ok(())
}

impl AuditEntry {
    #[allow(clippy::too_many_arguments)]
    pub fn new(
        rid: impl Into<String>,
        route: impl Into<String>,
        key_fp: impl Into<String>,
        method: impl Into<String>,
        path: impl Into<String>,
        bytes_in: usize,
        stream: bool,
    ) -> Self {
        Self {
            ts: util::now_ts(),
            rid: rid.into(),
            route: route.into(),
            key_fp: key_fp.into(),
            method: method.into(),
            path: path.into(),
            status: 0,
            ms: 0,
            bytes_in,
            stream,
            attempts: 1,
            rejected: false,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn is_enabled_reflects_the_writer() {
        assert!(!Audit::disabled().is_enabled());
        assert!(
            Audit::disabled().healthy(),
            "audit off is not an unhealthy audit"
        );
        let dir = tempfile::tempdir().unwrap();
        let audit = Audit::new(dir.path().join("audit.jsonl"));
        assert!(audit.is_enabled());
        assert!(audit.healthy());
        audit.close();
        assert!(!audit.is_enabled(), "close() takes the writer out");
    }

    #[test]
    fn dead_writer_is_reported_unhealthy() {
        // Simulate the writer thread dying under a still-enabled audit (a
        // panic in the writer): `is_enabled` keeps saying yes, so health must
        // be the thing that notices.
        let dir = tempfile::tempdir().unwrap();
        let audit = Audit::new(dir.path().join("audit.jsonl"));
        audit.state.alive.store(false, Ordering::Relaxed);
        assert!(audit.is_enabled());
        assert!(!audit.healthy(), "a dead writer must not look healthy");
        audit.close();
    }

    #[test]
    fn write_failure_is_reported_unhealthy_until_recovery() {
        // The parent directory does not exist, so the writer's idle recovery
        // probe can never reopen the chains: the flag this test sets by hand
        // stays set until the test clears it, keeping the assertions race-free.
        let dir = tempfile::tempdir().unwrap();
        let audit = Audit::new(dir.path().join("missing").join("audit.jsonl"));
        assert!(audit.healthy());
        audit.state.write_failed.store(true, Ordering::Relaxed);
        assert!(
            !audit.healthy(),
            "a writer that cannot write must not look healthy"
        );
        audit.state.write_failed.store(false, Ordering::Relaxed);
        assert!(audit.healthy(), "a recovered write restores health");
        audit.close();
    }

    /// Regression: after a real write failure, `write_failed` used to clear
    /// only on the next successful record. With no traffic at all — the exact
    /// situation of an idle gateway whose audit disk hiccuped — the flag stuck
    /// forever. The idle probe must restore health once the path is writable
    /// again, without any new record arriving.
    #[cfg(unix)]
    #[test]
    fn write_failure_recovers_via_the_idle_probe_without_new_traffic() {
        if unsafe { libc::geteuid() } == 0 {
            // root bypasses directory permissions, so the failure injection
            // below would not fail.
            return;
        }
        use std::os::unix::fs::PermissionsExt as _;

        fn wait_until(limit: Duration, mut cond: impl FnMut() -> bool) -> bool {
            let deadline = Instant::now() + limit;
            while Instant::now() < deadline {
                if cond() {
                    return true;
                }
                std::thread::sleep(Duration::from_millis(10));
            }
            cond()
        }

        let dir = tempfile::tempdir().unwrap();
        let audit = Audit::new(dir.path().join("audit.jsonl"));

        // Read-only parent: every open (and with it every write) fails.
        std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o555)).unwrap();
        audit.record(AuditEntry::new("R1", "-", "fp", "GET", "/x", 0, false));
        assert!(
            wait_until(Duration::from_secs(2), || !audit.healthy()),
            "the failed write must turn the audit unhealthy"
        );

        // Repair the directory. No further record is queued: only the idle
        // probe may bring the flag back.
        std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o755)).unwrap();
        assert!(
            wait_until(Duration::from_secs(2), || audit.healthy()),
            "the idle probe must recover health once the path is writable again"
        );
        audit.close();
    }

    #[test]
    fn probe_creates_both_rotation_chains() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("audit.jsonl");
        probe(&path).unwrap();
        assert!(path.exists());
        assert!(dir.path().join("audit.jsonl.rej").exists());
    }

    #[cfg(unix)]
    #[test]
    fn probe_refuses_a_symlink() {
        let dir = tempfile::tempdir().unwrap();
        let target = dir.path().join("target.jsonl");
        std::fs::write(&target, b"").unwrap();
        let link = dir.path().join("audit.jsonl");
        std::os::unix::fs::symlink(&target, &link).unwrap();
        let err = probe(&link).unwrap_err().to_string();
        assert!(err.contains("cannot be opened for append"), "{err}");
    }

    #[test]
    fn close_detaches_a_stuck_writer_instead_of_hanging() {
        // A writer thread parked in a write must not be able to hold shutdown:
        // close() waits at most CLOSE_JOIN_TIMEOUT, then detaches.
        let dir = tempfile::tempdir().unwrap();
        let audit = Audit::new(dir.path().join("audit.jsonl"));
        // Replace the writer handle with one whose thread never finishes.
        let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
        let stuck = std::thread::spawn(move || {
            let _ = release_rx.recv();
        });
        *audit.handle.lock().unwrap() = Some(stuck);

        let started = Instant::now();
        audit.close();
        assert!(
            started.elapsed() < CLOSE_JOIN_TIMEOUT + Duration::from_secs(2),
            "close() must not wait for a stuck writer, took {:?}",
            started.elapsed()
        );
        let _ = release_tx.send(());
    }

    #[test]
    fn write_backoff_doubles_and_caps() {
        assert_eq!(write_backoff(1), WRITE_BACKOFF_START);
        assert_eq!(write_backoff(2), WRITE_BACKOFF_START * 2);
        assert_eq!(write_backoff(30), WRITE_BACKOFF_MAX);
    }

    #[test]
    fn recovery_wait_stops_on_close() {
        // close() drops the sender; the writer's backoff wait must notice and
        // exit instead of sleeping out the whole backoff (a shutdown would
        // otherwise hang for up to WRITE_BACKOFF_MAX).
        let (tx, rx) = sync_channel::<AuditEntry>(4);
        let state = AuditState {
            alive: AtomicBool::new(true),
            closed: AtomicBool::new(false),
            write_failed: AtomicBool::new(false),
            dropped: AtomicU64::new(0),
            dead_warned: AtomicBool::new(false),
        };
        drop(tx);
        assert!(!wait_for_recovery(&rx, Duration::from_secs(30), &state));
    }

    #[test]
    fn queued_entries_are_counted_as_dropped_while_backing_off() {
        let (tx, rx) = sync_channel::<AuditEntry>(4);
        let state = AuditState {
            alive: AtomicBool::new(true),
            closed: AtomicBool::new(false),
            write_failed: AtomicBool::new(false),
            dropped: AtomicU64::new(0),
            dead_warned: AtomicBool::new(false),
        };
        tx.try_send(AuditEntry::new("R9", "r", "fp", "GET", "/x", 0, false))
            .unwrap();
        assert!(wait_for_recovery(&rx, Duration::from_millis(1), &state));
        assert_eq!(state.dropped.load(Ordering::Relaxed), 1);
    }

    #[test]
    fn writes_one_json_line() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("audit.jsonl");
        let audit = Audit::new(path.clone());
        let mut entry =
            AuditEntry::new("R1", "claude", "abcd1234", "POST", "/v1/messages", 12, true);
        entry.status = 200;
        entry.ms = 42;
        audit.record(entry);
        audit.close(); // assert only after the queue is drained
        let text = std::fs::read_to_string(path).unwrap();
        assert_eq!(text.lines().count(), 1);
        let v: serde_json::Value = serde_json::from_str(text.lines().next().unwrap()).unwrap();
        assert_eq!(v["route"], "claude");
        assert_eq!(v["status"], 200);
        assert!(v.get("query").is_none());
    }

    #[test]
    fn rotates_after_size_limit() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("audit.jsonl");
        std::fs::write(&path, vec![b'x'; MAX_AUDIT_BYTES as usize + 1]).unwrap();
        let audit = Audit::new(path.clone());
        // The first record pushes written past the limit (forcing the next close); the second opens a new file.
        audit.record(AuditEntry::new("R3", "r", "fp", "GET", "/x", 0, false));
        audit.record(AuditEntry::new("R4", "r", "fp", "GET", "/x", 0, false));
        audit.close();

        let rotated = dir.path().join("audit.jsonl.1");
        assert!(rotated.exists());
        assert!(std::fs::metadata(&rotated).unwrap().len() > MAX_AUDIT_BYTES);
        let text = std::fs::read_to_string(&path).unwrap();
        assert_eq!(text.lines().count(), 1);
        assert!(
            text.contains("R4"),
            "new file after rotation should contain only the last entry: {text}"
        );
    }

    #[test]
    fn record_reject_does_not_touch_the_gate_when_audit_is_disabled() {
        let audit = Audit::disabled();
        audit.record_reject(AuditEntry::new("R1", "-", "fp", "GET", "/x", 0, false));
        assert!(
            audit.rejects.lock().unwrap().per_fp.is_empty(),
            "the reject gate must not be populated when audit is off"
        );
        assert_eq!(audit.dropped(), 0);
    }

    #[test]
    fn record_after_close_is_silent_noop() {
        let dir = tempfile::tempdir().unwrap();
        let audit = Audit::new(dir.path().join("audit.jsonl"));
        audit.close();
        audit.record(AuditEntry::new("R2", "r", "fp", "GET", "/x", 0, false));
        assert!(!dir.path().join("audit.jsonl").exists());
    }

    #[test]
    fn rotation_shifts_generations_in_order() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("audit.jsonl");
        std::fs::write(&path, "current").unwrap();
        std::fs::write(dir.path().join("audit.jsonl.1"), "gen1").unwrap();
        std::fs::write(dir.path().join("audit.jsonl.2"), "gen2").unwrap();
        let mut log = OpenLog {
            file: None,
            written: u64::MAX, // force rotation
        };
        rotate_if_needed(&path, &mut log).unwrap();
        assert_eq!(log.written, 0);
        assert_eq!(
            std::fs::read_to_string(dir.path().join("audit.jsonl.1")).unwrap(),
            "current"
        );
        assert_eq!(
            std::fs::read_to_string(dir.path().join("audit.jsonl.2")).unwrap(),
            "gen1"
        );
        assert_eq!(
            std::fs::read_to_string(dir.path().join("audit.jsonl.3")).unwrap(),
            "gen2"
        );
        assert!(!dir.path().join("audit.jsonl.4").exists());
        assert!(!path.exists(), "the live file moves into .1 on rotation");
    }

    #[test]
    fn rotation_drops_the_oldest_generation() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("audit.jsonl");
        std::fs::write(&path, "current").unwrap();
        for generation in 1..=AUDIT_GENERATIONS {
            std::fs::write(
                dir.path().join(format!("audit.jsonl.{generation}")),
                format!("gen{generation}"),
            )
            .unwrap();
        }
        let mut log = OpenLog {
            file: None,
            written: u64::MAX,
        };
        rotate_if_needed(&path, &mut log).unwrap();
        assert_eq!(
            std::fs::read_to_string(dir.path().join("audit.jsonl.5")).unwrap(),
            format!("gen{}", AUDIT_GENERATIONS - 1)
        );
        assert!(
            !dir.path()
                .join(format!("audit.jsonl.{}", AUDIT_GENERATIONS + 1))
                .exists()
        );
    }

    #[test]
    fn rejection_rows_throttle_per_fingerprint_and_globally() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("audit.jsonl");
        let rej = reject_chain_path(&path);
        let audit = Audit::new(path.clone());
        audit.record_reject(AuditEntry::new("A1", "-", "fpX", "GET", "/x", 0, false));
        audit.record_reject(AuditEntry::new("A2", "-", "fpX", "GET", "/x", 0, false));
        // Unique fingerprints (randomized-token flood): admitted only up to
        // the global window cap, which fpX's first row already counted against.
        for i in 0..150 {
            audit.record_reject(AuditEntry::new(
                format!("U{i}"),
                "-",
                format!("fp{i}"),
                "GET",
                "/x",
                0,
                false,
            ));
        }
        audit.record(AuditEntry::new(
            "F1",
            "claude",
            "fpOK",
            "POST",
            "/v1/messages",
            12,
            true,
        ));
        audit.close();

        let text = std::fs::read_to_string(&rej).unwrap();
        let count = text.lines().count();
        assert!(text.contains("\"A1\""));
        assert!(
            !text.contains("\"A2\""),
            "same-fingerprint repeat within the interval must be suppressed"
        );
        assert_eq!(
            count, REJECT_GLOBAL_MAX,
            "gate admits the window cap, all on the reject chain: {count}"
        );

        // The randomized flood never touches the main chain: F1 is the only
        // row there.
        let main_text = std::fs::read_to_string(&path).unwrap();
        assert!(main_text.contains("\"F1\""));
        assert_eq!(main_text.lines().count(), 1);
    }

    #[test]
    fn rejection_rows_land_on_the_reject_chain_and_main_format_is_unchanged() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("audit.jsonl");
        let audit = Audit::new(path.clone());
        let mut main = AuditEntry::new("M1", "claude", "fpOK", "POST", "/v1/messages", 12, true);
        main.status = 200;
        audit.record(main);
        audit.record_reject(AuditEntry::new("R1", "-", "fpX", "GET", "/x", 0, false));
        audit.close();

        let main_text = std::fs::read_to_string(&path).unwrap();
        assert_eq!(
            main_text.lines().count(),
            1,
            "main chain holds only real traffic"
        );
        assert!(
            !main_text.contains("rejected"),
            "main-chain line format must stay free of the reject marker: {main_text}"
        );
        assert!(main_text.contains("\"M1\""));

        let rej_text = std::fs::read_to_string(path.with_extension("jsonl.rej")).unwrap();
        assert_eq!(rej_text.lines().count(), 1);
        assert!(rej_text.contains("\"R1\""));
        assert!(
            rej_text.contains("\"rejected\":true"),
            "reject chain rows are self-describing: {rej_text}"
        );
    }

    #[test]
    fn reject_flood_can_only_ever_rotate_the_reject_chain() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("audit.jsonl");
        // Force the reject chain past its rotation threshold while the main
        // chain holds a single sentinel row: the flood must cycle its own
        // generations and leave the real-traffic trail untouched.
        let rej = reject_chain_path(&path);
        std::fs::write(&rej, vec![b'x'; MAX_AUDIT_BYTES as usize + 1]).unwrap();
        let audit = Audit::new(path.clone());
        audit.record(AuditEntry::new(
            "M1", "claude", "fpOK", "GET", "/v1/x", 0, false,
        ));
        for i in 0..(REJECT_GLOBAL_MAX + 5) {
            audit.record_reject(AuditEntry::new(
                format!("R{i}"),
                "-",
                format!("fp{i}"),
                "GET",
                "/x",
                0,
                false,
            ));
        }
        audit.close();

        let main_text = std::fs::read_to_string(&path).unwrap();
        assert!(
            main_text.contains("\"M1\"") && main_text.lines().count() == 1,
            "the sentinel row must survive a reject-chain rotation: {main_text}"
        );
        assert!(
            !path.with_extension("jsonl.1").exists(),
            "the main chain must not have rotated"
        );
        assert!(
            rej.with_file_name("audit.jsonl.rej.1").exists(),
            "the reject chain must have rotated"
        );
    }
}