zc2 0.0.42

P2P compute broker with credit-based billing, WAL, and broker mesh support
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
//! What a run measures, separated from how it is drawn.
//!
//! Nothing here prints. Every figure the live view and the report show comes
//! from this module, so the two can never tell different stories — the bug
//! that two implementations of "the 50th percentile" always eventually
//! produce.

use std::collections::{BTreeMap, BTreeSet};
use std::time::{Duration, Instant};

// ---------------------------------------------------------------------------
// percentiles
// ---------------------------------------------------------------------------

/// The `p`th percentile of `latencies`, in microseconds. One rule, used by
/// both the live line and the summary.
pub(crate) fn percentile_us(latencies: &[u64], p: f64) -> u64 {
    if latencies.is_empty() {
        return 0;
    }
    let mut sorted = latencies.to_vec();
    sorted.sort_unstable();
    sorted_percentile_us(&sorted, p)
}

/// The same, on samples already sorted — so a caller needing several
/// percentiles sorts once instead of once per percentile.
pub(crate) fn sorted_percentile_us(sorted: &[u64], p: f64) -> u64 {
    if sorted.is_empty() {
        return 0;
    }
    let idx = ((sorted.len() as f64 * p / 100.0) as usize).min(sorted.len() - 1);
    sorted[idx]
}

/// p50 and p99 of the samples so far, for the live line.
pub(crate) fn percentiles_us(latencies: &[u64]) -> (u64, u64) {
    (
        percentile_us(latencies, 50.0),
        percentile_us(latencies, 99.0),
    )
}

/// The mean of `latencies`, in microseconds.
pub(crate) fn mean_us(latencies: &[u64]) -> f64 {
    match latencies.len() {
        0 => 0.0,
        n => latencies.iter().sum::<u64>() as f64 / n as f64,
    }
}

/// Population standard deviation, in microseconds. Next to a mean it says
/// whether the mean means anything.
pub(crate) fn stdev_us(latencies: &[u64]) -> f64 {
    if latencies.len() < 2 {
        return 0.0;
    }
    let mean = mean_us(latencies);
    let var = latencies
        .iter()
        .map(|v| {
            let d = *v as f64 - mean;
            d * d
        })
        .sum::<f64>()
        / latencies.len() as f64;
    var.sqrt()
}

/// Mean, standard deviation and coefficient of variation of a set of
/// per-second rates. `None` until two seconds have closed, because a spread
/// over one sample is not a spread — and reporting it as `0` says the run was
/// perfectly steady when nothing has been measured yet.
pub(crate) fn rate_spread(rates: &[u64]) -> Option<(f64, f64, f64)> {
    if rates.len() < 2 {
        return None;
    }
    let mean = rates.iter().sum::<u64>() as f64 / rates.len() as f64;
    let var = rates
        .iter()
        .map(|v| {
            let d = *v as f64 - mean;
            d * d
        })
        .sum::<f64>()
        / rates.len() as f64;
    let stdev = var.sqrt();
    let cv = match mean > 0.0 {
        true => stdev / mean * 100.0,
        false => 0.0,
    };
    Some((mean, stdev, cv))
}

// ---------------------------------------------------------------------------
// buckets
// ---------------------------------------------------------------------------

/// How many per-second samples a throughput distribution needs before it is
/// telling the reader anything. Below this the "buckets" are an artefact of
/// there being one number to bucket: a 1.3 s run produced `≤2295 req/s` and
/// `≤2296 req/s`, which is noise wearing a chart's clothes.
pub(crate) const MIN_RATE_SAMPLES: usize = 3;

/// How many rows a distribution may occupy. Enough to show a shape, few
/// enough to keep the live block on one screen.
pub(crate) const MAX_BUCKET_ROWS: usize = 12;

/// A distribution: upper bounds and how many samples fell at or below each.
/// The last bound is [`u64::MAX`], so every sample lands in exactly one row
/// and the counts always sum to the sample count.
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct Buckets {
    pub bounds: Vec<u64>,
    pub counts: Vec<u64>,
}

/// The step sizes a reader can hold in their head. A bucket edge at 6 033 µs
/// is arithmetically fine and unreadable; one at 10 ms is neither.
fn nice_step(raw: f64) -> u64 {
    if raw <= 1.0 {
        return 1;
    }
    let mag = 10f64.powf(raw.log10().floor());
    let norm = raw / mag;
    let step = if norm <= 1.0 {
        1.0
    } else if norm <= 2.0 {
        2.0
    } else if norm <= 2.5 {
        2.5
    } else if norm <= 5.0 {
        5.0
    } else {
        10.0
    };
    ((step * mag).round() as u64).max(1)
}

/// `1 234 567` → `1 200 000`: a log bound rounded to two significant figures,
/// because `10^(3/8)` is a real number and nobody wants to read it.
fn two_sig_figs(v: f64) -> u64 {
    if v <= 0.0 {
        return 1;
    }
    let mag = 10f64.powf(v.log10().floor() - 1.0);
    ((v / mag).round() * mag).round() as u64
}

impl Buckets {
    /// Bucket `samples`, choosing the shape that shows this particular run.
    ///
    /// The fixed decade ladder this replaced — 1, 5, 10, 25, 50, 100, 250,
    /// 500, 1000 ms — puts a run whose p50 is 260 ms and p99 is 315 ms
    /// entirely in the `≤500ms` row. The chart is then a single full bar: a
    /// picture with no information in it. So: a tight spread gets linear bins
    /// over where the samples actually are, a wide one gets log bins, and
    /// either way the reader sees a shape.
    pub(crate) fn of(samples: &[u64], max_rows: usize) -> Buckets {
        let max_rows = max_rows.max(3);
        if samples.is_empty() {
            return Buckets {
                bounds: vec![u64::MAX],
                counts: vec![0],
            };
        }
        let mut sorted = samples.to_vec();
        sorted.sort_unstable();
        let lo = sorted_percentile_us(&sorted, 1.0).max(1);
        let hi = sorted_percentile_us(&sorted, 99.0).max(lo + 1);
        let bounds = match hi as f64 / lo as f64 >= 3.0 {
            true => log_bounds(lo, hi, max_rows),
            false => linear_bounds(lo, hi, max_rows),
        };
        Self::with_bounds(&sorted, bounds)
    }

    /// Bucket `samples` linearly whatever their spread — for a quantity whose
    /// reader thinks in even steps (requests per second) rather than in
    /// orders of magnitude (latency).
    pub(crate) fn linear(samples: &[u64], max_rows: usize) -> Buckets {
        let max_rows = max_rows.max(3);
        if samples.is_empty() {
            return Buckets {
                bounds: vec![u64::MAX],
                counts: vec![0],
            };
        }
        let mut sorted = samples.to_vec();
        sorted.sort_unstable();
        let lo = *sorted.first().unwrap();
        let hi = (*sorted.last().unwrap()).max(lo + 1);
        Self::with_bounds(&sorted, linear_bounds(lo.max(1), hi, max_rows))
    }

    fn with_bounds(sorted: &[u64], mut bounds: Vec<u64>) -> Buckets {
        bounds.push(u64::MAX);
        let mut counts = vec![0u64; bounds.len()];
        for v in sorted {
            let i = bounds
                .iter()
                .position(|b| v <= b)
                .unwrap_or(bounds.len() - 1);
            counts[i] += 1;
        }
        Buckets { bounds, counts }
    }

    /// How many samples went in.
    pub(crate) fn total(&self) -> u64 {
        self.counts.iter().sum()
    }

    /// The fullest row, for scaling the bars.
    pub(crate) fn peak(&self) -> u64 {
        self.counts.iter().copied().max().unwrap_or(0)
    }

    /// A row's label: `≤250ms` for a named bound, `>320ms` for the overflow.
    pub(crate) fn label(&self, i: usize, fmt: &dyn Fn(u64) -> String) -> String {
        match self.bounds.get(i) {
            Some(&b) if b != u64::MAX => format!("≤{}", fmt(b)),
            _ => match i {
                0 => "all".to_string(),
                i => format!(">{}", fmt(self.bounds[i - 1])),
            },
        }
    }

    /// The rows worth drawing: the run of buckets that hold something, plus
    /// one either side for scale. Ten rows of which nine are zero say less
    /// than four rows that frame where the times fell.
    pub(crate) fn interesting(&self) -> std::ops::Range<usize> {
        let first = self.counts.iter().position(|c| *c > 0);
        let last = self.counts.iter().rposition(|c| *c > 0);
        match (first, last) {
            (Some(f), Some(l)) => f.saturating_sub(1)..(l + 2).min(self.counts.len()),
            _ => 0..self.counts.len(),
        }
    }
}

fn linear_bounds(lo: u64, hi: u64, max_rows: usize) -> Vec<u64> {
    let rows = max_rows.saturating_sub(1).max(2);
    let step = nice_step((hi - lo) as f64 / rows as f64);
    let start = (lo / step) * step;
    let mut bounds = Vec::new();
    let mut b = start + step;
    while b <= hi + step && bounds.len() < rows {
        bounds.push(b);
        b += step;
    }
    if bounds.is_empty() {
        bounds.push(hi);
    }
    bounds
}

fn log_bounds(lo: u64, hi: u64, max_rows: usize) -> Vec<u64> {
    let decades = (hi as f64 / lo as f64).log10().max(0.05);
    // Eight steps per decade where that fits, coarser where it does not, so a
    // run spanning microseconds to seconds still fits on one screen.
    let per_decade = [8.0, 4.0, 2.0, 1.0]
        .into_iter()
        .find(|p| ((decades * p).ceil() as usize) < max_rows)
        .unwrap_or(1.0);
    let first = (lo as f64).log10() * per_decade;
    let last = (hi as f64).log10() * per_decade;
    let mut bounds = Vec::new();
    let mut k = first.floor() + 1.0;
    while k <= last.ceil() && bounds.len() + 1 < max_rows {
        let v = two_sig_figs(10f64.powf(k / per_decade));
        if bounds.last() != Some(&v) {
            bounds.push(v);
        }
        k += 1.0;
    }
    if bounds.is_empty() {
        bounds.push(hi);
    }
    bounds
}

// ---------------------------------------------------------------------------
// throughput over time
// ---------------------------------------------------------------------------

/// Completions per second of wall clock.
///
/// The old view reported `done / elapsed`, which is the mean since the run
/// started: by minute forty it takes a catastrophe to move it, so a broker
/// that halves its rate half way through looks like one that slowed by a few
/// percent. These are the samples that show it.
#[derive(Debug, Clone, Default)]
pub(crate) struct RateRing {
    /// One entry per completed second of the run.
    per_second: Vec<u64>,
    last_second: u64,
    last_done: u64,
}

impl RateRing {
    pub(crate) fn new() -> Self {
        Self::default()
    }

    /// Fold in the completion counter as the progress loop sees it. Called
    /// far more often than once a second; closes a bucket only when the clock
    /// crosses into a new one.
    pub(crate) fn sample(&mut self, done: u64, elapsed: Duration) {
        let sec = elapsed.as_secs();
        while self.last_second < sec {
            self.per_second.push(done.saturating_sub(self.last_done));
            self.last_done = done;
            self.last_second += 1;
            // Only the second that just closed gets the real delta; any whole
            // seconds skipped by a stalled sampler are recorded as empty
            // rather than credited to the one that happened to be sampled.
            if self.last_second < sec {
                self.per_second.pop();
                self.per_second.push(0);
            }
        }
    }

    /// Close the run's final, partial second, scaled to a full-second rate.
    ///
    /// A run lasting 0.98 s closes no second at all, and reporting no peak
    /// for a run that plainly had one reads as a gap in the instrument. Half
    /// a second is the floor: below that the scaling multiplies too little
    /// evidence by too much, and no figure is better than an invented one.
    pub(crate) fn finish(&mut self, done: u64, elapsed: Duration) {
        self.sample(done, elapsed);
        let tail = elapsed.as_secs_f64() - self.last_second as f64;
        if tail >= 0.5 {
            let counted = done.saturating_sub(self.last_done);
            self.per_second.push((counted as f64 / tail).round() as u64);
            self.last_done = done;
        }
    }

    /// The closed seconds, oldest first.
    pub(crate) fn per_second(&self) -> &[u64] {
        &self.per_second
    }

    /// The rate over the last `window` seconds — what "now" means on the live
    /// line. Falls back to the run mean before a window has passed.
    pub(crate) fn now(&self, window: usize, done: u64, elapsed: Duration) -> f64 {
        let tail: Vec<u64> = self.per_second.iter().rev().take(window).copied().collect();
        match tail.len() {
            0 => done as f64 / elapsed.as_secs_f64().max(0.001),
            n => tail.iter().sum::<u64>() as f64 / n as f64,
        }
    }

    /// The best second the run has had.
    pub(crate) fn peak(&self) -> u64 {
        self.per_second.iter().copied().max().unwrap_or(0)
    }

    /// The worst second the run has had, once there are enough to mean
    /// something. The first second is always partial, so it is skipped.
    pub(crate) fn dip(&self) -> Option<u64> {
        match self.per_second.len() {
            0..=2 => None,
            _ => self.per_second[1..].iter().copied().min(),
        }
    }

    /// The per-second samples as rates, for a sparkline.
    pub(crate) fn as_f64(&self) -> Vec<f64> {
        self.per_second.iter().map(|n| *n as f64).collect()
    }
}

// ---------------------------------------------------------------------------
// placement: where the work ran, and what it cost
// ---------------------------------------------------------------------------

/// Whether a request cost anything.
///
/// This is the split `zc bench` exists to show, and it is decided by the only
/// thing that cannot be wrong about it: what the broker said it charged. Work
/// on hardware you already own is free; work that is billed is billed.
///
/// It is deliberately NOT decided by the transport. A broker dispatches to
/// every worker registered with it — including workers on other machines
/// across the mesh — over its `local` path, so classing by transport reports
/// a job that ran on another node, over the network, as "local". True of the
/// code path, useless to the reader, and not the question being asked. The
/// transport is kept and shown per node as `via`, where it is a fact rather
/// than a claim.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) enum Placement {
    /// Cost nothing.
    Free,
    /// Cost credits.
    Billed,
}

impl Placement {
    pub(crate) fn as_str(&self) -> &'static str {
        match self {
            Placement::Free => "free",
            Placement::Billed => "billed",
        }
    }

    /// Classify one reply by what it actually charged.
    pub(crate) fn of(charged_zkcr: f64) -> Placement {
        match charged_zkcr > 0.0 {
            true => Placement::Billed,
            false => Placement::Free,
        }
    }
}

/// What the router expected of a worker against what it actually delivered.
/// An expectation that does not match reality is how a stale estimate gets
/// noticed — but only where one was actually named; `avg_expected_ms` is
/// `None` when nothing was.
#[derive(Debug, Clone, Copy, Default, PartialEq)]
pub(crate) struct Expectation {
    expected_ms_sum: f64,
    expected_n: u64,
    delivered_ms_sum: f64,
    delivered_n: u64,
}

impl Expectation {
    pub(crate) fn record(&mut self, expected_ms: Option<f64>, delivered_ms: f64) {
        if let Some(e) = expected_ms {
            self.expected_ms_sum += e;
            self.expected_n += 1;
        }
        self.delivered_ms_sum += delivered_ms;
        self.delivered_n += 1;
    }

    pub(crate) fn merge(&mut self, other: &Expectation) {
        self.expected_ms_sum += other.expected_ms_sum;
        self.expected_n += other.expected_n;
        self.delivered_ms_sum += other.delivered_ms_sum;
        self.delivered_n += other.delivered_n;
    }

    pub(crate) fn avg_expected_ms(&self) -> Option<f64> {
        (self.expected_n > 0).then(|| self.expected_ms_sum / self.expected_n as f64)
    }

    pub(crate) fn avg_delivered_ms(&self) -> f64 {
        match self.delivered_n {
            0 => 0.0,
            n => self.delivered_ms_sum / n as f64,
        }
    }

    /// How many times out of what the router promised the node actually took
    /// — above 1.0 the router was optimistic, below it pessimistic. `None`
    /// when nothing was promised or nothing delivered.
    pub(crate) fn ratio(&self) -> Option<f64> {
        let expected = self.avg_expected_ms()?;
        let delivered = self.avg_delivered_ms();
        (expected > 0.0 && delivered > 0.0).then_some(delivered / expected)
    }

    /// Whether the gap is wide enough to be worth a reader's attention: a
    /// factor of two either way. Routing decisions are made on these
    /// estimates, so one that is badly wrong is a finding, not a detail.
    pub(crate) fn is_miscalibrated(&self) -> bool {
        matches!(self.ratio(), Some(r) if !(0.5..=2.0).contains(&r))
    }
}

/// One node's share of a run.
#[derive(Debug, Clone, Default)]
pub(crate) struct NodeStat {
    pub(crate) requests: u64,
    pub(crate) charged_zkcr: f64,
    pub(crate) latencies_us: Vec<u64>,
    pub(crate) expectation: Expectation,
    pub(crate) workers: BTreeMap<String, u64>,
    /// What this node's workers advertise, in zkcr/hour. Shown beside what
    /// they actually charged: a node priced at 3.6 that bills 0.0000 is the
    /// whole of #297 on one line, and no amount of staring at a spend column
    /// alone would show it.
    pub(crate) price_per_hour: Option<f64>,
    /// How the broker reached this node, as its replies reported it:
    /// `local` for a worker registered with this broker (wherever that
    /// worker's machine is), or the peer transport for one dispatched away.
    pub(crate) transports: BTreeSet<String>,
}

impl NodeStat {
    pub(crate) fn p50_us(&self) -> u64 {
        percentile_us(&self.latencies_us, 50.0)
    }

    /// What this node charges per hour, as it advertises it.
    pub(crate) fn price(&self) -> String {
        match self.price_per_hour {
            None => "—".to_string(),
            Some(p) if p <= 0.0 => "free".to_string(),
            Some(p) => format!("{p} zkcr/h"),
        }
    }

    /// A node that advertises a price and charged nothing. Not an accusation
    /// — a broker's own workers are priced and rightly free to their owner —
    /// but on a node that is not yours it is #297, and worth marking.
    pub(crate) fn priced_but_unbilled(&self) -> bool {
        matches!(self.price_per_hour, Some(p) if p > 0.0) && self.charged_zkcr == 0.0
    }

    /// The transports this node was reached over, for the `via` column.
    pub(crate) fn via(&self) -> String {
        match self.transports.len() {
            0 => "—".to_string(),
            _ => self
                .transports
                .iter()
                .cloned()
                .collect::<Vec<_>>()
                .join("+"),
        }
    }
}

/// Where a run's work went, and what each part of it cost.
#[derive(Debug, Clone, Default)]
pub(crate) struct Ledger {
    nodes: BTreeMap<(Placement, String), NodeStat>,
}

impl Ledger {
    pub(crate) fn new() -> Self {
        Self::default()
    }

    /// Fold in one successful request.
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn record(
        &mut self,
        worker: &str,
        transport: Option<&str>,
        price_per_hour: Option<f64>,
        latency_us: u64,
        charged_zkcr: f64,
        expected_ms: Option<f64>,
    ) {
        let placement = Placement::of(charged_zkcr);
        let node = node_of(worker);
        let stat = self.nodes.entry((placement, node)).or_default();
        if let Some(t) = transport.map(|t| t.trim().to_ascii_lowercase()) {
            if !t.is_empty() {
                stat.transports.insert(t);
            }
        }
        if let Some(p) = price_per_hour {
            stat.price_per_hour = Some(p);
        }
        stat.requests += 1;
        stat.charged_zkcr += charged_zkcr;
        stat.latencies_us.push(latency_us);
        stat.expectation
            .record(expected_ms, latency_us as f64 / 1000.0);
        *stat.workers.entry(worker.to_string()).or_insert(0) += 1;
    }

    pub(crate) fn is_empty(&self) -> bool {
        self.nodes.is_empty()
    }

    /// Every node, busiest first, so the row that explains the run is the row
    /// at the top.
    pub(crate) fn rows(&self) -> Vec<(Placement, String, &NodeStat)> {
        let mut v: Vec<(Placement, String, &NodeStat)> = self
            .nodes
            .iter()
            .map(|((p, n), s)| (*p, n.clone(), s))
            .collect();
        v.sort_by(|a, b| {
            a.0.cmp(&b.0)
                .then_with(|| b.2.requests.cmp(&a.2.requests))
                .then_with(|| a.1.cmp(&b.1))
        });
        v
    }

    /// Every node the run touched, with how many requests landed there,
    /// busiest first. One entry per node however the split fell — a node that
    /// ran some work free and some billed is still one node.
    ///
    /// This is what a pinned run is judged on: a pin that held leaves exactly
    /// one entry.
    pub(crate) fn nodes_touched(&self) -> Vec<(String, u64)> {
        let mut by_node: BTreeMap<String, u64> = BTreeMap::new();
        for ((_, node), stat) in &self.nodes {
            *by_node.entry(node.clone()).or_default() += stat.requests;
        }
        let mut v: Vec<(String, u64)> = by_node.into_iter().collect();
        v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
        v
    }

    /// Requests and credits for one side of the split.
    pub(crate) fn totals(&self, placement: Placement) -> (u64, f64) {
        self.nodes
            .iter()
            .filter(|((p, _), _)| *p == placement)
            .fold((0, 0.0), |(n, c), (_, s)| {
                (n + s.requests, c + s.charged_zkcr)
            })
    }

    /// Every latency sample recorded against one side of the split.
    pub(crate) fn latencies(&self, placement: Placement) -> Vec<u64> {
        self.nodes
            .iter()
            .filter(|((p, _), _)| *p == placement)
            .flat_map(|(_, s)| s.latencies_us.iter().copied())
            .collect()
    }

    /// Total requests placed, either side.
    pub(crate) fn placed(&self) -> u64 {
        self.nodes.values().map(|s| s.requests).sum()
    }

    /// Total credits charged, either side.
    pub(crate) fn charged(&self) -> f64 {
        self.nodes.values().map(|s| s.charged_zkcr).sum()
    }

    /// Nodes whose delivered time is far from what the router predicted.
    pub(crate) fn miscalibrated(&self) -> Vec<(String, Expectation)> {
        self.rows()
            .into_iter()
            .filter(|(_, _, s)| s.expectation.is_miscalibrated())
            .map(|(_, n, s)| (n, s.expectation))
            .collect()
    }

    /// Distinct worker names seen, across every node.
    pub(crate) fn worker_names(&self) -> BTreeSet<String> {
        self.nodes
            .values()
            .flat_map(|s| s.workers.keys().cloned())
            .collect()
    }
}

/// The node a worker belongs to, for the "where did the work go" view.
/// `worker-zc-worker-i9-1` is node `i9`; `dd715b7f-w0` is this machine's
/// broker, whose workers all share the node fingerprint before `-w`.
pub(crate) fn node_of(worker: &str) -> String {
    if let Some(rest) = worker.strip_prefix("worker-zc-worker-") {
        return rest
            .rsplit_once('-')
            .map_or(rest, |(node, _)| node)
            .to_string();
    }
    if let Some((node, tail)) = worker.rsplit_once("-w") {
        if tail.chars().all(|c| c.is_ascii_digit()) && !node.is_empty() {
            return node.to_string();
        }
    }
    worker.to_string()
}

/// #267: the millisecond figure a `RoutingDecision`'s reason names, when it
/// names one at all -- `12.4` from "Best latency: 12.4ms", `254.0` from
/// "Adaptive: expected 254ms". `None` for a reason with no latency in it.
pub(crate) fn parse_expected_ms(reason: &str) -> Option<f64> {
    let bytes = reason.as_bytes();
    if bytes.len() < 2 {
        return None;
    }
    for i in 0..=bytes.len() - 2 {
        if &bytes[i..i + 2] != b"ms" {
            continue;
        }
        let mut j = i;
        let mut seen_digit = false;
        let mut seen_dot = false;
        while j > 0 {
            let c = bytes[j - 1];
            if c.is_ascii_digit() {
                seen_digit = true;
                j -= 1;
            } else if c == b'.' && !seen_dot {
                seen_dot = true;
                j -= 1;
            } else {
                break;
            }
        }
        if seen_digit {
            if let Ok(v) = reason[j..i].parse::<f64>() {
                return Some(v);
            }
        }
    }
    None
}

/// A monotonic stopwatch that the live view and the report agree on.
pub(crate) fn since(start: Instant) -> Duration {
    start.elapsed()
}

#[cfg(test)]
mod tests {
    use super::*;

    // ---- buckets ---------------------------------------------------------

    #[test]
    fn a_run_too_short_to_close_two_seconds_has_no_spread_to_report() {
        // A 0.1s run closes no seconds. Reporting stdev 0 / cv 0 would say
        // the throughput was perfectly steady, which is not what was seen —
        // nothing was.
        assert!(rate_spread(&[]).is_none());
        assert!(rate_spread(&[40]).is_none());
        let (mean, stdev, cv) = rate_spread(&[30, 50]).unwrap();
        assert_eq!(mean, 40.0);
        assert_eq!(stdev, 10.0);
        assert_eq!(cv, 25.0);
    }

    #[test]
    fn a_tight_spread_gets_bins_that_show_its_shape() {
        // The run in issue #293: every sample between 241 ms and 402 ms. The
        // old decade ladder put all 786 of them in one row.
        let samples: Vec<u64> = (0..786).map(|i| 241_000 + (i * 137) % 161_000).collect();
        let b = Buckets::of(&samples, MAX_BUCKET_ROWS);
        let occupied = b.counts.iter().filter(|c| **c > 0).count();
        assert!(
            occupied >= 4,
            "a 241-402ms spread must fill several rows, filled {occupied}"
        );
        assert_eq!(b.total(), 786, "every sample lands in exactly one bucket");
    }

    #[test]
    fn a_wide_spread_gets_log_bins_and_still_fits_on_a_screen() {
        let mut samples: Vec<u64> = (0..500).map(|i| 200 + i * 3).collect();
        samples.extend((0..500).map(|i| 2_000_000 + i * 900));
        let b = Buckets::of(&samples, MAX_BUCKET_ROWS);
        assert!(
            b.counts.len() <= MAX_BUCKET_ROWS,
            "{} rows is more than a screen",
            b.counts.len()
        );
        assert_eq!(b.total(), 1000);
    }

    #[test]
    fn the_buckets_account_for_every_sample_once() {
        for n in [1usize, 2, 17, 500] {
            let samples: Vec<u64> = (0..n as u64).map(|i| 1000 + i * i).collect();
            let b = Buckets::of(&samples, MAX_BUCKET_ROWS);
            assert_eq!(b.total(), n as u64, "n={n}");
            assert_eq!(b.bounds.len(), b.counts.len());
        }
    }

    #[test]
    fn no_samples_is_one_empty_row_not_a_panic() {
        let b = Buckets::of(&[], MAX_BUCKET_ROWS);
        assert_eq!(b.total(), 0);
        assert_eq!(b.counts.len(), 1);
    }

    #[test]
    fn identical_samples_do_not_divide_by_zero() {
        let b = Buckets::of(&[5_000; 40], MAX_BUCKET_ROWS);
        assert_eq!(b.total(), 40);
    }

    #[test]
    fn bucket_edges_are_numbers_a_reader_can_hold() {
        assert_eq!(nice_step(6_033.0), 10_000);
        assert_eq!(nice_step(1.4), 2);
        assert_eq!(nice_step(230.0), 250);
        assert_eq!(nice_step(0.2), 1);
    }

    #[test]
    fn the_last_row_is_the_overflow_and_says_so() {
        let b = Buckets::of(&[1000, 2000, 3000], MAX_BUCKET_ROWS);
        let last = b.counts.len() - 1;
        assert_eq!(b.bounds[last], u64::MAX);
        assert!(b.label(last, &|v| v.to_string()).starts_with('>'));
    }

    #[test]
    fn one_second_of_samples_is_not_a_distribution() {
        // 2 500 requests in a 1.3 s run closed one second, and the chart drew
        // "<=2295 req/s" and "<=2296 req/s" — bucket edges invented from a
        // single number.
        let rates = [2295];
        assert!(
            rates.len() < MIN_RATE_SAMPLES,
            "one closed second must fall below the gate every view reads"
        );
        let b = Buckets::linear(&rates, MAX_BUCKET_ROWS);
        assert_eq!(b.total(), 1, "it still buckets; the view is what declines");
    }

    #[test]
    fn a_rate_is_bucketed_in_even_steps_not_orders_of_magnitude() {
        let rates: Vec<u64> = vec![11, 19, 28, 37, 44, 51, 58, 63, 69, 74, 78, 81, 91];
        let b = Buckets::linear(&rates, MAX_BUCKET_ROWS);
        assert_eq!(b.total(), 13);
        let widths: Vec<u64> = b.bounds[..b.bounds.len() - 1]
            .windows(2)
            .map(|w| w[1] - w[0])
            .collect();
        assert!(
            widths.windows(2).all(|w| w[0] == w[1]),
            "linear bins are evenly spaced, got {widths:?}"
        );
    }

    // ---- rate ring -------------------------------------------------------

    #[test]
    fn the_rate_ring_closes_one_bucket_per_second() {
        let mut r = RateRing::new();
        r.sample(10, Duration::from_millis(500));
        assert!(r.per_second().is_empty(), "half a second closes nothing");
        r.sample(40, Duration::from_millis(1_100));
        assert_eq!(r.per_second(), &[40]);
        r.sample(70, Duration::from_millis(2_050));
        assert_eq!(r.per_second(), &[40, 30]);
    }

    #[test]
    fn now_is_the_recent_rate_not_the_run_average() {
        // Fast for five seconds, then almost stopped. The mean is still high;
        // "now" must not be.
        let mut r = RateRing::new();
        let mut done = 0;
        for sec in 1..=5 {
            done += 100;
            r.sample(done, Duration::from_secs(sec));
        }
        for sec in 6..=10 {
            done += 2;
            r.sample(done, Duration::from_secs(sec));
        }
        let now = r.now(5, done, Duration::from_secs(10));
        assert!(now < 10.0, "recent rate was {now}, should be ~2");
        assert_eq!(r.peak(), 100);
    }

    #[test]
    fn a_sub_second_run_still_reports_the_rate_it_achieved() {
        let mut r = RateRing::new();
        r.finish(4000, Duration::from_millis(980));
        assert_eq!(r.per_second().len(), 1);
        let peak = r.peak();
        assert!(
            (4000..4200).contains(&peak),
            "4000 requests in 0.98s is ~4082/s, got {peak}"
        );
    }

    #[test]
    fn a_run_too_short_to_measure_invents_nothing() {
        let mut r = RateRing::new();
        r.finish(60, Duration::from_millis(100));
        assert!(r.per_second().is_empty(), "0.1s is not a measurement");
    }

    #[test]
    fn finishing_does_not_double_count_the_last_closed_second() {
        let mut r = RateRing::new();
        r.sample(100, Duration::from_secs(1));
        r.finish(180, Duration::from_millis(1_800));
        assert_eq!(r.per_second().len(), 2);
        assert_eq!(r.per_second()[0], 100);
        assert_eq!(r.per_second()[1], 100, "80 requests in 0.8s is 100/s");
    }

    #[test]
    fn a_stalled_sampler_does_not_credit_one_second_with_ten() {
        let mut r = RateRing::new();
        r.sample(0, Duration::from_secs(1));
        r.sample(1000, Duration::from_secs(5));
        assert_eq!(r.per_second().len(), 5);
        assert!(
            r.peak() <= 1000,
            "the skipped seconds must not all land on one"
        );
    }

    // ---- placement -------------------------------------------------------

    #[test]
    fn what_was_charged_decides_the_split() {
        assert_eq!(Placement::of(0.0), Placement::Free);
        assert_eq!(Placement::of(0.004), Placement::Billed);
    }

    #[test]
    fn a_worker_on_another_machine_is_still_free_when_nothing_was_charged() {
        // The broker reaches every worker registered with it over its
        // `local` path, wherever that worker's machine is. Classing by
        // transport reported a job that ran on another node, over the
        // network, as "local" — true of the code path and useless to read.
        let mut l = Ledger::new();
        l.record(
            "worker-zc-worker-i9-1",
            Some("local"),
            Some(3.6),
            750_000,
            0.0,
            None,
        );
        assert_eq!(l.totals(Placement::Free), (1, 0.0));
        let rows = l.rows();
        assert_eq!(rows[0].1, "i9");
        assert_eq!(rows[0].2.via(), "local", "the transport is still reported");
    }

    #[test]
    fn a_node_that_advertises_a_price_and_billed_nothing_is_flagged() {
        // zc#297 on one line. Free is correct on your own hardware; the flag
        // is what makes it visible on anyone else's.
        let mut l = Ledger::new();
        l.record("i9-w0", Some("local"), Some(3.6), 750_000, 0.0, None);
        assert!(l.rows()[0].2.priced_but_unbilled());
        assert_eq!(l.rows()[0].2.price(), "3.6 zkcr/h");
    }

    #[test]
    fn a_node_that_charged_what_it_advertises_is_not_flagged() {
        let mut l = Ledger::new();
        l.record("i9-w0", Some("quic"), Some(3.6), 750_000, 0.001, None);
        assert!(!l.rows()[0].2.priced_but_unbilled());
    }

    #[test]
    fn a_worker_that_prices_itself_at_zero_is_not_a_finding() {
        let mut l = Ledger::new();
        l.record("a-w0", Some("local"), Some(0.0), 1000, 0.0, None);
        assert!(!l.rows()[0].2.priced_but_unbilled());
        assert_eq!(l.rows()[0].2.price(), "free");
    }

    #[test]
    fn a_price_the_broker_never_reported_is_not_a_finding_either() {
        let mut l = Ledger::new();
        l.record("a-w0", Some("local"), None, 1000, 0.0, None);
        assert!(!l.rows()[0].2.priced_but_unbilled());
        assert_eq!(l.rows()[0].2.price(), "—");
    }

    #[test]
    fn a_node_reached_two_ways_reports_both() {
        let mut l = Ledger::new();
        l.record("i9-w0", Some("quic"), Some(3.6), 400_000, 0.001, None);
        l.record("i9-w0", Some("http"), Some(3.6), 400_000, 0.001, None);
        assert_eq!(l.rows()[0].2.via(), "http+quic");
    }

    #[test]
    fn a_reply_with_no_transport_still_lands_on_the_right_side() {
        let mut l = Ledger::new();
        l.record("a-w0", None, None, 1000, 0.0, None);
        l.record("b-w0", None, None, 1000, 0.004, None);
        assert_eq!(l.totals(Placement::Free).0, 1);
        assert_eq!(l.totals(Placement::Billed).0, 1);
    }

    // ---- ledger ----------------------------------------------------------

    fn ledger() -> Ledger {
        let mut l = Ledger::new();
        for _ in 0..7 {
            l.record(
                "dd715b7f-w0",
                Some("local"),
                Some(28.0),
                260_000,
                0.0,
                Some(2500.0),
            );
        }
        for _ in 0..3 {
            l.record(
                "dd715b7f-w1",
                Some("local"),
                Some(28.0),
                270_000,
                0.0,
                Some(2500.0),
            );
        }
        for _ in 0..4 {
            l.record(
                "worker-zc-worker-i9-1",
                Some("quic"),
                Some(3.6),
                410_000,
                0.0012,
                Some(408.0),
            );
        }
        l
    }

    #[test]
    fn the_ledger_separates_what_was_free_from_what_was_billed() {
        let l = ledger();
        assert_eq!(l.totals(Placement::Free), (10, 0.0));
        let (n, charged) = l.totals(Placement::Billed);
        assert_eq!(n, 4);
        assert!((charged - 0.0048).abs() < 1e-9, "charged {charged}");
        assert_eq!(l.placed(), 14);
    }

    #[test]
    fn workers_on_one_node_are_totalled_under_it() {
        let l = ledger();
        let rows = l.rows();
        let local = rows
            .iter()
            .find(|(p, _, _)| *p == Placement::Free)
            .expect("a free row");
        assert_eq!(local.1, "dd715b7f");
        assert_eq!(local.2.requests, 10);
        assert_eq!(local.2.workers.len(), 2, "both workers under one node");
    }

    #[test]
    fn a_node_the_router_badly_misjudged_is_flagged() {
        let l = ledger();
        let bad = l.miscalibrated();
        assert_eq!(bad.len(), 1, "only the 2.5s-vs-264ms node");
        assert_eq!(bad[0].0, "dd715b7f");
        let ratio = bad[0].1.ratio().unwrap();
        assert!(ratio < 0.2, "delivered/expected was {ratio}");
    }

    #[test]
    fn a_node_the_router_judged_well_is_not_flagged() {
        let mut l = Ledger::new();
        l.record(
            "i9-w0",
            Some("quic"),
            Some(3.6),
            410_000,
            0.001,
            Some(408.0),
        );
        assert!(l.miscalibrated().is_empty());
    }

    #[test]
    fn a_decision_that_named_no_latency_is_never_called_miscalibrated() {
        // best_price says "0.001 credits", not a time. There is nothing to
        // compare, so there is nothing to warn about.
        let mut l = Ledger::new();
        l.record("a-w0", Some("local"), Some(0.0), 900_000, 0.0, None);
        assert!(l.miscalibrated().is_empty());
    }

    #[test]
    fn an_empty_ledger_reports_nothing_rather_than_dividing_by_zero() {
        let l = Ledger::new();
        assert!(l.is_empty());
        assert_eq!(l.placed(), 0);
        assert_eq!(l.charged(), 0.0);
        assert_eq!(l.totals(Placement::Billed), (0, 0.0));
    }

    // ---- shared helpers --------------------------------------------------

    #[test]
    fn a_worker_is_attributed_to_its_node() {
        assert_eq!(node_of("worker-zc-worker-i9-1"), "i9");
        assert_eq!(node_of("dd715b7f-w0"), "dd715b7f");
        assert_eq!(node_of("odd-name"), "odd-name");
    }

    #[test]
    fn parse_expected_ms_reads_the_number_immediately_before_ms() {
        assert_eq!(parse_expected_ms("Best latency: 12.4ms"), Some(12.4));
        assert_eq!(parse_expected_ms("Adaptive: expected 254ms"), Some(254.0));
        assert_eq!(parse_expected_ms("Best price: 0.001 credits"), None);
        assert_eq!(parse_expected_ms(""), None);
    }

    #[test]
    fn percentiles_come_from_the_samples_so_far() {
        let s: Vec<u64> = (1..=100).collect();
        assert_eq!(percentile_us(&s, 50.0), 51);
        assert_eq!(percentile_us(&s, 99.0), 100);
        assert_eq!(percentile_us(&[], 50.0), 0);
    }

    #[test]
    fn a_spread_is_summarised_without_dividing_by_zero() {
        assert_eq!(stdev_us(&[]), 0.0);
        assert_eq!(stdev_us(&[5]), 0.0);
        assert!(stdev_us(&[1, 3]) > 0.0);
        assert_eq!(mean_us(&[]), 0.0);
    }
}