exatok 0.1.1

Exact GPU-resident BPE tokenizer whose token ids match tiktoken byte for byte
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
//! Where `DeviceTokenizer::encode` spends host time, per corpus shape.
//!
//! This is a MEASUREMENT harness, not a benchmark and not an optimization. It
//! answers one question: of the wall a caller observes, how much lands in each
//! named component of the encode, and how much lands in none of them.
//!
//! # The rule this obeys
//!
//! A component table is only an explanation when its rows sum to the observed
//! wall. Four named costs adding to 4 ms against a 13 ms encode have not
//! explained the encode, they have found four things that are not it, so every
//! table below prints the named total, the observed wall, and the remainder
//! between them as three separate lines. The remainder is never spread over the
//! named rows and never omitted.
//!
//! # What is measured how
//!
//! * Per-phase host time comes from [`DeviceTokenizer::encode_batch_profiled`],
//!   whose phases are disjoint leaves that add. See [`exatok::device::EncodePhase`]
//!   for where each boundary falls and why.
//! * The split of the dispatch phase between vyre's host path and the device
//!   comes from the CUDA backend's own counters, read through
//!   [`VyreBackend::backend_metric_snapshot`] before and after one encode.
//!   `cuda_timed_enqueue_ns_total` is host time spent preparing and launching,
//!   `cuda_timed_device_ns_total` is CUDA event elapsed on the kernel, and
//!   `cuda_timed_wait_ns_total` is host time blocked on completion and copying
//!   outputs back. Those are vyre's own instruments on vyre's own path, so the
//!   ownership answer does not rest on a wrapper this crate wrote.
//! * The resident staging inside a dispatch, which is neither in the timed
//!   window above nor this crate's code, is priced by [`stage_cost`]: it times
//!   the same `allocate_resident`, `upload_resident` and `free_resident` calls
//!   at a sweep of sizes and fits a fixed-plus-per-byte model, which is then
//!   applied to the operation and byte counts the CUDA counters report.
//!
//! # Box state
//!
//! Every wall is warm, repeated, and reported as a median with its full spread,
//! with `/proc/loadavg` printed beside it. Counts and byte totals are printed
//! too, because those reproduce on a contended box and walls do not: a ratio
//! built from a duration and a count survives a change of box state where the
//! duration alone does not.
//!
//! # Running it
//!
//! ```text
//! EXATOK_BENCH_CORPUS=<dir a throughput bench printed as `corpora:`> \
//!   cargo run --release --features cuda --example encode_profile
//! ```
//!
//! The corpus directory is required rather than rebuilt here, so this measures
//! the identical bytes the throughput benches measured instead of a corpus
//! assembled the same way twice from a tree that moved in between.

#[cfg(not(feature = "cuda"))]
fn main() {
    println!(
        "encode_profile measured nothing: it needs --features cuda and a CUDA device. \
         Rerun with `cargo run --release --features cuda --example encode_profile`."
    );
}

#[cfg(feature = "cuda")]
fn main() -> Result<(), String> {
    profile::run()
}

#[cfg(feature = "cuda")]
mod profile {
    use std::collections::BTreeMap;
    use std::path::PathBuf;
    use std::time::{Duration, Instant};

    use vyre_driver::VyreBackend;
    use exatok::device::{DeviceTokenizer, EncodePhase, EncodeProfile};
    use exatok::vocab::{self, Artifact, CL100K_BASE, Vocabulary};
    use exatok::{Encoding, TokenId};

    /// Shapes measured, hardest first.
    ///
    /// `short_pretokens` leads deliberately. Its kernel runs for tens of
    /// microseconds while the whole encode takes milliseconds, so almost all of
    /// its time is the host work this harness is hunting and the signal is not
    /// competing with device time. Prose is the comparison: it sits far closer
    /// to a host encoder, which makes it the WORST shape to look for host cost
    /// in first, because box-state noise is the same size as the effect.
    ///
    /// All four rather than those two, because two shapes cannot tell a cost per
    /// pretoken from a cost per output token: prose produces 1.08 tokens per
    /// pretoken and `short_pretokens` produces 1.00, so both units predict the
    /// same thing to within a few percent and a row can be charged to either.
    /// `cjk` breaks that tie, producing several tokens per pretoken, and `code`
    /// fills in between. Every corpus is the same 262144 bytes, so input size is
    /// held fixed while pretoken count and token count vary independently.
    const SHAPES: [&str; 4] = ["short_pretokens", "prose", "cjk", "code"];

    /// Encodes run before any figure is taken.
    ///
    /// The first encode of a shape pays the cooperative residency bisection, the
    /// 16 MiB table pack, and the backend's compilation of that shape's program,
    /// all one-time and none of them what a warm caller experiences. Two rather
    /// than one, because the second is the first to find every cache already
    /// warm.
    const WARMUP: usize = 3;

    /// Timed encodes per shape.
    ///
    /// Enough that a median has a spread to report rather than a single number
    /// that a scheduling hiccup can own. Not so many that the run holds a shared
    /// device for minutes.
    const REPEATS: usize = 15;

    /// Sizes the resident staging round trip is measured at, in bytes.
    ///
    /// Spans the buffers a dispatch stages, from the single-word pass counter
    /// upward, and continues past them to the 16 MiB merge table so the sweep
    /// shows the shape of the whole curve rather than only the part being
    /// fitted.
    const STAGE_SIZES: [usize; 6] = [4, 4 << 10, 64 << 10, 1 << 20, 4 << 20, 16 << 20];

    /// Largest measured size admitted into the fitted model, in bytes.
    ///
    /// The model is applied to the buffers a dispatch stages, and the largest of
    /// those is the padded state array, bounded by the whole encode's state word
    /// count times four: under 2 MiB on both shapes measured here, spread across
    /// several dispatches. Points above that bound are outside the range the
    /// model is used in, and admitting them does real damage rather than merely
    /// wasting effort: an ordinary least squares line over an x range spanning
    /// four orders of magnitude is decided almost entirely by its largest point,
    /// which drove the fixed term to zero and left the model under-predicting a
    /// small round trip by its entire cost. Restricting the fit to the range it
    /// is applied over is not curve shopping, it is the condition under which a
    /// linear fit means anything.
    const FIT_MAX_BYTES: usize = 1 << 20;

    /// Repeats per staging size.
    const STAGE_REPEATS: usize = 12;

    pub fn run() -> Result<(), String> {
        let corpora = corpus_dir()?;
        let vocabulary = load_cl100k()?;
        let backend = vyre_driver_cuda::cuda_factory()
            .map_err(|error| format!("CUDA backend acquisition failed: {error}"))?;
        let backend: &dyn VyreBackend = &*backend;
        let tokenizer = DeviceTokenizer::new(&vocabulary, &Encoding::cl100k_base())
            .map_err(|error| format!("device merge table build failed: {error}"))?;

        println!("exatok encode profile");
        println!("  vocabulary   cl100k_base");
        println!("  corpora      {}", corpora.display());
        println!("  loadavg      {}", loadavg());
        // The GPU's own state, not just the CPU's. This box ran a resident
        // 27B inference server on the card for most of the effort that produced
        // this harness, and `/proc/loadavg` reported a quiet machine throughout,
        // because it describes the run queue and knows nothing about who else is
        // holding the device. A capture without this line cannot be told apart
        // from a capture that was time slicing with a neighbour.
        println!("  gpu          {}", gpu_state());
        println!(
            "  table        {} wire bytes",
            tokenizer.table().slot_words().len() * 4
        );

        // Fitted once here on a cold driver pool, purely to record the gap
        // against the warm fits below. `allocate_resident` on a pool that has
        // never served a block of that size is several times more expensive than
        // the call a dispatch makes, and a dispatch always meets a warm pool, so
        // this model is NOT the one any ownership figure uses.
        println!();
        println!("staging model, COLD: no encode has run in this process yet");
        let cold = stage_cost(backend)?;

        let mut splits = Vec::new();
        for shape in SHAPES {
            let path = corpora.join(format!("{shape}.txt"));
            let bytes = std::fs::read(&path).map_err(|error| {
                format!(
                    "cannot read {}: {error}. Fix: point EXATOK_BENCH_CORPUS at a directory a \
                     throughput bench printed as `corpora:`.",
                    path.display()
                )
            })?;
            splits.push(measure_shape(&tokenizer, backend, shape, &bytes)?);
        }

        println!();
        println!(
            "cold round trip {:.2}us, warm round trips per shape above",
            cold.fixed.as_nanos() as f64 / 1000.0
        );
        for split in &splits {
            println!(
                "  {:<17} warm {:.2}us per round trip, cold is {:.1}x that",
                split.shape,
                split.staging.fixed.as_nanos() as f64 / 1000.0,
                cold.fixed.as_nanos() as f64 / split.staging.fixed.as_nanos().max(1) as f64
            );
        }

        report_ownership(&splits);
        Ok(())
    }

    /// The cost of one resident staging round trip, as a fixed part and a
    /// per-byte part.
    ///
    /// Fitted rather than assumed because the two parts answer different
    /// questions: a dispatch stages eight buffers whose sizes span four orders
    /// of magnitude, so whether staging is a per-operation cost or a bandwidth
    /// cost decides whether the count of buffers or their total size is the
    /// thing to look at.
    #[derive(Clone, Copy)]
    struct StageModel {
        /// Cost of a staging round trip carrying no payload.
        fixed: Duration,
        /// Marginal cost of one payload byte, in nanoseconds.
        per_byte: f64,
    }

    impl StageModel {
        /// What `operations` round trips carrying `bytes` in total should cost.
        fn predict(self, operations: u64, bytes: u64) -> Duration {
            let nanos =
                self.fixed.as_nanos() as f64 * operations as f64 + self.per_byte * bytes as f64;
            Duration::from_nanos(nanos.max(0.0) as u64)
        }
    }

    /// One shape's component table, its vyre split, and its instrumentation
    /// overhead.
    fn measure_shape(
        tokenizer: &DeviceTokenizer,
        backend: &dyn VyreBackend,
        shape: &str,
        bytes: &[u8],
    ) -> Result<ShapeSplit, String> {
        let inputs = [bytes];
        for _ in 0..WARMUP {
            encode(tokenizer, backend, &inputs)?;
        }
        let reference = encode(tokenizer, backend, &inputs)?;

        // Plain encodes FIRST and profiled ones second, both warm, so the two
        // medians answer whether the instrumentation costs anything. A profiler
        // that slows what it profiles reports its own overhead as a finding.
        let mut plain: Vec<Duration> = Vec::with_capacity(REPEATS);
        for _ in 0..REPEATS {
            let started = Instant::now();
            let ids = encode(tokenizer, backend, &inputs)?;
            plain.push(started.elapsed());
            same_ids(&reference, &ids, shape)?;
        }

        let mut walls: Vec<Duration> = Vec::with_capacity(REPEATS);
        let mut runs: Vec<EncodeProfile> = Vec::with_capacity(REPEATS);
        for _ in 0..REPEATS {
            let started = Instant::now();
            let (encoded, profile) = tokenizer
                .encode_batch_profiled(backend, &inputs)
                .map_err(|error| format!("{shape}: profiled encode failed: {error}"))?;
            walls.push(started.elapsed());
            same_ids(
                &reference,
                encoded.ids().first().unwrap_or(&Vec::new()),
                shape,
            )?;
            runs.push(profile);
        }

        let last = runs.last().ok_or("no profiled run was taken")?;
        println!();
        println!(
            "shape {shape}: {} bytes, {} pretokens on device, {} on host, {} dispatches, \
             {} state words, {} ids",
            last.input_bytes(),
            last.device_pretokens(),
            last.host_pretokens(),
            last.dispatches(),
            last.state_words(),
            last.output_ids()
        );
        println!("  loadavg      {}   gpu {}", loadavg(), gpu_state());
        println!(
            "  {:<17}{:>11}{:>11}{:>11}{:>9}",
            "component", "median", "min", "max", "share"
        );

        let totals: Vec<Duration> = runs.iter().map(EncodeProfile::total).collect();
        let total_median = median(&totals);
        for phase in EncodePhase::ALL {
            let samples: Vec<Duration> = runs.iter().map(|run| run.phase(phase)).collect();
            row(phase.label(), &samples, total_median);
        }
        let named: Vec<Duration> = runs.iter().map(EncodeProfile::named).collect();
        let remainder: Vec<Duration> = runs.iter().map(EncodeProfile::unattributed).collect();
        println!("  {:-<17}{:->42}", "", "");
        row("NAMED TOTAL", &named, total_median);
        row("UNATTRIBUTED", &remainder, total_median);
        row("observed wall", &totals, total_median);
        println!("  {:-<17}{:->42}", "", "");
        row("caller wall", &walls, total_median);
        row("caller wall plain", &plain, total_median);

        let mut split = vyre_split(tokenizer, backend, shape, &inputs, &runs)?;
        // Fitted HERE, immediately after this shape's encodes and before the
        // next shape runs, so every term in this shape's ownership split comes
        // from the same few hundred milliseconds of box state. A single model
        // fitted once at the end of the run was tried and discarded: it prices
        // the first shape's staging using a machine minutes away from the one
        // that shape actually met, and when a neighbour appeared on the card
        // partway through, it priced staging above the whole region it belongs
        // to. A common-mode contaminant cancels in an in-run comparison and does
        // not cancel across one.
        println!("  staging model, WARM and in-run for this shape");
        split.staging = stage_cost(backend)?;
        Ok(split)
    }

    /// How much of one encode belongs to vyre's host path, the device, and this
    /// crate.
    ///
    /// Taken from the CUDA backend's own cumulative counters, so the numbers
    /// come from vyre's instruments on vyre's code rather than from anything
    /// this crate wrapped around it. The counters are process-wide, which is why
    /// exactly one encode runs between each pair of snapshots and nothing else
    /// touches the device in between.
    ///
    /// Every figure is a median of [`REPEATS`] such encodes, not one. A single
    /// draw was tried first and had to be thrown away: on the `cjk` and `code`
    /// corpora the vyre share it reported swung from 43 percent to 86 percent
    /// across three captures of the identical binary on the identical bytes,
    /// which is not a measurement of anything. The per-encode delta is taken
    /// separately each time rather than one delta divided by the count, so the
    /// spread is visible instead of averaged away.
    fn vyre_split(
        tokenizer: &DeviceTokenizer,
        backend: &dyn VyreBackend,
        shape: &str,
        inputs: &[&[u8]],
        runs: &[EncodeProfile],
    ) -> Result<ShapeSplit, String> {
        let keys = [
            "cuda_timed_wall_ns_total",
            "cuda_timed_enqueue_ns_total",
            "cuda_timed_device_ns_total",
            "cuda_timed_wait_ns_total",
            "cuda_timed_dispatches",
            "cuda_timed_dispatches_missing_device_time",
            "cuda_host_upload_operations",
            "cuda_host_to_device_bytes",
            "cuda_device_readback_operations",
            "cuda_device_to_host_bytes",
            "cuda_kernel_launches",
            "cuda_sync_points",
        ];
        let mut samples: BTreeMap<&str, Vec<u64>> = BTreeMap::new();
        let mut dispatches: Vec<Duration> = Vec::with_capacity(REPEATS);
        let mut table_bytes = 0;
        for _ in 0..REPEATS {
            let before = metrics(backend);
            let (_, profile) = tokenizer
                .encode_batch_profiled(backend, inputs)
                .map_err(|error| format!("{shape}: counted encode failed: {error}"))?;
            let after = metrics(backend);
            for key in keys {
                let step =
                    after.get(key).copied().unwrap_or(0) - before.get(key).copied().unwrap_or(0);
                samples.entry(key).or_default().push(step);
            }
            dispatches.push(profile.phase(EncodePhase::Dispatch));
            table_bytes = profile.table_bytes() as u64;
        }
        let delta = |key: &str| {
            let mut values = samples.get(key).cloned().unwrap_or_default();
            values.sort_unstable();
            values.get(values.len() / 2).copied().unwrap_or(0)
        };

        let dispatch = median(&dispatches);
        let timed_wall = Duration::from_nanos(delta("cuda_timed_wall_ns_total"));
        let enqueue = Duration::from_nanos(delta("cuda_timed_enqueue_ns_total"));
        let device = Duration::from_nanos(delta("cuda_timed_device_ns_total"));
        let wait = Duration::from_nanos(delta("cuda_timed_wait_ns_total"));
        // The spread of the one figure everything downstream divides by, so a
        // reader can see whether the median is describing a stable path or
        // standing in the middle of a swing.
        let window_span = {
            let mut values = samples
                .get("cuda_timed_wall_ns_total")
                .cloned()
                .unwrap_or_default();
            values.sort_unstable();
            (
                Duration::from_nanos(values.first().copied().unwrap_or(0)),
                Duration::from_nanos(values.last().copied().unwrap_or(0)),
            )
        };

        println!("  vyre split of the dispatch phase, from the CUDA backend's own counters");
        println!(
            "    {:<28}{:>11}   median of {REPEATS} counted encodes",
            "dispatch phase",
            millis(dispatch)
        );
        println!(
            "    {:<28}{:>11}   spread {} to {}",
            "vyre timed window",
            millis(timed_wall),
            millis(window_span.0),
            millis(window_span.1)
        );
        println!(
            "    {:<28}{:>11}   vyre host: prepare, PTX lookup, launch",
            "  enqueue",
            millis(enqueue)
        );
        println!(
            "    {:<28}{:>11}   host blocked plus output readback",
            "  wait",
            millis(wait)
        );
        println!(
            "    {:<28}{:>11}   CUDA events: kernel only",
            "  of which device",
            millis(device)
        );
        // Everything the dispatch phase spent that vyre's timed window did not
        // see. It is two things at once, this crate's `gpu_loop` packing and
        // unpacking word arrays and vyre's own resident staging of the buffers
        // that marshalling produces. Splitting it needs the staging model, and
        // the model that applies is the WARM one fitted after every encode has
        // run, so the split is printed in the rollup at the end rather than
        // here.
        let outside = dispatch.saturating_sub(timed_wall);
        // The table's own upload is excluded: `table_upload` measured it
        // directly, so pricing it again from the model would count it twice.
        let table_operations = 1;
        let staged_operations =
            delta("cuda_host_upload_operations").saturating_sub(table_operations);
        let staged_bytes = delta("cuda_host_to_device_bytes").saturating_sub(table_bytes);
        println!(
            "    {:<28}{:>11}   gpu_loop marshalling plus resident staging",
            "outside the timed window",
            millis(outside)
        );
        println!(
            "    {:<28}{:>11}",
            "timed dispatches",
            delta("cuda_timed_dispatches")
        );
        println!(
            "    {:<28}{:>11}",
            "missing device time",
            delta("cuda_timed_dispatches_missing_device_time")
        );
        println!(
            "    {:<28}{:>11}",
            "host upload operations",
            delta("cuda_host_upload_operations")
        );
        println!(
            "    {:<28}{:>11}",
            "host to device bytes",
            delta("cuda_host_to_device_bytes")
        );
        println!(
            "    {:<28}{:>11}",
            "readback operations",
            delta("cuda_device_readback_operations")
        );
        println!(
            "    {:<28}{:>11}",
            "device to host bytes",
            delta("cuda_device_to_host_bytes")
        );
        println!(
            "    {:<28}{:>11}",
            "kernel launches",
            delta("cuda_kernel_launches")
        );
        println!("    {:<28}{:>11}", "sync points", delta("cuda_sync_points"));

        let median_phase = |phase: EncodePhase| {
            let samples: Vec<Duration> = runs.iter().map(|run| run.phase(phase)).collect();
            median(&samples)
        };
        let observed: Vec<Duration> = runs.iter().map(EncodeProfile::total).collect();
        Ok(ShapeSplit {
            shape: shape.to_owned(),
            observed: median(&observed),
            timed_wall,
            device,
            outside,
            staged_operations,
            staged_bytes,
            table: median_phase(EncodePhase::TableUpload) + median_phase(EncodePhase::TableFree),
            // Replaced by the caller with a fit taken immediately after this
            // shape's encodes. Zero here would price staging at nothing, which
            // is why the caller overwrites it rather than defaulting to it.
            staging: StageModel {
                fixed: Duration::ZERO,
                per_byte: 0.0,
            },
        })
    }

    /// One shape's dispatch split, with the staging model fitted beside it.
    struct ShapeSplit {
        shape: String,
        /// Median observed wall of the profiled encodes.
        observed: Duration,
        /// Wall of vyre's own timed dispatch window.
        timed_wall: Duration,
        /// CUDA event time inside that window.
        device: Duration,
        /// Dispatch phase time the timed window did not cover.
        outside: Duration,
        /// Resident staging round trips, excluding the merge table's.
        staged_operations: u64,
        /// Bytes those round trips carried.
        staged_bytes: u64,
        /// Merge table upload plus free, median of the table.
        table: Duration,
        /// Staging round trip cost, fitted in-run right after this shape.
        staging: StageModel,
    }

    /// Charges every millisecond of each encode to vyre, the device, or this
    /// crate.
    ///
    /// The question the ticket actually asks. Built from the median table rather
    /// than from a single counted encode, because one draw of a wall on this box
    /// is not a measurement, and from phase rows rather than from a second set
    /// of timers, so nothing here is a figure the tables above do not contain.
    ///
    /// Charged to vyre: the resident merge table's upload and free, the part of
    /// the timed dispatch window that is not CUDA event time, and the modelled
    /// staging. Charged to the device: CUDA event time. Everything else is this
    /// crate's host code.
    ///
    /// Each shape uses the model fitted in-run beside its own encodes, never one
    /// shared model. Resident allocation is several times cheaper once the
    /// driver's pool is warm, and a shared fit also carries whatever the box was
    /// doing when it was taken into shapes measured minutes away from it.
    fn report_ownership(splits: &[ShapeSplit]) {
        println!();
        println!("ownership of the whole encode, in-run staging model per shape");
        for split in splits {
            let staged = split
                .staging
                .predict(split.staged_operations, split.staged_bytes);
            // A staging estimate larger than the whole region it is a part of
            // means the model does not describe this run, which happens when the
            // box moves under the fit: the sweep and the encodes then meet
            // different machines. Saying so is the only honest output. Clamping
            // to zero would print a gpu_loop share of nothing and a vyre share
            // inflated by the overrun, and it would look like a finding.
            if staged > split.outside {
                println!("  {}", split.shape);
                println!(
                    "    staging model REFUSED: it prices {} staging round trips over {} bytes \
                     at {}, which exceeds the {} the dispatch phase spent outside vyre's timed \
                     window. The fit does not describe this run, so no ownership split is \
                     reported for this shape. Rerun on a quieter box.",
                    split.staged_operations,
                    split.staged_bytes,
                    millis(staged),
                    millis(split.outside)
                );
                continue;
            }
            let gpu_loop = split.outside - staged;
            let vyre = split.timed_wall.saturating_sub(split.device) + staged + split.table;
            let exatok = split
                .observed
                .saturating_sub(vyre)
                .saturating_sub(split.device);
            println!("  {}", split.shape);
            println!(
                "    {:<28}{:>11}{:>8}   table upload and free, dispatch window less device, staging",
                "vyre",
                millis(vyre),
                share(vyre, split.observed)
            );
            println!(
                "    {:<28}{:>11}{:>8}   CUDA event time on the kernel",
                "device",
                millis(split.device),
                share(split.device, split.observed)
            );
            println!(
                "    {:<28}{:>11}{:>8}   this crate, host",
                "exatok",
                millis(exatok),
                share(exatok, split.observed)
            );
            println!(
                "    {:<28}{:>11}{:>8}",
                "observed wall (median)",
                millis(split.observed),
                share(split.observed, split.observed)
            );
            println!(
                "    staging {} round trips over {} bytes priced at {}, leaving {} for gpu_loop",
                split.staged_operations,
                split.staged_bytes,
                millis(staged),
                millis(gpu_loop)
            );
        }
    }

    /// Price one resident staging operation, as a fixed cost plus a per-byte one.
    ///
    /// The six harness buffers and the pick seed are allocated, uploaded and
    /// freed inside every dispatch, by `gpu_loop` calling vyre. That work is
    /// vyre's, and it sits OUTSIDE the timed window the counters above report,
    /// so without this the residue of the dispatch phase could be read either as
    /// vyre's staging or as this crate's marshalling. Timing the identical three
    /// calls at a sweep of sizes says which.
    fn stage_cost(backend: &dyn VyreBackend) -> Result<StageModel, String> {
        println!();
        println!("resident staging: allocate + upload + free, measured at each size");
        println!("  loadavg      {}   gpu {}", loadavg(), gpu_state());
        println!(
            "  {:<14}{:>11}{:>11}{:>11}",
            "bytes", "median", "min", "max"
        );
        let mut fit: Vec<(f64, f64)> = Vec::with_capacity(STAGE_SIZES.len());
        for size in STAGE_SIZES {
            let payload = vec![0_u8; size];
            // One untimed round trip per size, so the first allocation of a size
            // class does not charge this figure for warming a driver pool.
            stage_once(backend, &payload)?;
            let mut samples = Vec::with_capacity(STAGE_REPEATS);
            for _ in 0..STAGE_REPEATS {
                samples.push(stage_once(backend, &payload)?);
            }
            samples.sort_unstable();
            let center = median(&samples);
            // Medians, not means, into the fit. One scheduling hiccup at one
            // size would otherwise tilt a line that every ownership figure below
            // rests on.
            if size <= FIT_MAX_BYTES {
                fit.push((size as f64, center.as_nanos() as f64));
            }
            println!(
                "  {size:<14}{:>11}{:>11}{:>11}",
                micros(center),
                micros(samples.first().copied().unwrap_or_default()),
                micros(samples.last().copied().unwrap_or_default())
            );
        }

        let model = least_squares(&fit);
        println!(
            "  fitted over {} of {} points, up to {FIT_MAX_BYTES} bytes: {:.2}us per round trip \
             + {:.4}us per KiB",
            fit.len(),
            STAGE_SIZES.len(),
            model.fixed.as_nanos() as f64 / 1000.0,
            model.per_byte * 1024.0 / 1000.0
        );
        // The residual at each fitted point, so a reader can see whether the two
        // term model actually describes this path before trusting what it is
        // used for. A staging cost that is neither fixed nor proportional would
        // show up here as a residual comparable to the value.
        for (bytes, measured) in &fit {
            let predicted = model.predict(1, *bytes as u64).as_nanos() as f64;
            println!(
                "    {:<12} measured {:>9.2}us  predicted {:>9.2}us  residual {:>+8.2}us",
                *bytes as u64,
                measured / 1000.0,
                predicted / 1000.0,
                (predicted - measured) / 1000.0
            );
        }
        Ok(model)
    }

    /// Ordinary least squares through `points`, each an `(x, y)` pair.
    ///
    /// Written out rather than pulled in, because this is six points and one
    /// line and a dependency added to an example is a dependency the crate
    /// carries. A degenerate set (every x identical) yields a pure fixed cost
    /// rather than a division by zero.
    fn least_squares(points: &[(f64, f64)]) -> StageModel {
        let count = points.len() as f64;
        if count == 0.0 {
            return StageModel {
                fixed: Duration::ZERO,
                per_byte: 0.0,
            };
        }
        let mean_x = points.iter().map(|point| point.0).sum::<f64>() / count;
        let mean_y = points.iter().map(|point| point.1).sum::<f64>() / count;
        let covariance: f64 = points
            .iter()
            .map(|point| (point.0 - mean_x) * (point.1 - mean_y))
            .sum();
        let variance: f64 = points
            .iter()
            .map(|point| (point.0 - mean_x) * (point.0 - mean_x))
            .sum();
        let per_byte = if variance > 0.0 {
            covariance / variance
        } else {
            0.0
        };
        let fixed = (mean_y - per_byte * mean_x).max(0.0);
        StageModel {
            fixed: Duration::from_nanos(fixed as u64),
            per_byte,
        }
    }

    /// One figure as a percentage of another, for a rollup column.
    fn share(part: Duration, whole: Duration) -> String {
        if whole.is_zero() {
            return "  n/a".to_owned();
        }
        format!("{:.1}%", 100.0 * part.as_secs_f64() / whole.as_secs_f64())
    }

    /// One allocate, upload and free of `payload`, timed end to end.
    fn stage_once(backend: &dyn VyreBackend, payload: &[u8]) -> Result<Duration, String> {
        let started = Instant::now();
        let resource = backend
            .allocate_resident(payload.len())
            .map_err(|error| format!("resident allocation of {} failed: {error}", payload.len()))?;
        let uploaded = backend.upload_resident(&resource, payload);
        let freed = backend.free_resident(resource);
        let elapsed = started.elapsed();
        uploaded.map_err(|error| format!("resident upload failed: {error}"))?;
        freed.map_err(|error| format!("resident free failed: {error}"))?;
        Ok(elapsed)
    }

    /// Encode one batch and hand back the single input's ids.
    fn encode(
        tokenizer: &DeviceTokenizer,
        backend: &dyn VyreBackend,
        inputs: &[&[u8]],
    ) -> Result<Vec<TokenId>, String> {
        let encoded = tokenizer
            .encode_batch(backend, inputs)
            .map_err(|error| format!("encode failed: {error}"))?;
        Ok(encoded.into_ids().pop().unwrap_or_default())
    }

    /// Every timed encode must have computed the same thing, or the figures
    /// above describe two different computations.
    fn same_ids(reference: &[TokenId], ids: &[TokenId], shape: &str) -> Result<(), String> {
        if reference == ids {
            return Ok(());
        }
        Err(format!(
            "{shape}: two encodes of one input returned different ids, {} against {}",
            reference.len(),
            ids.len()
        ))
    }

    /// One table row: median, min, max, and the median's share of the encode.
    fn row(label: &str, samples: &[Duration], total: Duration) {
        let mut sorted = samples.to_vec();
        sorted.sort_unstable();
        let middle = median(&sorted);
        let share = if total.is_zero() {
            0.0
        } else {
            100.0 * middle.as_secs_f64() / total.as_secs_f64()
        };
        println!(
            "  {label:<17}{:>11}{:>11}{:>11}{:>8.1}%",
            millis(middle),
            millis(sorted.first().copied().unwrap_or_default()),
            millis(sorted.last().copied().unwrap_or_default()),
            share
        );
    }

    /// Middle sample, or zero for an empty set.
    fn median(samples: &[Duration]) -> Duration {
        let mut sorted = samples.to_vec();
        sorted.sort_unstable();
        sorted.get(sorted.len() / 2).copied().unwrap_or_default()
    }

    fn millis(value: Duration) -> String {
        format!("{:.4}ms", value.as_secs_f64() * 1e3)
    }

    fn micros(value: Duration) -> String {
        format!("{:.2}us", value.as_secs_f64() * 1e6)
    }

    /// The CUDA backend's cumulative counters, by name.
    fn metrics(backend: &dyn VyreBackend) -> BTreeMap<&'static str, u64> {
        backend.backend_metric_snapshot().into_iter().collect()
    }

    /// Who else is on the card, printed beside every wall this harness reports.
    ///
    /// Necessary because `/proc/loadavg` cannot see it. A resident inference
    /// server held 24 GiB and half the card's duty cycle through most of the
    /// work that produced this harness while loadavg read under 2, and the
    /// captures taken in that window were time slicing with it. Utilization well
    /// above zero here, or used memory far above this process's own, means the
    /// absolutes in the run are contaminated and only in-run ratios survive.
    fn gpu_state() -> String {
        let output = std::process::Command::new("nvidia-smi")
            .args([
                "--query-gpu=utilization.gpu,memory.used,memory.total",
                "--format=csv,noheader,nounits",
            ])
            .output();
        match output {
            Ok(done) if done.status.success() => {
                let text = String::from_utf8_lossy(&done.stdout);
                let fields: Vec<&str> = text.trim().split(',').map(str::trim).collect();
                match fields.as_slice() {
                    [used_pct, used_mib, total_mib] => {
                        format!("{used_pct}% busy, {used_mib} of {total_mib} MiB in use")
                    }
                    _ => text.trim().to_owned(),
                }
            }
            _ => "unavailable (nvidia-smi did not answer)".to_owned(),
        }
    }

    /// The three load figures, printed beside every wall this harness reports.
    fn loadavg() -> String {
        std::fs::read_to_string("/proc/loadavg").map_or_else(
            |_| "unavailable".to_owned(),
            |text| {
                text.split_whitespace()
                    .take(3)
                    .collect::<Vec<_>>()
                    .join(" ")
            },
        )
    }

    /// Directory holding `<shape>.txt`, from the same variable the throughput
    /// benches replay from.
    fn corpus_dir() -> Result<PathBuf, String> {
        std::env::var_os("EXATOK_BENCH_CORPUS")
            .map(PathBuf::from)
            .ok_or_else(|| {
                "EXATOK_BENCH_CORPUS is unset. Fix: set it to a directory a throughput bench \
                 printed as `corpora:`, which holds prose.txt and short_pretokens.txt."
                    .to_owned()
            })
    }

    /// The real `cl100k_base` vocabulary.
    fn load_cl100k() -> Result<Vocabulary, String> {
        match vocab::locate(CL100K_BASE) {
            Artifact::Present(path) => Vocabulary::load_tiktoken_path(&path)
                .map_err(|error| format!("cannot load {}: {error}", path.display())),
            other => Err(format!(
                "cl100k_base rank file is not provisioned: {other:?}. Fix: place it under \
                 $EXATOK_VOCAB_DIR (default ~/.cache/exatok-vocab)."
            )),
        }
    }
}