exatok 0.1.1

Exact GPU-resident BPE tokenizer whose token ids match tiktoken byte for byte
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
//! Every route to an id must produce the same id. OWNER: Main.
//!
//! One concern: the crate has several ways to ask for token ids, and they must be
//! the same answer, not merely similar. The routes are
//!
//! - `exatok::Tokenizer::encode`, the shipping host entry point,
//! - `exatok::tokenizer::encode_bytes`, the free function it wraps,
//! - `exatok::parity::encode_bytes`, the full-rescan oracle,
//! - `exatok::reference::encode`, the per-pretoken reference merge loop,
//! - `exatok::specials::encode_allowed`, the special-token-aware wrapper,
//! - the `exatok` binary's `encode` subcommand.
//!
//! # Why this is a real defect class and not tautology
//!
//! Each route was written at a different time for a different purpose, and the
//! refactor that introduced `Tokenizer` made three of them delegate to a fourth.
//! Delegation is exactly what makes divergence quiet: a route that stops
//! delegating still compiles, still returns plausible ids, and only differs on
//! inputs nobody happened to pin. The parity suite pins the oracle against real
//! tiktoken; nothing pins the other routes against the oracle across the whole
//! corpus, which is what this file does.
//!
//! The corpus is `exatok::parity::corpus()` plus `byte_corpus()`, over all four
//! shipped vocabularies, so the comparison covers astral scalars, unpaired
//! surrogate encodings, invalid UTF-8, contractions, digit-run caps, and
//! whitespace runs rather than the ASCII a hand-written case list would reach.

#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]

use std::process::Command;

use exatok::parity;
use exatok::pretokenize::pretokens;
use exatok::specials::specials_for;
use exatok::vocab::{
    self, Artifact, CL100K_BASE, NamedVocabulary, O200K_BASE, P50K_BASE, R50K_BASE, Vocabulary,
};
use exatok::{Encoding, TokenId, Tokenizer};

/// The four shipped encodings.
const ENCODINGS: [NamedVocabulary; 4] = [CL100K_BASE, R50K_BASE, P50K_BASE, O200K_BASE];

/// A loaded vocabulary with its tokenizer and byte table.
struct Loaded {
    vocab: Vocabulary,
    tokenizer: Tokenizer,
    bytes: [TokenId; 256],
}

/// Set to `1` to declare a machine that deliberately holds no rank artifacts.
///
/// Only an explicit statement counts. Inferring "unprovisioned" from a missing
/// directory is what let a moved cache read as a bare machine and turned nine
/// artifact-gated suites green while they exercised nothing.
const ALLOW_UNPROVISIONED: &str = "EXATOK_ALLOW_UNPROVISIONED";

/// Whether the operator declared this box unprovisioned.
///
/// Anything other than exactly `1` is a provisioned box, so a typo or an empty
/// value fails loudly rather than quietly disabling the comparisons.
fn unprovisioned_is_allowed() -> bool {
    std::env::var_os(ALLOW_UNPROVISIONED).is_some_and(|value| value == "1")
}

/// Load one encoding, or refuse to run unless the operator declared the box bare.
///
/// A directory that exists but lacks the file is an interrupted download and has
/// always panicked here. An ABSENT directory used to skip SILENTLY, and that was
/// the hole: with `EXATOK_VOCAB_DIR` pointed at a path that does not exist this
/// suite printed `7 passed` in 0.00s having invoked not one of the routes it
/// exists to compare, on a result line byte-identical to a full run. Measured,
/// not supposed.
///
/// The trigger was one command away from firing for real: the crate rename to
/// `exatok` moved `ARTIFACT_DIR_DEFAULT` to `.cache/exatok-vocab`, and only a
/// matching `mv` of the cache directory kept this gate honest. Rename the
/// constant, forget the move, and every route-agreement claim in this file
/// reports success while comparing nothing.
///
/// So skipping is now OPT-IN. Absent artifacts fail loudly by default, and a
/// genuinely unprovisioned machine sets [`ALLOW_UNPROVISIONED`] to state that as
/// a deliberate choice rather than having it inferred from a missing path. Same
/// rule and same `Fix:` shape as `tests/device_determinism.rs`, which never
/// tolerated a skip.
fn load(encoding: &NamedVocabulary) -> Option<Loaded> {
    let located = vocab::locate(*encoding);
    let path = match &located {
        Artifact::Present(path) => path.clone(),
        Artifact::Incomplete { .. } => panic!("{}", located.report()),
        Artifact::NoCache { .. } => {
            assert!(
                unprovisioned_is_allowed(),
                "{}\nFix: this suite pins every public route to a token id against the \
                 full-rescan oracle and must not be skipped past. Populate {} or point {} at a \
                 directory holding the rank tables. Set {}=1 only to declare a deliberately \
                 unprovisioned machine, which makes these comparisons skip and report nothing.",
                located.report(),
                vocab::artifact_dir().display(),
                vocab::ARTIFACT_DIR_ENV,
                ALLOW_UNPROVISIONED
            );
            eprintln!("{}", located.report());
            return None;
        }
    };
    let vocab = vocab::load(*encoding, &path).expect("the pinned counts must hold");
    let bytes = exatok::byte_token_ids(encoding.name, &vocab).expect("every byte must resolve");
    let tokenizer = Tokenizer::new(vocab.clone(), Encoding::new(encoding.name, encoding.pattern))
        .unwrap_or_else(|error| panic!("{} must carry every byte: {error}", encoding.name));
    Some(Loaded {
        vocab,
        tokenizer,
        bytes,
    })
}

/// Every text and byte case of the parity corpus, as raw bytes.
///
/// Both corpora, because the text corpus is valid UTF-8 by construction and the
/// byte corpus is where a route that assumes `str` diverges.
fn all_cases() -> Vec<Vec<u8>> {
    let mut cases: Vec<Vec<u8>> = parity::corpus()
        .into_iter()
        .map(|case| case.text.into_bytes())
        .collect();
    cases.extend(parity::byte_corpus().into_iter().map(|case| case.bytes));
    cases
}

/// The four host routes must agree on every corpus case, for every vocabulary.
///
/// Asserted case by case rather than on a digest of the whole corpus, so a
/// failure names the input and both id streams instead of reporting that two
/// hashes differ.
#[test]
fn every_host_route_produces_identical_ids_on_the_whole_corpus() {
    let cases = all_cases();
    let mut compared = 0_usize;
    for encoding in &ENCODINGS {
        let Some(loaded) = load(encoding) else {
            continue;
        };
        let table = loaded.vocab.merge_table();
        for (index, case) in cases.iter().enumerate() {
            let oracle = parity::encode_bytes(encoding.pattern, case, &loaded.bytes, table);
            let shipping = loaded.tokenizer.encode(case);
            let free = exatok::tokenizer::encode_bytes(
                encoding.pattern,
                case,
                &loaded.bytes,
                table,
            );
            assert_eq!(
                shipping, oracle,
                "{} case {index}: Tokenizer::encode diverged from the full-rescan oracle on \
                 {case:?}",
                encoding.name
            );
            assert_eq!(
                free, oracle,
                "{} case {index}: tokenizer::encode_bytes diverged from the full-rescan oracle \
                 on {case:?}",
                encoding.name
            );

            // The per-pretoken reference loop, joined by hand. This is the route
            // that catches a pretoken-boundary defect rather than a merge defect:
            // if the shipping join ever merged across a boundary, the joined
            // per-pretoken answer would be the one that differs.
            let mut joined = Vec::new();
            for span in pretokens(encoding.pattern, case) {
                let pretoken = span
                    .bytes(case)
                    .expect("a span this module produced must lie inside its own input");
                let start: Vec<TokenId> = pretoken
                    .iter()
                    .map(|byte| loaded.bytes[*byte as usize])
                    .collect();
                joined.extend(exatok::reference::encode_pretoken(&start, table));
            }
            assert_eq!(
                joined, oracle,
                "{} case {index}: the per-pretoken reference loop joined by hand diverged, which \
                 means a merge crossed a pretoken boundary on {case:?}",
                encoding.name
            );
            compared += 1;
        }
    }
    assert!(
        compared == 0 || compared >= 200,
        "either no artifacts are present or the whole corpus ran; a partial count means a \
         vocabulary was skipped silently, got {compared}"
    );
    if compared == 0 {
        eprintln!("no vocabulary artifacts present, route agreement not exercised");
    }
}

/// True when `haystack` contains `needle` as a contiguous run of bytes.
///
/// Byte-level and not `str::contains`, because the corpus holds cases that are
/// not valid UTF-8 and a lossy conversion would compare different bytes than the
/// encoder sees.
fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
    !needle.is_empty() && haystack.windows(needle.len()).any(|window| window == needle)
}

/// The special-token wrapper must equal the plain path when no special appears.
///
/// `exatok::specials::encode_allowed` now delegates to
/// `exatok::tokenizer::encode_bytes` for the ordinary spans between specials.
/// On an input with no special token that reduces to one span covering
/// everything, so it must be byte-identical to the plain route. If it ever
/// re-splits, drops a boundary byte, or re-encodes a span twice, this is where it
/// shows, and every specials test that only uses inputs CONTAINING specials would
/// still pass.
#[test]
fn the_specials_wrapper_equals_the_plain_route_when_no_special_is_present() {
    let cases = all_cases();
    let mut compared = 0_usize;
    for encoding in &ENCODINGS {
        let Some(loaded) = load(encoding) else {
            continue;
        };
        let named = specials_for(encoding.name)
            .unwrap_or_else(|| panic!("{} must declare its specials", encoding.name));
        let allowed = named.allowed();
        assert!(
            !allowed.is_empty(),
            "{} declares at least one special, so allowing none would make this test vacuous",
            encoding.name
        );
        for (index, case) in cases.iter().enumerate() {
            // Skip a case that spells a special out, because there the two routes
            // are SUPPOSED to differ: the wrapper emits the special's id and the
            // plain route encodes the same bytes as text. The parity corpus does
            // contain such cases, and asserting equality over them would be
            // asserting that allowing a special does nothing.
            //
            // Checked against the literal spellings rather than a fixed case list,
            // so a corpus that grows another special-bearing case is excluded
            // automatically instead of turning into a false divergence report.
            if allowed
                .iter()
                .any(|(text, _)| contains_subslice(case, text.as_bytes()))
            {
                continue;
            }
            let with_specials = exatok::specials::encode_allowed(
                encoding.pattern,
                case,
                &loaded.bytes,
                loaded.vocab.merge_table(),
                &allowed,
            );
            assert_eq!(
                with_specials,
                loaded.tokenizer.encode(case),
                "{} case {index}: the specials wrapper diverged from the plain route on an input \
                 with no special token: {case:?}",
                encoding.name
            );
            compared += 1;
        }
    }
    if compared == 0 {
        eprintln!("no vocabulary artifacts present, the specials wrapper was not exercised");
    } else {
        // The skip filter above could silently swallow the whole corpus, leaving a
        // test that loops over nothing and passes. Pinned to a floor per loaded
        // vocabulary so a corpus edit that adds special-bearing cases has to be
        // acknowledged here rather than quietly shrinking the comparison.
        let per_vocab = compared / ENCODINGS.len().max(1);
        assert!(
            per_vocab >= 38,
            "only {per_vocab} of the {} corpus cases are special-free, so the skip filter is \
             swallowing the corpus rather than excluding a handful of cases",
            cases.len()
        );
    }
}

/// The wrapper must emit a single id for a declared special, not its bytes.
///
/// The companion to the test above, and the reason that one is not vacuous: if
/// `encode_allowed` degenerated into the plain route, the previous test would
/// still pass on every case while special handling was gone entirely. Here the
/// input CONTAINS `<|endoftext|>`, so the two routes must DIFFER, and the
/// difference must be exactly the special's own id in place of its 13 bytes.
#[test]
fn a_declared_special_becomes_one_id_and_the_plain_route_disagrees() {
    let Some(loaded) = load(&CL100K_BASE) else {
        return;
    };
    let named = specials_for("cl100k_base").expect("cl100k_base declares specials");
    let allowed = named.allowed();
    let endoftext = allowed
        .iter()
        .find(|(text, _)| *text == "<|endoftext|>")
        .map(|(_, id)| *id)
        .expect("cl100k_base declares <|endoftext|>");
    assert_eq!(
        endoftext,
        TokenId(100_257),
        "the cl100k_base end-of-text id is 100257"
    );

    let input = b"a<|endoftext|>b";
    let with_specials = exatok::specials::encode_allowed(
        CL100K_BASE.pattern,
        input,
        &loaded.bytes,
        loaded.vocab.merge_table(),
        &allowed,
    );
    let plain = loaded.tokenizer.encode(input);
    assert_eq!(
        with_specials,
        vec![
            loaded.tokenizer.encode(b"a")[0],
            endoftext,
            loaded.tokenizer.encode(b"b")[0],
        ],
        "the special must collapse to one id between the ids of the surrounding text"
    );
    assert_ne!(
        with_specials, plain,
        "the plain route must encode the special's literal bytes, so the two routes have to \
         differ here; if they agree, special handling has been lost"
    );
    assert!(
        !plain.contains(&endoftext),
        "the plain route must never emit a special id: {plain:?}"
    );
}

/// Encoding then decoding must return the exact input bytes, on every case.
///
/// The round trip is exact for a byte-level BPE with a complete single-byte table
/// even when the input is not valid UTF-8, because ids carry bytes and not
/// characters. A decoder that went through `String` would corrupt the byte corpus
/// while leaving the text corpus perfect, and the text corpus is what a
/// hand-written test uses.
#[test]
fn every_case_round_trips_through_decode_byte_for_byte() {
    let cases = all_cases();
    for encoding in &ENCODINGS {
        let Some(loaded) = load(encoding) else {
            continue;
        };
        for (index, case) in cases.iter().enumerate() {
            let ids = loaded.tokenizer.encode(case);
            let decoded = loaded.tokenizer.decode(&ids).unwrap_or_else(|error| {
                panic!(
                    "{} case {index}: decode rejected ids the encoder produced: {error}",
                    encoding.name
                )
            });
            assert_eq!(
                decoded, *case,
                "{} case {index}: round trip changed the bytes",
                encoding.name
            );
        }
    }
}

/// Decoding must reject an id the vocabulary does not define.
///
/// A decoder that silently substituted a replacement character, or skipped the
/// id, would turn a caller's off-by-one id stream into plausible text. The error
/// must name the id. Asserted on the exact reported id, because an error that
/// names the wrong one is nearly as unhelpful as no error.
#[test]
fn decoding_an_undefined_id_is_an_error_that_names_the_id() {
    let Some(loaded) = load(&CL100K_BASE) else {
        return;
    };
    // One past the largest id any shipped vocabulary defines. cl100k_base tops
    // out at 100276 including specials, so this cannot be a real token.
    let bogus = TokenId(1_000_000);
    let error = loaded
        .tokenizer
        .decode(&[TokenId(9906), bogus, TokenId(1917)])
        .expect_err("an undefined id must not decode");
    let text = error.to_string();
    assert!(
        text.contains("1000000"),
        "the error must name the offending id, got: {text}"
    );
    assert!(
        loaded
            .tokenizer
            .decode(&[TokenId(9906), TokenId(1917)])
            .is_ok(),
        "the surrounding ids are valid, so removing the bogus one must decode"
    );
}

/// The binary must print exactly the ids the library returns.
///
/// The CLI is a separate route to the same answer, and it was just migrated onto
/// `Tokenizer`. A CLI that read a stale helper, applied a different pattern, or
/// reordered output would be invisible to every library test. This drives the
/// built binary and parses its stdout, so the assertion covers argument parsing,
/// vocabulary resolution, pattern selection, and formatting together.
///
/// Skipped only when the artifact cache is absent, never when the binary is
/// missing: `CARGO_BIN_EXE_exatok` is set by cargo for an integration test in
/// this crate, so an absent binary is a build defect and must panic.
#[test]
fn the_binary_prints_exactly_the_library_ids() {
    let Some(loaded) = load(&CL100K_BASE) else {
        return;
    };
    // Inputs passed as --text must be valid UTF-8, so this uses text cases only.
    // The byte corpus reaches the CLI through --file, which tests/cli.rs owns.
    for (index, case) in parity::corpus().into_iter().take(12).enumerate() {
        let output = Command::new(env!("CARGO_BIN_EXE_exatok"))
            .args(["encode", "--vocab", "cl100k_base", "--text", &case.text])
            .output()
            .expect("the binary must be runnable");
        assert!(
            output.status.success(),
            "case {index}: the binary failed: {}",
            String::from_utf8_lossy(&output.stderr)
        );
        let printed: Vec<TokenId> = String::from_utf8(output.stdout)
            .expect("id output must be UTF-8")
            .split_whitespace()
            .map(|field| TokenId(field.parse().expect("each id field must be a u32")))
            .collect();
        assert_eq!(
            printed,
            loaded.tokenizer.encode(case.text.as_bytes()),
            "case {index}: the binary printed different ids than the library returned for {:?}",
            case.text
        );
    }
}

/// A tokenizer built from a path must equal one built from the named vocabulary.
///
/// The CLI accepts `--vocab-path` with an explicit `--pattern` as well as
/// `--vocab`. Those are two different loaders reaching the same artifact, and the
/// path loader is the one a consumer with their own rank file uses. If it ever
/// applied a different byte table or skipped the specials pass, the named route
/// would stay green and every consumer-supplied artifact would silently encode
/// differently.
#[test]
fn the_path_loader_and_the_named_loader_agree() {
    let located = vocab::locate(CL100K_BASE);
    let path = match &located {
        Artifact::Present(path) => path.clone(),
        // An interrupted download is not a bare machine. The single `else` arm
        // this replaces swallowed Incomplete along with NoCache, so a half
        // written rank file skipped the only test that drives `--vocab-path`.
        Artifact::Incomplete { .. } => panic!("{}", located.report()),
        Artifact::NoCache { .. } => {
            assert!(
                unprovisioned_is_allowed(),
                "{}\nFix: the path loader is the route a consumer with their own rank file \
                 takes, and it must not be skipped past. Populate {} or point {} at a directory \
                 holding the rank tables. Set {}=1 only to declare a deliberately unprovisioned \
                 machine, which makes this comparison skip and report nothing.",
                located.report(),
                vocab::artifact_dir().display(),
                vocab::ARTIFACT_DIR_ENV,
                ALLOW_UNPROVISIONED
            );
            eprintln!("{}", located.report());
            return;
        }
    };
    let Some(loaded) = load(&CL100K_BASE) else {
        return;
    };
    let output = Command::new(env!("CARGO_BIN_EXE_exatok"))
        .args([
            "encode",
            "--vocab-path",
            &path.display().to_string(),
            "--pattern",
            "cl100k-base",
            "--text",
            "the quick brown fox jumps over the lazy dog",
        ])
        .output()
        .expect("the binary must be runnable");
    assert!(
        output.status.success(),
        "the path loader failed: {}",
        String::from_utf8_lossy(&output.stderr)
    );
    let printed: Vec<TokenId> = String::from_utf8(output.stdout)
        .expect("id output must be UTF-8")
        .split_whitespace()
        .map(|field| TokenId(field.parse().expect("each id field must be a u32")))
        .collect();
    assert_eq!(
        printed,
        loaded
            .tokenizer
            .encode(b"the quick brown fox jumps over the lazy dog"),
        "the path loader produced different ids than the named loader"
    );
}

/// Public library entry points driven against every loaded vocabulary by the
/// count pin below.
///
/// In order: [`parity::encode_bytes`], `exatok::tokenizer::encode_bytes`,
/// [`Tokenizer::encode`], `exatok::reference::encode_pretoken`,
/// `exatok::specials::encode_allowed`, [`Tokenizer::decode`]. Every one of them
/// is a route this file exists to hold to the same answer, so the count is the
/// shape of the public surface under test and not an incidental loop bound.
const LIBRARY_ENTRY_POINTS: usize = 6;

/// Text cases fed through the binary's `encode` subcommand by the count pin.
///
/// The CLI reparses and re-derives the rank file on every run, so driving all
/// four vocabularies over the whole corpus would cost minutes to pin ids the
/// library routes already pin. Twelve cases through one vocabulary is enough to
/// prove the process route ran at all, which is the property at issue.
const CLI_CASES: usize = 12;

/// Entry-point invocations a provisioned box must make: six library routes over
/// each of the four vocabularies, plus the binary once.
const ENTRY_POINT_INVOCATIONS: usize = LIBRARY_ENTRY_POINTS * ENCODINGS.len() + 1;

/// Ids the pinned entry points emit in total, recomputed by invoking them.
///
/// 32696 = 7841 + 9330 + 8485 + 6874 + 166: the five id-producing library
/// routes over the whole of [`all_cases`], per vocabulary in `ENCODINGS` order,
/// plus the binary over the first [`CLI_CASES`] text cases of `cl100k_base`.
/// Written out so the arithmetic is reviewable rather than recomputed from the
/// same routes it is meant to check. The per-vocabulary figures are five-route
/// sums, not per-route figures: the specials wrapper emits ONE id where the
/// other four spell a special out as ordinary text, so no single per-case
/// total divides them.
const API_SHAPE_IDS_TOTAL: usize = 32696;

/// Bytes [`Tokenizer::decode`] returns in total, recomputed by invoking it.
///
/// 15908 = 4 * 3977: [`all_cases`] is 3977 bytes of input and every vocabulary
/// decodes its own encoding of every case back to exactly those bytes. The
/// equality is re-checked per case in
/// [`every_case_round_trips_through_decode_byte_for_byte`], so this total
/// cannot be reached by decoding to the wrong bytes.
///
/// Pinned separately from the id total because decode is the one entry point in
/// the set that answers in bytes. A decode that started dropping ids, or a
/// corpus that lost cases, changes this and nothing else.
const DECODED_BYTES_TOTAL: usize = 15908;

/// The suite invoked a pinned set of entry points and produced a pinned, NONZERO
/// number of ids, or the box was declared bare.
///
/// This is the durable half of the vacuous-green fix and it outlives the trigger
/// that exposed it. Making an absent artifact directory loud closes the one door
/// that was measured, where this file printed `7 passed` in 0.00s having invoked
/// none of the six routes it compares. It does not make a vacuous pass
/// IMPOSSIBLE: any future skip, filter, early return or corpus that silently
/// loses its cases reintroduces the same silence through a different door, and
/// every one of those still reads as `ok` on the result line.
///
/// A pass count is not evidence. `7 passed` carries no information about whether
/// a single public entry point was called, which is why this asserts a count of
/// invocations and an id total both RECOMPUTED by calling the routes here rather
/// than read back from a fixture. If a route is deleted, stops being driven
/// against one vocabulary, or the corpus shrinks, the numbers move and this
/// fails; the other seven tests would keep passing on whatever was left.
///
/// Zero is admissible only when the operator set [`ALLOW_UNPROVISIONED`], which
/// makes an empty run a declared choice instead of an accident nobody can see.
#[test]
fn the_suite_invokes_a_pinned_set_of_entry_points_over_a_pinned_id_total() {
    let loaded: Vec<(&NamedVocabulary, Loaded)> = ENCODINGS
        .iter()
        .filter_map(|encoding| load(encoding).map(|entry| (encoding, entry)))
        .collect();
    if loaded.is_empty() {
        assert!(
            unprovisioned_is_allowed(),
            "no vocabulary loaded and {ALLOW_UNPROVISIONED} is unset, so this suite invoked \
             zero public entry points while reporting success. Fix: populate {} or set {}=1 to \
             declare a bare machine.",
            vocab::artifact_dir().display(),
            ALLOW_UNPROVISIONED
        );
        return;
    }
    assert_eq!(
        loaded.len(),
        ENCODINGS.len(),
        "a provisioned box carries all four vocabularies, so a partial set means the routes are \
         not being driven against one of them at all"
    );

    let cases = all_cases();
    let mut invocations = 0_usize;
    let mut ids_total = 0_usize;
    let mut decoded_bytes_total = 0_usize;

    for (encoding, entry) in &loaded {
        let table = entry.vocab.merge_table();
        let allowed = specials_for(encoding.name)
            .unwrap_or_else(|| panic!("{} must declare its specials", encoding.name))
            .allowed();

        for case in &cases {
            ids_total += parity::encode_bytes(encoding.pattern, case, &entry.bytes, table).len();
        }
        invocations += 1;

        for case in &cases {
            ids_total +=
                exatok::tokenizer::encode_bytes(encoding.pattern, case, &entry.bytes, table).len();
        }
        invocations += 1;

        for case in &cases {
            ids_total += entry.tokenizer.encode(case).len();
        }
        invocations += 1;

        for case in &cases {
            for span in pretokens(encoding.pattern, case) {
                let pretoken = span
                    .bytes(case)
                    .expect("a span this module produced must lie inside its own input");
                let start: Vec<TokenId> = pretoken
                    .iter()
                    .map(|byte| entry.bytes[*byte as usize])
                    .collect();
                ids_total += exatok::reference::encode_pretoken(&start, table).len();
            }
        }
        invocations += 1;

        for case in &cases {
            ids_total += exatok::specials::encode_allowed(
                encoding.pattern,
                case,
                &entry.bytes,
                table,
                &allowed,
            )
            .len();
        }
        invocations += 1;

        for case in &cases {
            let ids = entry.tokenizer.encode(case);
            decoded_bytes_total += entry
                .tokenizer
                .decode(&ids)
                .unwrap_or_else(|error| {
                    panic!("{}: decode rejected ids its own encoder produced: {error}", encoding.name)
                })
                .len();
        }
        invocations += 1;
    }

    for (index, case) in parity::corpus().into_iter().take(CLI_CASES).enumerate() {
        let output = Command::new(env!("CARGO_BIN_EXE_exatok"))
            .args(["encode", "--vocab", "cl100k_base", "--text", &case.text])
            .output()
            .expect("the binary must be runnable");
        assert!(
            output.status.success(),
            "case {index}: the binary failed: {}",
            String::from_utf8_lossy(&output.stderr)
        );
        ids_total += String::from_utf8(output.stdout)
            .expect("id output must be UTF-8")
            .split_whitespace()
            .count();
    }
    invocations += 1;

    assert_eq!(
        invocations, ENTRY_POINT_INVOCATIONS,
        "the suite drove {invocations} entry-point invocations where \
         {ENTRY_POINT_INVOCATIONS} are pinned, so a public route is no longer being exercised \
         against every vocabulary"
    );
    assert!(
        ids_total > 0,
        "the pinned entry points produced zero ids on a provisioned box, which is the vacuous \
         pass this test exists to catch"
    );
    assert_eq!(
        ids_total, API_SHAPE_IDS_TOTAL,
        "the entry points produced {ids_total} ids where {API_SHAPE_IDS_TOTAL} are pinned, so \
         this suite is driving a different amount of text than it records"
    );
    assert_eq!(
        decoded_bytes_total, DECODED_BYTES_TOTAL,
        "decode returned {decoded_bytes_total} bytes where {DECODED_BYTES_TOTAL} are pinned"
    );
}