exatok 0.1.1

Exact GPU-resident BPE tokenizer whose token ids match tiktoken byte for byte
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
//! Command line interface behavior. OWNER: Main.
//!
//! One concern: what the binary does with its arguments and what it writes to
//! stdout, stderr, and its exit status. Tokenization correctness is proven in
//! the reference and parity suites; this suite proves the CLI reaches that path
//! with the arguments it was given, and reports honestly when it cannot.
//!
//! These tests drive the real binary as a subprocess rather than calling library
//! functions. That is deliberate: the defects this suite exists to catch are
//! argument plumbing, exit codes, and output formatting, none of which are
//! reachable from a library call.
//!
//! The binary's own source is ALSO linked in, as [`binary`], for two things a
//! subprocess cannot supply: the parser, so the expected flag set is derived
//! from `clap` rather than from a hardcoded list that would rot, and the
//! exit-code constants, so the numbers this suite asserts are the numbers the
//! binary uses. The literal values are pinned once, in
//! [`the_exit_code_contract_is_the_documented_numbers`], so importing the
//! constants elsewhere cannot turn an assertion into a tautology.

#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]

use std::collections::BTreeSet;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};
use std::time::{Duration, Instant};

use clap::CommandFactory;
use exatok::vocab::{self, Artifact, CL100K_BASE, NAMED_VOCABULARIES};
use exatok::{TokenId, ExatokError};

/// The binary under test, as its own source.
///
/// Compiled a second time into this test rather than reached through the
/// process, because a subprocess cannot hand back its parser or its constants.
#[path = "../src/main.rs"]
#[allow(dead_code)]
mod binary;

use binary::{
    CliError, EXIT_DEVICE, EXIT_INPUT, EXIT_INTERNAL, EXIT_OUTPUT, EXIT_USAGE, EXIT_VOCABULARY,
};

/// Path to the built binary.
///
/// Derived from the package name rather than spelled out, so a crate rename is
/// one edit in `Cargo.toml` and not a broken test target.
const BIN: &str = env!(concat!("CARGO_BIN_EXE_", env!("CARGO_PKG_NAME")));

/// Run the binary with `args` and an immediately closed stdin.
fn run(args: &[&str]) -> Output {
    run_full(args, &[], &[])
}

/// Run the binary with `stdin` piped in.
fn run_stdin(args: &[&str], stdin: &[u8]) -> Output {
    run_full(args, stdin, &[])
}

/// Run the binary with extra environment variables.
fn run_env(args: &[&str], env: &[(&str, &str)]) -> Output {
    run_full(args, &[], env)
}

/// Run the binary, controlling stdin and the environment.
///
/// stdin is a pipe that is written and then closed, never the harness's own
/// stdin: a test that inherited a terminal would block instead of failing.
fn run_full(args: &[&str], stdin: &[u8], env: &[(&str, &str)]) -> Output {
    let mut child = Command::new(BIN)
        .args(args)
        .envs(env.iter().copied())
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()
        .expect("the binary under test must be runnable");
    // A failing run may exit before reading, so a broken pipe here is not a
    // test failure. Dropping the handle is what signals end of input.
    let _ = child
        .stdin
        .take()
        .expect("stdin was piped")
        .write_all(stdin);
    child
        .wait_with_output()
        .expect("the process must be waitable")
}

/// Stdout as UTF-8 with the trailing newline removed.
fn stdout_of(output: &Output) -> String {
    String::from_utf8(output.stdout.clone())
        .expect("stdout must be UTF-8 for these cases")
        .trim_end()
        .to_owned()
}

fn stderr_of(output: &Output) -> String {
    String::from_utf8(output.stderr.clone()).expect("stderr must be UTF-8")
}

fn exit_code(output: &Output) -> i32 {
    output.status.code().expect("the process must exit normally")
}

/// Whether the rank artifacts are provisioned on this machine.
///
/// Mirrors the parity suite's three-way classification rather than a bare
/// `is_file` check, so a HALF-populated cache fails loudly instead of skipping.
/// A skip that also fires on a broken cache is how a gate stops gating.
fn artifacts_available() -> bool {
    let mut present = 0_usize;
    for vocabulary in NAMED_VOCABULARIES {
        match vocab::locate(vocabulary) {
            Artifact::Present(_) => present += 1,
            Artifact::NoCache { .. } => {}
            // The verdict, not the evidence behind it. This arm covers a
            // zero-length rank file as well as an absent one, so "{path} is
            // missing" would be a confident falsehood about a file the operator
            // can see sitting there, and a wrong fact is worse than a vague one
            // because it makes someone act.
            Artifact::Incomplete { dir, path } => panic!(
                "vocabulary cache is incomplete, which is a broken machine rather than an \
                 unprovisioned one: there is no usable rank file at {}, where a zero-length \
                 file counts as none, in the cache at {}",
                path.display(),
                dir.display()
            ),
        }
    }
    assert!(
        present == 0 || present == NAMED_VOCABULARIES.len(),
        "expected either no artifacts or all {}, found {present}",
        NAMED_VOCABULARIES.len()
    );
    present == NAMED_VOCABULARIES.len()
}

/// A temporary path unique to `label`, removed first if a previous run left it.
fn temp_path(label: &str) -> PathBuf {
    let path = std::env::temp_dir().join(format!("exatok-cli-{label}"));
    if path.is_dir() {
        std::fs::remove_dir_all(&path).ok();
    } else {
        std::fs::remove_file(&path).ok();
    }
    path
}

/// Bytes that are not valid UTF-8, for the byte-oriented paths.
///
/// Deliberately mixed: two lone high bytes, a truncated two-byte sequence, and
/// a stray continuation byte. A decoder that only tripped on one of those
/// shapes would pass a single-byte fixture.
const NOT_UTF8: &[u8] = b"hi\xff\xfe\xc3(\x80hi";

// ---------------------------------------------------------------------------
// The exit-code contract
// ---------------------------------------------------------------------------

/// The exit codes are these numbers, and this is the only place they are typed.
///
/// Every other test names the constants, so this one pins the values: without
/// it, renumbering `EXIT_USAGE` to 9 would leave the whole suite green while
/// breaking every script that branches on 2.
///
/// `EXIT_INTERNAL` has no subprocess case anywhere in this suite, and that is
/// the honest state rather than an omission. It reports a violated invariant of
/// the binary itself, and both of its sites are unreachable by construction: a
/// span outside the input `spans` just partitioned, and a divergence between the
/// device and the host oracle, which would mean the ids were wrong. What CAN be
/// reached is the classification, so it is reached here, with a real
/// `ExatokError` value rather than a fabricated fault.
///
/// `ExatokError::Backend` deliberately does NOT land on `EXIT_INTERNAL`. It did
/// while the CLI ran only the host path, where a dispatch cannot happen; wiring
/// `--device gpu` made it reachable, and a machine refusing the work is the
/// caller's to retry rather than a defect here. That reclassification is the
/// reason this test pins the mapping per variant instead of counting codes.
#[test]
fn the_exit_code_contract_is_the_documented_numbers() {
    assert_eq!(EXIT_USAGE, 2, "usage");
    assert_eq!(EXIT_VOCABULARY, 3, "vocabulary");
    assert_eq!(EXIT_INPUT, 4, "input");
    assert_eq!(EXIT_OUTPUT, 5, "output");
    assert_eq!(EXIT_DEVICE, 6, "device");
    assert_eq!(EXIT_INTERNAL, 70, "internal");

    assert_eq!(CliError::Usage(String::new()).exit_code(), EXIT_USAGE);
    assert_eq!(CliError::Vocabulary(String::new()).exit_code(), EXIT_VOCABULARY);
    assert_eq!(CliError::Input(String::new()).exit_code(), EXIT_INPUT);
    assert_eq!(CliError::Output(String::new()).exit_code(), EXIT_OUTPUT);
    assert_eq!(CliError::Device(String::new()).exit_code(), EXIT_DEVICE);
    assert_eq!(CliError::Internal(String::new()).exit_code(), EXIT_INTERNAL);

    // A backend fault is the device refusing the work. It must be the device
    // code, so a caller can retry on the host, and must NOT be internal: this
    // assertion is what caught the stale classification when --device gpu made
    // the variant reachable.
    let backend = CliError::from(ExatokError::Backend("dispatch refused".to_owned()));
    assert_eq!(backend.exit_code(), EXIT_DEVICE);
    assert!(
        backend.to_string().contains("Fix:"),
        "a device failure must say what to do: {backend}"
    );
    assert!(
        backend.to_string().contains("--device gpu"),
        "a device failure must name the flag whose removal encodes on the host: {backend}"
    );

    // A divergence means the ids would have been wrong, which is the one thing
    // this crate promises never to be, so it stays internal. This is the value
    // that keeps EXIT_INTERNAL's classification exercised now that Backend has
    // moved off it.
    let divergence = CliError::from(ExatokError::Divergence {
        byte_offset: 7,
        host: Some(TokenId(11)),
        device: Some(TokenId(12)),
        fix: "report this input".to_owned(),
    });
    assert_eq!(divergence.exit_code(), EXIT_INTERNAL);
    assert!(
        divergence.to_string().contains("Fix:"),
        "even an internal error must say what to do: {divergence}"
    );

    // An artifact fault is the caller's to fix, so it must NOT be internal.
    let artifact = CliError::from(ExatokError::Artifact {
        artifact: "cl100k_base".to_owned(),
        location: "line 3".to_owned(),
        detail: "rank is not a number".to_owned(),
        fix: "re-download the rank file".to_owned(),
    });
    assert_eq!(artifact.exit_code(), EXIT_VOCABULARY);
}

/// `--help` must document exactly the codes the binary can return.
///
/// Scraped from the rendered help and compared to the constants, so a new
/// failure class with a new code cannot ship undocumented, and a code cannot be
/// documented into the help text without existing.
#[test]
fn help_documents_exactly_the_exit_codes_that_exist() {
    let output = run(&["--help"]);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    let text = String::from_utf8(output.stdout.clone()).expect("help must be UTF-8");

    let mut documented = BTreeSet::new();
    let mut in_table = false;
    for line in text.lines() {
        if line == "Exit codes:" {
            in_table = true;
            continue;
        }
        if !in_table {
            continue;
        }
        let indent = line.len() - line.trim_start().len();
        if !line.is_empty() && indent == 0 {
            break;
        }
        // Table rows are indented two and start with the code. Wrapped
        // continuations are indented further, and the prose below the table
        // starts with a word.
        if indent == 2 {
            if let Ok(code) = line
                .split_whitespace()
                .next()
                .unwrap_or_default()
                .parse::<i32>()
            {
                documented.insert(code);
            }
        }
    }

    let expected = BTreeSet::from([
        0,
        EXIT_USAGE,
        EXIT_VOCABULARY,
        EXIT_INPUT,
        EXIT_OUTPUT,
        EXIT_DEVICE,
        EXIT_INTERNAL,
    ]);
    assert_eq!(
        documented, expected,
        "the exit-code table in --help must match the codes the binary has"
    );
    assert!(
        text.contains("Status 1 is never returned"),
        "the reserved status must stay documented: {text}"
    );
}

// ---------------------------------------------------------------------------
// Help coherence
// ---------------------------------------------------------------------------

/// Long flags the rendered help for `args` shows.
///
/// Reads only the entry line of each option, which is indented two or six, and
/// takes the first long flag on it. Descriptions are indented ten, so a
/// description that mentions another flag, as `--pattern`'s does, cannot be
/// mistaken for one.
fn help_flags(args: &[&str]) -> BTreeSet<String> {
    let output = run(args);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    let text = String::from_utf8(output.stdout.clone()).expect("help must be UTF-8");

    let mut flags = BTreeSet::new();
    let mut in_options = false;
    for line in text.lines() {
        if line == "Options:" {
            in_options = true;
            continue;
        }
        if !in_options {
            continue;
        }
        let trimmed = line.trim_start();
        let indent = line.len() - trimmed.len();
        if !line.is_empty() && indent == 0 {
            break;
        }
        if indent > 6 || !trimmed.starts_with('-') {
            continue;
        }
        if let Some(rest) = trimmed.split("--").nth(1) {
            flags.insert(
                rest.chars()
                    .take_while(|c| c.is_ascii_alphanumeric() || *c == '-')
                    .collect(),
            );
        }
    }
    flags
}

/// Long flags `command`'s parser accepts.
fn parser_flags(command: &clap::Command) -> BTreeSet<String> {
    command
        .get_arguments()
        .filter_map(|arg| arg.get_long().map(str::to_owned))
        .collect()
}

/// Help must list exactly the flags the parser accepts, everywhere.
///
/// The expected set comes from `clap` itself, so the two sides cannot drift: a
/// flag added to the parser and forgotten in the help, or a flag deleted from
/// the parser while its help entry survives, both fail here. A hardcoded list
/// would only prove the list matches itself.
///
/// Both directions have been real defects in this file's history. `--vocab`'s
/// help once enumerated four vocabulary names by hand, and `--pattern`'s help
/// claimed it was required for a `.tiktoken` path when the code required it for
/// a `.json` bundle too.
#[test]
fn help_lists_exactly_the_flags_the_parser_accepts() {
    let mut root = binary::Cli::command();
    root.build();

    assert_eq!(
        help_flags(&["--help"]),
        parser_flags(&root),
        "root command flags"
    );

    let mut checked = 0_usize;
    for sub in root.get_subcommands() {
        // Skip the `help` subcommand clap generates: it has no help flag of its
        // own, takes a positional, and is not part of this binary's surface.
        // Identified by that property rather than by name, so the skip cannot
        // silently swallow one of ours.
        if sub.is_disable_help_flag_set() {
            continue;
        }
        let name = sub.get_name();
        assert_eq!(
            help_flags(&[name, "--help"]),
            parser_flags(sub),
            "flags of subcommand {name}"
        );
        checked += 1;
    }
    assert_eq!(
        checked, 6,
        "every subcommand must be covered; found {checked}"
    );
}

/// `--vocab`'s help must name every vocabulary the binary accepts.
///
/// The help string is built from `NAMED_VOCABULARIES`, and this is what proves it:
/// a fifth vocabulary that did not reach the help text would leave an operator
/// unable to discover it from the tool.
#[test]
fn help_names_every_vocabulary_the_binary_accepts() {
    let output = run(&["encode", "--help"]);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    let text = stdout_of(&output);
    for vocabulary in NAMED_VOCABULARIES {
        assert!(
            text.contains(vocabulary.name),
            "--vocab help must name {}: {text}",
            vocabulary.name
        );
    }
}

/// `--version` must work and name this package's version, exactly.
///
/// The one flag the help lists that no behavioral test would otherwise
/// exercise, so "every flag shown works" would have a hole without it.
///
/// Asserted against the WHOLE stdout rather than `contains`: a substring check
/// passes on any output that happens to carry the version, including an error
/// or a decorated line, while breaking every script that parses the line.
#[test]
fn version_reports_the_package_version() {
    let output = run(&["--version"]);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    assert_eq!(
        stdout_of(&output),
        format!("{} {}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")),
        "the version output must be exactly the package name and version"
    );
}

// ---------------------------------------------------------------------------
// Encoding
// ---------------------------------------------------------------------------

/// Every split pattern must reach the encoder and produce the reference ids.
///
/// These three id sequences are the reference tokenizer's own output for the
/// same input under the three patterns. They differ, which is the point: a CLI
/// that ignored `--vocab` and always loaded one vocabulary, or that resolved the
/// wrong pattern for a name, would produce one of these three for all three
/// cases and would otherwise look entirely healthy.
#[test]
fn encode_produces_the_reference_ids_for_every_pattern() {
    if !artifacts_available() {
        return;
    }
    for (name, expected) in [
        ("cl100k_base", "15339 1917\n"),
        ("r50k_base", "31373 995\n"),
        ("o200k_base", "24912 2375\n"),
    ] {
        let output = run(&["encode", "--vocab", name, "--text", "hello world"]);
        assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
        assert_eq!(output.stdout, expected.as_bytes(), "vocabulary {name}");
    }
}

/// `--format json` must emit a parseable array, not ids with brackets glued on.
///
/// Pins the exact bytes because the formats share a code path: a stray space
/// after the comma would still "look like JSON" in a terminal while breaking
/// every consumer that parses it.
#[test]
fn json_format_is_exactly_a_json_array() {
    if !artifacts_available() {
        return;
    }
    let output = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--text",
        "hello world",
        "--format",
        "json",
    ]);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    assert_eq!(output.stdout, b"[15339,1917]\n");
}

/// `count` must agree with the number of ids `encode` prints.
///
/// The two commands run the same encoder, so this is a dedup check rather than
/// an arithmetic one: if `count` ever grows its own counting path, this fails.
/// The input is chosen to be longer than one token so a hardcoded count cannot
/// pass.
#[test]
fn count_equals_the_number_of_ids_encode_prints() {
    if !artifacts_available() {
        return;
    }
    let text = "The quick brown fox jumps over the lazy dog, 1234567890 times.";
    let encoded = run(&["encode", "--vocab", "cl100k_base", "--text", text]);
    let counted = run(&["count", "--vocab", "cl100k_base", "--text", text]);
    let ids = stdout_of(&encoded);
    let id_count = ids.split_whitespace().count();
    assert!(id_count > 1, "fixture must not be a single token: {ids}");
    assert_eq!(counted.stdout, format!("{id_count}\n").into_bytes());
}

// ---------------------------------------------------------------------------
// Input handling
// ---------------------------------------------------------------------------

/// `--text`, `--file` and stdin must produce the same bytes for the same input.
///
/// Three plumbing paths into one encoder. A test that only used `--text` would
/// pass with a `--file` arm that read the wrong path or a stdin arm that
/// dropped the last buffer.
#[test]
fn the_three_input_sources_produce_identical_output() {
    if !artifacts_available() {
        return;
    }
    let text = "hello world, and a tail long enough to fill more than one token";
    let path = temp_path("three-sources.txt");
    std::fs::write(&path, text).expect("temp file must be writable");

    let by_text = run(&["encode", "--vocab", "cl100k_base", "--text", text]);
    let by_file = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--file",
        path.to_str().unwrap(),
    ]);
    let by_stdin = run_stdin(&["encode", "--vocab", "cl100k_base"], text.as_bytes());

    assert_eq!(exit_code(&by_text), 0, "{}", stderr_of(&by_text));
    assert_eq!(exit_code(&by_file), 0, "{}", stderr_of(&by_file));
    assert_eq!(exit_code(&by_stdin), 0, "{}", stderr_of(&by_stdin));
    assert_eq!(by_file.stdout, by_text.stdout, "--file against --text");
    assert_eq!(by_stdin.stdout, by_text.stdout, "stdin against --text");
    std::fs::remove_file(&path).ok();
}

/// Binary input on stdin must encode, not panic and not be mangled.
///
/// The encoder is defined on bytes, so input that is not valid UTF-8 is
/// ORDINARY input rather than an error. A stdin path that read a `String` would
/// either refuse this or substitute U+FFFD, and the second failure produces
/// plausible ids for the wrong bytes.
///
/// The id literal is a regression pin. What proves the ids are RIGHT is the
/// round trip below, which recovers the original bytes exactly.
#[test]
fn non_utf8_input_on_stdin_encodes_without_a_panic() {
    if !artifacts_available() {
        return;
    }
    let output = run_stdin(&["encode", "--vocab", "cl100k_base"], NOT_UTF8);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    assert_eq!(output.stdout, b"6151 187 186 127 7 222 6151\n");
    assert_eq!(
        stderr_of(&output),
        "",
        "a byte-oriented encoder must not complain about bytes"
    );
}

/// An empty input must produce empty output and exit 0, on every path.
///
/// Zero tokens is a legitimate answer, not a failure, and an empty id line is
/// not an empty output: a consumer reading lines would see one blank record.
/// `count` is the exception on purpose, because a count is always a number.
#[test]
fn empty_input_produces_empty_output_and_exit_zero() {
    if !artifacts_available() {
        return;
    }
    for args in [
        vec!["encode", "--vocab", "cl100k_base"],
        vec!["split", "--vocab", "cl100k_base"],
        vec!["decode", "--vocab", "cl100k_base"],
    ] {
        let output = run_stdin(&args, b"");
        assert_eq!(exit_code(&output), 0, "{args:?}: {}", stderr_of(&output));
        assert_eq!(output.stdout, b"", "{args:?} must write nothing at all");
    }

    let by_flag = run(&["encode", "--vocab", "cl100k_base", "--text", ""]);
    assert_eq!(exit_code(&by_flag), 0, "{}", stderr_of(&by_flag));
    assert_eq!(by_flag.stdout, b"");

    // JSON has to stay parseable, so its empty case is an empty array.
    let json = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--text",
        "",
        "--format",
        "json",
    ]);
    assert_eq!(json.stdout, b"[]\n");

    let counted = run(&["count", "--vocab", "cl100k_base", "--text", ""]);
    assert_eq!(exit_code(&counted), 0, "{}", stderr_of(&counted));
    assert_eq!(counted.stdout, b"0\n");
}

/// A large input must encode in one pass and agree with the file path.
///
/// The binary reads its input whole into memory, which `--help` states, so this
/// guards the consequence that matters: the buffered read must not truncate at a
/// pipe boundary. 256 KiB is many `read` calls, and a stdin path that stopped at
/// the first short read would report a smaller count than `--file`.
#[test]
fn a_large_input_encodes_whole_through_stdin() {
    if !artifacts_available() {
        return;
    }
    let line = "The quick brown fox jumps over the lazy dog 1234567890, don't stop.\n";
    let corpus = line.repeat(4096);
    assert!(corpus.len() > 256 * 1024, "fixture must exceed 256 KiB");
    let path = temp_path("large-input.txt");
    std::fs::write(&path, &corpus).expect("temp file must be writable");

    let by_stdin = run_stdin(&["count", "--vocab", "cl100k_base"], corpus.as_bytes());
    let by_file = run(&[
        "count",
        "--vocab",
        "cl100k_base",
        "--file",
        path.to_str().unwrap(),
    ]);
    assert_eq!(exit_code(&by_stdin), 0, "{}", stderr_of(&by_stdin));
    assert_eq!(by_stdin.stdout, by_file.stdout, "stdin must not truncate");
    let counted: usize = stdout_of(&by_stdin).parse().expect("count must be a number");
    assert!(
        counted > 4096,
        "4096 lines must exceed 4096 tokens, got {counted}"
    );
    std::fs::remove_file(&path).ok();
}

// ---------------------------------------------------------------------------
// Decoding and round trips
// ---------------------------------------------------------------------------

/// Decoding must return the exact bytes, including bytes that are not UTF-8.
///
/// The CLI writes raw bytes rather than a string precisely so this case works. A
/// decoder that round-tripped through `String` would replace the invalid byte
/// with U+FFFD and silently corrupt output, which no text-only test would see.
#[test]
fn decode_returns_exact_bytes_for_invalid_utf8() {
    if !artifacts_available() {
        return;
    }
    let path = temp_path("invalid-utf8.bin");
    std::fs::write(&path, NOT_UTF8).expect("temp file must be writable");

    let encoded = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--file",
        path.to_str().unwrap(),
    ]);
    assert_eq!(exit_code(&encoded), 0, "{}", stderr_of(&encoded));
    let ids = stdout_of(&encoded);

    let decoded = run(&["decode", "--vocab", "cl100k_base", "--ids", &ids]);
    assert_eq!(exit_code(&decoded), 0, "{}", stderr_of(&decoded));
    assert_eq!(
        decoded.stdout, NOT_UTF8,
        "decode must reproduce the original bytes exactly"
    );
    std::fs::remove_file(&path).ok();
}

/// Encode then decode must return the original bytes, for every vocabulary.
///
/// Byte-exact round tripping is the whole reason both directions exist, and it
/// is per-vocabulary because each has its own byte-token ids and its own special
/// table: a reverse lookup that worked for `cl100k_base` and silently produced
/// nothing for `o200k_base` would pass a single-vocabulary test. The fixture is
/// not valid UTF-8 on purpose, because that is the case a text pipeline breaks
/// and a byte pipeline must not.
#[test]
fn encode_then_decode_round_trips_non_utf8_for_every_vocabulary() {
    if !artifacts_available() {
        return;
    }
    let path = temp_path("round-trip.bin");
    std::fs::write(&path, NOT_UTF8).expect("temp file must be writable");

    for vocabulary in NAMED_VOCABULARIES {
        let name = vocabulary.name;
        let encoded = run(&[
            "encode",
            "--vocab",
            name,
            "--file",
            path.to_str().unwrap(),
        ]);
        assert_eq!(exit_code(&encoded), 0, "{name}: {}", stderr_of(&encoded));
        let ids = stdout_of(&encoded);
        assert!(!ids.is_empty(), "{name} must encode to something");

        let decoded = run(&["decode", "--vocab", name, "--ids", &ids]);
        assert_eq!(exit_code(&decoded), 0, "{name}: {}", stderr_of(&decoded));
        assert_eq!(
            decoded.stdout, NOT_UTF8,
            "{name} must round-trip the exact bytes, ids were {ids}"
        );
    }
    std::fs::remove_file(&path).ok();
}

/// A registered special token must decode to its text.
///
/// Specials live outside the ordinary id table, so an ordinary-only reverse
/// lookup reports a perfectly valid `<|endoftext|>` as unknown. That was a real
/// defect in this CLI: `.tiktoken` files contain no specials, so nothing was
/// registered until the named loader did it.
#[test]
fn decode_names_a_registered_special_token() {
    if !artifacts_available() {
        return;
    }
    let output = run(&["decode", "--vocab", "cl100k_base", "--ids", "100257"]);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    assert_eq!(output.stdout, b"<|endoftext|>");
}

/// Id lists must accept commas and whitespace interchangeably.
///
/// Both spellings appear in the wild, and a parser that split on only one would
/// read "15339, 1917" as a single malformed field.
#[test]
fn id_lists_accept_commas_and_whitespace_identically() {
    if !artifacts_available() {
        return;
    }
    for spelling in ["15339,1917", "15339 1917", "15339, 1917", "15339\t1917"] {
        let output = run(&["decode", "--vocab", "cl100k_base", "--ids", spelling]);
        assert_eq!(exit_code(&output), 0, "{spelling}: {}", stderr_of(&output));
        assert_eq!(output.stdout, b"hello world", "spelling {spelling}");
    }
}

/// Ids may also arrive on stdin, and must decode the same way.
#[test]
fn decode_reads_ids_from_stdin_when_the_flag_is_absent() {
    if !artifacts_available() {
        return;
    }
    let output = run_stdin(&["decode", "--vocab", "cl100k_base"], b"15339 1917\n");
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    assert_eq!(output.stdout, b"hello world");
}

// ---------------------------------------------------------------------------
// Reporting
// ---------------------------------------------------------------------------

/// `info` must reproduce each vocabulary's published `n_vocab`.
///
/// This is the check that the CLI's special-token table is CORRECT rather than
/// merely present. `n_vocab` is derived as the exclusive upper bound over
/// ordinary and special ids, so a mistyped special id changes it. All four
/// values below are the reference tokenizer's published `n_vocab`, and note
/// that none equals entries plus specials: `cl100k_base` has 100256 entries and
/// 5 specials yet 100277, because 16 ids between them are reserved and unused.
/// A table of plausible-looking wrong ids fails here.
#[test]
fn info_reproduces_the_published_n_vocab_for_every_vocabulary() {
    if !artifacts_available() {
        return;
    }
    for (name, n_vocab, specials) in [
        ("cl100k_base", "100277", "5"),
        ("r50k_base", "50257", "1"),
        ("p50k_base", "50281", "1"),
        ("o200k_base", "200019", "2"),
    ] {
        let output = run(&["info", "--vocab", name]);
        assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
        let text = stdout_of(&output);
        let field = |key: &str| -> String {
            text.lines()
                .find_map(|line| line.strip_prefix(key))
                .map(|rest| rest.trim().to_owned())
                .unwrap_or_else(|| panic!("{name} info must report {key}: {text}"))
        };
        assert_eq!(field("n_vocab"), n_vocab, "vocabulary {name}");
        assert_eq!(field("specials"), specials, "vocabulary {name}");
    }
}

/// `split` output must reconstruct the input exactly.
///
/// Pretokens partition the input, so concatenating them recovers it byte for
/// byte. This drives that invariant through the CLI, where a formatting bug
/// could drop or duplicate a pretoken without any library test noticing. The
/// fixture includes a contraction, a digit run past the three-digit cap, and a
/// double space, so a wrong pattern changes the line breakdown.
#[test]
fn split_output_reconstructs_the_input() {
    if !artifacts_available() {
        return;
    }
    let text = "don't stop  1234567 now";
    let output = run(&["split", "--vocab", "cl100k_base", "--text", text]);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    let body = String::from_utf8(output.stdout.clone()).unwrap();
    let joined: String = body.lines().collect();
    assert_eq!(joined, text, "pretokens must partition the input");
}

/// The digit-run cap is a real difference between the patterns.
///
/// `cl100k_base` caps a digit run at three, `gpt2` does not cap it at all. This
/// pins that the CLI resolves a NAME to the right pattern, which a test using
/// only one vocabulary cannot see.
#[test]
fn digit_runs_are_capped_for_cl100k_and_uncapped_for_gpt2() {
    if !artifacts_available() {
        return;
    }
    let capped = run(&["split", "--vocab", "cl100k_base", "--text", "1234567"]);
    let uncapped = run(&["split", "--vocab", "r50k_base", "--text", "1234567"]);
    assert_eq!(capped.stdout, b"123\n456\n7\n");
    assert_eq!(uncapped.stdout, b"1234567\n");
}

/// A pretoken holding a newline must be written as it is, spanning lines.
///
/// `split --help` says the line breakdown is not a parseable framing, and this
/// is the case that makes that true: a run of blank lines is ONE pretoken, so
/// the output has more lines than pretokens and no consumer can recover the
/// partition from it. Pinned to exact bytes so the statement in the help text
/// stays a description of what happens rather than an excuse, and so a future
/// escaping scheme has to change this test deliberately.
#[test]
fn a_pretoken_containing_newlines_is_written_verbatim() {
    if !artifacts_available() {
        return;
    }
    let output = run(&["split", "--vocab", "cl100k_base", "--text", "a\n\nb"]);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    // Pretokens are "a", "\n\n", "b": three pretokens, five output lines.
    assert_eq!(output.stdout, b"a\n\n\n\nb\n");
    let body = String::from_utf8(output.stdout.clone()).unwrap();
    assert_eq!(
        body.lines().count(),
        5,
        "the ambiguity is the point: more lines than pretokens"
    );
}

/// A rejected id must leave no output at all, not a truncated stream.
///
/// `decode` validates every id before writing any byte, which `--help` states
/// as a guarantee. Streaming as it went would hand a caller 499 good tokens and
/// then a failure, and a shell redirect would keep the truncated file: the exit
/// code would be right and the file would be silently wrong.
#[test]
fn a_rejected_id_leaves_no_partial_output() {
    if !artifacts_available() {
        return;
    }
    let output = run(&[
        "decode",
        "--vocab",
        "cl100k_base",
        "--ids",
        "15339,1917,999999",
    ]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    assert_eq!(
        output.stdout, b"",
        "the two valid ids must not reach stdout before the third is refused"
    );
    assert!(stderr_of(&output).contains("999999"));
}

/// `parity-status` must name every vocabulary and the directory it searched.
///
/// A status command that does not say WHERE it looked is indistinguishable from
/// one that found nothing, which is the specific failure the parity suite's
/// artifact reporting exists to prevent.
#[test]
fn parity_status_names_every_vocabulary_and_the_search_directory() {
    let output = run(&["parity-status"]);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    let text = stdout_of(&output);
    assert!(
        text.contains(&vocab::artifact_dir().display().to_string()),
        "must name the directory it searched: {text}"
    );
    for vocabulary in NAMED_VOCABULARIES {
        assert!(
            text.contains(vocabulary.name),
            "must list {}: {text}",
            vocabulary.name
        );
    }
}

/// `parity-status` must follow the directory override, not a compiled-in path.
///
/// Asserted on the whole stdout rather than `contains(dir)` plus
/// `contains("absent")`: those two substrings can be satisfied by DIFFERENT
/// rows of the same report, so the substring form passes even if no row
/// reports the overridden directory as absent. The exact form pins the header
/// line and every vocabulary's status row, so nothing unrelated can satisfy it.
#[test]
fn parity_status_reports_the_overridden_directory() {
    let dir = temp_path("status-dir");
    let output = run_env(
        &["parity-status"],
        &[(vocab::ARTIFACT_DIR_ENV, dir.to_str().unwrap())],
    );
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    let mut expected = format!("vocabulary directory: {}\n", dir.display());
    for vocabulary in NAMED_VOCABULARIES {
        expected.push_str(&format!(
            "{:<12} {:<14} absent   no rank files\n",
            vocabulary.name,
            vocabulary.pattern.name()
        ));
    }
    assert_eq!(
        stdout_of(&output),
        expected.trim_end(),
        "every row must report the overridden directory's absence, exactly"
    );
}

// ---------------------------------------------------------------------------
// Usage errors, exit 2
// ---------------------------------------------------------------------------

/// A `.tiktoken` path without `--pattern` must be refused, not guessed.
///
/// A rank file does not record its pretokenizer, and the three patterns produce
/// DIFFERENT ids for the same bytes. Defaulting to one would give a wrong answer
/// that only a parity run would ever catch, so refusing is the correct behavior
/// and this test pins it against a future convenience default.
#[test]
fn a_vocabulary_path_without_a_pattern_is_refused() {
    let output = run(&[
        "encode",
        "--vocab-path",
        "/nonexistent/ranks.tiktoken",
        "--text",
        "hi",
    ]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    let message = stderr_of(&output);
    assert!(message.contains("--pattern"), "{message}");
    assert!(
        message.contains("Fix:"),
        "every usage error must carry an actionable Fix clause: {message}"
    );
    // The missing flag must be settled before the file is opened, or the
    // message would be about a path when the command line was the problem.
    assert!(
        !message.contains("No such file"),
        "the flag is missing regardless of the path: {message}"
    );
}

/// The same path WITH `--pattern` must succeed and agree with the named load.
///
/// Pairs with the test above so the refusal is proven to be about the missing
/// pattern rather than about paths being unsupported.
#[test]
fn a_tiktoken_path_with_a_pattern_matches_the_named_vocabulary() {
    if !artifacts_available() {
        return;
    }
    let Artifact::Present(path) = vocab::locate(CL100K_BASE) else {
        return;
    };
    let by_path = run(&[
        "encode",
        "--vocab-path",
        path.to_str().unwrap(),
        "--pattern",
        "cl100k-base",
        "--text",
        "hello world",
    ]);
    assert_eq!(exit_code(&by_path), 0, "{}", stderr_of(&by_path));
    assert_eq!(by_path.stdout, b"15339 1917\n");
}

/// `--pattern` with a named vocabulary must be refused rather than ignored.
///
/// A name already states its pattern. Silently ignoring a contradicting
/// `--pattern` would let someone believe they had overridden it, and silently
/// honoring it would let a name mean two different things.
#[test]
fn a_pattern_with_a_named_vocabulary_is_refused() {
    let output = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--pattern",
        "gpt2",
        "--text",
        "hi",
    ]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    assert!(stderr_of(&output).contains("--pattern"));
}

/// An unknown vocabulary name must list the names that would work.
///
/// An error naming only what was wrong makes the reader go looking; listing the
/// four accepted names makes it self-correcting. A name is a command line value,
/// so it is a usage error and not a vocabulary one: nothing was searched for.
#[test]
fn an_unknown_vocabulary_name_lists_the_accepted_names() {
    let output = run(&["encode", "--vocab", "gpt5_base", "--text", "hi"]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    let message = stderr_of(&output);
    assert!(message.contains("gpt5_base"), "{message}");
    for vocabulary in NAMED_VOCABULARIES {
        assert!(
            message.contains(vocabulary.name),
            "must list {}: {message}",
            vocabulary.name
        );
    }
}

/// Two input sources at once must be refused rather than one silently winning.
#[test]
fn text_and_file_together_are_refused() {
    let output = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--text",
        "hi",
        "--file",
        "/etc/hostname",
    ]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    let message = stderr_of(&output);
    assert!(message.contains("mutually exclusive"), "{message}");
    assert!(message.contains("/etc/hostname"), "{message}");
}

/// Two vocabulary sources at once must be refused the same way.
#[test]
fn vocab_and_vocab_path_together_are_refused() {
    let output = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--vocab-path",
        "/nonexistent/ranks.tiktoken",
        "--pattern",
        "gpt2",
        "--text",
        "hi",
    ]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    assert!(stderr_of(&output).contains("mutually exclusive"));
}

/// A path whose extension names no known format must be refused.
#[test]
fn an_unrecognized_vocabulary_extension_is_refused() {
    let output = run(&[
        "encode",
        "--vocab-path",
        "/tmp/vocab.txt",
        "--pattern",
        "gpt2",
        "--text",
        "hi",
    ]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    assert!(stderr_of(&output).contains(".tiktoken"));
}

/// No vocabulary at all must be refused with the ways to supply one.
#[test]
fn a_missing_vocabulary_argument_is_refused() {
    let output = run(&["encode", "--text", "hi"]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    let message = stderr_of(&output);
    assert!(message.contains("--vocab"), "{message}");
    assert!(message.contains("--vocab-path"), "{message}");
}

/// A non-numeric id in `--ids` must be refused as a usage error, quoted.
///
/// The value came from the command line, so it is the invocation that is wrong
/// and 2 is the code: the caller fixes it by editing the command. The same text
/// arriving on stdin is data and exits 4, which
/// [`a_bad_id_on_stdin_is_an_input_error`] pins.
#[test]
fn a_non_numeric_id_argument_is_refused_and_quoted() {
    if !artifacts_available() {
        return;
    }
    let output = run(&["decode", "--vocab", "cl100k_base", "--ids", "15339,abc"]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    let message = stderr_of(&output);
    assert!(message.contains("abc"), "{message}");
    assert!(message.contains("Fix:"), "{message}");
}

/// An id above the vocabulary must be refused, and the ceiling reported must be
/// the DECODABLE one.
///
/// The first version of this error reported `max_token_id`, which excludes
/// specials, so it told the user the highest id was 100255 immediately after
/// successfully decoding 100276. An error message that contradicts the tool's
/// own behavior sends the reader hunting the wrong problem.
#[test]
fn decode_rejects_an_out_of_range_id_and_reports_the_decodable_ceiling() {
    if !artifacts_available() {
        return;
    }
    let output = run(&["decode", "--vocab", "cl100k_base", "--ids", "999999"]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    let message = stderr_of(&output);
    assert!(message.contains("999999"), "{message}");
    assert!(
        message.contains("100276"),
        "must report the highest DECODABLE id, specials included: {message}"
    );
}

/// An unknown flag must be refused by the parser and still carry a Fix clause.
///
/// clap writes this message, not the binary, so without the appended clause the
/// "every failure says what to do" contract would hold everywhere except the
/// most common mistake there is.
#[test]
fn an_unknown_flag_is_refused_with_a_fix_clause() {
    let output = run(&["encode", "--vocab", "cl100k_base", "--txt", "hi"]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    let message = stderr_of(&output);
    assert!(message.contains("--txt"), "{message}");
    assert!(message.contains("Fix:"), "{message}");
}

/// An invalid enum value must name the value and the accepted set.
#[test]
fn an_invalid_format_value_names_the_alternatives() {
    let output = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--format",
        "yaml",
        "--text",
        "hi",
    ]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    let message = stderr_of(&output);
    assert!(message.contains("yaml"), "{message}");
    assert!(message.contains("ids"), "{message}");
    assert!(message.contains("json"), "{message}");
}

/// No subcommand at all must be refused, not treated as a default.
#[test]
fn a_bare_invocation_is_refused() {
    let output = run(&[]);
    assert_eq!(exit_code(&output), EXIT_USAGE);
    assert!(stderr_of(&output).contains("Fix:"));
}

// ---------------------------------------------------------------------------
// Vocabulary errors, exit 3
// ---------------------------------------------------------------------------

/// An unprovisioned vocabulary must exit 3 and say where it looked.
///
/// Distinct from a usage error on purpose: the command was well formed and the
/// name was one this binary knows, so a script retrying with different flags is
/// wasting its time. It needs to fetch an artifact instead.
///
/// Three shapes, two verdicts, because the verdicts call for different actions.
/// A missing directory and a directory holding no rank table are both
/// UNPROVISIONED: nothing was ever fetched there, and the remedy is to
/// provision. A directory holding another vocabulary's rank table is
/// INCOMPLETE: that machine IS provisioned and this one artifact is not usable,
/// so the remedy is to re-fetch a single file. The empty directory used to
/// report incomplete, and this test asserted that wording, so it was pinning
/// the misrouting rather than the contract: it sent an operator to repair a
/// machine that was merely fresh. The third arm exists because the second one
/// believed it was covering the incomplete verdict and never did. Until a
/// sibling rank file is on disk, that verdict is unreachable from here.
#[test]
fn an_unprovisioned_vocabulary_exits_three() {
    let missing = temp_path("no-such-cache");
    let absent = run_env(
        &["encode", "--vocab", "cl100k_base", "--text", "hi"],
        &[(vocab::ARTIFACT_DIR_ENV, missing.to_str().unwrap())],
    );
    assert_eq!(exit_code(&absent), EXIT_VOCABULARY, "{}", stderr_of(&absent));
    let message = stderr_of(&absent);
    assert!(message.contains("cl100k_base"), "{message}");
    assert!(message.contains(missing.to_str().unwrap()), "{message}");
    assert!(message.contains(vocab::ARTIFACT_DIR_ENV), "{message}");
    assert!(message.contains("Fix:"), "{message}");
    // The parity gate's own wording says SKIPPED, which is true of a gate and
    // false of a CLI that just refused to run.
    assert!(
        !message.contains("SKIPPED"),
        "a failed command did not skip anything: {message}"
    );

    let empty = temp_path("empty-cache");
    std::fs::create_dir_all(&empty).expect("temp dir must be creatable");
    let nothing_in_it = run_env(
        &["encode", "--vocab", "cl100k_base", "--text", "hi"],
        &[(vocab::ARTIFACT_DIR_ENV, empty.to_str().unwrap())],
    );
    assert_eq!(exit_code(&nothing_in_it), EXIT_VOCABULARY);
    let message = stderr_of(&nothing_in_it);
    assert!(message.contains("is not provisioned"), "{message}");
    assert!(
        !message.contains("incomplete"),
        "a directory holding no rank table was never provisioned, so there is \
         nothing to repair: {message}"
    );
    assert!(message.contains("Fix:"), "{message}");
    std::fs::remove_dir_all(&empty).ok();

    let partial = temp_path("partial-cache");
    std::fs::create_dir_all(&partial).expect("temp dir must be creatable");
    // One OTHER vocabulary's rank file is what makes this cache populated, and
    // it is the only thing that separates this case from the one above.
    std::fs::write(partial.join("r50k_base.tiktoken"), b"YQ== 0\n")
        .expect("a sibling artifact must be writable");
    let half = run_env(
        &["encode", "--vocab", "cl100k_base", "--text", "hi"],
        &[(vocab::ARTIFACT_DIR_ENV, partial.to_str().unwrap())],
    );
    assert_eq!(exit_code(&half), EXIT_VOCABULARY);
    let message = stderr_of(&half);
    assert!(message.contains("incomplete"), "{message}");
    assert!(
        !message.contains("is not provisioned"),
        "a populated cache is repaired, not provisioned: {message}"
    );
    assert!(
        message.contains(
            partial
                .join("cl100k_base.tiktoken")
                .to_str()
                .expect("the temporary directory must be UTF-8")
        ),
        "the one file to re-fetch must be named: {message}"
    );
    assert!(message.contains("Fix:"), "{message}");
    std::fs::remove_dir_all(&partial).ok();
}

/// An unreadable artifact path must exit 3 and name the path and the reason.
///
/// Three ways a path can fail to be an artifact, all of them exit 3 because all
/// of them are fixed by producing a readable file. A bare
/// "No such file or directory" would be a defect here: the operator needs to
/// know WHICH path, and the loader's own message called an absent file
/// "malformed", a claim about contents it never read.
#[test]
fn an_unreadable_vocabulary_artifact_exits_three() {
    let missing = temp_path("absent-ranks.tiktoken");
    let output = run(&[
        "encode",
        "--vocab-path",
        missing.to_str().unwrap(),
        "--pattern",
        "gpt2",
        "--text",
        "hi",
    ]);
    assert_eq!(exit_code(&output), EXIT_VOCABULARY, "{}", stderr_of(&output));
    let message = stderr_of(&output);
    assert!(message.contains(missing.to_str().unwrap()), "{message}");
    assert!(message.contains("Fix:"), "{message}");
    assert!(
        !message.contains("malformed"),
        "a file that is not there is not malformed: {message}"
    );

    let directory = temp_path("directory-ranks.tiktoken");
    std::fs::create_dir_all(&directory).expect("temp dir must be creatable");
    let output = run(&[
        "encode",
        "--vocab-path",
        directory.to_str().unwrap(),
        "--pattern",
        "gpt2",
        "--text",
        "hi",
    ]);
    assert_eq!(exit_code(&output), EXIT_VOCABULARY);
    assert!(stderr_of(&output).contains("directory"), "{}", stderr_of(&output));
    std::fs::remove_dir_all(&directory).ok();

    // A file present but not readable. Skipped rather than asserted when the
    // mode does not deny this user, because root reads it regardless and a test
    // that passes only for non-root is a test that lies about what it proved.
    let forbidden = temp_path("forbidden-ranks.tiktoken");
    std::fs::write(&forbidden, "IQ== 0\n").expect("temp file must be writable");
    if set_unreadable(&forbidden) {
        let output = run(&[
            "encode",
            "--vocab-path",
            forbidden.to_str().unwrap(),
            "--pattern",
            "gpt2",
            "--text",
            "hi",
        ]);
        assert_eq!(exit_code(&output), EXIT_VOCABULARY);
        let message = stderr_of(&output);
        assert!(message.contains(forbidden.to_str().unwrap()), "{message}");
        assert!(message.contains("Fix:"), "{message}");
    }
    std::fs::remove_file(&forbidden).ok();
}

/// Make `path` unreadable, reporting whether it actually became unreadable.
fn set_unreadable(path: &Path) -> bool {
    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt;
        std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o000)).ok();
        return std::fs::read(path).is_err();
    }
    #[cfg(not(unix))]
    {
        let _ = path;
        false
    }
}

// ---------------------------------------------------------------------------
// Input errors, exit 4
// ---------------------------------------------------------------------------

/// An unreadable input file must exit 4 and name the path.
///
/// Deliberately NOT 2. The command line is well formed and the flag is spelled
/// correctly; what failed is reading the data it pointed at, which is the same
/// class of thing as a broken pipe on stdin and a different class from a
/// misspelled flag. A caller retrying with different arguments gets nowhere,
/// while a caller who fixes the file succeeds, and the exit code is what tells
/// those two apart.
#[test]
fn an_unreadable_input_file_exits_four_and_names_the_path() {
    if !artifacts_available() {
        return;
    }
    let missing = temp_path("absent-input.txt");
    let output = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--file",
        missing.to_str().unwrap(),
    ]);
    assert_eq!(exit_code(&output), EXIT_INPUT, "{}", stderr_of(&output));
    let message = stderr_of(&output);
    assert!(message.contains(missing.to_str().unwrap()), "{message}");
    assert!(message.contains("Fix:"), "{message}");
}

/// A bad id arriving on stdin must exit 4, where the same id in `--ids` exits 2.
///
/// The pair is the contract: the code says WHERE to go look. An argument is
/// fixed by editing the command, a stream is fixed by fixing its producer, and
/// collapsing both to one code would tell a script nothing it could act on.
#[test]
fn a_bad_id_on_stdin_is_an_input_error() {
    if !artifacts_available() {
        return;
    }
    let piped = run_stdin(&["decode", "--vocab", "cl100k_base"], b"15339 abc\n");
    assert_eq!(exit_code(&piped), EXIT_INPUT, "{}", stderr_of(&piped));
    assert!(stderr_of(&piped).contains("abc"), "{}", stderr_of(&piped));

    let argument = run(&["decode", "--vocab", "cl100k_base", "--ids", "15339 abc"]);
    assert_eq!(exit_code(&argument), EXIT_USAGE);
    assert_eq!(
        stderr_of(&piped),
        stderr_of(&argument),
        "the same fault must read the same either way; only the code differs"
    );
}

/// An id stream that is not text must exit 4, naming the offending byte.
///
/// `decode` WRITES raw bytes, so piping raw bytes back into it is a plausible
/// mistake. `read_to_string` would report "stream did not contain valid UTF-8",
/// which names neither the offset nor the byte.
#[test]
fn a_non_text_id_stream_exits_four_and_names_the_byte() {
    if !artifacts_available() {
        return;
    }
    let output = run_stdin(&["decode", "--vocab", "cl100k_base"], NOT_UTF8);
    assert_eq!(exit_code(&output), EXIT_INPUT, "{}", stderr_of(&output));
    let message = stderr_of(&output);
    assert!(message.contains("0xff"), "must name the byte: {message}");
    assert!(message.contains("byte 2"), "must name the offset: {message}");
    assert!(message.contains("Fix:"), "{message}");
}

// ---------------------------------------------------------------------------
// Output errors, exit 5
// ---------------------------------------------------------------------------

/// Output that cannot be written must exit 5, never 0.
///
/// `println!` PANICS on a write failure, so the version of this binary that used
/// it exited 101 with a panic message for a full disk or a closed pipe. Worse,
/// the raw-byte paths wrote through a buffer that Rust flushes at exit and
/// whose error it discards, so a truncated `decode` could exit 0. Every
/// subcommand is covered because each one had its own writer.
#[test]
fn output_that_cannot_be_written_exits_five() {
    if !artifacts_available() {
        return;
    }
    // /dev/full accepts writes and fails every one with ENOSPC, which makes
    // this deterministic where a closed pipe would race the child's startup.
    if std::fs::OpenOptions::new()
        .write(true)
        .open("/dev/full")
        .is_err()
    {
        return;
    }
    for args in [
        vec!["encode", "--vocab", "cl100k_base", "--text", "hi"],
        vec!["count", "--vocab", "cl100k_base", "--text", "hi"],
        vec!["decode", "--vocab", "cl100k_base", "--ids", "15339"],
        vec!["split", "--vocab", "cl100k_base", "--text", "hi"],
        vec!["info", "--vocab", "cl100k_base"],
        vec!["parity-status"],
        vec!["--help"],
    ] {
        let full = std::fs::OpenOptions::new()
            .write(true)
            .open("/dev/full")
            .expect("checked above");
        let status = Command::new(BIN)
            .args(&args)
            .stdin(Stdio::null())
            .stdout(Stdio::from(full))
            .stderr(Stdio::piped())
            .spawn()
            .expect("the binary under test must be runnable")
            .wait_with_output()
            .expect("the process must be waitable");
        assert_eq!(
            status.status.code(),
            Some(EXIT_OUTPUT),
            "{args:?} must report a failed write: {}",
            stderr_of(&status)
        );
    }
}

// ---------------------------------------------------------------------------
// Cross-cutting contracts
// ---------------------------------------------------------------------------

/// Every failure must name what offended it and say what to do about it.
///
/// One table over every failure class, because the contract is uniform: a
/// message with no `Fix:` clause leaves the reader to guess, and a message that
/// does not quote the offending value leaves them to hunt. Checked together so a
/// new failure path cannot land with neither.
#[test]
fn every_failure_names_the_offender_and_a_fix() {
    let missing_cache = temp_path("fix-clause-cache");
    let cases: [(&[&str], &[(&str, &str)], &str); 8] = [
        (&["encode", "--text", "hi"], &[], "--vocab"),
        (
            &["encode", "--vocab", "gpt5_base", "--text", "hi"],
            &[],
            "gpt5_base",
        ),
        (
            &["encode", "--vocab", "cl100k_base", "--txt", "hi"],
            &[],
            "--txt",
        ),
        (
            &["encode", "--vocab-path", "/tmp/v.txt", "--pattern", "gpt2", "--text", "hi"],
            &[],
            "/tmp/v.txt",
        ),
        (
            &["encode", "--vocab-path", "/tmp/v.tiktoken", "--text", "hi"],
            &[],
            "--pattern",
        ),
        (
            &["encode", "--vocab", "cl100k_base", "--text", "hi"],
            &[(vocab::ARTIFACT_DIR_ENV, "")],
            "cl100k_base",
        ),
        (
            &["decode", "--vocab", "cl100k_base", "--ids", "zz"],
            &[],
            "zz",
        ),
        (
            &["encode", "--vocab", "cl100k_base", "--format", "yaml", "--text", "hi"],
            &[],
            "yaml",
        ),
    ];
    for (args, env, offender) in cases {
        let env: Vec<(&str, &str)> = env
            .iter()
            .map(|(key, value)| {
                if value.is_empty() {
                    (*key, missing_cache.to_str().unwrap())
                } else {
                    (*key, *value)
                }
            })
            .collect();
        let output = run_env(args, &env);
        let code = exit_code(&output);
        let message = stderr_of(&output);
        assert_ne!(code, 0, "{args:?} must fail");
        assert_ne!(code, 1, "{args:?} must not return the reserved status 1");
        assert!(
            message.contains(offender),
            "{args:?} must quote {offender}: {message}"
        );
        assert!(
            message.contains("Fix:"),
            "{args:?} must carry a Fix clause: {message}"
        );
    }
}

/// The failure a caller sees first must be the one they can act on first.
///
/// With three things wrong at once the code must name the earliest: a wrong
/// command line before an absent artifact, and an absent artifact before an
/// unreadable input. Without this order a script gets sent to fetch a
/// vocabulary because of a typo in a flag.
#[test]
fn usage_is_reported_before_vocabulary_and_vocabulary_before_input() {
    let missing_cache = temp_path("precedence-cache");
    let cache = [(vocab::ARTIFACT_DIR_ENV, missing_cache.to_str().unwrap())];

    let usage_first = run_env(
        &["encode", "--vocab", "gpt5_base", "--file", "/nonexistent"],
        &cache,
    );
    assert_eq!(
        exit_code(&usage_first),
        EXIT_USAGE,
        "{}",
        stderr_of(&usage_first)
    );

    let vocabulary_next = run_env(
        &["encode", "--vocab", "cl100k_base", "--file", "/nonexistent"],
        &cache,
    );
    assert_eq!(
        exit_code(&vocabulary_next),
        EXIT_VOCABULARY,
        "{}",
        stderr_of(&vocabulary_next)
    );

    if artifacts_available() {
        let input_last = run(&[
            "encode",
            "--vocab",
            "cl100k_base",
            "--file",
            "/nonexistent",
        ]);
        assert_eq!(
            exit_code(&input_last),
            EXIT_INPUT,
            "{}",
            stderr_of(&input_last)
        );
    }
}

/// A failure that needs no input must not wait for one.
///
/// The original code read the input BEFORE validating anything, so
/// `encode --vocab typo` on a terminal sat waiting for end of input and only
/// then reported a typo it already knew about. Polling with a deadline rather
/// than waiting, because a test that hangs to prove a hang never reports it.
#[test]
fn a_failure_that_needs_no_input_does_not_wait_on_stdin() {
    let missing_cache = temp_path("no-wait-cache");
    let cases: [(&[&str], &[(&str, &str)], i32); 3] = [
        (&["encode", "--vocab", "gpt5_base"], &[], EXIT_USAGE),
        (&["count", "--text", "hi", "--file", "/etc/hostname"], &[], EXIT_USAGE),
        (
            &["encode", "--vocab", "cl100k_base"],
            &[(vocab::ARTIFACT_DIR_ENV, "")],
            EXIT_VOCABULARY,
        ),
    ];
    for (args, env, expected) in cases {
        let env: Vec<(&str, &str)> = env
            .iter()
            .map(|(key, _)| (*key, missing_cache.to_str().unwrap()))
            .collect();
        let mut child = Command::new(BIN)
            .args(args)
            .envs(env)
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .spawn()
            .expect("the binary under test must be runnable");
        // stdin stays open and empty, the way an idle terminal leaves it.
        let deadline = Instant::now() + Duration::from_secs(10);
        loop {
            match child.try_wait().expect("the child must be waitable") {
                Some(status) => {
                    assert_eq!(status.code(), Some(expected), "{args:?}");
                    break;
                }
                None if Instant::now() >= deadline => {
                    child.kill().ok();
                    child.wait().ok();
                    panic!("{args:?} waited on stdin for a failure it already knew about");
                }
                None => std::thread::sleep(Duration::from_millis(10)),
            }
        }
    }
}

/// The reserved status must never appear, for any failure.
///
/// The binary keeps 1 unused so a shell test can tell "ran and said no" from
/// "could not run". This sweeps several unrelated failures at once, because a
/// single-case check would not notice one path drifting.
#[test]
fn no_failure_returns_the_reserved_status_one() {
    let known = BTreeSet::from([
        EXIT_USAGE,
        EXIT_VOCABULARY,
        EXIT_INPUT,
        EXIT_OUTPUT,
        EXIT_INTERNAL,
    ]);
    let failures: [&[&str]; 6] = [
        &[],
        &["encode", "--text", "hi"],
        &["encode", "--vocab", "nope", "--text", "hi"],
        &["decode", "--vocab", "cl100k_base", "--ids", "xyz"],
        &["encode", "--vocab-path", "/tmp/v.txt", "--pattern", "gpt2", "--text", "hi"],
        &["frobnicate"],
    ];
    for args in failures {
        let output = run(args);
        let code = exit_code(&output);
        assert_ne!(code, 1, "{args:?} must not return the reserved status 1");
        assert!(
            known.contains(&code),
            "{args:?} returned undocumented status {code}: {}",
            stderr_of(&output)
        );
    }
}

/// A successful run must write nothing to stderr.
///
/// Progress chatter on stderr is how a tool becomes unusable in a pipeline that
/// treats any stderr output as a failure signal.
#[test]
fn a_successful_run_is_silent_on_stderr() {
    if !artifacts_available() {
        return;
    }
    for args in [
        vec!["encode", "--vocab", "cl100k_base", "--text", "hi"],
        vec!["count", "--vocab", "cl100k_base", "--text", "hi"],
        vec!["decode", "--vocab", "cl100k_base", "--ids", "15339"],
        vec!["split", "--vocab", "cl100k_base", "--text", "hi"],
        vec!["info", "--vocab", "cl100k_base"],
        vec!["parity-status"],
    ] {
        let output = run(&args);
        assert_eq!(exit_code(&output), 0, "{args:?}");
        assert_eq!(stderr_of(&output), "", "{args:?} must be silent on success");
    }
}

/// The ids `encode` prints must be the ids `TokenId` carries.
///
/// Formatting is the one place the CLI can disagree with the library without
/// either being wrong on its own: a printer that emitted the debug form would
/// produce `TokenId(15339)` and no round trip would survive it.
#[test]
fn printed_ids_are_bare_decimal_numbers() {
    if !artifacts_available() {
        return;
    }
    let output = run(&["encode", "--vocab", "cl100k_base", "--text", "hello"]);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    let printed = stdout_of(&output);
    let parsed: Vec<TokenId> = printed
        .split_whitespace()
        .map(|field| TokenId(field.parse().expect("every field must be a bare number")))
        .collect();
    assert_eq!(parsed, vec![TokenId(15339)]);
}

/// `--device` must default to the host, and the default must be the same run.
///
/// Exists because a default that drifts from its named value is invisible: every
/// other test in this suite omits `--device`, so if the default ever became `gpu`
/// the whole suite would silently start testing a different code path, and on a
/// machine without a device it would fail everywhere at once with no clue why.
/// Comparing the two spellings byte for byte pins the default to the host path.
#[test]
fn the_default_device_is_the_host_and_matches_an_explicit_cpu() {
    if !artifacts_available() {
        return;
    }
    let text = "don't stop, 1234567890 times, and a tail to fill more than one token";
    let implicit = run(&["encode", "--vocab", "cl100k_base", "--text", text]);
    let explicit = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--text",
        text,
        "--device",
        "cpu",
    ]);
    assert_eq!(exit_code(&implicit), 0, "{}", stderr_of(&implicit));
    assert_eq!(exit_code(&explicit), 0, "{}", stderr_of(&explicit));
    assert_eq!(
        implicit.stdout, explicit.stdout,
        "omitting --device must be exactly --device cpu"
    );
    assert!(
        !implicit.stdout.is_empty(),
        "the comparison is only meaningful if ids were actually printed"
    );
    assert_eq!(
        stderr_of(&explicit),
        "",
        "a host run must not print a routing report, which belongs to device runs only"
    );
}

/// `--help` must document the device flag AND that it refuses rather than
/// falling back.
///
/// The refusal is the load-bearing half. An operator scripting `--device gpu`
/// across a fleet needs to know from `--help` that a machine without a device
/// fails with a status instead of quietly encoding on the host, because a silent
/// host encode would report success for work they believed ran on the device.
/// Pinning it here means the promise cannot be dropped from the help text while
/// the behavior stays.
#[test]
fn help_documents_the_device_flag_and_its_refusal_to_fall_back() {
    let output = run(&["--help"]);
    assert_eq!(exit_code(&output), 0, "{}", stderr_of(&output));
    let text = String::from_utf8(output.stdout.clone()).expect("help must be UTF-8");
    for needle in [
        "--device",
        "Device:",
        "THE SAME IDS",
        "FAILS with status 6",
    ] {
        assert!(
            text.contains(needle),
            "--help must document {needle:?} so the device contract is readable without the \
             source"
        );
    }
}

/// `--device gpu` must refuse, print no ids, and name a remedy, in EVERY build.
///
/// This is the exact case where a fallback would be tempting: the binary CAN
/// encode the input, just not where it was asked to. Encoding on the host anyway
/// would exit 0 and print ids, and the caller would have no way to learn the
/// device never ran. So this pins all three facts together, the status, the
/// empty stdout, and a message naming the remedy, because any one of them alone
/// would still permit a silent fallback.
///
/// THE ARM IS READ OFF THE MESSAGE THE BINARY PRINTED, never predicted from this
/// test crate's `cfg!(feature = "cuda")`. Those two can disagree, because the cargo
/// target dir is shared and a concurrent `--features cuda` build can replace the
/// binary under a test crate compiled without it. The `#[cfg(not(feature = "cuda"))]`
/// gate this replaces was worse than merely wrong: it SKIPPED, so it reported green
/// while pinning a message the binary could no longer emit. Emptying
/// `CUDA_VISIBLE_DEVICES` makes both builds refuse, so one ungated test covers both
/// arms and neither can silently disappear.
///
/// NEGATIVE CONTROL, run and recorded rather than asserted: swapping the emptied
/// `CUDA_VISIBLE_DEVICES` for an unrelated variable lets the cuda build find a real
/// device, and the exit assertion fires with `left: 0, right: 6` alongside the success
/// line `routed 2 pretoken(s) to the device, 0 to the host`. So this test depends on the
/// refusal actually happening, and the cuda arm is genuinely exercised rather than
/// passing vacuously on a message it never triggers. Re-run that injection if the env
/// handling here changes.
#[test]
fn device_gpu_refuses_and_produces_no_ids_in_every_build() {
    if !artifacts_available() {
        return;
    }
    let output = run_env(
        &[
            "encode",
            "--vocab",
            "cl100k_base",
            "--text",
            "hello world",
            "--device",
            "gpu",
        ],
        &[("CUDA_VISIBLE_DEVICES", "")],
    );
    assert_eq!(
        exit_code(&output),
        EXIT_DEVICE,
        "--device gpu with no usable device must exit EXIT_DEVICE: {}",
        stderr_of(&output)
    );
    assert!(
        output.stdout.is_empty(),
        "a refused device run must print no ids, or the caller cannot tell it was refused: {:?}",
        String::from_utf8_lossy(&output.stdout)
    );
    let message = stderr_of(&output);
    assert!(
        message.contains("Fix:"),
        "every failure in this binary carries a Fix clause: {message}"
    );
    let unsupported = message.contains("no CUDA support");
    let no_device = message.contains("no usable CUDA device");
    assert!(
        unsupported ^ no_device,
        "the refusal must be exactly one of the two known arms and not a third shape, \
         or an unrecognized message would be filed under whichever arm happened to run: \
         {message}"
    );
    if unsupported {
        assert!(
            message.contains("--features cuda"),
            "a build without cuda support must name the rebuild that would honor the flag: \
             {message}"
        );
    }
}

/// With the cuda feature and a device present, `--device gpu` must produce the
/// SAME ids as the host and report its routing.
///
/// This is the exactness contract at the CLI boundary. The library suites prove
/// the device encoder matches the host oracle; this proves the binary actually
/// reaches that encoder, because a flag can be parsed, accepted, and then ignored
/// while every library test stays green. It also pins the routing report, since a
/// device run that silently encoded some pretokens on the host would otherwise be
/// indistinguishable from one that did not.
#[cfg(feature = "cuda")]
#[test]
fn device_gpu_produces_the_host_ids_and_reports_its_routing() {
    if !artifacts_available() {
        return;
    }
    let text = "don't stop, 1234567890 times, and a tail to fill more than one token";
    let host = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--text",
        text,
        "--device",
        "cpu",
    ]);
    let device = run(&[
        "encode",
        "--vocab",
        "cl100k_base",
        "--text",
        text,
        "--device",
        "gpu",
    ]);
    assert_eq!(exit_code(&host), 0, "{}", stderr_of(&host));
    assert_eq!(
        exit_code(&device),
        0,
        "the device run must succeed on a machine with a device rather than skip: {}",
        stderr_of(&device)
    );
    assert_eq!(
        device.stdout, host.stdout,
        "the device path is exact, so its ids must equal the host's byte for byte"
    );
    assert!(
        !host.stdout.is_empty(),
        "the comparison is only meaningful if ids were actually printed"
    );

    let report = stderr_of(&device);
    assert!(
        report.contains("routed") && report.contains("to the device"),
        "a device run must report where the work happened: {report}"
    );
    assert!(
        report.contains("0 to the host"),
        "this input has no pretoken above the device cap, so nothing should have been routed to \
         the host: {report}"
    );
}

/// `count` must agree across devices.
///
/// The count is derived from the ids, so this would be redundant if `count`
/// shared one code path with `encode`. It exists precisely to prove that it
/// does: the two subcommands take `--device` separately, and a wiring mistake
/// that left `count` always on the host would be invisible to the encode test
/// above.
#[cfg(feature = "cuda")]
#[test]
fn count_agrees_between_the_host_and_the_device() {
    if !artifacts_available() {
        return;
    }
    let text = "The quick brown fox jumps over the lazy dog, 1234567890 times.";
    let host = run(&[
        "count",
        "--vocab",
        "cl100k_base",
        "--text",
        text,
        "--device",
        "cpu",
    ]);
    let device = run(&[
        "count",
        "--vocab",
        "cl100k_base",
        "--text",
        text,
        "--device",
        "gpu",
    ]);
    assert_eq!(exit_code(&host), 0, "{}", stderr_of(&host));
    assert_eq!(exit_code(&device), 0, "{}", stderr_of(&device));
    assert_eq!(
        device.stdout, host.stdout,
        "the token count cannot depend on where the encode ran"
    );
    assert!(
        stderr_of(&device).contains("routed"),
        "count must report its routing too, so a device count is as auditable as a device encode"
    );
}