exatok 0.1.1

Exact GPU-resident BPE tokenizer whose token ids match tiktoken byte for byte
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
//! Command line interface for this crate.
//!
//! Wraps the exact host encoder over the vocabulary loaders. Every subcommand
//! runs the library path and nothing else: `encode` is [`Tokenizer::encode`],
//! `decode` is [`Tokenizer::decode`], and `split` is [`spans`]. Two entry points
//! to one algorithm is where divergence hides, so `tests/api_shape.rs` compares
//! this binary's printed ids against the library's ids over the whole parity
//! corpus rather than trusting that call graph.
//!
//! # Vocabulary resolution
//!
//! A vocabulary is named or it is a path, and the two are deliberately not
//! conflated.
//!
//! A NAME is one of the four vocabularies this crate ships. It resolves through
//! [`exatok::vocab::locate`], carries its own split pattern, and is loaded
//! through [`exatok::vocab::load`], which verifies the token and pair counts. A
//! name therefore cannot silently load a truncated artifact.
//!
//! A PATH is loaded by extension: `.tiktoken` as a rank file, `.json` as a
//! HuggingFace `tokenizer.json` bundle. Neither form tells this binary which
//! pretokenizer produced it, so `--pattern` is REQUIRED with any
//! `--vocab-path`, and its absence is an error rather than a guess. Guessing
//! here would produce ids that are wrong in a way only a parity run would
//! notice.
//!
//! # Failure order
//!
//! A usage error is settled before any artifact is opened, and an artifact
//! error before any input is read. That order is what makes the exit codes
//! worth branching on, and it is also why a mistyped vocabulary name no longer
//! blocks on stdin: the old code read the input first, so `encode --vocab typo`
//! on a terminal waited for EOF and only then reported the typo.
//!
//! # Exit codes
//!
//! [`EXIT_USAGE`], [`EXIT_VOCABULARY`], [`EXIT_INPUT`], [`EXIT_OUTPUT`] and
//! [`EXIT_INTERNAL`], one class of failure each, published in [`LONG_ABOUT`]
//! because an operator scripting against this binary reads `--help` rather
//! than this file. `1` is deliberately never returned, so a shell test can
//! tell "ran and said no" from "could not run".
//!
//! # Memory
//!
//! The input is read whole into memory; it is not streamed, and that is a
//! choice rather than an oversight. A pretoken can span any byte position, so
//! no chunk boundary is safe to cut at: cutting inside one changes the ids,
//! which is the single thing this binary promises not to do. Deciding a safe
//! boundary needs a resumable pretokenizer, which is a library concern and not
//! a CLI one. What the CLI does do is stream its OUTPUT, so the peak is the
//! input plus its id vector rather than those plus a formatted copy of them.

use std::io::{BufWriter, Read, Write};
use std::path::{Path, PathBuf};

use clap::{Parser, Subcommand, ValueEnum};
use exatok::pretokenize::{SplitPattern, unicode_version};
use exatok::specials;
use exatok::vocab::{self, Artifact, NAMED_VOCABULARIES, NamedVocabulary, Vocabulary};
use exatok::{Encoding, TokenId, Tokenizer, UnknownTokenId, ExatokError};

/// This binary's own name, for messages and help text.
///
/// Read from the package name so a crate rename does not leave a stale string
/// in every error message.
const NAME: &str = env!("CARGO_PKG_NAME");

/// The command line itself was wrong.
///
/// Matches the status `clap` returns for its own parse failures, so a caller
/// sees one code for "you asked for something that is not a command" whether
/// the parser or this binary noticed.
pub const EXIT_USAGE: i32 = 2;

/// A vocabulary artifact was missing, unreadable, or not the table it claims.
pub const EXIT_VOCABULARY: i32 = 3;

/// The input could not be read, or an id stream could not be decoded.
pub const EXIT_INPUT: i32 = 4;

/// Standard output could not be written.
pub const EXIT_OUTPUT: i32 = 5;

/// The device asked for with `--device` could not be used.
///
/// Separate from [`EXIT_USAGE`] because the flag was spelled correctly and this
/// binary would honor it on a machine that had the device. An operator
/// scripting a device run needs to tell "no device here" from "wrong flag", and
/// encoding on the host instead would report success for work the caller asked
/// to have done somewhere else.
pub const EXIT_DEVICE: i32 = 6;

/// An invariant this binary maintains was violated, so it is a defect here.
///
/// `70` is `EX_SOFTWARE` from `sysexits`, chosen because it is far away from
/// the small codes: a caller branching on 2, 3, 4 and 5 gets a value it does
/// not recognize instead of one that looks like an ordinary refusal.
pub const EXIT_INTERNAL: i32 = 70;

/// Long help for the root command.
///
/// Carries the exit-code contract, the input rules, the memory bound, and
/// where the vocabularies live, because an operator scripting against this
/// binary reads `--help` and not this source file. `tests/cli.rs` scrapes the exit-code table out of the
/// rendered help and compares it to the constants above, so the table cannot
/// drift away from the codes it documents.
pub const LONG_ABOUT: &str = concat!(
    "Exact byte-pair encoding. Token ids match the reference tokenizer byte for byte.\n",
    "Run `",
    env!("CARGO_PKG_NAME"),
    " parity-status` to see which vocabularies are provisioned on this machine.\n",
    "\n",
    "Input:\n",
    "  --text takes the input as an argument, --file reads it from a path, and with neither the\n",
    "  input is read from stdin. Input is bytes and not text: a file or a stdin stream that is\n",
    "  not valid UTF-8 encodes normally, because the encoder is defined on bytes. An input that\n",
    "  encodes to no tokens produces no output and exit 0.\n",
    "\n",
    "  The input is read whole into memory rather than streamed, because a pretoken can span any\n",
    "  byte position and cutting inside one would change the ids. Peak memory is a few times the\n",
    "  input size.\n",
    "\n",
    "Output:\n",
    "  Everything goes to stdout and nothing else is written. encode streams its ids as it\n",
    "  formats them; decode and split assemble their bytes first, so a rejected id leaves no\n",
    "  partial output behind.\n",
    "\n",
    "Device:\n",
    "  --device cpu, the default, encodes on the host. --device gpu encodes on a CUDA device and\n",
    "  produces THE SAME IDS: the device path is exact, not an approximation of the host one.\n",
    "\n",
    "  A device run reports on stderr how the work was routed. A pretoken longer than the\n",
    "  device's per-pretoken cap is encoded on the host and spliced back in order, which is\n",
    "  scheduling rather than a fallback, and the line is printed on every device run so that\n",
    "  \"no line\" and \"the line said zero\" are not the same observation. Ids go to stdout, so\n",
    "  the report cannot corrupt what a program parses.\n",
    "\n",
    "  --device gpu FAILS with status 6 rather than encoding on the host when no CUDA device is\n",
    "  usable, or when this binary was built without its cuda feature. Encoding on the host\n",
    "  instead would report success for work you asked to run on the device.\n",
    "\n",
    "Vocabulary:\n",
    "  --vocab resolves a name to one .tiktoken rank table in the artifact directory:\n",
    "  ~/.cache/exatok-vocab by default, or the directory EXATOK_VOCAB_DIR names when\n",
    "  it is set. The override relocates the whole cache for the library, this binary,\n",
    "  and every artifact-dependent test at once, so one export points all of them at\n",
    "  the same directory.\n",
    "\n",
    "  A missing or unreadable artifact is a vocabulary error, status 3, and a\n",
    "  zero-length one is a broken cache rather than nothing provisioned. `",
    env!("CARGO_PKG_NAME"),
    " parity-status` reports what is provisioned on this machine, so it is the check\n",
    "  to run on a fresh machine or after changing the override.\n",
    "\n",
    "Exit codes:\n",
    "  0   success.\n",
    "  2   usage error. An unknown flag, a contradictory pair, a missing one, or a value given\n",
    "      on the command line that this binary will not accept.\n",
    "  3   vocabulary error. An artifact was missing, unreadable, or not the table it claims.\n",
    "  4   input error. An input file or stream could not be read, or its contents could not be\n",
    "      decoded. A bad id on the command line is a usage error; a bad id arriving on stdin is\n",
    "      this, because one is fixed by editing the command and the other by fixing the\n",
    "      producer.\n",
    "  5   output error. Standard output could not be written, a closed pipe for instance.\n",
    "  6   device error. --device gpu was asked for, but no CUDA device is usable or this build\n",
    "      has no cuda feature. No ids were produced and nothing was encoded on the host\n",
    "      instead.\n",
    "  70  internal error. A defect in this binary, so the run proves nothing.\n",
    "\n",
    "  Status 1 is never returned, so a caller can tell \"ran and said no\" from \"could not run\".\n",
    "  Any other status, 101 for instance, means this binary crashed instead of reporting, which\n",
    "  is a defect and not a refusal.",
);

/// A failure, carrying the exit code it is reported with.
///
/// One variant per exit code and the message built at the point of failure,
/// rather than a wrapped source chain: a CLI prints one message, and the
/// variant that owns the code is also the one that knows which value offended
/// and what would fix it.
#[derive(Debug, thiserror::Error)]
pub enum CliError {
    /// The command line was wrong, including a value given on it that this
    /// binary will not accept. [`EXIT_USAGE`].
    #[error("{0}")]
    Usage(String),
    /// A vocabulary artifact could not be found, read, or trusted.
    /// [`EXIT_VOCABULARY`].
    #[error("{0}")]
    Vocabulary(String),
    /// An input file or stream could not be read, or its contents could not be
    /// decoded. [`EXIT_INPUT`].
    ///
    /// The line between this and [`CliError::Usage`] is where the offending
    /// value came from, not what is wrong with it: a bad id in `--ids` is part
    /// of the invocation and a bad id on stdin is data. They send the reader to
    /// different places, so they get different codes. See [`IdOrigin`].
    #[error("{0}")]
    Input(String),
    /// Standard output could not be written. [`EXIT_OUTPUT`].
    #[error("{0}")]
    Output(String),
    /// The device named by `--device` could not be used. [`EXIT_DEVICE`].
    ///
    /// Deliberately not [`CliError::Usage`]: the flag was accepted, and only the
    /// machine or this build's feature set could not honor it.
    #[error("{0}")]
    Device(String),
    /// An invariant of this binary was violated. [`EXIT_INTERNAL`].
    #[error("{0}")]
    Internal(String),
}

impl CliError {
    /// The documented exit status for this failure.
    ///
    /// This mapping is the interface: [`LONG_ABOUT`] publishes it and
    /// `tests/cli.rs` pins both the numbers and this table, so changing an arm
    /// is a deliberate contract change rather than an edit.
    #[must_use]
    pub const fn exit_code(&self) -> i32 {
        match self {
            Self::Usage(_) => EXIT_USAGE,
            Self::Vocabulary(_) => EXIT_VOCABULARY,
            Self::Input(_) => EXIT_INPUT,
            Self::Output(_) => EXIT_OUTPUT,
            Self::Device(_) => EXIT_DEVICE,
            Self::Internal(_) => EXIT_INTERNAL,
        }
    }
}

impl From<ExatokError> for CliError {
    fn from(error: ExatokError) -> Self {
        match &error {
            // Every library error this binary can provoke comes out of loading
            // or checking an artifact, and that message already names the
            // artifact and carries its own Fix clause.
            ExatokError::Artifact { .. } => Self::Vocabulary(error.to_string()),
            // A dispatch failure is the machine refusing the work rather than
            // this binary misbehaving, so it carries the device code and tells
            // the caller the host path is still available. It became reachable
            // when --device gpu was wired; on the host path it cannot arise.
            ExatokError::Backend(detail) => Self::Device(format!(
                "the CUDA device failed to run the encode: {detail}\nFix: re-run without \
                 --device gpu to encode on the host, and report this with the input if the host \
                 path succeeds."
            )),
            // A divergence means the ids would have been wrong, which is the one
            // thing this crate promises never to be. That is a defect here and
            // not something the caller can act on.
            other => Self::Internal(format!(
                "{other}\nFix: this is a defect in {NAME}; report it with the command you ran."
            )),
        }
    }
}

/// Report a failed write to standard output.
///
/// Named separately because every output path shares it: a closed pipe or a
/// full destination must be a reported failure with a status, never a silent
/// truncation.
fn output_error(source: &std::io::Error) -> CliError {
    CliError::Output(format!(
        "cannot write to standard output: {source}\nFix: check the pipe or file stdout is \
         redirected to; the reader may have exited or the destination may be full."
    ))
}

/// Write `bytes` to standard output and flush them.
///
/// The flush is not optional. Rust's process exit flushes stdout and DISCARDS
/// the error, so a run that could not deliver its bytes would still exit 0
/// without it.
fn emit(bytes: &[u8]) -> Result<(), CliError> {
    let stdout = std::io::stdout();
    let mut out = stdout.lock();
    out.write_all(bytes).map_err(|source| output_error(&source))?;
    out.flush().map_err(|source| output_error(&source))
}

/// Split pattern, as a CLI value.
///
/// Mirrors [`SplitPattern`] rather than deriving [`ValueEnum`] on it, because
/// the library type must not grow a CLI dependency.
#[derive(Debug, Clone, Copy, ValueEnum)]
pub enum PatternArg {
    /// `gpt2`, shared with `r50k_base` and `p50k_base`.
    Gpt2,
    /// `cl100k_base`.
    Cl100kBase,
    /// `o200k_base`.
    O200kBase,
}

impl From<PatternArg> for SplitPattern {
    fn from(value: PatternArg) -> Self {
        match value {
            PatternArg::Gpt2 => Self::Gpt2,
            PatternArg::Cl100kBase => Self::Cl100kBase,
            PatternArg::O200kBase => Self::O200kBase,
        }
    }
}

/// How to print an id stream.
#[derive(Debug, Clone, Copy, ValueEnum)]
pub enum IdFormat {
    /// Space-separated decimal ids on one line, or nothing at all when the
    /// input encodes to no ids.
    Ids,
    /// JSON array of numbers, `[]` when the input encodes to no ids.
    Json,
}

/// Which device encodes the input.
///
/// Mirrors nothing in the library on purpose: the library takes a backend, and
/// choosing one from a flag is a CLI concern.
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum, Default)]
pub enum DeviceArg {
    /// Encode on the host.
    #[default]
    Cpu,
    /// Encode on a CUDA device, producing the same ids.
    Gpu,
}

#[derive(Debug, Parser)]
#[command(about = "Exact BPE tokenization.", long_about = LONG_ABOUT, version)]
pub struct Cli {
    #[command(subcommand)]
    pub command: Command,
}

#[derive(Debug, Subcommand)]
pub enum Command {
    /// Encode input to token ids.
    Encode {
        #[command(flatten)]
        vocab: VocabArgs,
        #[command(flatten)]
        input: InputArgs,
        /// Device to encode on. A device run reports its routing on stderr, and
        /// fails rather than encoding on the host when the device is unusable.
        #[arg(long, value_enum, default_value_t = DeviceArg::Cpu)]
        device: DeviceArg,
        /// Output format.
        #[arg(long, value_enum, default_value_t = IdFormat::Ids)]
        format: IdFormat,
    },
    /// Print the token count, without the ids.
    Count {
        #[command(flatten)]
        vocab: VocabArgs,
        #[command(flatten)]
        input: InputArgs,
        /// Device to encode on. A device run reports its routing on stderr, and
        /// fails rather than encoding on the host when the device is unusable.
        #[arg(long, value_enum, default_value_t = DeviceArg::Cpu)]
        device: DeviceArg,
    },
    /// Decode token ids back to bytes, written raw to stdout.
    Decode {
        #[command(flatten)]
        vocab: VocabArgs,
        /// Ids to decode, separated by commas or whitespace. Read from stdin
        /// when absent.
        #[arg(long)]
        ids: Option<String>,
    },
    /// Show the pretokenizer split, one pretoken per line.
    #[command(long_about = "Show the pretokenizer split, one pretoken per line.\n\n\
                            A pretoken may itself contain a newline, a run of blank lines being \
                            one pretoken, so a pretoken can span output lines and the line \
                            breakdown is not a parseable framing. Read it, and take ids from \
                            `encode` when a program needs the partition.")]
    Split {
        #[command(flatten)]
        vocab: VocabArgs,
        #[command(flatten)]
        input: InputArgs,
    },
    /// Report a loaded vocabulary: sizes, specials, pattern.
    Info {
        #[command(flatten)]
        vocab: VocabArgs,
    },
    /// List the named vocabularies and whether each is present on disk.
    ParityStatus,
}

#[derive(Debug, clap::Args)]
pub struct VocabArgs {
    // Help is built from NAMED_VOCABULARIES rather than written out, so adding a
    // fifth vocabulary shows up in --help without a second edit. A hardcoded
    // list here would rot silently, which is the whole failure mode help text
    // has.
    #[arg(long, value_name = "NAME", help = vocab_help())]
    pub vocab: Option<String>,
    /// Path to a `.tiktoken` rank file or a `tokenizer.json` bundle. Requires
    /// --pattern.
    #[arg(long)]
    pub vocab_path: Option<PathBuf>,
    /// Split pattern for the artifact --vocab-path names. Required with
    /// --vocab-path, and rejected with --vocab, which states its own pattern.
    #[arg(long, value_enum)]
    pub pattern: Option<PatternArg>,
}

#[derive(Debug, clap::Args)]
pub struct InputArgs {
    /// Input text, encoded as its UTF-8 bytes. With neither --text nor --file,
    /// the input is read from stdin.
    #[arg(long)]
    pub text: Option<String>,
    /// Input file, read as raw bytes. With neither --text nor --file, the
    /// input is read from stdin.
    #[arg(long)]
    pub file: Option<PathBuf>,
}

/// Where input bytes come from, once the flags are known to agree.
///
/// Deciding this before reading is what lets a usage error be reported before
/// the binary touches a file or blocks on stdin.
#[derive(Debug)]
pub enum InputSource<'args> {
    /// The bytes of a `--text` argument.
    Text(&'args str),
    /// The contents of a `--file` path.
    File(&'args Path),
    /// Everything on stdin, to end of stream.
    Stdin,
}

impl InputArgs {
    /// Settle which source the input comes from.
    fn source(&self) -> Result<InputSource<'_>, CliError> {
        match (self.text.as_deref(), self.file.as_deref()) {
            (Some(text), None) => Ok(InputSource::Text(text)),
            (None, Some(path)) => Ok(InputSource::File(path)),
            (None, None) => Ok(InputSource::Stdin),
            (Some(text), Some(path)) => Err(CliError::Usage(format!(
                "--text {text:?} and --file {} are mutually exclusive, and neither silently \
                 wins.\nFix: pass one of them, or neither to read stdin.",
                path.display()
            ))),
        }
    }
}

impl InputSource<'_> {
    /// Read the input bytes.
    ///
    /// Bytes rather than a `String` throughout: the encoder is defined on bytes
    /// and invalid UTF-8 has exact expected ids, so decoding to a `String` here
    /// would reject input the tokenizer is required to handle.
    fn read(&self) -> Result<Vec<u8>, CliError> {
        match *self {
            Self::Text(text) => Ok(text.as_bytes().to_vec()),
            Self::File(path) => std::fs::read(path).map_err(|source| {
                CliError::Input(format!(
                    "cannot read the input file {}: {source}\nFix: check the path exists and is \
                     readable, or pass --text to give the input on the command line.",
                    path.display()
                ))
            }),
            Self::Stdin => {
                let mut buffer = Vec::new();
                std::io::stdin().read_to_end(&mut buffer).map_err(|source| {
                    CliError::Input(format!(
                        "cannot read the input from stdin: {source}\nFix: redirect a file or a \
                         pipe into this command, or pass --text or --file instead."
                    ))
                })?;
                Ok(buffer)
            }
        }
    }
}

/// The named vocabulary matching `name`, if it is one.
fn named(name: &str) -> Option<NamedVocabulary> {
    NAMED_VOCABULARIES.into_iter().find(|v| v.name == name)
}

/// Comma-separated list of every name [`named`] accepts.
fn known_names() -> String {
    NAMED_VOCABULARIES
        .iter()
        .map(|v| v.name)
        .collect::<Vec<_>>()
        .join(", ")
}

/// Help for `--vocab`, listing the names it accepts.
fn vocab_help() -> String {
    format!("Vocabulary name, one of {}", known_names())
}

// The special-token table lives in the library, at `exatok::specials`, so the
// test suite can guard the ids. It was here first and nothing under `tests/`
// could reach it, which left four published `n_vocab` values resting on manual
// CLI runs.

impl VocabArgs {
    /// Load the vocabulary, settle its split pattern, and resolve its byte table.
    fn load(&self) -> Result<Tokenizer, CliError> {
        match (self.vocab.as_deref(), self.vocab_path.as_deref()) {
            (None, None) => Err(CliError::Usage(format!(
                "no vocabulary given.\nFix: pass --vocab <{}> or --vocab-path <file>.",
                known_names()
            ))),
            (Some(name), Some(path)) => Err(CliError::Usage(format!(
                "--vocab {name:?} and --vocab-path {} are mutually exclusive, because a name \
                 carries its own pattern and count checks while a path carries neither.\nFix: \
                 pass one of them.",
                path.display()
            ))),
            (Some(name), None) => self.load_named(name),
            (None, Some(path)) => self.load_path(path),
        }
    }

    /// Load a named vocabulary, with its counts verified.
    fn load_named(&self, name: &str) -> Result<Tokenizer, CliError> {
        if self.pattern.is_some() {
            return Err(CliError::Usage(format!(
                "--pattern is not accepted with --vocab, because {name} already states its \
                 pattern.\nFix: drop --pattern, or use --vocab-path to override deliberately."
            )));
        }
        let Some(vocabulary) = named(name) else {
            return Err(CliError::Usage(format!(
                "unknown vocabulary {name:?}.\nFix: use one of {}, or --vocab-path for an \
                 artifact elsewhere.",
                known_names()
            )));
        };
        // Every arm states the classifier's VERDICT, and names the path as
        // detail rather than as the claim. Restating the evidence instead, as
        // "the directory does not exist" and "{path} is absent" both did, makes
        // the message false the moment the classifier reaches that arm by a
        // second route: the verdict survives a broadened classifier, the
        // filesystem predicate behind it does not.
        match vocab::locate(vocabulary) {
            Artifact::Present(path) => {
                let mut loaded = vocab::load(vocabulary, &path)?;
                specials::register(&mut loaded, vocabulary.name)?;
                Ok(Tokenizer::new(
                    loaded,
                    Encoding::new(vocabulary.name, vocabulary.pattern),
                )?)
            }
            Artifact::NoCache { dir, path } => Err(CliError::Vocabulary(format!(
                "{name} is not provisioned: the cache holds no tiktoken rank table at all. \
                 Looked for {} in {}.\nFix: put the tiktoken rank files in that directory, \
                 creating it if it does not exist, or set {} to a directory that already holds \
                 them.",
                path.display(),
                dir.display(),
                vocab::ARTIFACT_DIR_ENV
            ))),
            Artifact::Incomplete { dir, path } => Err(CliError::Vocabulary(format!(
                "{name} has no usable rank file, so this cache is incomplete rather than \
                 unprovisioned: nothing loadable at {}, where an empty file counts as nothing.\n\
                 Fix: fetch that one rank file into {} again, replacing a zero-length one if \
                 that is what is there, or set {} to a complete directory.",
                path.display(),
                dir.display(),
                vocab::ARTIFACT_DIR_ENV
            ))),
        }
    }

    /// Load a vocabulary from a path, dispatching on the extension.
    fn load_path(&self, path: &Path) -> Result<Tokenizer, CliError> {
        // The pattern is settled before the file is opened, so a missing
        // --pattern is one message for both formats. Checking it per format
        // let the `.json` arm load a 2 MiB bundle and only then report a flag
        // that was already absent.
        let Some(pattern) = self.pattern else {
            return Err(CliError::Usage(format!(
                "--pattern is required with --vocab-path {}, because a rank file does not record \
                 its pretokenizer and this binary does not yet read the one a tokenizer.json \
                 bundle records, and the three patterns give different ids for the same \
                 bytes.\nFix: pass --pattern gpt2, cl100k-base, or o200k-base.",
                path.display()
            )));
        };
        let extension = path
            .extension()
            .and_then(|e| e.to_str())
            .unwrap_or_default()
            .to_ascii_lowercase();
        if extension != "tiktoken" && extension != "json" {
            return Err(CliError::Usage(format!(
                "cannot tell the format of {} from its extension.\nFix: use a .tiktoken rank \
                 file or a .json tokenizer bundle.",
                path.display()
            )));
        }
        // Classify the path by OPENING it, not by stat, because a mode-000
        // file stats fine and the loader's own message then calls it
        // "malformed at file read": a claim about the contents of a file it
        // never read. A directory does open, so the kind is checked from the
        // handle rather than assumed.
        match std::fs::File::open(path) {
            Ok(file) => {
                if file.metadata().is_ok_and(|found| found.is_dir()) {
                    return Err(CliError::Vocabulary(format!(
                        "the vocabulary artifact {} is a directory, not a rank file or a \
                         bundle.\nFix: name the file inside it.",
                        path.display()
                    )));
                }
            }
            Err(source) => {
                return Err(CliError::Vocabulary(format!(
                    "cannot read the vocabulary artifact {}: {source}\nFix: check the path exists \
                     and is readable, or pass --vocab <{}> for a provisioned vocabulary.",
                    path.display(),
                    known_names()
                )));
            }
        }
        let loaded = if extension == "tiktoken" {
            Vocabulary::load_tiktoken_path(path)?
        } else {
            Vocabulary::load_tokenizer_json_path(path)?
        };
        Ok(Tokenizer::new(
            loaded,
            Encoding::new(path.display().to_string(), pattern.into()),
        )?)
    }
}

/// Write `ids` in `format`.
fn write_ids(out: &mut impl Write, ids: &[TokenId], format: IdFormat) -> std::io::Result<()> {
    match format {
        // Nothing at all for an empty stream, so an empty input gives empty
        // output rather than a blank line a consumer has to know to discard.
        IdFormat::Ids => {
            for (index, id) in ids.iter().enumerate() {
                if index > 0 {
                    out.write_all(b" ")?;
                }
                write!(out, "{}", id.0)?;
            }
            if !ids.is_empty() {
                out.write_all(b"\n")?;
            }
        }
        IdFormat::Json => {
            out.write_all(b"[")?;
            for (index, id) in ids.iter().enumerate() {
                if index > 0 {
                    out.write_all(b",")?;
                }
                write!(out, "{}", id.0)?;
            }
            out.write_all(b"]\n")?;
        }
    }
    out.flush()
}

/// Print `ids` in `format`, streaming rather than building one string.
fn print_ids(ids: &[TokenId], format: IdFormat) -> Result<(), CliError> {
    let stdout = std::io::stdout();
    let mut out = BufWriter::new(stdout.lock());
    write_ids(&mut out, ids, format).map_err(|source| output_error(&source))
}

/// Where an id list came from, which decides the code a bad one exits with.
///
/// An id in `--ids` is part of the invocation, so a bad one is a usage error
/// fixed by editing the command. An id arriving on stdin is data, so a bad one
/// is an input error fixed by fixing whatever produced it. Same text, two
/// different places to go look, so two codes.
#[derive(Debug, Clone, Copy)]
pub enum IdOrigin {
    /// The `--ids` argument.
    Argument,
    /// The stdin stream.
    Stdin,
}

impl IdOrigin {
    /// Classify a fault in an id list that came from this origin.
    fn fault(self, message: String) -> CliError {
        match self {
            Self::Argument => CliError::Usage(message),
            Self::Stdin => CliError::Input(message),
        }
    }
}

/// Read an id list from stdin.
///
/// Reads bytes and then checks UTF-8, rather than `read_to_string`, so binary
/// on stdin is reported with the offset and value of the offending byte instead
/// of "stream did not contain valid UTF-8", which names nothing.
fn read_ids_from_stdin() -> Result<String, CliError> {
    let mut buffer = Vec::new();
    std::io::stdin().read_to_end(&mut buffer).map_err(|source| {
        CliError::Input(format!(
            "cannot read the id list from stdin: {source}\nFix: pipe the ids in, or pass --ids."
        ))
    })?;
    String::from_utf8(buffer).map_err(|error| {
        let offset = error.utf8_error().valid_up_to();
        let byte = error.as_bytes().get(offset).copied().unwrap_or_default();
        CliError::Input(format!(
            "the id list on stdin is not text: byte {offset} is {byte:#04x}, which is not valid \
             UTF-8.\nFix: pipe decimal ids separated by commas or spaces; raw bytes are what \
             decode WRITES, not what it reads."
        ))
    })
}

/// Parse a comma or whitespace separated id list.
fn parse_ids(text: &str, origin: IdOrigin) -> Result<Vec<TokenId>, CliError> {
    let mut ids = Vec::new();
    for field in text.split([',', ' ', '\t', '\n', '\r']) {
        if field.is_empty() {
            continue;
        }
        let id: u32 = field.parse().map_err(|_| {
            origin.fault(format!(
                "{field:?} is not a token id.\nFix: pass decimal ids separated by commas or \
                 spaces, as encode prints them."
            ))
        })?;
        ids.push(TokenId(id));
    }
    Ok(ids)
}

/// Report an id the loaded vocabulary cannot decode.
///
/// [`Tokenizer::decode`] reports the id and its position. This adds which
/// vocabulary was asked and what to do about it, because an operator reading
/// stderr has neither of those in front of them.
fn undecodable(error: &UnknownTokenId, artifact: &str, origin: IdOrigin) -> CliError {
    origin.fault(format!(
        "cannot decode with {artifact}: {error}.\nFix: decode with the vocabulary the ids were \
         produced by."
    ))
}

/// Render a loaded vocabulary as the `info` report.
fn info_report(tokenizer: &Tokenizer) -> String {
    let vocabulary = tokenizer.vocabulary();
    let mut text = String::new();
    text.push_str(&format!(
        "pattern         {}\n",
        tokenizer.encoding().pattern().name()
    ));
    text.push_str(&format!("unicode         {}\n", unicode_version()));
    text.push_str(&format!("tokens          {}\n", vocabulary.len()));
    text.push_str(&format!("pairs           {}\n", vocabulary.merge_count()));
    text.push_str(&format!("n_vocab         {}\n", vocabulary.n_vocab()));
    text.push_str(&format!("specials        {}\n", vocabulary.special_len()));
    for (special, id) in vocabulary.specials() {
        text.push_str(&format!("  {id:>7} {special}\n", id = id.0));
    }
    text
}

/// Render the `parity-status` report.
fn parity_status_report() -> String {
    let mut text = format!("vocabulary directory: {}\n", vocab::artifact_dir().display());
    for vocabulary in NAMED_VOCABULARIES {
        let status = match vocab::locate(vocabulary) {
            Artifact::Present(path) => format!("present  {}", path.display()),
            // Not "no cache directory": this state also covers a directory that
            // exists and holds no rank table, which is the common fresh-machine
            // shape, and a status line that named a cause it had not checked
            // would be read as one it had.
            Artifact::NoCache { .. } => "absent   no rank files".to_owned(),
            Artifact::Incomplete { .. } => "MISSING  cache is incomplete".to_owned(),
        };
        text.push_str(&format!(
            "{:<12} {:<14} {status}\n",
            vocabulary.name,
            vocabulary.pattern.name()
        ));
    }
    text
}

/// Encode `bytes` on the device `device` names.
///
/// One function for both `encode` and `count`, so the two subcommands cannot
/// drift into disagreeing about which device ran or about whether routing was
/// reported. `count` reports routing too: the token count is the same either
/// way, but where it was computed is still worth knowing.
fn encode_on(
    tokenizer: &Tokenizer,
    bytes: &[u8],
    device: DeviceArg,
) -> Result<Vec<TokenId>, CliError> {
    match device {
        DeviceArg::Cpu => Ok(tokenizer.encode(bytes)),
        DeviceArg::Gpu => encode_on_cuda(tokenizer, bytes),
    }
}

/// Report how a device encode was routed, on stderr.
///
/// Stderr and not stdout, so the id stream stays exactly what a program parses.
/// Printed on every device run rather than only when a pretoken went to the
/// host, because an operator checking whether the device did the work should
/// not have to infer it from the absence of a line.
#[cfg(feature = "cuda")]
fn report_routing(routing: exatok::device::Routing) {
    eprintln!(
        "{NAME}: routed {} pretoken(s) to the device, {} to the host (longest host pretoken {} \
         tokens)",
        routing.on_device(),
        routing.on_host(),
        routing.largest_host_tokens()
    );
}

/// Encode on a CUDA device.
///
/// Builds the backend BEFORE the device merge table, so a machine with no
/// device reports that rather than spending the table build first.
#[cfg(feature = "cuda")]
fn encode_on_cuda(tokenizer: &Tokenizer, bytes: &[u8]) -> Result<Vec<TokenId>, CliError> {
    let backend = vyre_driver_cuda::cuda_factory().map_err(|error| {
        CliError::Device(format!(
            "no usable CUDA device: {error}\nFix: run without --device gpu to encode on the \
             host, or check that a CUDA device and its driver are present."
        ))
    })?;
    let device =
        exatok::device::DeviceTokenizer::new(tokenizer.vocabulary(), tokenizer.encoding())?;
    let encoded = device.encode_batch(backend.as_ref(), &[bytes])?;
    report_routing(encoded.routing());
    // One input in, so one id vector out. Take it positionally rather than
    // indexing, and treat an empty result as no tokens, which is what an empty
    // input produces.
    Ok(encoded.into_ids().pop().unwrap_or_default())
}

/// Refuse a device encode in a build without the cuda feature.
///
/// A build with no CUDA support must say so rather than encode on the host, for
/// the same reason a machine with no device must: the caller asked for the
/// device, and a host encode would report success for something they did not
/// ask for.
#[cfg(not(feature = "cuda"))]
fn encode_on_cuda(_tokenizer: &Tokenizer, _bytes: &[u8]) -> Result<Vec<TokenId>, CliError> {
    Err(CliError::Device(format!(
        "this build of {NAME} has no CUDA support, so --device gpu cannot be honored.\nFix: \
         rebuild with `--features cuda`, or run without --device gpu to encode on the host."
    )))
}

/// Run one subcommand.
///
/// Each arm settles usage, then the vocabulary, then the input, in that order.
/// The order is a contract: it decides which exit code a caller sees when more
/// than one thing is wrong at once.
pub fn run(cli: Cli) -> Result<(), CliError> {
    match cli.command {
        Command::Encode {
            vocab,
            input,
            format,
            device,
        } => {
            let source = input.source()?;
            let tokenizer = vocab.load()?;
            let ids = encode_on(&tokenizer, &source.read()?, device)?;
            print_ids(&ids, format)
        }
        Command::Count {
            vocab,
            input,
            device,
        } => {
            let source = input.source()?;
            let tokenizer = vocab.load()?;
            let ids = encode_on(&tokenizer, &source.read()?, device)?;
            emit(format!("{}\n", ids.len()).as_bytes())
        }
        Command::Decode { vocab, ids } => {
            let tokenizer = vocab.load()?;
            let (text, origin) = match ids {
                Some(text) => (text, IdOrigin::Argument),
                None => (read_ids_from_stdin()?, IdOrigin::Stdin),
            };
            let ids = parse_ids(&text, origin)?;
            let decoded = tokenizer
                .decode(&ids)
                .map_err(|error| undecodable(&error, tokenizer.encoding().artifact(), origin))?;
            emit(&decoded)
        }
        Command::Split { vocab, input } => {
            let source = input.source()?;
            let tokenizer = vocab.load()?;
            let bytes = source.read()?;
            let mut out = Vec::new();
            for span in tokenizer.split(&bytes) {
                // `Tokenizer::split` partitions this same input, so every span
                // is in bounds. Report the violation rather than skipping the
                // span: silently dropping it would turn a partition defect into
                // output that merely looks a little short.
                let Some(pretoken) = span.bytes(&bytes) else {
                    return Err(CliError::Internal(format!(
                        "the pretokenizer returned {span:?}, which is outside the {} bytes it \
                         was given.\nFix: this is a defect in {NAME}; report it with the input \
                         that produced it.",
                        bytes.len()
                    )));
                };
                out.extend_from_slice(pretoken);
                out.push(b'\n');
            }
            emit(&out)
        }
        Command::Info { vocab } => {
            let tokenizer = vocab.load()?;
            emit(info_report(&tokenizer).as_bytes())
        }
        Command::ParityStatus => emit(parity_status_report().as_bytes()),
    }
}

fn main() {
    let cli = match Cli::try_parse() {
        Ok(cli) => cli,
        Err(error) => {
            // clap's own message already names the offending value and shows
            // the usage line, so keep it and add the Fix clause every other
            // failure here carries. Help and version are not failures: they
            // go to stdout and exit 0.
            let usage = error.use_stderr();
            if error.print().is_err() {
                // The message could not be delivered, so the status is the
                // only true thing left.
                std::process::exit(EXIT_OUTPUT);
            }
            if usage {
                eprintln!("Fix: run `{NAME} --help` for the flag list and the exit-code table.");
                std::process::exit(EXIT_USAGE);
            }
            std::process::exit(0);
        }
    };
    if let Err(error) = run(cli) {
        eprintln!("{NAME}: {error}");
        std::process::exit(error.exit_code());
    }
}