wordchipper_cli_util/
tokenizer_mode.rs

1/// The tokenizer mode.
2#[derive(Debug, Clone, Copy)]
3pub enum TokenizerMode {
4    /// Encode from text to tokens.
5    Encode,
6
7    /// Decode from tokens to text.
8    Decode,
9}
10
11/// Tokenizer mode argument group.
12#[derive(clap::Args, Debug)]
13#[group(required = true, multiple = false)]
14pub struct TokenizerModeArgs {
15    /// Encode from text to tokens.
16    #[arg(long, action=clap::ArgAction::SetTrue)]
17    encode: bool,
18
19    /// Decode from tokens to text.
20    #[arg(long, action=clap::ArgAction::SetTrue)]
21    decode: bool,
22}
23
24impl TokenizerModeArgs {
25    /// Get the tokenizer mode.
26    pub fn mode(&self) -> TokenizerMode {
27        if self.encode {
28            TokenizerMode::Encode
29        } else if self.decode {
30            TokenizerMode::Decode
31        } else {
32            panic!("No tokenizer mode specified.");
33        }
34    }
35}

wordchipper_cli_util/tokenizer_mode.rs

wordchipper_cli_util/
tokenizer_mode.rs