toklen 0.2.0

A single-threaded, lightweight, and fast token counter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

use crate::added_tokens::AddedTokenConfig;
use crate::models::ModelConfig;
use crate::normalizers::NormalizerConfig;
use crate::pre_tokenizers::PreTokenizerConfig;
use serde::Deserialize;

// ── TokenizerJson ────────────────────────────────────────────────────────────

/// Parsed `tokenizer.json`.
#[derive(Debug, Deserialize)]
pub struct TokenizerJson {
    #[serde(default)]
    pub added_tokens: Vec<AddedTokenConfig>,
    pub normalizer: Option<NormalizerConfig>,
    pub pre_tokenizer: Option<PreTokenizerConfig>,
    pub model: ModelConfig,
}