tokie 0.0.9

Blazingly fast tokenizer - 50x faster tokenization, 10x smaller model files, 100% accurate drop-in replacement for HuggingFace
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

//! Core types for BPE tokenization

/// Token identifier - corresponds to position in vocabulary
pub type TokenId = u32;

/// Represents the two tokens that were merged to form a token.
/// For base tokens (single bytes), both values point to the token itself.
#[derive(Debug, Clone, Copy)]
pub struct Split {
    pub left: TokenId,
    pub right: TokenId,
}

impl Split {
    /// Create a split for a base token (points to itself)
    pub fn base(id: TokenId) -> Self {
        Self { left: id, right: id }
    }

    /// Create a split from a merge of two tokens
    pub fn merge(left: TokenId, right: TokenId) -> Self {
        Self { left, right }
    }
}