kitoken 0.11.0

Fast tokenizer for language models, supporting BPE, Unigram and WordPiece tokenization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

//! Encoder for the tokenizer.

use alloc::borrow::Cow;
use alloc::vec::Vec;
use core::borrow::Borrow;
use core::fmt::Debug;
use core::ops::Deref;

use crate::{Model, TokenId};

mod bytepair;
mod unigram;
mod wordpiece;

pub(crate) use bytepair::*;
pub(crate) use unigram::*;
pub(crate) use wordpiece::*;

/// Errors encountered during encoding.
#[non_exhaustive]
#[derive(Debug, thiserror::Error)]
pub enum EncodeError {
    /// A piece could not be encoded.
    #[error("invalid piece {0:?}")]
    InvalidPiece(Vec<u8>),
}

/// Part of a text.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct TextPart<'a> {
    pub text:    Cow<'a, str>,
    pub special: TokenId,
}
impl Borrow<[u8]> for TextPart<'_> {
    #[inline(always)]
    fn borrow(&self) -> &[u8] {
        self.text.as_bytes()
    }
}
impl Deref for TextPart<'_> {
    type Target = [u8];

    #[inline(always)]
    fn deref(&self) -> &Self::Target {
        self.text.as_bytes()
    }
}

/// Encoder for the tokenizer.
pub(crate) trait Encoder: Debug + Send + Sync + 'static {
    /// Encodes the given parts into a sequence of tokens.
    ///
    /// If `encode_specials` is `true`, control tokens are tokenized with their ids, otherwise they are tokenized with the regular vocabulary.
    ///
    /// Returns an error if no token for a part exists in the encoder, and the configuration has no unknown token or skip fallback set.
    fn encode(&self, text: &str, parts: &mut [TextPart]) -> Result<Vec<TokenId>, EncodeError>;

    /// Returns the vocabulary and scores.
    fn model(&self) -> Model;
}