tokenizers 0.23.1

Provides an implementation of today's most used tokenizers, with a focus on performances and versatility.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

use std::time::{Duration, Instant};

use std::hint::black_box;

use tokenizers::{
    Decoder, EncodeInput, Model, Normalizer, PostProcessor, PreTokenizer, TokenizerImpl, Trainer,
};

#[allow(dead_code)]
pub fn iter_bench_encode<M, N, PT, PP, D>(
    iters: u64,
    tokenizer: &TokenizerImpl<M, N, PT, PP, D>,
    lines: &[EncodeInput],
) -> Duration
where
    M: Model,
    N: Normalizer,
    PT: PreTokenizer,
    PP: PostProcessor,
    D: Decoder,
{
    let mut duration = Duration::new(0, 0);
    for _i in 0..iters {
        for line in lines {
            let input = line.clone();
            let start = Instant::now();
            let _ = black_box(tokenizer.encode(input, false));
            duration = duration.checked_add(start.elapsed()).unwrap();
        }
    }
    duration
}

#[allow(dead_code)]
pub fn iter_bench_encode_batch<M, N, PT, PP, D>(
    iters: u64,
    tokenizer: &TokenizerImpl<M, N, PT, PP, D>,
    batches: &[Vec<EncodeInput>],
) -> Duration
where
    M: Model + Send + Sync,
    N: Normalizer + Send + Sync,
    PT: PreTokenizer + Send + Sync,
    PP: PostProcessor + Send + Sync,
    D: Decoder + Send + Sync,
{
    let mut duration = Duration::new(0, 0);
    for _i in 0..iters {
        for batch in batches {
            let batch = batch.clone();
            let start = Instant::now();
            let _ = black_box(tokenizer.encode_batch(batch, false));
            duration = duration.checked_add(start.elapsed()).unwrap();
        }
    }
    duration
}

#[allow(dead_code)]
pub fn iter_bench_train<T, M, N, PT, PP, D>(
    iters: u64,
    tokenizer: &mut TokenizerImpl<M, N, PT, PP, D>,
    trainer: &mut T,
    files: Vec<String>,
) -> Duration
where
    T: Trainer<Model = M> + Sync,
    M: Model + Send + Sync,
    N: Normalizer + Send + Sync,
    PT: PreTokenizer + Send + Sync,
    PP: PostProcessor + Send + Sync,
    D: Decoder + Send + Sync,
{
    let mut duration = Duration::new(0, 0);
    for _i in 0..iters {
        let start = Instant::now();
        tokenizer.train_from_files(trainer, files.clone()).unwrap();
        duration = duration.checked_add(start.elapsed()).unwrap();
    }
    duration
}