pretokie 0.0.4

Fast, zero-allocation pretokenizers for BPE tokenizers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

//! Configuration trait and dimension enums for BPE pretokenizers.

#[derive(Clone, Copy, PartialEq, Eq)]
pub enum ContractionCase {
    Sensitive,
    Insensitive,
}

#[derive(Clone, Copy, PartialEq, Eq)]
pub enum ContractionMode {
    Standalone,
    Suffix,
}

#[derive(Clone, Copy, PartialEq, Eq)]
pub enum DigitMode {
    Unlimited,
    Chunked3,
    Single,
}

#[derive(Clone, Copy, PartialEq, Eq)]
pub enum LetterMode {
    Plain,
    PlainWithMarks,
    CamelCase,
}

#[derive(Clone, Copy, PartialEq, Eq)]
pub enum WsPattern {
    Gpt2,
    Cl100k,
}

#[derive(Clone, Copy, PartialEq, Eq)]
pub enum PunctTrailing {
    None,
    Newlines,
    NewlinesAndSlashes,
}

#[derive(Clone, Copy, PartialEq, Eq)]
pub enum WsException {
    None,
    Digits,
    Cjk,
}

#[derive(Clone, Copy, PartialEq, Eq)]
pub enum PunctPrefixMode {
    SpaceOnly,
    Any,
    AsciiOnly,
}

pub trait PretokConfig {
    const CONTRACTION_CASE: ContractionCase;
    const CONTRACTION_MODE: ContractionMode;
    const DIGIT_MODE: DigitMode;
    const LETTER_MODE: LetterMode;
    const SPACE_PREFIXES_DIGITS: bool;
    const WS_PATTERN: WsPattern;
    const PUNCT_TRAILING: PunctTrailing;
    const WS_EXCEPTION: WsException;
    const PUNCT_PREFIX_MODE: PunctPrefixMode;
}