pretokie 0.0.4

Fast, zero-allocation pretokenizers for BPE tokenizers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

//! Zero-sized config types for each BPE pretokenizer.

use crate::core::config::*;

pub struct Gpt2Config;
impl PretokConfig for Gpt2Config {
    const CONTRACTION_CASE: ContractionCase = ContractionCase::Sensitive;
    const CONTRACTION_MODE: ContractionMode = ContractionMode::Standalone;
    const DIGIT_MODE: DigitMode = DigitMode::Unlimited;
    const LETTER_MODE: LetterMode = LetterMode::Plain;
    const SPACE_PREFIXES_DIGITS: bool = true;
    const WS_PATTERN: WsPattern = WsPattern::Gpt2;
    const PUNCT_TRAILING: PunctTrailing = PunctTrailing::None;
    const WS_EXCEPTION: WsException = WsException::None;
    const PUNCT_PREFIX_MODE: PunctPrefixMode = PunctPrefixMode::SpaceOnly;
}

pub struct Cl100kConfig;
impl PretokConfig for Cl100kConfig {
    const CONTRACTION_CASE: ContractionCase = ContractionCase::Insensitive;
    const CONTRACTION_MODE: ContractionMode = ContractionMode::Standalone;
    const DIGIT_MODE: DigitMode = DigitMode::Chunked3;
    const LETTER_MODE: LetterMode = LetterMode::Plain;
    const SPACE_PREFIXES_DIGITS: bool = false;
    const WS_PATTERN: WsPattern = WsPattern::Cl100k;
    const PUNCT_TRAILING: PunctTrailing = PunctTrailing::Newlines;
    const WS_EXCEPTION: WsException = WsException::None;
    const PUNCT_PREFIX_MODE: PunctPrefixMode = PunctPrefixMode::Any;
}

pub struct O200kConfig;
impl PretokConfig for O200kConfig {
    const CONTRACTION_CASE: ContractionCase = ContractionCase::Insensitive;
    const CONTRACTION_MODE: ContractionMode = ContractionMode::Suffix;
    const DIGIT_MODE: DigitMode = DigitMode::Chunked3;
    const LETTER_MODE: LetterMode = LetterMode::CamelCase;
    const SPACE_PREFIXES_DIGITS: bool = false;
    const WS_PATTERN: WsPattern = WsPattern::Cl100k;
    const PUNCT_TRAILING: PunctTrailing = PunctTrailing::NewlinesAndSlashes;
    const WS_EXCEPTION: WsException = WsException::None;
    const PUNCT_PREFIX_MODE: PunctPrefixMode = PunctPrefixMode::Any;
}

pub struct VoyageConfig;
impl PretokConfig for VoyageConfig {
    const CONTRACTION_CASE: ContractionCase = ContractionCase::Insensitive;
    const CONTRACTION_MODE: ContractionMode = ContractionMode::Standalone;
    const DIGIT_MODE: DigitMode = DigitMode::Single;
    const LETTER_MODE: LetterMode = LetterMode::Plain;
    const SPACE_PREFIXES_DIGITS: bool = false;
    const WS_PATTERN: WsPattern = WsPattern::Cl100k;
    const PUNCT_TRAILING: PunctTrailing = PunctTrailing::Newlines;
    const WS_EXCEPTION: WsException = WsException::None;
    const PUNCT_PREFIX_MODE: PunctPrefixMode = PunctPrefixMode::Any;
}

pub struct SmolLMConfig;
impl PretokConfig for SmolLMConfig {
    const CONTRACTION_CASE: ContractionCase = ContractionCase::Sensitive;
    const CONTRACTION_MODE: ContractionMode = ContractionMode::Standalone;
    const DIGIT_MODE: DigitMode = DigitMode::Single;
    const LETTER_MODE: LetterMode = LetterMode::Plain;
    const SPACE_PREFIXES_DIGITS: bool = false;
    const WS_PATTERN: WsPattern = WsPattern::Gpt2;
    const PUNCT_TRAILING: PunctTrailing = PunctTrailing::None;
    const WS_EXCEPTION: WsException = WsException::Digits;
    const PUNCT_PREFIX_MODE: PunctPrefixMode = PunctPrefixMode::SpaceOnly;
}

pub struct DeepSeekConfig;
impl PretokConfig for DeepSeekConfig {
    const CONTRACTION_CASE: ContractionCase = ContractionCase::Insensitive;
    const CONTRACTION_MODE: ContractionMode = ContractionMode::Standalone;
    const DIGIT_MODE: DigitMode = DigitMode::Chunked3;
    const LETTER_MODE: LetterMode = LetterMode::PlainWithMarks;
    const SPACE_PREFIXES_DIGITS: bool = false;
    const WS_PATTERN: WsPattern = WsPattern::Cl100k;
    const PUNCT_TRAILING: PunctTrailing = PunctTrailing::Newlines;
    const WS_EXCEPTION: WsException = WsException::Cjk;
    const PUNCT_PREFIX_MODE: PunctPrefixMode = PunctPrefixMode::AsciiOnly;
}

pub struct QwenConfig;
impl PretokConfig for QwenConfig {
    const CONTRACTION_CASE: ContractionCase = ContractionCase::Insensitive;
    const CONTRACTION_MODE: ContractionMode = ContractionMode::Standalone;
    const DIGIT_MODE: DigitMode = DigitMode::Single;
    const LETTER_MODE: LetterMode = LetterMode::PlainWithMarks;
    const SPACE_PREFIXES_DIGITS: bool = false;
    const WS_PATTERN: WsPattern = WsPattern::Cl100k;
    const PUNCT_TRAILING: PunctTrailing = PunctTrailing::Newlines;
    const WS_EXCEPTION: WsException = WsException::None;
    const PUNCT_PREFIX_MODE: PunctPrefixMode = PunctPrefixMode::Any;
}