mako 0.3.0

Mako is the main Sidekick AI data processing library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

pub mod bert;
pub mod byte_level;
pub mod delimiter;
pub mod digits;
pub mod metaspace;
pub mod punctuation;
pub mod sequence;
pub mod split;
pub mod unicode_scripts;
pub mod whitespace;

use serde::{Deserialize, Serialize};

use crate::tokenization::hf_tokenizers::pre_tokenizers::bert::BertPreTokenizer;
use crate::tokenization::hf_tokenizers::pre_tokenizers::byte_level::ByteLevel;
use crate::tokenization::hf_tokenizers::pre_tokenizers::delimiter::CharDelimiterSplit;
use crate::tokenization::hf_tokenizers::pre_tokenizers::digits::Digits;
use crate::tokenization::hf_tokenizers::pre_tokenizers::metaspace::Metaspace;
use crate::tokenization::hf_tokenizers::pre_tokenizers::punctuation::Punctuation;
use crate::tokenization::hf_tokenizers::pre_tokenizers::sequence::Sequence;
use crate::tokenization::hf_tokenizers::pre_tokenizers::split::Split;
use crate::tokenization::hf_tokenizers::pre_tokenizers::unicode_scripts::UnicodeScripts;
use crate::tokenization::hf_tokenizers::pre_tokenizers::whitespace::{Whitespace, WhitespaceSplit};
use crate::tokenization::hf_tokenizers::{PreTokenizedString, PreTokenizer};

#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(untagged)]
pub enum PreTokenizerWrapper {
    BertPreTokenizer(BertPreTokenizer),
    ByteLevel(ByteLevel),
    Delimiter(CharDelimiterSplit),
    Metaspace(Metaspace),
    Whitespace(Whitespace),
    Sequence(Sequence),
    Split(Split),
    Punctuation(Punctuation),
    WhitespaceSplit(WhitespaceSplit),
    Digits(Digits),
    UnicodeScripts(UnicodeScripts),
}

impl PreTokenizer for PreTokenizerWrapper {
    fn pre_tokenize(&self, normalized: &mut PreTokenizedString) -> crate::tokenization::hf_tokenizers::Result<()> {
        match self {
            PreTokenizerWrapper::BertPreTokenizer(bpt) => bpt.pre_tokenize(normalized),
            PreTokenizerWrapper::ByteLevel(bpt) => bpt.pre_tokenize(normalized),
            PreTokenizerWrapper::Delimiter(dpt) => dpt.pre_tokenize(normalized),
            PreTokenizerWrapper::Metaspace(mspt) => mspt.pre_tokenize(normalized),
            PreTokenizerWrapper::Whitespace(wspt) => wspt.pre_tokenize(normalized),
            PreTokenizerWrapper::Punctuation(tok) => tok.pre_tokenize(normalized),
            PreTokenizerWrapper::Sequence(tok) => tok.pre_tokenize(normalized),
            PreTokenizerWrapper::Split(tok) => tok.pre_tokenize(normalized),
            PreTokenizerWrapper::WhitespaceSplit(wspt) => wspt.pre_tokenize(normalized),
            PreTokenizerWrapper::Digits(wspt) => wspt.pre_tokenize(normalized),
            PreTokenizerWrapper::UnicodeScripts(us) => us.pre_tokenize(normalized),
        }
    }
}

impl_enum_from!(BertPreTokenizer, PreTokenizerWrapper, BertPreTokenizer);
impl_enum_from!(ByteLevel, PreTokenizerWrapper, ByteLevel);
impl_enum_from!(CharDelimiterSplit, PreTokenizerWrapper, Delimiter);
impl_enum_from!(Whitespace, PreTokenizerWrapper, Whitespace);
impl_enum_from!(Punctuation, PreTokenizerWrapper, Punctuation);
impl_enum_from!(Sequence, PreTokenizerWrapper, Sequence);
impl_enum_from!(Split, PreTokenizerWrapper, Split);
impl_enum_from!(Metaspace, PreTokenizerWrapper, Metaspace);
impl_enum_from!(WhitespaceSplit, PreTokenizerWrapper, WhitespaceSplit);
impl_enum_from!(Digits, PreTokenizerWrapper, Digits);
impl_enum_from!(UnicodeScripts, PreTokenizerWrapper, UnicodeScripts);