unitoken 0.1.1

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

#[macro_use]
extern crate tracing;

pub mod bpe;
pub mod spec;
pub mod pretokenizer;
pub mod traits;
#[cfg(feature = "py")]
pub mod py;

#[derive(thiserror::Error, Debug)]
pub enum MyError {
  #[error("IO error: {0}")]
  Io(#[from] std::io::Error),
  #[error("Regex error: {0}")]
  Regex(#[from] fancy_regex::Error),
  #[error("Json error: {0}")]
  Json(#[from] serde_json::Error),
  #[error("Merge txt error: {0} at line {1}")]
  MergeTxt(&'static str, usize),
  #[error("UTF-8 error: {0}")]
  Utf8(#[from] std::str::Utf8Error),
  #[error("Character not in printable set: {0}")]
  InvalidPrintableChar(char),
  #[error("Character not in printable set: {0}")]
  InvalidPrintableEscape(String),
  #[error("Out of vocabulary: {0}")]
  Oov(String),
  #[error("Out of vocabulary idx: {0}")]
  OovIdx(u64),
  #[error("Out of vocabulary bytes: {0}")]
  OovBytes(String),
  #[error("No more step could be performed")]
  TrainStep,
  #[error("Specification error: {0}")]
  SpecError(String),
  #[error("Bpe builder: {0}")]
  BpeBuilder(String),
}

pub type MyResult<T> = Result<T, MyError>;

#[cfg(test)]
#[allow(dead_code)]
mod tests {
  pub enum TestData {
    /// fixtures/tinystories_sample_5M.txt
    TinyStories5M,
    /// fixtures/_words.TinyStoriesV2-GPT4-train.json
    TinyStroies,
    /// https://huggingface.co/datasets/52AI/TinyStoriesZh
    TinyStoriesZh
  }
}