piecer 2.0.0

Unicode codepoint BPE tokenizer featuring byte fallback and optional byte-level-only tokenization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12

use serde_json::Error;
use thiserror::Error;

#[derive(Error, Debug)]
pub enum TokenizerError {
    #[error("I/O failure")]
    Io(#[from] std::io::Error),
    #[error("Deserialization failure")]
    Deserialization(#[from] Error),
    #[error("Vocabulary version mismatch")]
    Vocabulary(String),
}