pub const PAD: u32 = 0;
pub const BOS: u32 = 1;
pub const EOS: u32 = 2;
pub const FIRST_BYTE_ID: u32 = 3;
pub const BYTE_VOCAB_SIZE: usize = 256;
pub trait Tokenizer: Send + Sync {
fn encode(&self, text: &str) -> Vec<u32>;
fn decode(&self, tokens: &[u32]) -> String;
fn eos_token_id(&self) -> u32;
fn bos_token_id(&self) -> Option<u32>;
fn vocab_size(&self) -> usize;
fn name(&self) -> &'static str;
fn is_byte_level(&self) -> bool {
false
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct ByteTokenizer;
impl ByteTokenizer {
pub fn new() -> Self {
ByteTokenizer
}
}
impl Tokenizer for ByteTokenizer {
fn encode(&self, text: &str) -> Vec<u32> {
text.as_bytes()
.iter()
.map(|&b| u32::from(b) + FIRST_BYTE_ID)
.collect()
}
fn decode(&self, tokens: &[u32]) -> String {
let bytes: Vec<u8> = tokens
.iter()
.filter(|&&t| t >= FIRST_BYTE_ID && t < FIRST_BYTE_ID + BYTE_VOCAB_SIZE as u32)
.map(|&t| (t - FIRST_BYTE_ID) as u8)
.collect();
String::from_utf8_lossy(&bytes).into_owned()
}
fn eos_token_id(&self) -> u32 {
EOS
}
fn bos_token_id(&self) -> Option<u32> {
Some(BOS)
}
fn vocab_size(&self) -> usize {
3 + BYTE_VOCAB_SIZE
}
fn name(&self) -> &'static str {
"byte"
}
fn is_byte_level(&self) -> bool {
true
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn byte_tokenizer_roundtrip() {
let tk = ByteTokenizer::new();
for text in ["", "hello", "hello world", "中文 文本 ✓", "What is 1+1?"] {
let tokens = tk.encode(text);
assert_eq!(tk.decode(&tokens), text);
}
}
#[test]
fn byte_tokenizer_specials() {
let tk = ByteTokenizer::new();
assert_eq!(tk.eos_token_id(), EOS);
assert_eq!(tk.bos_token_id(), Some(BOS));
assert_eq!(tk.vocab_size(), 259);
assert_eq!(tk.encode("A"), vec![68]); }
}