use crate::profile;
use memmap2::MmapOptions;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use std::cmp::Reverse;
use std::collections::{BinaryHeap, HashMap, HashSet};
use std::error::Error;
use std::fmt;
use std::fs::{self, File};
use std::io::{BufReader, BufWriter, Write};
use std::path::{Path, PathBuf};
pub const BYTE_VOCAB_SIZE: usize = 256;
pub const PAD_ID: usize = 256;
pub const UNK_ID: usize = 257;
pub const BOS_ID: usize = 258;
pub const EOS_ID: usize = 259;
pub const FIRST_MERGE_ID: usize = 260;
const MODEL_VERSION: u32 = 1;
type Pair = (usize, usize);
type MergeCandidate = Reverse<(usize, usize, usize, usize, usize)>;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SpecialTokens {
pub pad: usize,
pub unk: usize,
pub bos: usize,
pub eos: usize,
}
impl Default for SpecialTokens {
fn default() -> Self {
Self {
pad: PAD_ID,
unk: UNK_ID,
bos: BOS_ID,
eos: EOS_ID,
}
}
}
fn default_document_separator() -> String {
"\n\n".to_string()
}
fn default_true() -> bool {
true
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum PreTokenization {
#[default]
Raw,
Documents {
#[serde(default = "default_document_separator")]
separator: String,
},
Whitespace {
#[serde(default = "default_true")]
space: bool,
#[serde(default = "default_true")]
tab: bool,
#[serde(default = "default_true")]
newline: bool,
},
}
impl PreTokenization {
pub fn validate(&self) -> Result<(), TokenizerError> {
match self {
Self::Raw => Ok(()),
Self::Documents { separator } => {
if separator.is_empty() {
return Err(TokenizerError::InvalidModel(
"document separator must not be empty".into(),
));
}
Ok(())
}
Self::Whitespace {
space,
tab,
newline,
} => {
if !(*space || *tab || *newline) {
return Err(TokenizerError::InvalidModel(
"whitespace pre-tokenization requires at least one selected class".into(),
));
}
Ok(())
}
}
}
pub fn split<'a>(&self, bytes: &'a [u8]) -> Vec<&'a [u8]> {
match self {
Self::Raw => {
if bytes.is_empty() {
Vec::new()
} else {
vec![bytes]
}
}
Self::Documents { separator } => split_documents(bytes, separator.as_bytes()),
Self::Whitespace {
space,
tab,
newline,
} => split_whitespace_classes(bytes, *space, *tab, *newline),
}
}
}
#[derive(Clone, Debug)]
pub struct BpeTokenizer {
pub vocab: Vec<Vec<u8>>,
pub encoder: HashMap<Vec<u8>, usize>,
pub merges: HashMap<Pair, usize>,
pub merge_ranks: HashMap<Pair, usize>,
pub special_tokens: SpecialTokens,
pub pre_tokenization: PreTokenization,
}
#[derive(Debug)]
pub enum TokenizerError {
Io(std::io::Error),
Json(serde_json::Error),
InvalidVocabSize { requested: usize, minimum: usize },
InvalidModel(String),
}
impl fmt::Display for TokenizerError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Io(error) => write!(formatter, "tokenizer I/O error: {error}"),
Self::Json(error) => write!(formatter, "tokenizer JSON error: {error}"),
Self::InvalidVocabSize { requested, minimum } => write!(
formatter,
"vocabulary size {requested} is too small; minimum is {minimum}"
),
Self::InvalidModel(message) => write!(formatter, "invalid tokenizer model: {message}"),
}
}
}
impl Error for TokenizerError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Io(error) => Some(error),
Self::Json(error) => Some(error),
Self::InvalidVocabSize { .. } | Self::InvalidModel(_) => None,
}
}
}
impl From<std::io::Error> for TokenizerError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
impl From<serde_json::Error> for TokenizerError {
fn from(error: serde_json::Error) -> Self {
Self::Json(error)
}
}
#[derive(Serialize, Deserialize)]
struct PersistedTokenizer {
version: u32,
vocab: Vec<PersistedToken>,
merges: Vec<PersistedMerge>,
special_tokens: SpecialTokens,
#[serde(default)]
pre_tokenization: PreTokenization,
}
#[derive(Serialize, Deserialize)]
struct PersistedToken {
id: usize,
bytes: Vec<u8>,
}
#[derive(Serialize, Deserialize)]
struct PersistedMerge {
left: usize,
right: usize,
token_id: usize,
rank: usize,
}
#[derive(Deserialize)]
struct HfTokenizerFile {
model: HfBpeModel,
#[serde(default)]
added_tokens: Vec<HfAddedToken>,
#[serde(default)]
decoder: Option<serde_json::Value>,
}
#[derive(Deserialize)]
struct HfBpeModel {
#[serde(rename = "type")]
model_type: String,
vocab: HashMap<String, usize>,
merges: serde_json::Value,
#[serde(default)]
byte_fallback: Option<bool>,
#[serde(default)]
continuing_subword_prefix: Option<String>,
#[serde(default)]
end_of_word_suffix: Option<String>,
}
#[derive(Clone, Deserialize)]
struct HfAddedToken {
content: String,
#[serde(default)]
special: bool,
}
impl Default for BpeTokenizer {
fn default() -> Self {
Self::new()
}
}
impl BpeTokenizer {
pub fn new() -> Self {
let special_tokens = SpecialTokens::default();
let mut vocab = Vec::with_capacity(FIRST_MERGE_ID);
let mut encoder = HashMap::with_capacity(BYTE_VOCAB_SIZE);
for byte in 0..BYTE_VOCAB_SIZE {
let bytes = vec![byte as u8];
vocab.push(bytes.clone());
encoder.insert(bytes, byte);
}
vocab.push(b"<PAD>".to_vec());
vocab.push(b"<UNK>".to_vec());
vocab.push(b"<BOS>".to_vec());
vocab.push(b"<EOS>".to_vec());
encoder.insert(b"<PAD>".to_vec(), special_tokens.pad);
encoder.insert(b"<UNK>".to_vec(), special_tokens.unk);
encoder.insert(b"<BOS>".to_vec(), special_tokens.bos);
encoder.insert(b"<EOS>".to_vec(), special_tokens.eos);
Self {
vocab,
encoder,
merges: HashMap::new(),
merge_ranks: HashMap::new(),
special_tokens,
pre_tokenization: PreTokenization::Raw,
}
}
pub fn train_from_file(
path: impl AsRef<Path>,
target_vocab_size: usize,
) -> Result<Self, TokenizerError> {
Self::train_from_file_with_options(path, target_vocab_size, 1, PreTokenization::Raw)
}
pub fn train_from_file_with_min_frequency(
path: impl AsRef<Path>,
target_vocab_size: usize,
minimum_frequency: usize,
) -> Result<Self, TokenizerError> {
Self::train_from_file_with_options(
path,
target_vocab_size,
minimum_frequency,
PreTokenization::Raw,
)
}
pub fn train_from_file_with_options(
path: impl AsRef<Path>,
target_vocab_size: usize,
minimum_frequency: usize,
pre_tokenization: PreTokenization,
) -> Result<Self, TokenizerError> {
Self::train_from_shards_with_options(
std::iter::once(path.as_ref()),
target_vocab_size,
minimum_frequency,
pre_tokenization,
)
}
pub fn train_from_shards<P>(
paths: &[P],
target_vocab_size: usize,
) -> Result<Self, TokenizerError>
where
P: AsRef<Path>,
{
Self::train_from_shards_with_options(paths, target_vocab_size, 1, PreTokenization::Raw)
}
pub fn train_from_shards_with_min_frequency<P>(
paths: &[P],
target_vocab_size: usize,
minimum_frequency: usize,
) -> Result<Self, TokenizerError>
where
P: AsRef<Path>,
{
Self::train_from_shards_with_options(
paths,
target_vocab_size,
minimum_frequency,
PreTokenization::Raw,
)
}
pub fn train_from_shards_with_options<I, P>(
paths: I,
target_vocab_size: usize,
minimum_frequency: usize,
pre_tokenization: PreTokenization,
) -> Result<Self, TokenizerError>
where
I: IntoIterator<Item = P>,
P: AsRef<Path>,
{
let mut mapped = Vec::new();
for path in paths {
mapped.push(MappedTextShard::open(path.as_ref())?);
}
let shards: Vec<&[u8]> = mapped.iter().map(MappedTextShard::bytes).collect();
Self::train_from_byte_shards_with_options(
&shards,
target_vocab_size,
minimum_frequency,
pre_tokenization,
)
}
pub fn train_from_shard_directory(
directory: impl AsRef<Path>,
target_vocab_size: usize,
) -> Result<Self, TokenizerError> {
Self::train_from_shard_directory_with_options(
directory,
target_vocab_size,
1,
PreTokenization::Raw,
)
}
pub fn train_from_shard_directory_with_options(
directory: impl AsRef<Path>,
target_vocab_size: usize,
minimum_frequency: usize,
pre_tokenization: PreTokenization,
) -> Result<Self, TokenizerError> {
let paths = list_shard_paths(directory.as_ref())?;
Self::train_from_shards_with_options(
&paths,
target_vocab_size,
minimum_frequency,
pre_tokenization,
)
}
pub fn train_from_bytes(
bytes: &[u8],
target_vocab_size: usize,
) -> Result<Self, TokenizerError> {
Self::train_from_bytes_with_options(bytes, target_vocab_size, 1, PreTokenization::Raw)
}
pub fn train_from_bytes_with_min_frequency(
bytes: &[u8],
target_vocab_size: usize,
minimum_frequency: usize,
) -> Result<Self, TokenizerError> {
Self::train_from_bytes_with_options(
bytes,
target_vocab_size,
minimum_frequency,
PreTokenization::Raw,
)
}
pub fn train_from_bytes_with_options(
bytes: &[u8],
target_vocab_size: usize,
minimum_frequency: usize,
pre_tokenization: PreTokenization,
) -> Result<Self, TokenizerError> {
Self::train_from_byte_shards_with_options(
&[bytes],
target_vocab_size,
minimum_frequency,
pre_tokenization,
)
}
pub fn train_from_byte_shards_with_options(
shards: &[&[u8]],
target_vocab_size: usize,
minimum_frequency: usize,
pre_tokenization: PreTokenization,
) -> Result<Self, TokenizerError> {
let _span = profile::span("tokenizer.train");
pre_tokenization.validate()?;
if target_vocab_size < FIRST_MERGE_ID {
return Err(TokenizerError::InvalidVocabSize {
requested: target_vocab_size,
minimum: FIRST_MERGE_ID,
});
}
let mut tokenizer = Self::new();
tokenizer.pre_tokenization = pre_tokenization;
let segments: Vec<&[u8]> = shards
.iter()
.flat_map(|shard| tokenizer.pre_tokenization.split(shard))
.collect();
let token_segments: Vec<Vec<usize>> = segments
.par_iter()
.map(|segment| segment.iter().map(|byte| *byte as usize).collect())
.collect();
let mut next_token_id = FIRST_MERGE_ID;
let mut state = IncrementalPairState::from_segments(token_segments);
let min_frequency = minimum_frequency.max(1);
while tokenizer.vocab.len() < target_vocab_size {
let Some((best_pair, count)) = state.most_frequent_pair() else {
break;
};
if count < min_frequency {
break;
}
let merged_bytes = {
let left = tokenizer.vocab.get(best_pair.0).ok_or_else(|| {
TokenizerError::InvalidModel("training pair has unknown left token".into())
})?;
let right = tokenizer.vocab.get(best_pair.1).ok_or_else(|| {
TokenizerError::InvalidModel("training pair has unknown right token".into())
})?;
let mut merged = Vec::with_capacity(left.len() + right.len());
merged.extend_from_slice(left);
merged.extend_from_slice(right);
merged
};
let merged_token_id = if let Some(existing_id) = tokenizer.encoder.get(&merged_bytes) {
*existing_id
} else {
let token_id = next_token_id;
next_token_id += 1;
push_vocab_token(&mut tokenizer, token_id, merged_bytes.clone())?;
tokenizer.encoder.insert(merged_bytes, token_id);
token_id
};
let rank = tokenizer.merges.len();
tokenizer.merges.insert(best_pair, merged_token_id);
tokenizer.merge_ranks.insert(best_pair, rank);
state.merge_pair(best_pair, merged_token_id);
}
Ok(tokenizer)
}
pub fn vocab_size(&self) -> usize {
self.vocab.len()
}
pub fn encode(&self, text: &str) -> Vec<usize> {
self.encode_bytes(text.as_bytes())
}
pub fn encode_bytes(&self, bytes: &[u8]) -> Vec<usize> {
let _span = profile::span("tokenizer.encode");
let segments = self.pre_tokenization.split(bytes);
if segments.is_empty() {
return Vec::new();
}
if segments.len() == 1 {
return self.encode_contiguous(segments[0]);
}
let mut encoded = Vec::with_capacity(bytes.len());
for segment in segments {
encoded.extend(self.encode_contiguous(segment));
}
encoded
}
fn encode_contiguous(&self, bytes: &[u8]) -> Vec<usize> {
if bytes.is_empty() {
return Vec::new();
}
let mut tokens: Vec<usize> = bytes.iter().map(|byte| *byte as usize).collect();
if tokens.len() == 1 || self.merges.is_empty() {
return tokens;
}
let length = tokens.len();
let mut previous: Vec<Option<usize>> =
(0..length).map(|index| index.checked_sub(1)).collect();
let mut next: Vec<Option<usize>> = (0..length)
.map(|index| (index + 1 < length).then_some(index + 1))
.collect();
let mut generation = vec![0usize; length];
let mut alive = vec![true; length];
let mut heap = BinaryHeap::new();
for left_index in 0..length - 1 {
push_candidate(
left_index,
&tokens,
&next,
&generation,
&self.merges,
&self.merge_ranks,
&mut heap,
);
}
while let Some(Reverse((rank, left, left_generation, right, right_generation))) = heap.pop()
{
if !alive[left]
|| !alive[right]
|| generation[left] != left_generation
|| generation[right] != right_generation
|| next[left] != Some(right)
{
continue;
}
let pair = (tokens[left], tokens[right]);
if self.merge_ranks.get(&pair) != Some(&rank) {
continue;
}
let Some(&merged_token) = self.merges.get(&pair) else {
continue;
};
tokens[left] = merged_token;
generation[left] += 1;
alive[right] = false;
generation[right] += 1;
let right_neighbor = next[right];
next[left] = right_neighbor;
if let Some(neighbor) = right_neighbor {
previous[neighbor] = Some(left);
}
if let Some(left_neighbor) = previous[left] {
push_candidate(
left_neighbor,
&tokens,
&next,
&generation,
&self.merges,
&self.merge_ranks,
&mut heap,
);
}
push_candidate(
left,
&tokens,
&next,
&generation,
&self.merges,
&self.merge_ranks,
&mut heap,
);
}
let mut encoded = Vec::with_capacity(length);
let mut current = Some(0usize);
while let Some(index) = current {
if alive[index] {
encoded.push(tokens[index]);
}
current = next[index];
}
encoded
}
pub fn encode_batch(&self, texts: &[&str]) -> Vec<Vec<usize>> {
texts.par_iter().map(|text| self.encode(text)).collect()
}
pub fn encode_with_special_tokens(
&self,
text: &str,
add_bos: bool,
add_eos: bool,
) -> Vec<usize> {
let mut output =
Vec::with_capacity(text.len() + usize::from(add_bos) + usize::from(add_eos));
if add_bos {
output.push(self.special_tokens.bos);
}
output.extend(self.encode(text));
if add_eos {
output.push(self.special_tokens.eos);
}
output
}
pub fn decode_bytes(&self, token_ids: &[usize]) -> Vec<u8> {
let _span = profile::span("tokenizer.decode");
let estimated_size = token_ids
.iter()
.filter_map(|&token_id| self.vocab.get(token_id))
.map(Vec::len)
.sum();
let mut bytes = Vec::with_capacity(estimated_size);
let unknown = self
.vocab
.get(self.special_tokens.unk)
.map(Vec::as_slice)
.unwrap_or(b"<UNK>");
for &token_id in token_ids {
bytes.extend_from_slice(
self.vocab
.get(token_id)
.map(Vec::as_slice)
.unwrap_or(unknown),
);
}
bytes
}
pub fn decode(&self, token_ids: &[usize]) -> String {
String::from_utf8_lossy(&self.decode_bytes(token_ids)).into_owned()
}
pub fn token_bytes(&self, token_id: usize) -> Option<&[u8]> {
self.vocab.get(token_id).map(Vec::as_slice)
}
pub fn save(&self, path: impl AsRef<Path>) -> Result<(), TokenizerError> {
let bytes = self.to_json_bytes()?;
let mut writer = BufWriter::new(File::create(path)?);
writer.write_all(&bytes)?;
Ok(())
}
pub fn to_json_bytes(&self) -> Result<Vec<u8>, TokenizerError> {
let vocab: Vec<PersistedToken> = self
.vocab
.iter()
.enumerate()
.map(|(id, bytes)| PersistedToken {
id,
bytes: bytes.clone(),
})
.collect();
let mut merges = Vec::with_capacity(self.merges.len());
for (&(left, right), &token_id) in &self.merges {
let rank = *self.merge_ranks.get(&(left, right)).ok_or_else(|| {
TokenizerError::InvalidModel(format!(
"merge ({left}, {right}) does not have a priority rank"
))
})?;
merges.push(PersistedMerge {
left,
right,
token_id,
rank,
});
}
merges.sort_unstable_by_key(|merge| merge.rank);
let model = PersistedTokenizer {
version: MODEL_VERSION,
vocab,
merges,
special_tokens: self.special_tokens.clone(),
pre_tokenization: self.pre_tokenization.clone(),
};
Ok(serde_json::to_vec(&model)?)
}
pub fn load(path: impl AsRef<Path>) -> Result<Self, TokenizerError> {
let reader = BufReader::new(File::open(path)?);
let model: PersistedTokenizer = serde_json::from_reader(reader)?;
Self::from_persisted(model)
}
pub fn from_json_bytes(bytes: &[u8]) -> Result<Self, TokenizerError> {
let model: PersistedTokenizer = serde_json::from_slice(bytes)?;
Self::from_persisted(model)
}
pub fn import_gpt2(
vocab_path: impl AsRef<Path>,
merges_path: impl AsRef<Path>,
) -> Result<Self, TokenizerError> {
let vocab = read_gpt2_vocab(vocab_path.as_ref())?;
let merges = read_gpt2_merges_file(merges_path.as_ref())?;
Self::from_gpt2_vocab_and_merges(&vocab, &merges)
}
pub fn from_gpt2_vocab_and_merges(
vocab: &HashMap<String, usize>,
merges: &[(String, String)],
) -> Result<Self, TokenizerError> {
from_byte_level_bpe(vocab, merges, &[])
}
pub fn import_huggingface(path: impl AsRef<Path>) -> Result<Self, TokenizerError> {
let reader = BufReader::new(File::open(path)?);
let file: HfTokenizerFile = serde_json::from_reader(reader)?;
Self::from_huggingface_file(file)
}
fn from_huggingface_file(file: HfTokenizerFile) -> Result<Self, TokenizerError> {
if file.model.model_type != "BPE" {
return Err(TokenizerError::InvalidModel(format!(
"unsupported Hugging Face model type '{}'; only BPE is supported",
file.model.model_type
)));
}
if file.model.byte_fallback == Some(true) {
return Err(TokenizerError::InvalidModel(
"Hugging Face BPE with byte_fallback is not supported".into(),
));
}
if let Some(prefix) = file.model.continuing_subword_prefix.as_deref() {
if !prefix.is_empty() {
return Err(TokenizerError::InvalidModel(
"Hugging Face BPE continuing_subword_prefix is not supported".into(),
));
}
}
if let Some(suffix) = file.model.end_of_word_suffix.as_deref() {
if !suffix.is_empty() {
return Err(TokenizerError::InvalidModel(
"Hugging Face BPE end_of_word_suffix is not supported".into(),
));
}
}
if let Some(decoder) = &file.decoder {
if let Some(decoder_type) = decoder.get("type").and_then(|value| value.as_str()) {
if decoder_type != "ByteLevel" {
return Err(TokenizerError::InvalidModel(format!(
"unsupported Hugging Face decoder type '{decoder_type}'; only ByteLevel is supported"
)));
}
}
}
let merges = parse_hf_merges(&file.model.merges)?;
let added: Vec<HfAddedToken> = file
.added_tokens
.into_iter()
.filter(|token| token.special)
.collect();
from_byte_level_bpe(&file.model.vocab, &merges, &added)
}
fn from_persisted(model: PersistedTokenizer) -> Result<Self, TokenizerError> {
if model.version != MODEL_VERSION {
return Err(TokenizerError::InvalidModel(format!(
"unsupported model version {}",
model.version
)));
}
let special_ids = [
model.special_tokens.pad,
model.special_tokens.unk,
model.special_tokens.bos,
model.special_tokens.eos,
];
let unique_special_ids: HashSet<usize> = special_ids.into_iter().collect();
if unique_special_ids.len() != 4 {
return Err(TokenizerError::InvalidModel(
"special token IDs must be distinct".into(),
));
}
let mut entries = model.vocab;
entries.sort_unstable_by_key(|token| token.id);
let mut vocab = Vec::with_capacity(entries.len());
for (expected_id, token) in entries.into_iter().enumerate() {
if token.id != expected_id {
return Err(TokenizerError::InvalidModel(format!(
"vocabulary IDs must be contiguous from 0; expected {expected_id}, found {}",
token.id
)));
}
vocab.push(token.bytes);
}
validate_base_vocab(&vocab)?;
for special_id in special_ids {
if special_id >= vocab.len() {
return Err(TokenizerError::InvalidModel(format!(
"missing special token ID {special_id}"
)));
}
}
let mut encoder = HashMap::with_capacity(vocab.len());
for (token_id, bytes) in vocab.iter().enumerate() {
if encoder.insert(bytes.clone(), token_id).is_some() {
return Err(TokenizerError::InvalidModel(
"vocabulary contains duplicate byte sequences".into(),
));
}
}
let mut merges = HashMap::with_capacity(model.merges.len());
let mut merge_ranks = HashMap::with_capacity(model.merges.len());
let mut seen_ranks = HashSet::with_capacity(model.merges.len());
for merge in model.merges {
let pair = (merge.left, merge.right);
if merges.insert(pair, merge.token_id).is_some() {
return Err(TokenizerError::InvalidModel(format!(
"duplicate merge pair {pair:?}"
)));
}
if !seen_ranks.insert(merge.rank) {
return Err(TokenizerError::InvalidModel(format!(
"duplicate merge rank {}",
merge.rank
)));
}
merge_ranks.insert(pair, merge.rank);
let left = vocab.get(merge.left).ok_or_else(|| {
TokenizerError::InvalidModel(format!(
"merge references missing token {}",
merge.left
))
})?;
let right = vocab.get(merge.right).ok_or_else(|| {
TokenizerError::InvalidModel(format!(
"merge references missing token {}",
merge.right
))
})?;
let output = vocab.get(merge.token_id).ok_or_else(|| {
TokenizerError::InvalidModel(format!(
"merge output token {} is missing",
merge.token_id
))
})?;
if output.len() != left.len() + right.len()
|| !output.starts_with(left)
|| !output.ends_with(right)
{
return Err(TokenizerError::InvalidModel(format!(
"merge {pair:?} has inconsistent output bytes"
)));
}
}
model.pre_tokenization.validate()?;
Ok(Self {
vocab,
encoder,
merges,
merge_ranks,
special_tokens: model.special_tokens,
pre_tokenization: model.pre_tokenization,
})
}
}
struct MappedTextShard {
_file: File,
mapping: Option<memmap2::Mmap>,
}
impl MappedTextShard {
fn open(path: &Path) -> Result<Self, TokenizerError> {
let file = File::open(path)?;
if file.metadata()?.len() == 0 {
return Ok(Self {
_file: file,
mapping: None,
});
}
let mapping = unsafe { MmapOptions::new().map(&file)? };
Ok(Self {
_file: file,
mapping: Some(mapping),
})
}
fn bytes(&self) -> &[u8] {
self.mapping.as_deref().unwrap_or(&[])
}
}
fn list_shard_paths(directory: &Path) -> Result<Vec<PathBuf>, TokenizerError> {
let mut paths = Vec::new();
for entry in fs::read_dir(directory)? {
let entry = entry?;
let path = entry.path();
if entry.file_type()?.is_file() {
paths.push(path);
}
}
paths.sort();
if paths.is_empty() {
return Err(TokenizerError::InvalidModel(format!(
"shard directory {} contains no regular files",
directory.display()
)));
}
Ok(paths)
}
fn split_documents<'a>(bytes: &'a [u8], separator: &[u8]) -> Vec<&'a [u8]> {
if bytes.is_empty() {
return Vec::new();
}
if separator.is_empty() {
return vec![bytes];
}
let mut segments = Vec::new();
let mut start = 0usize;
let mut index = 0usize;
while index + separator.len() <= bytes.len() {
if &bytes[index..index + separator.len()] == separator {
if index > start {
segments.push(&bytes[start..index]);
}
segments.push(&bytes[index..index + separator.len()]);
index += separator.len();
start = index;
} else {
index += 1;
}
}
if start < bytes.len() {
segments.push(&bytes[start..]);
}
segments
}
fn whitespace_class(byte: u8, space: bool, tab: bool, newline: bool) -> Option<u8> {
match byte {
b' ' if space => Some(0),
b'\t' if tab => Some(1),
b'\n' | b'\r' if newline => Some(2),
_ => None,
}
}
fn split_whitespace_classes(
bytes: &[u8],
space: bool,
tab: bool,
newline: bool,
) -> Vec<&[u8]> {
if bytes.is_empty() {
return Vec::new();
}
let mut segments = Vec::new();
let mut start = 0usize;
for index in 0..bytes.len().saturating_sub(1) {
let left = whitespace_class(bytes[index], space, tab, newline);
let right = whitespace_class(bytes[index + 1], space, tab, newline);
if left != right {
segments.push(&bytes[start..=index]);
start = index + 1;
}
}
segments.push(&bytes[start..]);
segments
}
fn validate_base_vocab(vocab: &[Vec<u8>]) -> Result<(), TokenizerError> {
if vocab.len() < BYTE_VOCAB_SIZE {
return Err(TokenizerError::InvalidModel(
"vocabulary is missing base byte tokens".into(),
));
}
for byte in 0..BYTE_VOCAB_SIZE {
if vocab[byte].as_slice() != [byte as u8] {
return Err(TokenizerError::InvalidModel(format!(
"base byte token {byte} is missing or invalid"
)));
}
}
Ok(())
}
fn push_vocab_token(
tokenizer: &mut BpeTokenizer,
token_id: usize,
bytes: Vec<u8>,
) -> Result<(), TokenizerError> {
if token_id != tokenizer.vocab.len() {
return Err(TokenizerError::InvalidModel(format!(
"vocabulary IDs must be contiguous from 0; expected {}, found {token_id}",
tokenizer.vocab.len()
)));
}
tokenizer.vocab.push(bytes);
Ok(())
}
fn replace_vocab_token(
tokenizer: &mut BpeTokenizer,
token_id: usize,
bytes: Vec<u8>,
) -> Result<Vec<u8>, TokenizerError> {
let slot = tokenizer.vocab.get_mut(token_id).ok_or_else(|| {
TokenizerError::InvalidModel(format!("vocabulary is missing token ID {token_id}"))
})?;
Ok(std::mem::replace(slot, bytes))
}
fn gpt2_byte_to_unicode() -> [char; BYTE_VOCAB_SIZE] {
let mut bytes: Vec<u8> = Vec::with_capacity(BYTE_VOCAB_SIZE);
bytes.extend(b'!'..=b'~');
bytes.extend(0xA1..=0xAC);
bytes.extend(0xAE..=0xFF);
let mut codepoints: Vec<u32> = bytes.iter().map(|byte| u32::from(*byte)).collect();
let mut next = 0u32;
for byte in 0..=255u8 {
if !bytes.contains(&byte) {
bytes.push(byte);
codepoints.push(BYTE_VOCAB_SIZE as u32 + next);
next += 1;
}
}
let mut table = ['\0'; BYTE_VOCAB_SIZE];
for (byte, codepoint) in bytes.into_iter().zip(codepoints) {
table[byte as usize] =
char::from_u32(codepoint).expect("GPT-2 bytes_to_unicode code point");
}
table
}
fn gpt2_unicode_to_byte() -> HashMap<char, u8> {
let byte_to_unicode = gpt2_byte_to_unicode();
let mut unicode_to_byte = HashMap::with_capacity(BYTE_VOCAB_SIZE);
for (byte, ch) in byte_to_unicode.iter().enumerate() {
unicode_to_byte.insert(*ch, byte as u8);
}
unicode_to_byte
}
fn gpt2_decode_token(token: &str, unicode_to_byte: &HashMap<char, u8>) -> Option<Vec<u8>> {
let mut bytes = Vec::with_capacity(token.len());
for ch in token.chars() {
bytes.push(*unicode_to_byte.get(&ch)?);
}
Some(bytes)
}
fn read_gpt2_vocab(path: &Path) -> Result<HashMap<String, usize>, TokenizerError> {
let reader = BufReader::new(File::open(path)?);
let vocab: HashMap<String, usize> = serde_json::from_reader(reader)?;
if vocab.is_empty() {
return Err(TokenizerError::InvalidModel(
"GPT-2 vocab.json is empty".into(),
));
}
Ok(vocab)
}
fn read_gpt2_merges_file(path: &Path) -> Result<Vec<(String, String)>, TokenizerError> {
let text = fs::read_to_string(path)?;
let mut merges = Vec::new();
for (index, line) in text.lines().enumerate() {
let line = line.trim_end();
if line.is_empty() {
continue;
}
if index == 0 && line.starts_with('#') {
continue;
}
merges.push(parse_merge_line(line)?);
}
Ok(merges)
}
fn parse_merge_line(line: &str) -> Result<(String, String), TokenizerError> {
let Some((left, right)) = line.split_once(' ') else {
return Err(TokenizerError::InvalidModel(format!(
"merge line must contain two symbols separated by a space: {line:?}"
)));
};
if left.is_empty() || right.is_empty() || right.contains(' ') {
return Err(TokenizerError::InvalidModel(format!(
"invalid merge pair: {line:?}"
)));
}
Ok((left.to_string(), right.to_string()))
}
fn parse_hf_merges(value: &serde_json::Value) -> Result<Vec<(String, String)>, TokenizerError> {
let Some(entries) = value.as_array() else {
return Err(TokenizerError::InvalidModel(
"Hugging Face model.merges must be a JSON array".into(),
));
};
let mut merges = Vec::with_capacity(entries.len());
for entry in entries {
if let Some(line) = entry.as_str() {
merges.push(parse_merge_line(line)?);
continue;
}
let Some(parts) = entry.as_array() else {
return Err(TokenizerError::InvalidModel(
"Hugging Face merge entries must be strings or [left, right] arrays".into(),
));
};
if parts.len() != 2 {
return Err(TokenizerError::InvalidModel(
"Hugging Face merge arrays must have exactly two strings".into(),
));
}
let left = parts[0].as_str().ok_or_else(|| {
TokenizerError::InvalidModel("Hugging Face merge left symbol must be a string".into())
})?;
let right = parts[1].as_str().ok_or_else(|| {
TokenizerError::InvalidModel("Hugging Face merge right symbol must be a string".into())
})?;
if left.is_empty() || right.is_empty() {
return Err(TokenizerError::InvalidModel(
"Hugging Face merge symbols must be non-empty".into(),
));
}
merges.push((left.to_string(), right.to_string()));
}
Ok(merges)
}
fn from_byte_level_bpe(
vocab: &HashMap<String, usize>,
merges: &[(String, String)],
added_tokens: &[HfAddedToken],
) -> Result<BpeTokenizer, TokenizerError> {
let unicode_to_byte = gpt2_unicode_to_byte();
let mut tokenizer = BpeTokenizer::new();
tokenizer.merges.reserve(merges.len());
tokenizer.merge_ranks.reserve(merges.len());
tokenizer.vocab.reserve(merges.len());
tokenizer.encoder.reserve(merges.len());
for (rank, (left_symbol, right_symbol)) in merges.iter().enumerate() {
let left_bytes = gpt2_decode_token(left_symbol, &unicode_to_byte).ok_or_else(|| {
TokenizerError::InvalidModel(format!(
"merge left symbol is not byte-level GPT-2 unicode: {left_symbol:?}"
))
})?;
let right_bytes = gpt2_decode_token(right_symbol, &unicode_to_byte).ok_or_else(|| {
TokenizerError::InvalidModel(format!(
"merge right symbol is not byte-level GPT-2 unicode: {right_symbol:?}"
))
})?;
let left_id = *tokenizer.encoder.get(&left_bytes).ok_or_else(|| {
TokenizerError::InvalidModel(format!(
"merge references unknown left token {left_symbol:?}"
))
})?;
let right_id = *tokenizer.encoder.get(&right_bytes).ok_or_else(|| {
TokenizerError::InvalidModel(format!(
"merge references unknown right token {right_symbol:?}"
))
})?;
let mut merged_bytes = left_bytes;
merged_bytes.extend_from_slice(&right_bytes);
if tokenizer.encoder.contains_key(&merged_bytes) {
return Err(TokenizerError::InvalidModel(format!(
"merge rank {rank} produces a duplicate byte sequence"
)));
}
let token_id = FIRST_MERGE_ID + rank;
push_vocab_token(&mut tokenizer, token_id, merged_bytes.clone())?;
tokenizer.encoder.insert(merged_bytes, token_id);
tokenizer.merges.insert((left_id, right_id), token_id);
tokenizer.merge_ranks.insert((left_id, right_id), rank);
}
let mut claimed_specials = HashSet::new();
for token in added_tokens {
apply_special_content(&mut tokenizer, &token.content, &mut claimed_specials)?;
}
let mut vocab_entries: Vec<(&String, &usize)> = vocab.iter().collect();
vocab_entries.sort_unstable_by_key(|(_, id)| *id);
for (symbol, _) in vocab_entries {
let Some(bytes) = gpt2_decode_token(symbol, &unicode_to_byte) else {
apply_special_content(&mut tokenizer, symbol, &mut claimed_specials)?;
continue;
};
if tokenizer.encoder.contains_key(&bytes) {
continue;
}
if classify_special_token(symbol).is_some() {
apply_special_content(&mut tokenizer, symbol, &mut claimed_specials)?;
continue;
}
let token_id = next_vocab_id(&tokenizer);
push_vocab_token(&mut tokenizer, token_id, bytes.clone())?;
tokenizer.encoder.insert(bytes, token_id);
}
validate_base_vocab(&tokenizer.vocab)?;
Ok(tokenizer)
}
fn next_vocab_id(tokenizer: &BpeTokenizer) -> usize {
tokenizer.vocab.len()
}
fn apply_special_content(
tokenizer: &mut BpeTokenizer,
content: &str,
claimed: &mut HashSet<&'static str>,
) -> Result<(), TokenizerError> {
let role = classify_special_token(content);
let token_bytes = content.as_bytes().to_vec();
if let Some(existing_id) = tokenizer.encoder.get(&token_bytes).copied() {
if let Some(role) = role {
assign_special_id(&mut tokenizer.special_tokens, role, existing_id);
}
return Ok(());
}
let Some(role) = role else {
let token_id = next_vocab_id(tokenizer);
push_vocab_token(tokenizer, token_id, token_bytes.clone())?;
tokenizer.encoder.insert(token_bytes, token_id);
return Ok(());
};
if !claimed.insert(role) {
return Ok(());
}
let token_id = special_id(&tokenizer.special_tokens, role);
let previous = replace_vocab_token(tokenizer, token_id, token_bytes.clone())?;
tokenizer.encoder.remove(&previous);
tokenizer.encoder.insert(token_bytes, token_id);
Ok(())
}
fn classify_special_token(content: &str) -> Option<&'static str> {
match content {
"<pad>" | "<PAD>" | "[PAD]" | "<|pad|>" => Some("pad"),
"<unk>" | "<UNK>" | "[UNK]" | "<|unk|>" => Some("unk"),
"<s>" | "<bos>" | "<BOS>" | "[BOS]" | "<|begin_of_text|>" | "<|bos|>" => Some("bos"),
"</s>" | "<eos>" | "<EOS>" | "[EOS]" | "<|endoftext|>" | "<|eot_id|>" | "<|eos|>" => {
Some("eos")
}
_ => None,
}
}
fn special_id(special_tokens: &SpecialTokens, role: &str) -> usize {
match role {
"pad" => special_tokens.pad,
"unk" => special_tokens.unk,
"bos" => special_tokens.bos,
"eos" => special_tokens.eos,
_ => panic!("unknown special token role {role}"),
}
}
fn assign_special_id(special_tokens: &mut SpecialTokens, role: &str, token_id: usize) {
match role {
"pad" => special_tokens.pad = token_id,
"unk" => special_tokens.unk = token_id,
"bos" => special_tokens.bos = token_id,
"eos" => special_tokens.eos = token_id,
_ => panic!("unknown special token role {role}"),
}
}
struct IncrementalPairState {
tokens: Vec<usize>,
previous: Vec<Option<usize>>,
next: Vec<Option<usize>>,
alive: Vec<bool>,
pair_counts: HashMap<Pair, usize>,
pair_occurrences: HashMap<Pair, HashSet<usize>>,
}
impl IncrementalPairState {
fn from_segments(segments: Vec<Vec<usize>>) -> Self {
let total_len = segments.iter().map(Vec::len).sum();
let mut tokens = Vec::with_capacity(total_len);
let mut previous = Vec::with_capacity(total_len);
let mut next = Vec::with_capacity(total_len);
let mut alive = Vec::with_capacity(total_len);
for segment in segments {
if segment.is_empty() {
continue;
}
let base = tokens.len();
let segment_len = segment.len();
for (offset, token) in segment.into_iter().enumerate() {
tokens.push(token);
previous.push(if offset == 0 {
None
} else {
Some(base + offset - 1)
});
next.push(if offset + 1 == segment_len {
None
} else {
Some(base + offset + 1)
});
alive.push(true);
}
}
let mut state = Self {
tokens,
previous,
next,
alive,
pair_counts: HashMap::new(),
pair_occurrences: HashMap::new(),
};
for index in 0..state.tokens.len() {
let Some(right) = state.next[index] else {
continue;
};
let pair = (state.tokens[index], state.tokens[right]);
state.add_occurrence(pair, index);
}
state
}
fn most_frequent_pair(&self) -> Option<(Pair, usize)> {
self.pair_counts
.iter()
.max_by(|(left_pair, left_count), (right_pair, right_count)| {
left_count
.cmp(right_count)
.then_with(|| right_pair.cmp(left_pair))
})
.map(|(&pair, &count)| (pair, count))
}
fn merge_pair(&mut self, pair: Pair, merged_token: usize) {
let Some(occurrences) = self.pair_occurrences.get(&pair) else {
self.pair_counts.remove(&pair);
return;
};
let mut positions: Vec<usize> = occurrences.iter().copied().collect();
positions.sort_unstable();
for left in positions {
let Some(right) = self.next[left] else {
continue;
};
if !self.alive[left]
|| !self.alive[right]
|| (self.tokens[left], self.tokens[right]) != pair
{
continue;
}
let previous = self.previous[left];
let next_after = self.next[right];
if let Some(previous_index) = previous {
let left_neighbor = (self.tokens[previous_index], self.tokens[left]);
self.remove_occurrence(left_neighbor, previous_index);
}
self.remove_occurrence(pair, left);
if let Some(next_index) = next_after {
let right_neighbor = (self.tokens[right], self.tokens[next_index]);
self.remove_occurrence(right_neighbor, right);
}
self.tokens[left] = merged_token;
self.alive[right] = false;
self.next[left] = next_after;
if let Some(next_index) = next_after {
self.previous[next_index] = Some(left);
}
if let Some(previous_index) = previous {
self.add_occurrence((self.tokens[previous_index], merged_token), previous_index);
}
if let Some(next_index) = next_after {
self.add_occurrence((merged_token, self.tokens[next_index]), left);
}
}
}
fn add_occurrence(&mut self, pair: Pair, left: usize) {
*self.pair_counts.entry(pair).or_insert(0) += 1;
self.pair_occurrences.entry(pair).or_default().insert(left);
}
fn remove_occurrence(&mut self, pair: Pair, left: usize) {
let Some(occurrences) = self.pair_occurrences.get_mut(&pair) else {
return;
};
if !occurrences.remove(&left) {
return;
}
if occurrences.is_empty() {
self.pair_occurrences.remove(&pair);
}
self.remove_count(pair);
}
fn remove_count(&mut self, pair: Pair) {
let Some(count) = self.pair_counts.get_mut(&pair) else {
return;
};
*count -= 1;
if *count == 0 {
self.pair_counts.remove(&pair);
}
}
}
#[allow(clippy::too_many_arguments)]
fn push_candidate(
left: usize,
tokens: &[usize],
next: &[Option<usize>],
generation: &[usize],
merges: &HashMap<Pair, usize>,
merge_ranks: &HashMap<Pair, usize>,
heap: &mut BinaryHeap<MergeCandidate>,
) {
let Some(right) = next[left] else {
return;
};
let pair = (tokens[left], tokens[right]);
if !merges.contains_key(&pair) {
return;
}
if let Some(&rank) = merge_ranks.get(&pair) {
heap.push(Reverse((
rank,
left,
generation[left],
right,
generation[right],
)));
}
}
#[cfg(test)]
mod tests {
use super::{
split_documents, split_whitespace_classes, BpeTokenizer, PreTokenization, FIRST_MERGE_ID,
};
use std::collections::HashMap;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
#[test]
fn learns_and_applies_basic_byte_pair_merge() {
let tokenizer = BpeTokenizer::train_from_bytes(b"abababab", FIRST_MERGE_ID + 1).unwrap();
let encoded = tokenizer.encode("abab");
assert_eq!(encoded, vec![FIRST_MERGE_ID, FIRST_MERGE_ID]);
assert_eq!(
tokenizer.token_bytes(FIRST_MERGE_ID),
Some(b"ab".as_slice())
);
assert_eq!(tokenizer.pre_tokenization, PreTokenization::Raw);
}
#[test]
fn incremental_training_merges_overlapping_pairs_left_to_right() {
let tokenizer = BpeTokenizer::train_from_bytes(b"aaaa", FIRST_MERGE_ID + 1).unwrap();
assert_eq!(
tokenizer.merges.get(&(b'a' as usize, b'a' as usize)),
Some(&FIRST_MERGE_ID)
);
assert_eq!(
tokenizer.encode_bytes(b"aaaa"),
vec![FIRST_MERGE_ID, FIRST_MERGE_ID]
);
}
#[test]
fn encode_decode_roundtrip_preserves_utf8() {
let corpus = "hello hello, héllo rust 🦀";
let tokenizer =
BpeTokenizer::train_from_bytes(corpus.as_bytes(), FIRST_MERGE_ID + 12).unwrap();
let prompt = "héllo 🦀";
assert_eq!(tokenizer.decode(&tokenizer.encode(prompt)), prompt);
}
#[test]
fn save_load_roundtrip_preserves_model() {
let tokenizer = BpeTokenizer::train_from_bytes_with_options(
b"low lower lowest low",
FIRST_MERGE_ID + 8,
1,
PreTokenization::Whitespace {
space: true,
tab: false,
newline: false,
},
)
.unwrap();
let path = temporary_model_path();
tokenizer.save(&path).unwrap();
let loaded = BpeTokenizer::load(&path).unwrap();
let prompt = "lowest low";
assert_eq!(loaded.encode(prompt), tokenizer.encode(prompt));
assert_eq!(loaded.decode(&loaded.encode(prompt)), prompt);
assert_eq!(loaded.special_tokens, tokenizer.special_tokens);
assert_eq!(loaded.pre_tokenization, tokenizer.pre_tokenization);
}
#[test]
fn whitespace_pre_tokenization_blocks_cross_space_merges() {
let corpus = b"a b a b a b a b a b a b a b a b";
let tokenizer = BpeTokenizer::train_from_bytes_with_options(
corpus,
FIRST_MERGE_ID + 8,
1,
PreTokenization::Whitespace {
space: true,
tab: false,
newline: false,
},
)
.unwrap();
assert!(tokenizer.merges.is_empty());
assert!(!tokenizer
.merges
.contains_key(&(b'a' as usize, b' ' as usize)));
}
#[test]
fn whitespace_pre_tokenization_still_merges_inside_words() {
let tokenizer = BpeTokenizer::train_from_bytes_with_options(
b"abab abab abab abab",
FIRST_MERGE_ID + 1,
1,
PreTokenization::Whitespace {
space: true,
tab: true,
newline: true,
},
)
.unwrap();
assert_eq!(
tokenizer.token_bytes(FIRST_MERGE_ID),
Some(b"ab".as_slice())
);
let encoded = tokenizer.encode_bytes(b"abab ab");
assert_eq!(
encoded,
vec![FIRST_MERGE_ID, FIRST_MERGE_ID, b' ' as usize, FIRST_MERGE_ID]
);
for token_id in encoded {
let bytes = tokenizer.token_bytes(token_id).unwrap();
let has_space = bytes.contains(&b' ');
let has_non_space = bytes.iter().any(|byte| *byte != b' ');
assert!(!(has_space && has_non_space));
}
}
#[test]
fn document_pre_tokenization_blocks_cross_separator_merges() {
let corpus = b"a\n\na\n\na\n\na\n\na\n\na\n\na\n\na";
let tokenizer = BpeTokenizer::train_from_bytes_with_options(
corpus,
FIRST_MERGE_ID + 4,
1,
PreTokenization::Documents {
separator: "\n\n".into(),
},
)
.unwrap();
assert!(!tokenizer
.merges
.contains_key(&(b'a' as usize, b'\n' as usize)));
assert!(!tokenizer
.merges
.contains_key(&(b'\n' as usize, b'a' as usize)));
assert_eq!(
tokenizer.merges.get(&(b'\n' as usize, b'\n' as usize)),
Some(&FIRST_MERGE_ID)
);
}
#[test]
fn raw_mode_still_allows_cross_whitespace_merges() {
let corpus = b"a b a b a b a b a b a b a b a b";
let tokenizer = BpeTokenizer::train_from_bytes_with_options(
corpus,
FIRST_MERGE_ID + 1,
1,
PreTokenization::Raw,
)
.unwrap();
assert!(!tokenizer.merges.is_empty());
let (left, right) = *tokenizer.merges.keys().next().unwrap();
assert!(left == b' ' as usize || right == b' ' as usize || left == b'a' as usize);
}
#[test]
fn split_helpers_respect_selected_boundaries() {
assert_eq!(
split_documents(b"ab\n\ncd", b"\n\n"),
vec![&b"ab"[..], &b"\n\n"[..], &b"cd"[..]]
);
assert_eq!(
split_whitespace_classes(b"ab \tcd\nef", true, true, true),
vec![
&b"ab"[..],
&b" "[..],
&b"\t"[..],
&b"cd"[..],
&b"\n"[..],
&b"ef"[..]
]
);
assert_eq!(
split_whitespace_classes(b"ab\tcd", true, false, false),
vec![&b"ab\tcd"[..]]
);
}
#[test]
fn byte_shards_never_merge_across_shard_boundaries() {
let concatenated =
BpeTokenizer::train_from_bytes(b"aa", FIRST_MERGE_ID + 1).unwrap();
assert!(concatenated
.merges
.contains_key(&(b'a' as usize, b'a' as usize)));
let sharded = BpeTokenizer::train_from_byte_shards_with_options(
&[b"a", b"a"],
FIRST_MERGE_ID + 1,
1,
PreTokenization::Raw,
)
.unwrap();
assert!(sharded.merges.is_empty());
}
#[test]
fn byte_shards_still_merge_inside_each_shard() {
let tokenizer = BpeTokenizer::train_from_byte_shards_with_options(
&[b"abab", b"abab"],
FIRST_MERGE_ID + 1,
1,
PreTokenization::Raw,
)
.unwrap();
assert_eq!(
tokenizer.token_bytes(FIRST_MERGE_ID),
Some(b"ab".as_slice())
);
assert!(!tokenizer
.merges
.contains_key(&(b'b' as usize, b'a' as usize)));
}
#[test]
fn byte_shards_respect_document_pre_tokenization_inside_shard() {
let tokenizer = BpeTokenizer::train_from_byte_shards_with_options(
&[b"a\n\na", b"a\n\na"],
FIRST_MERGE_ID + 4,
1,
PreTokenization::Documents {
separator: "\n\n".into(),
},
)
.unwrap();
assert!(!tokenizer
.merges
.contains_key(&(b'a' as usize, b'\n' as usize)));
assert!(!tokenizer
.merges
.contains_key(&(b'\n' as usize, b'a' as usize)));
assert_eq!(
tokenizer.merges.get(&(b'\n' as usize, b'\n' as usize)),
Some(&FIRST_MERGE_ID)
);
}
#[test]
fn train_from_shards_memory_maps_without_concatenating() {
let directory = temporary_directory("shards");
let shard_a = directory.join("a.txt");
let shard_b = directory.join("b.txt");
std::fs::write(&shard_a, b"a").unwrap();
std::fs::write(&shard_b, b"a").unwrap();
let tokenizer = BpeTokenizer::train_from_shards(
&[shard_a.as_path(), shard_b.as_path()],
FIRST_MERGE_ID + 1,
)
.unwrap();
assert!(tokenizer.merges.is_empty());
let from_directory = BpeTokenizer::train_from_shard_directory(
&directory,
FIRST_MERGE_ID + 1,
)
.unwrap();
assert!(from_directory.merges.is_empty());
let _ = std::fs::remove_dir_all(&directory);
}
#[test]
fn import_gpt2_vocab_and_merges_applies_byte_level_merge() {
let mut vocab = HashMap::new();
vocab.insert("a".into(), 0);
vocab.insert("b".into(), 1);
vocab.insert("ab".into(), 2);
vocab.insert("<|endoftext|>".into(), 3);
let merges = vec![("a".into(), "b".into())];
let tokenizer = BpeTokenizer::from_gpt2_vocab_and_merges(&vocab, &merges).unwrap();
assert_eq!(
tokenizer.token_bytes(FIRST_MERGE_ID),
Some(b"ab".as_slice())
);
assert_eq!(tokenizer.encode("abab"), vec![FIRST_MERGE_ID, FIRST_MERGE_ID]);
assert_eq!(
tokenizer.token_bytes(tokenizer.special_tokens.eos),
Some(b"<|endoftext|>".as_slice())
);
assert_eq!(tokenizer.decode(&tokenizer.encode("ab")), "ab");
}
#[test]
fn import_gpt2_files_roundtrip_through_native_json() {
let directory = temporary_directory("gpt2_import");
let vocab_path = directory.join("vocab.json");
let merges_path = directory.join("merges.txt");
let native_path = directory.join("tokenizer.json");
std::fs::write(
&vocab_path,
r#"{"a":0,"b":1,"c":2,"ab":3,"abc":4,"<|endoftext|>":5}"#,
)
.unwrap();
std::fs::write(&merges_path, "#version: 0.2\na b\nab c\n").unwrap();
let imported = BpeTokenizer::import_gpt2(&vocab_path, &merges_path).unwrap();
imported.save(&native_path).unwrap();
let loaded = BpeTokenizer::load(&native_path).unwrap();
assert_eq!(loaded.encode("abc"), imported.encode("abc"));
assert_eq!(loaded.decode(&loaded.encode("abc")), "abc");
let _ = std::fs::remove_dir_all(&directory);
}
#[test]
fn import_huggingface_tokenizer_json_subset() {
let directory = temporary_directory("hf_import");
let path = directory.join("tokenizer.json");
std::fs::write(
&path,
r#"{
"model": {
"type": "BPE",
"vocab": {"a": 0, "b": 1, "ab": 2, "<|endoftext|>": 3},
"merges": [["a", "b"]],
"byte_fallback": false,
"continuing_subword_prefix": null,
"end_of_word_suffix": null
},
"added_tokens": [
{
"id": 3,
"content": "<|endoftext|>",
"special": true
}
],
"decoder": {"type": "ByteLevel"}
}"#,
)
.unwrap();
let tokenizer = BpeTokenizer::import_huggingface(&path).unwrap();
assert_eq!(tokenizer.encode("ab"), vec![FIRST_MERGE_ID]);
assert_eq!(
tokenizer.token_bytes(tokenizer.special_tokens.eos),
Some(b"<|endoftext|>".as_slice())
);
let _ = std::fs::remove_dir_all(&directory);
}
#[test]
fn import_huggingface_rejects_non_byte_level_decoder() {
let directory = temporary_directory("hf_reject");
let path = directory.join("tokenizer.json");
std::fs::write(
&path,
r#"{
"model": {"type": "BPE", "vocab": {"a": 0}, "merges": []},
"decoder": {"type": "WordPiece"}
}"#,
)
.unwrap();
let error = BpeTokenizer::import_huggingface(&path).unwrap_err();
assert!(error.to_string().contains("ByteLevel"));
let _ = std::fs::remove_dir_all(&directory);
}
fn temporary_model_path() -> PathBuf {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
std::env::temp_dir().join(format!(
"oxidelm_tokenizer_{}_{}.json",
std::process::id(),
nonce
))
}
fn temporary_directory(label: &str) -> PathBuf {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let path = std::env::temp_dir().join(format!(
"oxidelm_tokenizer_{label}_{}_{}",
std::process::id(),
nonce
));
std::fs::create_dir_all(&path).unwrap();
path
}
}