use std::{
ffi::{OsStr, OsString},
fs::{self, OpenOptions},
io::Write,
path::{Path, PathBuf},
sync::atomic::{AtomicU64, Ordering},
};
use bincode::{Decode, Encode};
use serde::Deserialize;
use serde_json::Value;
use crate::{
AddedTokenConfig, DecoderConfig, Error, ModelConfig, NormalizerConfig, PostProcessorConfig,
PreTokenizerConfig, Tokenizer, TokenizerJson,
models::bpe::{Bpe, ResolvedBpe},
};
const MAGIC: &[u8; 8] = b"SNAPTKZ\0";
const VERSION: u32 = 4;
const HEADER_LEN: usize = 84;
const MAX_TKZ_BYTES: usize = 512 * 1024 * 1024;
static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
#[derive(Deserialize)]
struct TokenizerParts {
#[serde(default)]
added_tokens: Vec<AddedTokenConfig>,
normalizer: Option<NormalizerConfig>,
pre_tokenizer: Option<PreTokenizerConfig>,
post_processor: Option<PostProcessorConfig>,
decoder: Option<DecoderConfig>,
}
impl TokenizerParts {
fn with_model(self, model: ModelConfig) -> TokenizerJson {
TokenizerJson {
added_tokens: self.added_tokens,
normalizer: self.normalizer,
pre_tokenizer: self.pre_tokenizer,
model,
post_processor: self.post_processor,
decoder: self.decoder,
}
}
}
#[derive(Encode, Decode)]
struct Payload {
pipeline_json: Vec<u8>,
bpe: ResolvedBpe,
}
impl Payload {
fn into_tokenizer(self) -> Result<Tokenizer, Error> {
let parts: TokenizerParts = serde_json::from_slice(&self.pipeline_json)?;
let bpe = Bpe::from_resolved(self.bpe).map_err(Error::Model)?;
let json = parts.with_model(ModelConfig::Bpe(Box::new(bpe)));
Tokenizer::build(json)
}
}
pub(crate) fn load_or_create(path: &Path) -> Result<Tokenizer, Error> {
if path.extension() == Some(OsStr::new("tkz")) {
return load_tkz(path, None);
}
let sidecar = path.with_extension("tkz");
let source = match fs::read(path) {
Ok(source) => source,
Err(error) if error.kind() == std::io::ErrorKind::NotFound && sidecar.is_file() => {
return load_tkz(&sidecar, None);
}
Err(error) => return Err(error.into()),
};
let source_hash = *blake3::hash(&source).as_bytes();
if sidecar.is_file()
&& let Ok(tokenizer) = load_tkz(&sidecar, Some(source_hash))
{
return Ok(tokenizer);
}
let (tokenizer, payload) = from_json_bytes(&source)?;
let encoded = encode_file(&payload, source_hash)?;
write_atomic(&sidecar, &encoded)?;
Ok(tokenizer)
}
fn from_json_bytes(source: &[u8]) -> Result<(Tokenizer, Payload), Error> {
let mut json: Value = serde_json::from_slice(source)?;
let object = json
.as_object_mut()
.ok_or_else(|| Error::Tkz("tokenizer JSON root must be an object".into()))?;
let model_json = object
.remove("model")
.ok_or_else(|| Error::Tkz("tokenizer JSON is missing its model".into()))?;
let pipeline_json = serde_json::to_vec(&json)?;
let parts: TokenizerParts = serde_json::from_value(json)?;
let model: ModelConfig = serde_json::from_value(model_json)?;
let ModelConfig::Bpe(bpe) = model;
let resolved = bpe.resolved_config();
let tokenizer = Tokenizer::build(parts.with_model(ModelConfig::Bpe(bpe)))?;
let payload = Payload {
pipeline_json,
bpe: resolved,
};
Ok((tokenizer, payload))
}
fn load_tkz(path: &Path, source_hash: Option<[u8; 32]>) -> Result<Tokenizer, Error> {
let metadata = fs::metadata(path)?;
if metadata.len() > MAX_TKZ_BYTES as u64 {
return Err(Error::Tkz("file exceeds the 512 MiB limit".into()));
}
let file = fs::read(path)?;
decode_file(&file, source_hash)?.into_tokenizer()
}
fn encode_file(payload: &Payload, source_hash: [u8; 32]) -> Result<Vec<u8>, Error> {
let mut file = vec![0; HEADER_LEN];
bincode::encode_into_std_write(payload, &mut file, bincode_config())
.map_err(|error| Error::Tkz(format!("failed to encode payload: {error}")))?;
let payload_len = file.len() - HEADER_LEN;
let payload_hash = blake3::hash(&file[HEADER_LEN..]);
file[0..8].copy_from_slice(MAGIC);
file[8..12].copy_from_slice(&VERSION.to_le_bytes());
file[12..20].copy_from_slice(&(payload_len as u64).to_le_bytes());
file[20..52].copy_from_slice(&source_hash);
file[52..84].copy_from_slice(payload_hash.as_bytes());
Ok(file)
}
fn decode_file(file: &[u8], source_hash: Option<[u8; 32]>) -> Result<Payload, Error> {
if file.len() < HEADER_LEN {
return Err(Error::Tkz("file is shorter than its header".into()));
}
if &file[0..8] != MAGIC {
return Err(Error::Tkz("unrecognized file magic".into()));
}
let mut version = [0; 4];
version.copy_from_slice(&file[8..12]);
if u32::from_le_bytes(version) != VERSION {
return Err(Error::Tkz("unsupported format version".into()));
}
let mut encoded_len = [0; 8];
encoded_len.copy_from_slice(&file[12..20]);
let encoded_len = u64::from_le_bytes(encoded_len);
if encoded_len != (file.len() - HEADER_LEN) as u64 {
return Err(Error::Tkz("payload length mismatch".into()));
}
if source_hash.is_some_and(|hash| file[20..52] != hash) {
return Err(Error::Tkz("sidecar does not match its source JSON".into()));
}
let payload = &file[HEADER_LEN..];
if blake3::hash(payload).as_bytes() != &file[52..84] {
return Err(Error::Tkz("payload checksum mismatch".into()));
}
let (decoded, consumed) = bincode::decode_from_slice(payload, bincode_config())
.map_err(|error| Error::Tkz(format!("failed to decode payload: {error}")))?;
if consumed != payload.len() {
return Err(Error::Tkz("trailing payload bytes".into()));
}
Ok(decoded)
}
fn bincode_config() -> impl bincode::config::Config {
bincode::config::standard()
.with_little_endian()
.with_fixed_int_encoding()
.with_limit::<MAX_TKZ_BYTES>()
}
fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), Error> {
let temporary = temporary_path(path)?;
let result = (|| -> std::io::Result<()> {
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(&temporary)?;
file.write_all(bytes)?;
file.sync_all()?;
fs::rename(&temporary, path)
})();
if result.is_err() {
let _ = fs::remove_file(&temporary);
}
result.map_err(Error::Io)
}
fn temporary_path(path: &Path) -> Result<PathBuf, Error> {
let name = path
.file_name()
.ok_or_else(|| Error::Tkz("sidecar path has no file name".into()))?;
let counter = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed);
let mut temporary = OsString::from(".");
temporary.push(name);
temporary.push(format!(".{}.{}.tmp", std::process::id(), counter));
Ok(path.with_file_name(temporary))
}
#[cfg(test)]
mod tests {
use serde_json::{Value, json};
use super::*;
fn fixture(merged: bool) -> Value {
let merges = if merged {
json!([["a", "b"]])
} else {
json!([])
};
json!({
"added_tokens": [
{
"id": 5,
"content": "<s>",
"single_word": true,
"lstrip": true,
"rstrip": true,
"normalized": false,
"special": true
},
{
"id": 6,
"content": "e\u{301}!",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": false
}
],
"normalizer": {"type": "NFC"},
"pre_tokenizer": null,
"model": {
"type": "BPE",
"vocab": {
"a": 0,
"b": 1,
"ab": 2,
"<0xC3>": 3,
"<0xA9>": 4
},
"merges": merges,
"byte_fallback": true,
"ignore_merges": merged
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{"SpecialToken": {"id": "<s>", "type_id": 0}},
{"Sequence": {"id": "A", "type_id": 0}}
],
"pair": [],
"special_tokens": {
"<s>": {"id": "<s>", "ids": [5], "tokens": ["<s>"]}
}
},
"decoder": {
"type": "Sequence",
"decoders": [{"type": "ByteFallback"}, {"type": "Fuse"}]
}
})
}
fn test_directory(name: &str) -> PathBuf {
let counter = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed);
let path = std::env::temp_dir().join(format!(
"snaptokens-{name}-{}-{counter}",
std::process::id()
));
fs::create_dir(&path).unwrap();
path
}
fn write_fixture(path: &Path, merged: bool) {
fs::write(path, serde_json::to_vec(&fixture(merged)).unwrap()).unwrap();
}
#[test]
fn round_trip_preserves_pipeline_and_direct_loads() {
let directory = test_directory("tkz-round-trip");
let json_path = directory.join("tokenizer.json");
let tkz_path = directory.join("tokenizer.tkz");
write_fixture(&json_path, true);
let json = Tokenizer::from_file(&json_path).unwrap();
let disabled = Tokenizer::from_file_with_tkz_cache(&json_path, false).unwrap();
assert!(!tkz_path.exists());
assert_eq!(json.encode("ab").unwrap(), disabled.encode("ab").unwrap());
let cached = Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();
let direct = Tokenizer::from_file_with_tkz_cache(&tkz_path, true).unwrap();
assert!(tkz_path.is_file());
for input in ["ab", "é", "e\u{301}!", " <s> "] {
let expected = json.encode(input).unwrap();
assert_eq!(cached.encode(input).unwrap(), expected);
assert_eq!(direct.encode(input).unwrap(), expected);
}
let expected = json.encode_with_special_tokens("ab", true).unwrap();
assert_eq!(
cached.encode_with_special_tokens("ab", true).unwrap(),
expected
);
assert_eq!(direct.decode(&[3, 4], false).unwrap(), "é");
assert!(direct.is_special_token(5));
fs::remove_file(&json_path).unwrap();
let sidecar_only = Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();
assert_eq!(sidecar_only.encode("é").unwrap(), vec![3, 4]);
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn rebuilds_invalid_sidecars_and_handles_concurrent_creation() {
let directory = test_directory("tkz-recovery");
let json_path = directory.join("tokenizer.json");
let tkz_path = directory.join("tokenizer.tkz");
write_fixture(&json_path, true);
Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();
write_fixture(&json_path, false);
let refreshed = Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();
assert_eq!(refreshed.encode("ab").unwrap(), vec![0, 1]);
fs::write(&tkz_path, b"SNAPTKZ\0").unwrap();
let recovered = Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();
assert_eq!(recovered.encode("ab").unwrap(), vec![0, 1]);
fs::write(&tkz_path, b"SNAPTKZ\0").unwrap();
fs::remove_file(&json_path).unwrap();
assert!(Tokenizer::from_file_with_tkz_cache(&tkz_path, true).is_err());
fs::remove_dir_all(&directory).unwrap();
let directory = test_directory("tkz-concurrent");
let json_path = directory.join("tokenizer.json");
let tkz_path = directory.join("tokenizer.tkz");
write_fixture(&json_path, true);
let threads = (0..4)
.map(|_| {
let json_path = json_path.clone();
std::thread::spawn(move || {
Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();
})
})
.collect::<Vec<_>>();
for thread in threads {
thread.join().unwrap();
}
let final_tokenizer = Tokenizer::from_file_with_tkz_cache(&tkz_path, true).unwrap();
assert_eq!(final_tokenizer.encode("ab").unwrap(), vec![2]);
fs::remove_dir_all(directory).unwrap();
}
}