snaptokens 0.2.6

Fast BPE tokenization from local Hugging Face tokenizer.json files
Documentation
use std::{
    ffi::{OsStr, OsString},
    fs::{self, OpenOptions},
    io::Write,
    path::{Path, PathBuf},
    sync::atomic::{AtomicU64, Ordering},
};

use bincode::{Decode, Encode};
use serde::Deserialize;
use serde_json::Value;

use crate::{
    AddedTokenConfig, DecoderConfig, Error, ModelConfig, NormalizerConfig, PostProcessorConfig,
    PreTokenizerConfig, Tokenizer, TokenizerJson,
    models::bpe::{Bpe, ResolvedBpe},
};

const MAGIC: &[u8; 8] = b"SNAPTKZ\0";
const VERSION: u32 = 4;
const HEADER_LEN: usize = 84;
const MAX_TKZ_BYTES: usize = 512 * 1024 * 1024;

static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);

/// Top-level tokenizer fields whose compact sidecar model is stored separately.
#[derive(Deserialize)]
struct TokenizerParts {
    #[serde(default)]
    added_tokens: Vec<AddedTokenConfig>,
    normalizer: Option<NormalizerConfig>,
    pre_tokenizer: Option<PreTokenizerConfig>,
    post_processor: Option<PostProcessorConfig>,
    decoder: Option<DecoderConfig>,
}

impl TokenizerParts {
    /// Reattach a BPE model to the non-model tokenizer configuration.
    fn with_model(self, model: ModelConfig) -> TokenizerJson {
        TokenizerJson {
            added_tokens: self.added_tokens,
            normalizer: self.normalizer,
            pre_tokenizer: self.pre_tokenizer,
            model,
            post_processor: self.post_processor,
            decoder: self.decoder,
        }
    }
}

/// Portable sidecar payload: small pipeline JSON plus resolved BPE inputs.
#[derive(Encode, Decode)]
struct Payload {
    pipeline_json: Vec<u8>,
    bpe: ResolvedBpe,
}

impl Payload {
    /// Rebuild a tokenizer while assigning fresh model and cache identity.
    fn into_tokenizer(self) -> Result<Tokenizer, Error> {
        let parts: TokenizerParts = serde_json::from_slice(&self.pipeline_json)?;
        let bpe = Bpe::from_resolved(self.bpe).map_err(Error::Model)?;
        let json = parts.with_model(ModelConfig::Bpe(Box::new(bpe)));
        Tokenizer::build(json)
    }
}

/// Load a direct `.tkz`, or create and reuse a sibling for another local file.
pub(crate) fn load_or_create(path: &Path) -> Result<Tokenizer, Error> {
    if path.extension() == Some(OsStr::new("tkz")) {
        return load_tkz(path, None);
    }

    let sidecar = path.with_extension("tkz");
    let source = match fs::read(path) {
        Ok(source) => source,
        Err(error) if error.kind() == std::io::ErrorKind::NotFound && sidecar.is_file() => {
            return load_tkz(&sidecar, None);
        }
        Err(error) => return Err(error.into()),
    };
    let source_hash = *blake3::hash(&source).as_bytes();

    if sidecar.is_file()
        && let Ok(tokenizer) = load_tkz(&sidecar, Some(source_hash))
    {
        return Ok(tokenizer);
    }

    let (tokenizer, payload) = from_json_bytes(&source)?;
    let encoded = encode_file(&payload, source_hash)?;
    write_atomic(&sidecar, &encoded)?;
    Ok(tokenizer)
}

/// Parse source JSON once while separating compact pipeline and model data.
fn from_json_bytes(source: &[u8]) -> Result<(Tokenizer, Payload), Error> {
    let mut json: Value = serde_json::from_slice(source)?;
    let object = json
        .as_object_mut()
        .ok_or_else(|| Error::Tkz("tokenizer JSON root must be an object".into()))?;
    let model_json = object
        .remove("model")
        .ok_or_else(|| Error::Tkz("tokenizer JSON is missing its model".into()))?;

    let pipeline_json = serde_json::to_vec(&json)?;
    let parts: TokenizerParts = serde_json::from_value(json)?;
    let model: ModelConfig = serde_json::from_value(model_json)?;
    let ModelConfig::Bpe(bpe) = model;
    let resolved = bpe.resolved_config();

    let tokenizer = Tokenizer::build(parts.with_model(ModelConfig::Bpe(bpe)))?;
    let payload = Payload {
        pipeline_json,
        bpe: resolved,
    };
    Ok((tokenizer, payload))
}

/// Load and validate one native sidecar before rebuilding its tokenizer.
fn load_tkz(path: &Path, source_hash: Option<[u8; 32]>) -> Result<Tokenizer, Error> {
    let metadata = fs::metadata(path)?;
    if metadata.len() > MAX_TKZ_BYTES as u64 {
        return Err(Error::Tkz("file exceeds the 512 MiB limit".into()));
    }
    let file = fs::read(path)?;
    decode_file(&file, source_hash)?.into_tokenizer()
}

/// Encode a checksummed, source-bound sidecar using a fixed portable layout.
fn encode_file(payload: &Payload, source_hash: [u8; 32]) -> Result<Vec<u8>, Error> {
    let mut file = vec![0; HEADER_LEN];
    bincode::encode_into_std_write(payload, &mut file, bincode_config())
        .map_err(|error| Error::Tkz(format!("failed to encode payload: {error}")))?;

    let payload_len = file.len() - HEADER_LEN;
    let payload_hash = blake3::hash(&file[HEADER_LEN..]);
    file[0..8].copy_from_slice(MAGIC);
    file[8..12].copy_from_slice(&VERSION.to_le_bytes());
    file[12..20].copy_from_slice(&(payload_len as u64).to_le_bytes());
    file[20..52].copy_from_slice(&source_hash);
    file[52..84].copy_from_slice(payload_hash.as_bytes());
    Ok(file)
}

/// Validate one sidecar envelope and decode exactly one bounded payload.
fn decode_file(file: &[u8], source_hash: Option<[u8; 32]>) -> Result<Payload, Error> {
    if file.len() < HEADER_LEN {
        return Err(Error::Tkz("file is shorter than its header".into()));
    }
    if &file[0..8] != MAGIC {
        return Err(Error::Tkz("unrecognized file magic".into()));
    }

    let mut version = [0; 4];
    version.copy_from_slice(&file[8..12]);
    if u32::from_le_bytes(version) != VERSION {
        return Err(Error::Tkz("unsupported format version".into()));
    }

    let mut encoded_len = [0; 8];
    encoded_len.copy_from_slice(&file[12..20]);
    let encoded_len = u64::from_le_bytes(encoded_len);
    if encoded_len != (file.len() - HEADER_LEN) as u64 {
        return Err(Error::Tkz("payload length mismatch".into()));
    }
    if source_hash.is_some_and(|hash| file[20..52] != hash) {
        return Err(Error::Tkz("sidecar does not match its source JSON".into()));
    }

    let payload = &file[HEADER_LEN..];
    if blake3::hash(payload).as_bytes() != &file[52..84] {
        return Err(Error::Tkz("payload checksum mismatch".into()));
    }

    let (decoded, consumed) = bincode::decode_from_slice(payload, bincode_config())
        .map_err(|error| Error::Tkz(format!("failed to decode payload: {error}")))?;
    if consumed != payload.len() {
        return Err(Error::Tkz("trailing payload bytes".into()));
    }
    Ok(decoded)
}

/// Return the fixed little-endian codec for the current format version.
fn bincode_config() -> impl bincode::config::Config {
    bincode::config::standard()
        .with_little_endian()
        .with_fixed_int_encoding()
        .with_limit::<MAX_TKZ_BYTES>()
}

/// Persist a complete sidecar by renaming a unique file in the same directory.
fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), Error> {
    let temporary = temporary_path(path)?;
    let result = (|| -> std::io::Result<()> {
        let mut file = OpenOptions::new()
            .write(true)
            .create_new(true)
            .open(&temporary)?;
        file.write_all(bytes)?;
        file.sync_all()?;
        fs::rename(&temporary, path)
    })();

    if result.is_err() {
        let _ = fs::remove_file(&temporary);
    }
    result.map_err(Error::Io)
}

/// Build a collision-resistant temporary sibling name for atomic creation.
fn temporary_path(path: &Path) -> Result<PathBuf, Error> {
    let name = path
        .file_name()
        .ok_or_else(|| Error::Tkz("sidecar path has no file name".into()))?;
    let counter = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed);
    let mut temporary = OsString::from(".");
    temporary.push(name);
    temporary.push(format!(".{}.{}.tmp", std::process::id(), counter));
    Ok(path.with_file_name(temporary))
}

#[cfg(test)]
mod tests {
    use serde_json::{Value, json};

    use super::*;

    /// Return a small tokenizer that exercises every persisted metadata family.
    fn fixture(merged: bool) -> Value {
        let merges = if merged {
            json!([["a", "b"]])
        } else {
            json!([])
        };
        json!({
            "added_tokens": [
                {
                    "id": 5,
                    "content": "<s>",
                    "single_word": true,
                    "lstrip": true,
                    "rstrip": true,
                    "normalized": false,
                    "special": true
                },
                {
                    "id": 6,
                    "content": "e\u{301}!",
                    "single_word": false,
                    "lstrip": false,
                    "rstrip": false,
                    "normalized": true,
                    "special": false
                }
            ],
            "normalizer": {"type": "NFC"},
            "pre_tokenizer": null,
            "model": {
                "type": "BPE",
                "vocab": {
                    "a": 0,
                    "b": 1,
                    "ab": 2,
                    "<0xC3>": 3,
                    "<0xA9>": 4
                },
                "merges": merges,
                "byte_fallback": true,
                "ignore_merges": merged
            },
            "post_processor": {
                "type": "TemplateProcessing",
                "single": [
                    {"SpecialToken": {"id": "<s>", "type_id": 0}},
                    {"Sequence": {"id": "A", "type_id": 0}}
                ],
                "pair": [],
                "special_tokens": {
                    "<s>": {"id": "<s>", "ids": [5], "tokens": ["<s>"]}
                }
            },
            "decoder": {
                "type": "Sequence",
                "decoders": [{"type": "ByteFallback"}, {"type": "Fuse"}]
            }
        })
    }

    /// Create a unique temporary directory for one sidecar test.
    fn test_directory(name: &str) -> PathBuf {
        let counter = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed);
        let path = std::env::temp_dir().join(format!(
            "snaptokens-{name}-{}-{counter}",
            std::process::id()
        ));
        fs::create_dir(&path).unwrap();
        path
    }

    /// Write one compact tokenizer fixture as source JSON.
    fn write_fixture(path: &Path, merged: bool) {
        fs::write(path, serde_json::to_vec(&fixture(merged)).unwrap()).unwrap();
    }

    /// The opt-in path preserves full behavior and direct `.tkz` ownership.
    #[test]
    fn round_trip_preserves_pipeline_and_direct_loads() {
        let directory = test_directory("tkz-round-trip");
        let json_path = directory.join("tokenizer.json");
        let tkz_path = directory.join("tokenizer.tkz");
        write_fixture(&json_path, true);

        let json = Tokenizer::from_file(&json_path).unwrap();
        let disabled = Tokenizer::from_file_with_tkz_cache(&json_path, false).unwrap();
        assert!(!tkz_path.exists());
        assert_eq!(json.encode("ab").unwrap(), disabled.encode("ab").unwrap());

        let cached = Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();
        let direct = Tokenizer::from_file_with_tkz_cache(&tkz_path, true).unwrap();
        assert!(tkz_path.is_file());

        for input in ["ab", "é", "e\u{301}!", "  <s>  "] {
            let expected = json.encode(input).unwrap();
            assert_eq!(cached.encode(input).unwrap(), expected);
            assert_eq!(direct.encode(input).unwrap(), expected);
        }
        let expected = json.encode_with_special_tokens("ab", true).unwrap();
        assert_eq!(
            cached.encode_with_special_tokens("ab", true).unwrap(),
            expected
        );
        assert_eq!(direct.decode(&[3, 4], false).unwrap(), "é");
        assert!(direct.is_special_token(5));

        fs::remove_file(&json_path).unwrap();
        let sidecar_only = Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();
        assert_eq!(sidecar_only.encode("é").unwrap(), vec![3, 4]);
        fs::remove_dir_all(directory).unwrap();
    }

    /// Stale, torn, and concurrently produced sidecars never escape validation.
    #[test]
    fn rebuilds_invalid_sidecars_and_handles_concurrent_creation() {
        let directory = test_directory("tkz-recovery");
        let json_path = directory.join("tokenizer.json");
        let tkz_path = directory.join("tokenizer.tkz");
        write_fixture(&json_path, true);
        Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();

        write_fixture(&json_path, false);
        let refreshed = Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();
        assert_eq!(refreshed.encode("ab").unwrap(), vec![0, 1]);

        fs::write(&tkz_path, b"SNAPTKZ\0").unwrap();
        let recovered = Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();
        assert_eq!(recovered.encode("ab").unwrap(), vec![0, 1]);

        fs::write(&tkz_path, b"SNAPTKZ\0").unwrap();
        fs::remove_file(&json_path).unwrap();
        assert!(Tokenizer::from_file_with_tkz_cache(&tkz_path, true).is_err());
        fs::remove_dir_all(&directory).unwrap();

        let directory = test_directory("tkz-concurrent");
        let json_path = directory.join("tokenizer.json");
        let tkz_path = directory.join("tokenizer.tkz");
        write_fixture(&json_path, true);
        let threads = (0..4)
            .map(|_| {
                let json_path = json_path.clone();
                std::thread::spawn(move || {
                    Tokenizer::from_file_with_tkz_cache(&json_path, true).unwrap();
                })
            })
            .collect::<Vec<_>>();
        for thread in threads {
            thread.join().unwrap();
        }
        let final_tokenizer = Tokenizer::from_file_with_tkz_cache(&tkz_path, true).unwrap();
        assert_eq!(final_tokenizer.encode("ab").unwrap(), vec![2]);
        fs::remove_dir_all(directory).unwrap();
    }
}