[package]
name = "deepstrike-tokenizer"
description = "Multi-model token counting engine with SIMD acceleration"
version.workspace = true
edition.workspace = true
license-file = "LICENSE"
[dependencies]
tiktoken-rs = "0.6"
[dev-dependencies]
pretty_assertions = "1"