[package]
name = "tokenx"
version = "0.2.0"
edition = "2021"
description = "vocab-free heuristic LLM token-count estimation"
license = "MIT"
repository = "https://github.com/hey-jj/tokenx"
readme = "README.md"
keywords = ["tokenizer", "tokens", "llm", "gpt", "estimate"]
categories = ["text-processing", "parsing"]
[dependencies]
regex = "1.12.4"