llm-tokenizer 0.0.1

[dependencies]

[lib]
name = "llm_tokenizer"
path = "src/lib.rs"

[package]
autobenches = false
autobins = false
autoexamples = false
autolib = false
autotests = false
build = false
categories = ["development-tools"]
description = "Tokenizer utilities for LLM inference"
edition = "2021"
keywords = ["llm", "inference", "ml", "ai"]
license = "Apache-2.0"
name = "llm-tokenizer"
readme = "README.md"
repository = "https://github.com/shepherd-ai/shepherd"
version = "0.0.1"