utokenizer 0.1.1

CLI tool for building a local model-tokenizer registry and counting input tokens across model families.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

[package]
name = "utokenizer"
version = "0.1.1"
edition = "2021"
description = "CLI tool for building a local model-tokenizer registry and counting input tokens across model families."
license = "MIT"
readme = "README.md"
homepage = "https://crates.io/crates/utokenizer"
repository = "https://crates.io/crates/utokenizer"
documentation = "https://docs.rs/utokenizer"
keywords = ["tokenizer", "llm", "tokens", "cli", "ai"]
categories = ["command-line-utilities", "text-processing"]
exclude = [
  "validation_report*.json",
  "validation_report*.csv",
  "aihubmix_token_registry.json",
  "aihubmix_token_count_log.jsonl",
  "token_registry.json",
  "token_count_log.jsonl",
  "target/",
  ".venv/",
]

[dependencies]
anyhow = "1"
chrono = "0.4"
clap = { version = "4.5", features = ["derive"] }
dotenvy = "0.15"
futures = "0.3"
reqwest = { version = "0.12", features = ["json", "rustls-tls"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "fs"] }

[[bin]]
name = "utokenizer"
path = "src/main.rs"