tiktoken-stream 0.1.0

Streaming token counter for partial LLM responses. Accumulates token count across chunks without holding the full text. Pluggable estimator function. Zero deps.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

[package]
name = "tiktoken-stream"
version = "0.1.0"
edition = "2021"
rust-version = "1.74"
description = "Streaming token counter for partial LLM responses. Accumulates token count across chunks without holding the full text. Pluggable estimator function. Zero deps."
authors = ["Mukunda Rao Katta <mukunda.vjcs6@gmail.com>"]
license = "MIT OR Apache-2.0"
repository = "https://github.com/MukundaKatta/tiktoken-stream"
homepage = "https://github.com/MukundaKatta/tiktoken-stream"
documentation = "https://docs.rs/tiktoken-stream"
readme = "README.md"
keywords = ["llm", "tokens", "stream", "count", "incremental"]
categories = ["text-processing"]
exclude = [".github/*", "target/*"]