skimtoken 0.2.2

Fast token count estimation library
Documentation
1
2
3
4
5
6
from skimtoken import estimate_tokens

# Basic usage
text = "Hello, world!"
tokens = estimate_tokens(text)
print(f"'{text}' → {tokens} tokens")