riptoken 0.0.1

Fast BPE tokenizer for LLMs
Documentation
1
2
3
4
5
6
7
8
9
10
11
[project]
name = "riptoken"
version = "0.0.1"
description = "Fast BPE tokenizer for LLMs"
requires-python = ">=3.9"
license = "MIT"
authors = [{ name = "Dae Choi", email = "daechoi70@gmail.com" }]

[build-system]
build-backend = "setuptools.build_meta"
requires = ["setuptools>=62.4"]