smoltoken 0.2.0

A fast library for Byte Pair Encoding (BPE) tokenization.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

[dependencies.base64]
version = "0.22"

[dependencies.fancy-regex]
version = "0.14.0"

[dependencies.kdam]
version = "0.6.2"

[dependencies.rayon]
version = "1.10.0"

[dependencies.rustc-hash]
version = "2.0.0"

[dependencies.walkdir]
version = "2"

[lib]
name = "smoltoken"
path = "src/lib.rs"

[package]
authors = ["Arun S V <svarunid@gmail.com>"]
autobenches = false
autobins = false
autoexamples = false
autolib = false
autotests = false
build = false
categories = ["encoding", "text-processing"]
description = "A fast library for Byte Pair Encoding (BPE) tokenization."
edition = "2021"
exclude = ["src/bin/*"]
homepage = "https://github.com/svarunid/smoltoken"
keywords = ["tokenizer", "bpe", "ai"]
license-file = "LICENSE"
name = "smoltoken"
readme = "README.md"
repository = "https://github.com/svarunid/smoltoken/rust"
version = "0.2.0"