vil_tokenizer 0.3.0

VIL Tokenizer Engine — native Rust BPE tokenization for LLM token counting and text splitting
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

[package]
name = "vil_tokenizer"
version.workspace = true
edition = "2021"
license = "MIT OR Apache-2.0"
description = "VIL Tokenizer Engine — native Rust BPE tokenization for LLM token counting and text splitting"
repository = "https://github.com/OceanOS-id/VIL"
homepage = "https://vastar.id/products/vil"
documentation = "https://vastar.id/docs/vil"
readme = "README.md"

[dependencies]
vil_sdk = { version = "0.3", path = "../vil_sdk" }
vil_server_core = { version = "0.3", path = "../vil_server_core" }
vil_server = { version = "0.3", path = "../vil_server" }
vil_macros = { version = "0.3", path = "../vil_macros" }

serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
base64 = "0.22"
regex = "1"
axum = { version = "0.7", features = ["macros"] }