text-tokenizer 0.6.4

Custom text tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

[package]
name = "text-tokenizer"
version = "0.6.4"
authors = ["merl <merl.001.mia@gmail.com>"]
edition = "2024"
license = "MIT"
readme = "README.md"
description = "Custom text tokenizer"

repository = "https://github.com/merl-twin/text-tokenizer"
homepage = "https://github.com/merl-twin/text-tokenizer"

include = [
    "**/*.rs",
    "Cargo.toml",
]


[features]
default = []
strings = []

[dependencies]
text-parsing = "0.6"
unicode-segmentation = "1.2"
unicode-properties = "0.1"
lazy_static = "1.5"