[[bench]]
harness = false
name = "bpe_benchmark"
[[bin]]
bench = false
name = "cli"
path = "src/cli.rs"
[dependencies.clap]
version = "2.33.0"
[dependencies.indicatif]
version = "0.14.0"
[dependencies.lazy_static]
version = "1.3.0"
[dependencies.rand]
version = "0.7.2"
[dependencies.rayon]
version = "1.2.0"
[dependencies.regex]
version = "1.3.1"
[dependencies.regex-syntax]
version = "0.6.12"
[dependencies.serde]
version = "1.0"
[dependencies.serde_json]
version = "1.0"
[dependencies.unicode-normalization-alignments]
version = "0.1.12"
[dependencies.unicode_categories]
version = "0.1.1"
[dev-dependencies.criterion]
version = "0.3.0"
[dev-dependencies.tempfile]
version = "3.1"
[lib]
bench = false
name = "tokenizers"
path = "src/lib.rs"
[package]
authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
description = "Provides an implementation of today's most used tokenizers,\nwith a focus on performances and versatility.\n"
documentation = "https://docs.rs/tokenizers/"
edition = "2018"
exclude = ["rust-toolchain", "target/*", "Cargo.lock", "benches/*.txt", "benches/*.json", "data/*"]
homepage = "https://github.com/huggingface/tokenizers"
keywords = ["tokenizer", "NLP", "huggingface", "BPE", "WordPiece"]
license = "Apache-2.0"
name = "tokenizers"
readme = "./README.md"
repository = "https://github.com/huggingface/tokenizers"
version = "0.10.1"