[[test]]
name = "tests"
path = "tests/tests.rs"
[dependencies.anyhow]
version = "1.0.81"
[dependencies.bincode]
version = "1.3.3"
[dependencies.funty]
version = "2.0.0"
[dependencies.indicatif]
version = "0.17.8"
[dependencies.memmap2]
version = "0.9.4"
[dependencies.pyo3]
features = ["extension-module"]
version = "0.20.1"
[dependencies.rand]
version = "0.8.5"
[dependencies.rayon]
version = "1.10.0"
[dependencies.rayon-core]
version = "1.12.1"
[dependencies.serde]
features = ["derive"]
version = "1.0.197"
[dependencies.utf16_literal]
version = "0.2.1"
[dev-dependencies.quickcheck]
default-features = false
version = "0.9"
[dev-dependencies.rand]
version = "0.8.4"
[lib]
crate-type = ["cdylib", "rlib"]
name = "tokengrams"
[package]
description = "Compute n-gram statistics and model language over pre-tokenized text corpora used to train large language models."
edition = "2021"
license = "MIT"
name = "tokengrams"
readme = "README.md"
version = "0.3.0"