llm-tokenizer 1.3.2

# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.

[package]
edition = "2021"
name = "llm-tokenizer"
version = "1.3.2"
authors = [
    "Simo Lin <linsimo.mark@gmail.com>",
    "Chang Su <mckvtl@gmail.com>",
    "Keyang Ru <rukeyang@gmail.com>",
]
build = false
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "LLM tokenizer library with caching and chat template support"
readme = "README.md"
keywords = [
    "tokenizer",
    "llm",
    "huggingface",
    "tiktoken",
    "chat-template",
]
categories = [
    "text-processing",
    "parsing",
]
license = "Apache-2.0"
repository = "https://github.com/lightseekorg/smg"

[lib]
name = "llm_tokenizer"
path = "src/lib.rs"

[[test]]
name = "chat_template_format_detection"
path = "tests/chat_template_format_detection.rs"

[[test]]
name = "chat_template_integration"
path = "tests/chat_template_integration.rs"

[[test]]
name = "chat_template_loading"
path = "tests/chat_template_loading.rs"

[[test]]
name = "tiktoken_integration"
path = "tests/tiktoken_integration.rs"

[[test]]
name = "tokenizer_cache_correctness_test"
path = "tests/tokenizer_cache_correctness_test.rs"

[[test]]
name = "tokenizer_integration"
path = "tests/tokenizer_integration.rs"

[[bench]]
name = "stop_decoder"
path = "benches/stop_decoder.rs"
harness = false

[[bench]]
name = "stop_sequence_search"
path = "benches/stop_sequence_search.rs"
harness = false

[dependencies.aho-corasick]
version = "1.1"

[dependencies.anyhow]
version = "1.0"

[dependencies.base64]
version = "0.22"

[dependencies.blake3]
version = "1.5"

[dependencies.dashmap]
version = "6.1.0"

[dependencies.futures]
version = "0.3"

[dependencies.hf-hub]
version = "0.5.0"
features = ["tokio"]

[dependencies.minijinja]
version = "2.0"
features = [
    "unstable_machinery",
    "json",
    "builtins",
    "loader",
    "loop_controls",
]

[dependencies.minijinja-contrib]
version = "2.0"
features = ["pycompat"]

[dependencies.rayon]
version = "1.10"

[dependencies.rustc-hash]
version = "1.1"

[dependencies.serde]
version = "1.0"
features = ["derive"]

[dependencies.serde_json]
version = "1.0"

[dependencies.thiserror]
version = "2.0.12"

[dependencies.tiktoken-rs]
version = "0.9.1"

[dependencies.tokenizers]
version = "0.22.0"

[dependencies.tokio]
version = "1.42.0"
features = [
    "sync",
    "rt-multi-thread",
    "macros",
]

[dependencies.tracing]
version = "0.1"

[dependencies.uuid]
version = "1.10"
features = [
    "v7",
    "serde",
    "v4",
    "serde",
]

[dev-dependencies.criterion]
version = "0.8"

[dev-dependencies.openai-protocol]
version = "1.7.0"

[dev-dependencies.reqwest]
version = "0.12.8"
features = ["blocking"]
default-features = false

[dev-dependencies.tempfile]
version = "3.8"

[lints.clippy]
absolute_paths = "warn"
allow_attributes = "warn"
assigning_clones = "warn"
cloned_instead_of_copied = "warn"
dbg_macro = "deny"
default_trait_access = "allow"
disallowed_methods = "warn"
doc_lazy_continuation = "allow"
expect_used = "warn"
explicit_into_iter_loop = "warn"
explicit_iter_loop = "warn"
if_not_else = "warn"
ignored_unit_patterns = "warn"
inconsistent_struct_constructor = "warn"
inefficient_to_string = "warn"
large_futures = "warn"
manual_assert = "warn"
manual_string_new = "warn"
match_bool = "warn"
match_same_arms = "allow"
match_wildcard_for_single_variants = "warn"
must_use_candidate = "allow"
needless_raw_string_hashes = "warn"
or_fun_call = "warn"
panic = "warn"
print_stderr = "warn"
print_stdout = "warn"
ref_option = "warn"
semicolon_if_nothing_returned = "warn"
single_component_path_imports = "warn"
single_match_else = "allow"
stable_sort_primitive = "warn"
todo = "deny"
trivially_copy_pass_by_ref = "warn"
unimplemented = "deny"
uninlined_format_args = "warn"
unnecessary_wraps = "warn"
unreachable = "deny"
unused_async = "warn"
unused_self = "warn"
unwrap_used = "deny"

[lints.rust]
unsafe_code = "deny"
unused_qualifications = "warn"