eredu_text/lib.rs
1//! Backend-neutral tokenizer construction and chat-template rendering.
2//!
3//! This crate adapts Hugging Face tokenizers and Jinja chat templates for
4//! language-model runtimes. It can reconstruct tokenizers from GGUF metadata,
5//! import supported tiktoken vocabularies, and render structured conversations
6//! without depending on a model execution backend.
7
8#![warn(missing_docs)]
9
10/// Error types returned by tokenizer and template operations.
11pub mod error;
12/// Tokenizer reconstruction from GGUF metadata.
13pub mod gguf;
14/// Importers for rank-ordered tiktoken vocabularies.
15pub mod tiktoken;
16/// Tokenizer wrappers and chat-template rendering utilities.
17pub mod tokenizer;