tpt_tokenizer_core/
lib.rs

1//! # tpt-tokenizer-core
2//!
3//! A small, **dependency-free** implementation of the two tokenization schemes
4//! used by most open-weight LLMs:
5//!
6//! * [`BpeTokenizer`] — Byte-Pair Encoding (GPT-2 / Llama style).
7//! * [`WordPieceTokenizer`] — WordPiece (BERT style).
8//!
9//! Both implement the shared [`Tokenizer`] trait with `encode` / `decode`.
10//!
11//! [`BpeTokenizer`] additionally supports GPT-2 **byte-level** mode
12//! ([`with_byte_level`](BpeTokenizer::with_byte_level)) — encoding never fails
13//! on arbitrary UTF-8 and `decode(encode(s)) == s` — and atomic **special
14//! tokens** ([`with_special_tokens`](BpeTokenizer::with_special_tokens)).
15//! [`WordPieceTokenizer`] runs a BERT-style basic tokenizer (whitespace +
16//! punctuation splitting, CJK isolation, optional
17//! [`with_lowercase`](WordPieceTokenizer::with_lowercase)).
18//!
19//! The crate is `#![no_std]` compatible: all tokenization logic lives behind
20//! `alloc` and never touches the standard library. The `std` feature (enabled
21//! by default) only adds convenience constructors that load vocabularies and
22//! merge tables from disk.
23//!
24//! ## Example
25//!
26//! ```
27//! use tpt_tokenizer_core::{BpeTokenizer, Tokenizer};
28//!
29//! let mut vocab = std::collections::BTreeMap::new();
30//! vocab.insert("l".to_string(), 0u32);
31//! vocab.insert("o".to_string(), 1u32);
32//! vocab.insert("w".to_string(), 2u32);
33//! vocab.insert("lo".to_string(), 3u32);
34//! vocab.insert("low".to_string(), 4u32);
35//! let merges = vec![
36//!     ("l".to_string(), "o".to_string()),
37//!     ("lo".to_string(), "w".to_string()),
38//! ];
39//! let tok = BpeTokenizer::from_vocab_merges(vocab, merges);
40//! let ids = tok.encode("low").unwrap();
41//! assert_eq!(ids, vec![4]);
42//! ```
43#![cfg_attr(not(feature = "std"), no_std)]
44#![warn(missing_docs)]
45
46extern crate alloc;
47
48pub mod encoding;
49pub mod error;
50pub mod json;
51pub mod loader;
52#[cfg(feature = "normalization")]
53pub mod normalize;
54pub mod tokenizer;
55
56mod bpe;
57mod pretokenize;
58mod wordpiece;
59
60pub use bpe::BpeTokenizer;
61pub use encoding::{EncodeConfig, Encoding, Padding, TokenizerExt, Truncation};
62pub use error::TokenizerError;
63#[cfg(feature = "std")]
64pub use loader::from_tokenizer_json_file;
65pub use loader::{from_tokenizer_json_str, LoadedTokenizer};
66#[cfg(feature = "normalization")]
67pub use normalize::{normalize, NormalizationForm};
68pub use tokenizer::{TokenId, Tokenizer};
69pub use wordpiece::WordPieceTokenizer;
tpt_tokenizer_core/lib.rs

tpt_tokenizer_core/
lib.rs
