Skip to main content

tokenizers/
lib.rs

1#![cfg_attr(docsrs, feature(doc_cfg))]
2#![warn(clippy::all)]
3#![allow(clippy::upper_case_acronyms)]
4#![doc(html_favicon_url = "https://huggingface.co/favicon.ico")]
5#![doc(html_logo_url = "https://huggingface.co/landing/assets/huggingface_logo.svg")]
6
7//! The core of `tokenizers`, written in Rust.
8//! Provides an implementation of today's most used tokenizers, with a focus on performance and
9//! versatility.
10//!
11//! # What is a Tokenizer
12//!
13//! A Tokenizer works as a pipeline, it processes some raw text as input and outputs an `Encoding`.
14//! The various steps of the pipeline are:
15//!
16//! 1. The `Normalizer`: in charge of normalizing the text. Common examples of normalization are
17//!    the [unicode normalization standards](https://unicode.org/reports/tr15/#Norm_Forms), such as `NFD` or `NFKC`.
18//!    More details about how to use the `Normalizers` are available on the
19//!    [Hugging Face blog](https://huggingface.co/docs/tokenizers/components#normalizers)
20//! 2. The `PreTokenizer`: in charge of creating initial words splits in the text. The most common way of
21//!    splitting text is simply on whitespace.
22//! 3. The `Model`: in charge of doing the actual tokenization. An example of a `Model` would be
23//!    `BPE` or `WordPiece`.
24//! 4. The `PostProcessor`: in charge of post-processing the `Encoding` to add anything relevant
25//!    that, for example, a language model would need, such as special tokens.
26//!
27//! ## Loading a pretrained tokenizer from the Hub
28//! ```
29//! use tokenizers::tokenizer::{Result, Tokenizer};
30//!
31//! fn main() -> Result<()> {
32//!     # #[cfg(feature = "http")]
33//!     # {
34//!     // needs http feature enabled
35//!     let tokenizer = Tokenizer::from_pretrained("bert-base-cased", None)?;
36//!
37//!     let encoding = tokenizer.encode("Hey there!", false)?;
38//!     println!("{:?}", encoding.get_tokens());
39//!     # }
40//!     Ok(())
41//! }
42//! ```
43//!
44//! ## Deserialization and tokenization example
45//!
46//! ```no_run
47//! use tokenizers::tokenizer::{Result, Tokenizer, EncodeInput};
48//! use tokenizers::models::bpe::BPE;
49//!
50//! fn main() -> Result<()> {
51//!     let bpe_builder = BPE::from_file("./path/to/vocab.json", "./path/to/merges.txt");
52//!     let bpe = bpe_builder
53//!         .dropout(0.1)
54//!         .unk_token("[UNK]".into())
55//!         .build()?;
56//!
57//!     let mut tokenizer = Tokenizer::new(bpe);
58//!
59//!     let encoding = tokenizer.encode("Hey there!", false)?;
60//!     println!("{:?}", encoding.get_tokens());
61//!
62//!     Ok(())
63//! }
64//! ```
65//!
66//! ## Training and serialization example
67//!
68//! ```no_run
69//! use tokenizers::decoders::DecoderWrapper;
70//! use tokenizers::models::bpe::{BpeTrainerBuilder, BPE};
71//! use tokenizers::normalizers::{strip::Strip, unicode::NFC, utils::Sequence, NormalizerWrapper};
72//! use tokenizers::pre_tokenizers::byte_level::ByteLevel;
73//! use tokenizers::pre_tokenizers::PreTokenizerWrapper;
74//! use tokenizers::processors::PostProcessorWrapper;
75//! use tokenizers::{AddedToken, Model, Result, TokenizerBuilder};
76//!
77//! use std::path::Path;
78//!
79//! fn main() -> Result<()> {
80//!     let vocab_size: usize = 100;
81//!
82//!     let mut trainer = BpeTrainerBuilder::new()
83//!         .show_progress(true)
84//!         .vocab_size(vocab_size)
85//!         .min_frequency(0)
86//!         .special_tokens(vec![
87//!             AddedToken::from(String::from("<s>"), true),
88//!             AddedToken::from(String::from("<pad>"), true),
89//!             AddedToken::from(String::from("</s>"), true),
90//!             AddedToken::from(String::from("<unk>"), true),
91//!             AddedToken::from(String::from("<mask>"), true),
92//!         ])
93//!         .build();
94//!
95//!     let mut tokenizer = TokenizerBuilder::new()
96//!         .with_model(BPE::default())
97//!         .with_normalizer(Some(Sequence::new(vec![
98//!             Strip::new(true, true).into(),
99//!             NFC.into(),
100//!         ])))
101//!         .with_pre_tokenizer(Some(ByteLevel::default()))
102//!         .with_post_processor(Some(ByteLevel::default()))
103//!         .with_decoder(Some(ByteLevel::default()))
104//!         .build()?;
105//!
106//!     let pretty = false;
107//!     tokenizer
108//!         .train_from_files(
109//!             &mut trainer,
110//!             vec!["path/to/vocab.txt".to_string()],
111//!         )?
112//!         .save("tokenizer.json", pretty)?;
113//!
114//!     Ok(())
115//! }
116//! ```
117//!
118//! # Additional information
119//!
120//! - tokenizers is designed to leverage CPU parallelism when possible. The level of parallelism is determined
121//!   by the total number of core/threads your CPU provides but this can be tuned by setting the `RAYON_RS_NUM_THREADS`
122//!   environment variable. As an example setting `RAYON_RS_NUM_THREADS=4` will allocate a maximum of 4 threads.
123//!   **_Please note this behavior may evolve in the future_**
124//!
125//! # Features
126//!
127//! - **progressbar**: The progress bar visualization is enabled by default. It might be disabled if
128//!   compilation for certain targets is not supported by the [termios](https://crates.io/crates/termios)
129//!   dependency of the [indicatif](https://crates.io/crates/indicatif) progress bar.
130//!
131//! - **http**: This feature enables downloading the tokenizer via HTTP. It is disabled by default.
132//!   With this feature enabled, `Tokenizer::from_pretrained` becomes accessible.
133
134#[macro_use]
135extern crate log;
136
137#[macro_use]
138extern crate derive_builder;
139
140#[macro_use]
141pub mod utils;
142pub mod decoders;
143pub mod models;
144pub mod normalizers;
145pub mod pre_tokenizers;
146pub mod processors;
147pub mod tokenizer;
148
149// Re-export from tokenizer
150pub use tokenizer::*;
151
152// Re-export also parallelism utils
153pub use utils::parallelism;
154
155// Re-export ProgressFormat for trainer configuration
156pub use utils::ProgressFormat;
157
158// Re-export for from_pretrained
159#[cfg(feature = "http")]
160pub use utils::from_pretrained::FromPretrainedParameters;