Struct Tokenizer 

pub struct Tokenizer { /* private fields */ }

A tokenizer: normalizer, pre-tokenizer, model, post-processor and decoder, plus added tokens and truncation/padding settings.

Example

use std::collections::HashMap;
use morpheme::models::WordLevel;
use morpheme::pre_tokenizers::Whitespace;
use morpheme::Tokenizer;

let vocab: HashMap<String, u32> = [("[UNK]", 0), ("[PAD]", 1), ("hello", 2), ("world", 3), ("!", 4)]
    .map(|(t, i)| (t.to_string(), i))
    .into();
let model = WordLevel::builder().vocab(vocab).unk_token("[UNK]").build()?;
let tokenizer = Tokenizer::new(model).with_pre_tokenizer(Whitespace);

let encoding = tokenizer.encode("hello world!", false)?;
assert_eq!(encoding.ids(), [2, 3, 4]);
assert_eq!(tokenizer.decode(encoding.ids(), false)?, "hello world !");

// Save and reload as a Hugging Face tokenizer.json.
let reloaded = Tokenizer::from_json(&tokenizer.to_json(true)?)?;
assert_eq!(reloaded.encode("hello world!", false)?, encoding);

Implementations

impl Tokenizer

pub fn new(model: impl Into<ModelWrapper>) -> Self

A tokenizer with just a model (no normalizer, pre-tokenizer, post-processor or decoder).

pub fn from_json(json: &str) -> Result<Self>

Load from a tokenizer.json string.

pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result<Self>

Load from tokenizer.json bytes.

pub fn from_file(path: impl AsRef<Path>) -> Result<Self>

Load from a tokenizer.json file.

pub fn from_pretrained( identifier: &str, params: Option<FromPretrainedParameters>, ) -> Result<Self>

Available on crate feature hub only.

Download tokenizer.json for identifier (name or org/name) from the Hugging Face Hub and load it. Requires the hub feature.

Files are cached in the standard Hugging Face cache (shared with Python), honoring HF_HOME, HF_HUB_CACHE, HF_TOKEN, HF_ENDPOINT and HF_HUB_OFFLINE. When the Hub is unreachable a cached copy is used if there is one. See FromPretrainedParameters for the options.

use morpheme::{FromPretrainedParameters, Tokenizer};

let tokenizer = Tokenizer::from_pretrained("google-bert/bert-base-uncased", None)?;
let pinned = Tokenizer::from_pretrained(
    "openai-community/gpt2",
    Some(FromPretrainedParameters::default().revision("607a30d783dfa663caf39e06633721c8d4cfcd7e")),
)?;

Example

use morpheme::{FromPretrainedParameters, Tokenizer};

// Downloads into (or reuses) the Hugging Face cache shared with Python.
let bert = Tokenizer::from_pretrained("google-bert/bert-base-uncased", None)?;

// Pin a revision; use a token for gated models (or set HF_TOKEN).
let params = FromPretrainedParameters::default().revision("v1.0");
let pinned = Tokenizer::from_pretrained("my-org/my-model", Some(params))?;

pub fn to_json(&self, pretty: bool) -> Result<String>

Serialize to tokenizer.json.

pub fn save(&self, path: impl AsRef<Path>, pretty: bool) -> Result<()>

Available on non-WebAssembly only.

Save as tokenizer.json using atomic replacement. Readers see the previous complete file or the new complete file. A destination symlink is replaced, rather than writing through it.

Not available on wasm32, which has no filesystem; use to_json there.

pub fn with_normalizer(self, normalizer: impl Into<NormalizerWrapper>) -> Self

Set the normalizer (builder style).

Panics

Panics if an added token cannot be normalized with the new normalizer (see set_normalizer, which returns the error instead and leaves the tokenizer unchanged).

pub fn with_pre_tokenizer( self, pre_tokenizer: impl Into<PreTokenizerWrapper>, ) -> Self

Set the pre-tokenizer (builder style).

pub fn with_post_processor( self, post_processor: impl Into<PostProcessorWrapper>, ) -> Self

Set the post-processor (builder style).

pub fn with_decoder(self, decoder: impl Into<DecoderWrapper>) -> Self

Set the decoder (builder style).

pub fn set_normalizer( &mut self, normalizer: Option<NormalizerWrapper>, ) -> Result<()>

Replace the normalizer. Added tokens that match normalized text are re-normalized. On error, the tokenizer is left unchanged.

pub fn set_pre_tokenizer(&mut self, pre_tokenizer: Option<PreTokenizerWrapper>)

Replace the pre-tokenizer.

pub fn set_model(&mut self, model: impl Into<ModelWrapper>) -> Result<()>

Replace the model, rebinding every id that depends on its vocabulary.

Added tokens keep their options and are assigned ids against the new vocabulary: a token whose text the new model already has takes the model’s id, and the others get ids after the new vocabulary (in their previous order). Post-processor special-token ids and the padding id are then rebound by token text.

Errors

Fails if a token the post-processor or padding is configured with is neither an added token nor in the new vocabulary. On error the tokenizer is left unchanged (including its model).

Example

use std::collections::HashMap;
use morpheme::models::WordLevel;
use morpheme::{AddedToken, Tokenizer};

let wordlevel = |pairs: &[(&str, u32)]| {
    let vocab: HashMap<String, u32> = pairs.iter().map(|(t, i)| (t.to_string(), *i)).collect();
    WordLevel::builder().vocab(vocab).unk_token("[UNK]").build()
};
let mut tokenizer = Tokenizer::new(wordlevel(&[("[UNK]", 0), ("a", 1)])?);
tokenizer.add_special_tokens(&[AddedToken::new("<special>", true)])?;
assert_eq!(tokenizer.token_to_id("<special>"), Some(2));

tokenizer.set_model(wordlevel(&[("[UNK]", 0), ("a", 1), ("new", 2)])?)?;
assert_eq!(tokenizer.token_to_id("<special>"), Some(3)); // moved past the new vocabulary
assert_eq!(tokenizer.decode(&[2], false)?, "new");

pub fn set_post_processor( &mut self, post_processor: Option<PostProcessorWrapper>, )

Replace the post-processor.

pub fn set_decoder(&mut self, decoder: Option<DecoderWrapper>)

Replace the decoder.

pub fn set_truncation( &mut self, truncation: Option<TruncationParams>, ) -> Result<()>

Set truncation. Fails if the stride is not smaller than the effective max length (max length minus added special tokens).

Example

use std::collections::HashMap;
use morpheme::models::WordLevel;
use morpheme::pre_tokenizers::Whitespace;
use morpheme::Tokenizer;

let vocab: HashMap<String, u32> = [("[UNK]", 0), ("[PAD]", 1), ("hello", 2), ("world", 3), ("!", 4)]
    .map(|(t, i)| (t.to_string(), i))
    .into();
let model = WordLevel::builder().vocab(vocab).unk_token("[UNK]").build()?;
use morpheme::TruncationParams;

let mut tokenizer = Tokenizer::new(model).with_pre_tokenizer(Whitespace);
tokenizer.set_truncation(Some(TruncationParams { max_length: 2, ..Default::default() }))?;

let encoding = tokenizer.encode("hello world !", false)?;
assert_eq!(encoding.ids(), [2, 3]);
assert_eq!(encoding.overflowing()[0].ids(), [4]); // the rest

pub fn set_padding(&mut self, padding: Option<PaddingParams>)

Set padding.

pub fn normalizer(&self) -> Option<&NormalizerWrapper>

The normalizer.

pub fn pre_tokenizer(&self) -> Option<&PreTokenizerWrapper>

The pre-tokenizer.

pub fn model(&self) -> &ModelWrapper

The model.

pub fn model_mut(&mut self) -> &mut ModelWrapper

Mutable access to the model, for runtime settings such as Unigram sampling (Unigram::set_sampling / set_seed) or BPE dropout (Bpe::set_dropout).

Do not change the vocabulary through this accessor: added-token, post-processor and padding ids are bound to the current vocabulary, and swapping or retraining it in place leaves them stale. Use set_model, which rebinds them by token text, or train for that. Model-internal caches (the BPE word cache, the Unigram sentence cache) are managed by the models’ own setters.

use morpheme::models::{ModelWrapper, Unigram};
use morpheme::Tokenizer;

let mut tok = Tokenizer::new(Unigram::default());
if let ModelWrapper::Unigram(u) = tok.model_mut() {
    u.set_sampling(0.1, -1)?;
    u.set_seed(Some(42));
}

pub fn post_processor(&self) -> Option<&PostProcessorWrapper>

The post-processor.

pub fn decoder(&self) -> Option<&DecoderWrapper>

The decoder.

pub fn truncation(&self) -> Option<&TruncationParams>

Truncation settings.

pub fn padding(&self) -> Option<&PaddingParams>

Padding settings.

pub fn added_vocabulary(&self) -> &AddedVocabulary

The added tokens.

pub fn set_encode_special_tokens(&mut self, value: bool)

When true, special tokens in the input are tokenized as plain text instead of being matched.

pub fn vocab(&self, with_added_tokens: bool) -> HashMap<String, u32>

The vocabulary, optionally including added tokens.

pub fn vocab_size(&self, with_added_tokens: bool) -> usize

Vocabulary size, optionally counting added tokens that are not already in the model’s vocabulary.

pub fn token_to_id(&self, token: &str) -> Option<u32>

Id of token (added tokens take precedence).

pub fn id_to_token(&self, id: u32) -> Option<String>

Token for id (added tokens take precedence).

pub fn add_tokens(&mut self, tokens: &[AddedToken]) -> Result<usize>

Add regular tokens. Returns how many were added.

pub fn add_special_tokens(&mut self, tokens: &[AddedToken]) -> Result<usize>

Add special tokens (marked special). Returns how many were added.

pub fn normalize(&self, text: &str) -> Result<NormalizedString>

Run the normalizer (only) on text.

pub fn encode<'s>( &self, input: impl Into<EncodeInput<'s>>, add_special_tokens: bool, ) -> Result<Encoding>

Encode a sequence or a pair. Offsets are byte offsets into the original input(s).

Example

use std::collections::HashMap;
use morpheme::models::WordLevel;
use morpheme::pre_tokenizers::Whitespace;
use morpheme::Tokenizer;

let vocab: HashMap<String, u32> = [("[UNK]", 0), ("[PAD]", 1), ("hello", 2), ("world", 3), ("!", 4)]
    .map(|(t, i)| (t.to_string(), i))
    .into();
let model = WordLevel::builder().vocab(vocab).unk_token("[UNK]").build()?;
let tokenizer = Tokenizer::new(model).with_pre_tokenizer(Whitespace);

// A single sequence, a pair, or pre-split words.
assert_eq!(tokenizer.encode("hello world", true)?.ids(), [2, 3]);
let pair = tokenizer.encode(("hello", "world"), true)?;
assert_eq!(pair.type_ids(), [0, 1]);
let words = tokenizer.encode(vec!["hello", "world"], true)?;
assert_eq!(words.word_ids(), [Some(0), Some(1)]);

pub fn encode_char_offsets<'s>( &self, input: impl Into<EncodeInput<'s>>, add_special_tokens: bool, ) -> Result<Encoding>

Like encode, but offsets are char offsets (what the Python tokenizers API returns).

pub fn encode_batch<'s, E>( &self, inputs: Vec<E>, add_special_tokens: bool, ) -> Result<Vec<Encoding>>

where E: Into<EncodeInput<'s>> + Send,

Encode several inputs, in parallel with the parallel feature (on by default) and sequentially without it. With BatchLongest padding, all encodings are padded to the longest one. Byte offsets.

Example

use std::collections::HashMap;
use morpheme::models::WordLevel;
use morpheme::pre_tokenizers::Whitespace;
use morpheme::Tokenizer;

let vocab: HashMap<String, u32> = [("[UNK]", 0), ("[PAD]", 1), ("hello", 2), ("world", 3), ("!", 4)]
    .map(|(t, i)| (t.to_string(), i))
    .into();
let model = WordLevel::builder().vocab(vocab).unk_token("[UNK]").build()?;
use morpheme::PaddingParams;

let mut tokenizer = Tokenizer::new(model).with_pre_tokenizer(Whitespace);
tokenizer.set_padding(Some(PaddingParams { pad_id: 1, ..Default::default() }));

let batch = tokenizer.encode_batch(vec!["hello", "hello world !"], false)?;
assert_eq!(batch[0].ids(), [2, 1, 1]); // padded to the longest
assert_eq!(batch[0].attention_mask(), [1, 0, 0]);

pub fn encode_batch_char_offsets<'s, E>( &self, inputs: Vec<E>, add_special_tokens: bool, ) -> Result<Vec<Encoding>>

where E: Into<EncodeInput<'s>> + Send,

encode_batch with char offsets.

pub fn post_process( &self, encoding: Encoding, pair: Option<Encoding>, add_special_tokens: bool, ) -> Result<Encoding>

Truncate, post-process (add special tokens, merge pairs) and pad.

pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String>

Decode ids back to text. Unknown ids are skipped.

Example

use std::collections::HashMap;
use morpheme::models::WordLevel;
use morpheme::pre_tokenizers::Whitespace;
use morpheme::Tokenizer;

let vocab: HashMap<String, u32> = [("[UNK]", 0), ("[PAD]", 1), ("hello", 2), ("world", 3), ("!", 4)]
    .map(|(t, i)| (t.to_string(), i))
    .into();
let model = WordLevel::builder().vocab(vocab).unk_token("[UNK]").build()?;
use morpheme::decoders::WordPiece;
use morpheme::AddedToken;

let mut tokenizer = Tokenizer::new(model)
    .with_pre_tokenizer(Whitespace)
    .with_decoder(WordPiece::new("##", true));
tokenizer.add_special_tokens(&[AddedToken::new("[PAD]", true)])?;

assert_eq!(tokenizer.decode(&[2, 3, 4, 1], false)?, "hello world! [PAD]");
assert_eq!(tokenizer.decode(&[2, 3, 4, 1], true)?, "hello world!");

pub fn decode_batch( &self, sequences: &[&[u32]], skip_special_tokens: bool, ) -> Result<Vec<String>>

Decode several id sequences, in parallel with the parallel feature (on by default) and sequentially without it.

pub fn decode_stream(&self, skip_special_tokens: bool) -> DecodeStream<'_>

Start incremental decoding for ids that arrive one at a time (e.g. during generation). See DecodeStream.

pub fn train<T, I, S>(&mut self, trainer: T, sequences: I) -> Result<&mut Self>

where T: Into<TrainerWrapper>, I: Iterator<Item = S> + Send, S: AsRef<str> + Send,

Train the model on sequences. Each sequence is normalized and pre-tokenized with this tokenizer’s components before counting. The trainer’s special tokens are registered as added tokens. If the trainer is for a different kind of model, the model is replaced. Existing added tokens retain their options and are assigned ids against the new vocabulary. Post-processor and padding ids are rebound by token text. If a configured token is missing, training fails without changing this tokenizer.

pub fn train_from_files<T, P>( &mut self, trainer: T, files: &[P], ) -> Result<&mut Self>

where T: Into<TrainerWrapper>, P: AsRef<Path>,

Train on the lines of text files (line endings are kept, like Hugging Face). Lines are streamed rather than retaining the full corpus. I/O and UTF-8 errors are reported, not skipped.

Trait Implementations

impl Clone for Tokenizer

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Tokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for Tokenizer

fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error>

Deserialize this value from the given Serde deserializer.

impl FromStr for Tokenizer

type Err = Error

The associated error which can be returned from parsing.

fn from_str(s: &str) -> Result<Self>

Parses a string s to return a value of this type.

impl Serialize for Tokenizer

fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error>

Serialize this value into the given Serde serializer.