Struct SimpleTokenizerV2 

pub struct SimpleTokenizerV2 { /* private fields */ }

[Listing 2.4] A simple text tokenizer that handles unknown words

Implementations

impl SimpleTokenizerV2

pub fn from_vocab(vocab: HashMap<&str, i32>) -> Self

Creates a new SimpleTokenizerV2 from a vocab.

use llms_from_scratch_rs::listings::ch02::SimpleTokenizerV2;
use std::collections::HashMap;

let vocab: HashMap<&str, i32> = HashMap::from([
    ("this", 1_i32),
    ("is", 2_i32),
    ("a", 3_i32),
    ("test", 4_i32)
]);
// Any words not in the vocab will be encoded as "<|unk|>" token
let tokenizer = SimpleTokenizerV2::from_vocab(vocab);

pub fn encode(&self, text: &str) -> Vec<i32>

Encode a text into its token ids.

pub fn decode(&self, ids: Vec<i32>) -> String

Decode token ids into its text.

Trait Implementations

impl Debug for SimpleTokenizerV2

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for SimpleTokenizerV2

fn default() -> SimpleTokenizerV2

Returns the “default value” for a type.