Struct Vocabulary 

pub struct Vocabulary { /* private fields */ }

Vocabulary management

Implementations

impl Vocabulary

pub fn new(vocab_size: usize) -> Result<Self>

Create new vocabulary

pub fn tokenize(&self, text: &str) -> Result<Vec<usize>>

Tokenize text

pub fn get_embedding(&self, token_id: usize) -> Result<Array1<f64>>

Get token embedding

pub fn decode_token(&self, token_id: usize) -> Result<String>

Decode token

pub fn is_eos_token(&self, token_id: usize) -> bool

Check if token is end-of-sequence

Trait Implementations

impl Clone for Vocabulary

fn clone(&self) -> Vocabulary

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Vocabulary

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.