Struct mako::vocab::Vocab

pub struct Vocab {
    pub num_tokens: u32,
    pub token2index: HashMap<String, u32>,
    pub index2token: Vec<String>,
    pub PAD_token: u32,
    pub SOS_token: u32,
    pub EOS_token: u32,
    pub SEP_token: u32,
}

Railgun Vocab Object

Fields

num_tokens: u32

token2index: HashMap<String, u32>

index2token: Vec<String>

PAD_token: u32

SOS_token: u32

EOS_token: u32

SEP_token: u32

Implementations

impl Vocab

pub fn new() -> Vocab

Make a new vocab

pub fn add_token(&mut self, token: String)

Add token to vocab

pub fn add_tokens(&mut self, tokens: Vec<String>)

Add a vec of tokens to vocab

pub fn remove_tokens(&mut self, tokens: Vec<String>)

Remove a vec of tokens from vocab

pub fn remove_token(&mut self, token: String)

Remove token from vocab

pub fn tokens_from_indexes(&self, indexes: &Vec<u32>) -> Result<Vec<String>, ()>

Get vec of tokens from vec of indexes

pub fn batch_tokens_from_indexes(
    &self,
    indexes: &Vec<Vec<u32>>
) -> Result<Vec<Vec<String>>, ()>

Batched version of tokens_from_indexes

pub fn indexes_from_tokens(&self, tokens: &Vec<String>) -> Result<Vec<u32>, ()>

Get vec of indexes from vec of tokens

pub fn batch_indexes_from_tokens(
    &self,
    tokens: &Vec<Vec<String>>
) -> Result<Vec<Vec<u32>>, ()>

Batched version of indexes_from_tokens

Trait Implementations

impl<'de> Deserialize<'de> for Vocab

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error> where
    __D: Deserializer<'de>, 

Deserialize this value from the given Serde deserializer.

impl Serialize for Vocab

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error> where
    __S: Serializer, 

Serialize this value into the given Serde serializer.