Struct DylibTokenizer 

pub struct DylibTokenizer { /* private fields */ }

Implementations

impl DylibTokenizer

pub fn open(library: &Path, model: &Path) -> Result<DylibTokenizer, String>

pub fn library(&self) -> &Path

Trait Implementations

impl Drop for DylibTokenizer

fn drop(&mut self)

Executes the destructor for this type.

fn pin_drop(self: Pin<&mut Self>)

🔬This is a nightly-only experimental API. (pin_ergonomics)

Execute the destructor for this type, but different to Drop::drop, it requires self to be pinned.

impl Send for DylibTokenizer

impl Sync for DylibTokenizer

impl Tokenizer for DylibTokenizer

fn encode(&self, text: &str) -> Vec<u32>

fn token_bytes(&self, token: u32) -> Vec<u8>

fn vocab_size(&self) -> u32

fn special_tokens(&self) -> Vec<(String, u32)>

fn bos_token(&self) -> Option<u32>

fn eos_token(&self) -> u32

fn chat_template_jinja(&self) -> String

fn chat_template_named(&self, name: &str) -> Option<String>

fn encode_plain(&self, _text: &str) -> Option<Vec<u32>>

fn encode_pieces(&self, _pieces: &[(&str, bool)]) -> Option<Vec<u32>>