Struct HfTokenizer 

pub struct HfTokenizer { /* private fields */ }

HuggingFace-compatible tokenizer wrapper

Wraps aprender’s BPE tokenizer to provide training batch utilities.

Implementations

impl HfTokenizer

pub fn gpt2() -> Self

Create a GPT-2 tokenizer with base vocabulary

pub fn qwen2() -> Self

Create a Qwen2 tokenizer

pub fn from_file(path: impl AsRef<Path>) -> Result<Self>

Load tokenizer from HuggingFace tokenizer.json file

Errors

Returns error if file cannot be read or parsed.

pub fn from_json(json: &str) -> Result<Self>

Load tokenizer from JSON string

Errors

Returns error if JSON parsing fails.

pub fn vocab_size(&self) -> usize

Get vocabulary size

pub fn encode(&self, text: &str) -> Vec<u32>

Encode text to token IDs

pub fn encode_with_special(&self, text: &str) -> Vec<u32>

Encode text with special tokens (BOS/EOS)

pub fn decode(&self, ids: &[u32]) -> String

Decode token IDs to text

pub fn pad_id(&self) -> u32

Get padding token ID

pub fn eos_id(&self) -> Option<u32>

Get EOS token ID

pub fn bos_id(&self) -> Option<u32>

Get BOS token ID

pub fn batch_encode(&self, texts: &[&str], max_len: usize) -> Vec<Vec<u32>>

Batch encode texts with padding

pub fn create_batches( &self, pairs: &[(&str, &str)], max_len: usize, batch_size: usize, ) -> Vec<Batch>

Create training batches from text pairs

pub fn create_causal_batches( &self, texts: &[&str], max_len: usize, batch_size: usize, ) -> Vec<Batch>

Create causal LM batches (target = shifted input)

Trait Implementations

impl Clone for HfTokenizer

fn clone(&self) -> HfTokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for HfTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for HfTokenizer

fn default() -> Self

Returns the “default value” for a type.