Struct CodeTokenizer 

pub struct CodeTokenizer { /* private fields */ }

Code tokenizer implementation

Implementations

impl CodeTokenizer

pub fn new(config: CodeTokenizerConfig) -> Self

Create a new code tokenizer

pub fn for_language(language: Language) -> Self

Create tokenizer for a specific language

pub fn tokenize_code(&self, code: &str) -> Result<Vec<CodeToken>>

Tokenize code into structured tokens

pub fn vocab_size(&self) -> usize

Get vocabulary size

pub fn token_to_id(&self, token: &str) -> Option<u32>

Get token ID

pub fn id_to_token(&self, id: u32) -> Option<String>

Get token from ID

Trait Implementations

impl Tokenizer for CodeTokenizer

fn encode(&self, text: &str) -> Result<TokenizedInput>

Encodes a single text string into tokens.

fn decode(&self, ids: &[u32]) -> Result<String>

Decodes token IDs back into text.

fn encode_pair(&self, text_a: &str, text_b: &str) -> Result<TokenizedInput>

Encodes a pair of texts for sequence-pair tasks.

fn vocab_size(&self) -> usize

Returns the size of the tokenizer’s vocabulary.

fn get_vocab(&self) -> HashMap<String, u32>

Returns a copy of the vocabulary as a mapping from tokens to IDs.

fn token_to_id(&self, token: &str) -> Option<u32>

Converts a token string to its corresponding ID.

fn id_to_token(&self, id: u32) -> Option<String>

Converts a token ID to its corresponding token string.