Struct Qwen2BpeTokenizer 

pub struct Qwen2BpeTokenizer { /* private fields */ }

Qwen2-specific BPE tokenizer with chat template support.

Extends the base BPE tokenizer with Qwen2’s special tokens and chat formatting conventions.

Example

use aprender::text::bpe::Qwen2BpeTokenizer;

let tokenizer = Qwen2BpeTokenizer::new();

// Check special tokens
assert!(tokenizer.is_eos(151645)); // <|im_end|>

// Format a chat message
let formatted = tokenizer.format_chat("user", "Hello, world!");
assert!(formatted.contains("<|im_start|>user"));

Implementations

impl Qwen2BpeTokenizer

pub const IM_START_ID: u32 = 151644

Special token: <|im_start|>

pub const IM_END_ID: u32 = 151645

Special token: <|im_end|>

pub const ENDOFTEXT_ID: u32 = 151643

Special token: <|endoftext|>

pub fn new() -> Self

Create a new Qwen2 tokenizer.

pub fn is_eos(&self, token_id: u32) -> bool

Check if token is EOS (end of sequence).

pub fn is_bos(&self, token_id: u32) -> bool

Check if token is BOS (beginning of sequence).

pub fn vocab_size(&self) -> usize

Get vocabulary size.

pub fn encode(&self, text: &str) -> Vec<u32>

Encode text to token IDs.

pub fn decode(&self, ids: &[u32]) -> String

Decode token IDs to text.

pub fn format_chat(&self, role: &str, content: &str) -> String

Format a chat message with Qwen2 template.

Format: <|im_start|>role\nmessage<|im_end|>\n

pub fn format_conversation(&self, messages: &[(&str, &str)]) -> String

Format a complete chat conversation.

Arguments

• messages - List of (role, content) pairs

Returns

Formatted conversation string with chat template applied.

pub fn im_start_id(&self) -> u32

Get the im_start token ID.

pub fn im_end_id(&self) -> u32

Get the im_end token ID.

pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self>

Load tokenizer from tokenizer.json file path.

Arguments

• path - Path to tokenizer.json file

Returns

Loaded Qwen2 tokenizer with full vocabulary

Errors

Returns error if file cannot be read or parsed.

pub fn from_json(json: &str) -> Result<Self>

Load tokenizer from JSON string.

Arguments

• json - JSON string containing HuggingFace tokenizer format

Returns

Loaded Qwen2 tokenizer with full vocabulary

Errors

Returns error if JSON parsing fails.

Trait Implementations

impl Clone for Qwen2BpeTokenizer

fn clone(&self) -> Qwen2BpeTokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Qwen2BpeTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for Qwen2BpeTokenizer

fn default() -> Self

Returns the “default value” for a type.