Struct EncodingWithOffsets 

pub struct EncodingWithOffsets {
    pub ids: Vec<u32>,
    pub tokens: Vec<String>,
    pub offsets: Vec<(usize, usize)>,
}

Encoding result with tokens and their character offsets.

Produced by a tokenizer’s encode_with_offsets method (or equivalent). Used to map between character positions in source text and token indices.

Example

use candle_mi::EncodingWithOffsets;

let encoding = EncodingWithOffsets::new(
    vec![1, 2, 3],
    vec!["def".into(), " ".into(), "add".into()],
    vec![(0, 3), (3, 4), (4, 7)],
);

// Character 4 ('a' in "add") is in token 2
assert_eq!(encoding.char_to_token(4), Some(2));

Fields

ids: Vec<u32>

Token IDs.

tokens: Vec<String>

Token strings.

offsets: Vec<(usize, usize)>

Character offset for each token: (start, end).

Implementations

impl EncodingWithOffsets

pub const fn new( ids: Vec<u32>, tokens: Vec<String>, offsets: Vec<(usize, usize)>, ) -> Self

Create a new encoding with offsets.

pub fn tokens_with_offsets(&self) -> Vec<TokenWithOffset>

Get tokens with their character offsets.

pub fn char_to_token(&self, char_pos: usize) -> Option<usize>

Find the token index that contains the given character position.

Returns None if no token spans that position.

pub fn char_to_token_fuzzy(&self, char_pos: usize) -> Option<usize>

Find the token index for a character position, with fuzzy fallback.

If the exact position isn’t contained in any token, returns the index of the closest token by midpoint distance.

pub fn char_to_token_start(&self, char_pos: usize) -> Option<usize>

Find the token index that starts at or after the given character position.

pub fn char_range_to_tokens( &self, start_char: usize, end_char: usize, ) -> Vec<usize>

Find all token indices that overlap with the given character range.

pub fn token_to_char_range(&self, token_idx: usize) -> Option<(usize, usize)>

Get the character range for a token index.

pub const fn len(&self) -> usize

Number of tokens.

pub const fn is_empty(&self) -> bool

Whether the encoding is empty.

Trait Implementations

impl Clone for EncodingWithOffsets

fn clone(&self) -> EncodingWithOffsets

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for EncodingWithOffsets

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.