Struct Sequence 

pub struct Sequence { /* private fields */ }

Maintains state for an ongoing sequence of tokens and their decoded text This provides a cleaner abstraction for managing token sequences

Implementations

impl Sequence

pub fn new(tokenizer: Arc<dyn TokenizerTrait>) -> Self

Create a new empty sequence

pub fn new_with_options( tokenizer: Arc<dyn TokenizerTrait>, skip_special_tokens: bool, ) -> Self

Create a new empty sequence with skip_special_tokens option

pub fn with_tokens( tokenizer: Arc<dyn TokenizerTrait>, token_ids: Vec<TokenIdType>, ) -> Self

Create a sequence with initial tokens

pub fn with_tokens_and_options( tokenizer: Arc<dyn TokenizerTrait>, token_ids: Vec<TokenIdType>, skip_special_tokens: bool, ) -> Self

Create a sequence with initial tokens and skip_special_tokens option

pub fn is_empty(&self) -> bool

Check if the sequence is empty

pub fn len(&self) -> usize

Get the length of the sequence

pub fn clear(&mut self)

Clear the sequence

pub fn append_text( &mut self, input: &str, add_special_tokens: bool, ) -> Result<()>

Append text to the sequence by encoding it

Set add_special_tokens to true for embeddings, or false for chat completion where the chat template already handles special tokens.

pub fn append_token(&mut self, token_id: TokenIdType) -> Result<String>

Append a single token to the sequence and return newly decoded text Based on HuggingFace TGI incremental decoding

pub fn tokenizer(&self) -> &Arc<dyn TokenizerTrait>

Get a reference to the tokenizer

pub fn token_ids(&self) -> &[TokenIdType]

Get the current token ids

pub fn text(&self) -> Result<String>

Decode the entire sequence to text

pub fn prefix_offset(&self) -> usize

Get the prefix offset

pub fn read_offset(&self) -> usize

Get the read offset

pub fn skip_special_tokens(&self) -> bool

Get whether special tokens are skipped during decoding

Trait Implementations

impl Debug for Sequence

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.