Struct EncodeConfig 

pub struct EncodeConfig { /* private fields */ }

Configuration for encode_with / encode_batch.

Build it fluently:

use tpt_tokenizer_core::encoding::{EncodeConfig, Padding, Truncation};

let cfg = EncodeConfig::new()
    .with_bos(1)
    .with_eos(2)
    .with_pad(0)
    .padding(Padding::Longest)
    .truncation(Truncation::Fixed(128));

Implementations

impl EncodeConfig

pub fn new() -> Self

A config that adds no special tokens and applies no padding or truncation — equivalent to a plain Tokenizer::encode, but wrapped in an Encoding with an attention mask.

pub fn with_bos(self, id: TokenId) -> Self

Set the beginning-of-sequence token id to prepend (enables special-token insertion).

pub fn with_eos(self, id: TokenId) -> Self

Set the end-of-sequence token id to append (enables special-token insertion).

pub fn with_pad(self, id: TokenId) -> Self

Set the padding token id. Required for any Padding other than Padding::None.

pub fn add_special_tokens(self, yes: bool) -> Self

Enable or disable BOS/EOS insertion without changing the ids.

pub fn padding(self, padding: Padding) -> Self

Set the padding strategy.

pub fn truncation(self, truncation: Truncation) -> Self

Set the truncation strategy.

Trait Implementations

impl Clone for EncodeConfig

fn clone(&self) -> EncodeConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for EncodeConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for EncodeConfig

fn default() -> EncodeConfig

Returns the “default value” for a type.