Trait ChunkSizer

pub trait ChunkSizer {
    // Required method
    fn size(&self, chunk: &str) -> usize;
}

Determines the size of a given chunk.

Required Methods

fn size(&self, chunk: &str) -> usize

Determine the size of a given chunk to use for validation

Implementations on Foreign Types

impl ChunkSizer for &AlbertTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &BertTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &CtrlTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &DeBERTaTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &DeBERTaV2Tokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &FNetTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &Gpt2Tokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &M2M100Tokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &MarianTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &MBart50Tokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &NLLBTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &OpenAiGptTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &PegasusTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &ProphetNetTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &ReformerTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &RobertaTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &SentencePieceBpeTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &SentencePieceTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &T5Tokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &XLMRobertaTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &XLNetTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for &CoreBPE

Available on crate feature tiktoken-rs only.

fn size(&self, chunk: &str) -> usize

Returns the number of tokens in a given text after tokenization.

impl ChunkSizer for &Tokenizer

Available on crate feature tokenizers only.

fn size(&self, chunk: &str) -> usize

Returns the number of tokens in a given text after tokenization.

Panics

Will panic if you don’t have a byte-level tokenizer and the splitter encounters text it can’t tokenize.

impl ChunkSizer for AlbertTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for BertTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for CtrlTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for DeBERTaTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for DeBERTaV2Tokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for FNetTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for Gpt2Tokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for M2M100Tokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for MarianTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for MBart50Tokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for NLLBTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for OpenAiGptTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for PegasusTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for ProphetNetTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for ReformerTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for RobertaTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for SentencePieceBpeTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for SentencePieceTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for T5Tokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for XLMRobertaTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for XLNetTokenizer

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl ChunkSizer for CoreBPE

Available on crate feature tiktoken-rs only.

fn size(&self, chunk: &str) -> usize

Returns the number of tokens in a given text after tokenization.

impl ChunkSizer for Tokenizer

Available on crate feature tokenizers only.

fn size(&self, chunk: &str) -> usize

Returns the number of tokens in a given text after tokenization.

Panics

Will panic if you don’t have a byte-level tokenizer and the splitter encounters text it can’t tokenize.

impl<V> ChunkSizer for &BaseTokenizer<V>

where V: Vocab + Sync + Send,

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

impl<V> ChunkSizer for BaseTokenizer<V>

where V: Vocab + Sync + Send,

Available on crate feature rust-tokenizers only.

fn size(&self, chunk: &str) -> usize

Implementors

impl ChunkSizer for Characters