Trait ChunkSizer

pub trait ChunkSizer {
    // Required method
    fn size(&self, chunk: &str) -> usize;
}

Determines the size of a given chunk.

Required Methods

fn size(&self, chunk: &str) -> usize

Determine the size of a given chunk to use for validation

Implementations on Foreign Types

impl ChunkSizer for CoreBPE

Available on crate feature tiktoken-rs only.

fn size(&self, chunk: &str) -> usize

Returns the number of tokens in a given text after tokenization.

impl ChunkSizer for Tokenizer

Available on crate feature tokenizers only.

fn size(&self, chunk: &str) -> usize

Returns the number of tokens in a given text after tokenization.

Panics

Will panic if you don’t have a byte-level tokenizer and the splitter encounters text it can’t tokenize.

impl<T> ChunkSizer for Cow<'_, T>

where T: ChunkSizer + ToOwned + ?Sized, <T as ToOwned>::Owned: ChunkSizer,

fn size(&self, chunk: &str) -> usize

impl<T> ChunkSizer for &T

where T: ChunkSizer,

fn size(&self, chunk: &str) -> usize

impl<T> ChunkSizer for Box<T>

where T: ChunkSizer,

fn size(&self, chunk: &str) -> usize

impl<T> ChunkSizer for Rc<T>

where T: ChunkSizer,

fn size(&self, chunk: &str) -> usize

impl<T> ChunkSizer for Arc<T>

where T: ChunkSizer,

fn size(&self, chunk: &str) -> usize

impl<T> ChunkSizer for Ref<'_, T>

where T: ChunkSizer,

fn size(&self, chunk: &str) -> usize

impl<T> ChunkSizer for RefMut<'_, T>

where T: ChunkSizer,

fn size(&self, chunk: &str) -> usize

Implementors

impl ChunkSizer for Characters