Struct SplitterOptions

pub struct SplitterOptions {
    pub chunk_size: usize,
    pub chunk_overlap: usize,
    pub model_name: String,
    pub encoding_name: String,
    pub trim_chunks: bool,
}

Fields

chunk_size: usize

chunk_overlap: usize

model_name: String

encoding_name: String

trim_chunks: bool

Implementations

impl SplitterOptions

pub fn new() -> Self

impl SplitterOptions

pub fn with_chunk_size(self, chunk_size: usize) -> Self

pub fn with_chunk_overlap(self, chunk_overlap: usize) -> Self

pub fn with_model_name(self, model_name: &str) -> Self

pub fn with_encoding_name(self, encoding_name: &str) -> Self

pub fn with_trim_chunks(self, trim_chunks: bool) -> Self

pub fn get_tokenizer_from_str(s: &str) -> Option<Tokenizer>

Trait Implementations

impl Clone for SplitterOptions

fn clone(&self) -> SplitterOptions

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for SplitterOptions

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for SplitterOptions

fn default() -> Self

Returns the “default value” for a type.

impl TryFrom<&SplitterOptions> for ChunkConfig<CoreBPE>

type Error = TextSplitterError

The type returned in the event of a conversion error.

fn try_from(options: &SplitterOptions) -> Result<Self, Self::Error>

Performs the conversion.