langchain_rust/text_splitter/
options.rs

1use text_splitter::ChunkConfig;
2use tiktoken_rs::{bpe_for_model, bpe_for_tokenizer, tokenizer::Tokenizer, CoreBPE};
3
4use super::TextSplitterError;
5
6// Options is a struct that contains options for a text splitter.
7#[derive(Debug, Clone)]
8pub struct SplitterOptions {
9    pub chunk_size: usize,
10    pub chunk_overlap: usize,
11    pub model_name: String,
12    pub encoding_name: String,
13    pub trim_chunks: bool,
14}
15
16impl Default for SplitterOptions {
17    fn default() -> Self {
18        Self::new()
19    }
20}
21
22impl SplitterOptions {
23    pub fn new() -> Self {
24        SplitterOptions {
25            chunk_size: 512,
26            chunk_overlap: 0,
27            model_name: String::from("gpt-3.5-turbo"),
28            encoding_name: String::from("cl100k_base"),
29            trim_chunks: false,
30        }
31    }
32}
33
34// Builder pattern for Options struct
35impl SplitterOptions {
36    pub fn with_chunk_size(mut self, chunk_size: usize) -> Self {
37        self.chunk_size = chunk_size;
38        self
39    }
40
41    pub fn with_chunk_overlap(mut self, chunk_overlap: usize) -> Self {
42        self.chunk_overlap = chunk_overlap;
43        self
44    }
45
46    pub fn with_model_name(mut self, model_name: &str) -> Self {
47        self.model_name = String::from(model_name);
48        self
49    }
50
51    pub fn with_encoding_name(mut self, encoding_name: &str) -> Self {
52        self.encoding_name = String::from(encoding_name);
53        self
54    }
55
56    pub fn with_trim_chunks(mut self, trim_chunks: bool) -> Self {
57        self.trim_chunks = trim_chunks;
58        self
59    }
60
61    pub fn get_tokenizer_from_str(s: &str) -> Option<Tokenizer> {
62        match s.to_lowercase().as_str() {
63            "cl100k_base" => Some(Tokenizer::Cl100kBase),
64            "p50k_base" => Some(Tokenizer::P50kBase),
65            "r50k_base" => Some(Tokenizer::R50kBase),
66            "p50k_edit" => Some(Tokenizer::P50kEdit),
67            "gpt2" => Some(Tokenizer::Gpt2),
68            _ => None,
69        }
70    }
71}
72
73impl TryFrom<&SplitterOptions> for ChunkConfig<&'static CoreBPE> {
74    type Error = TextSplitterError;
75
76    fn try_from(options: &SplitterOptions) -> Result<Self, Self::Error> {
77        let tk = if !options.encoding_name.is_empty() {
78            let tokenizer = SplitterOptions::get_tokenizer_from_str(&options.encoding_name)
79                .ok_or(TextSplitterError::TokenizerNotFound)?;
80
81            bpe_for_tokenizer(tokenizer).map_err(|_| TextSplitterError::InvalidTokenizer)?
82        } else {
83            bpe_for_model(&options.model_name).map_err(|_| TextSplitterError::InvalidModel)?
84        };
85
86        Ok(ChunkConfig::new(options.chunk_size)
87            .with_sizer(tk)
88            .with_trim(options.trim_chunks)
89            .with_overlap(options.chunk_overlap)?)
90    }
91}
langchain_rust/text_splitter/options.rs

langchain_rust/text_splitter/
options.rs
