Skip to main content

recursive_text_splitter/
lib.rs

1/// A high-performance recursive character text splitter.
2///
3/// Mirrors Python LangChain's `RecursiveCharacterTextSplitter` API.
4/// Tries each separator in priority order; splits on the first viable separator,
5/// then recursively splits chunks that exceed `chunk_size` using the remaining
6/// separators. Merges splits with overlap, matching Python's `merge_splits` behavior.
7///
8/// # Efficiency advantages over Python version:
9/// - Uses `&str` byte slicing — no string cloning until final chunk output
10/// - Pre-allocates output capacity based on text length
11/// - Single-pass separator search using `str::find` (no regex overhead — Python uses `re.escape` + `re.search`)
12/// - Minimal allocations in the hot path
13#[derive(Clone)]
14pub struct RecursiveCharacterTextSplitter {
15    /// Ordered list of separators to try, e.g. ["\nINT.", "\nEXT.", "\n\n", "\n", " ", ""]
16    separators: Vec<&'static str>,
17    /// Maximum number of characters per chunk
18    chunk_size: usize,
19    /// Number of characters to overlap between chunks
20    chunk_overlap: usize,
21    /// Whether to add start_index metadata to each chunk
22    add_start_index: bool,
23    /// Whether to strip whitespace from chunk edges
24    strip_whitespace: bool,
25    /// Whether to keep separators attached to each piece
26    keep_separator: bool,
27    /// How to measure chunk size
28    length_function: LengthFunction,
29    /// Splitting strategy (sequential by default, parallel with rayon)
30    strategy: std::sync::Arc<dyn SplitterStrategy + Send + Sync>,
31}
32
33/// A chunk of text produced by the splitter.
34#[derive(Debug, Clone, PartialEq, Eq)]
35pub struct Chunk {
36    pub page_content: String,
37    pub start_index: Option<usize>,
38}
39
40impl Chunk {
41    pub fn new(content: String, start_index: Option<usize>) -> Self {
42        Self {
43            page_content: content,
44            start_index,
45        }
46    }
47}
48
49/// How to measure chunk size
50#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
51pub enum LengthFunction {
52    /// Count Unicode scalar values (characters)
53    #[default]
54    CharCount,
55    /// Count bytes
56    ByteCount,
57}
58
59impl LengthFunction {
60    #[inline]
61    fn len(&self, text: &str) -> usize {
62        match self {
63            LengthFunction::CharCount => text.chars().count(),
64            LengthFunction::ByteCount => text.len(),
65        }
66    }
67}
68
69/// Splitting strategy trait — allows selecting sequential or parallel
70/// implementation at compile time via feature flags.
71///
72/// When the `rayon` feature is enabled, `ParallelStrategy` is available and
73/// can be selected via `with_strategy::<ParallelStrategy>()`.
74/// When `rayon` is disabled, only `SequentialStrategy` is available.
75pub trait SplitterStrategy {
76    /// Split text into chunks using the strategy's algorithm.
77    fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk>;
78
79    /// Split text into content strings using the strategy's algorithm.
80    fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String>;
81}
82
83/// Default sequential splitting strategy — single-threaded.
84#[derive(Debug, Clone, Default)]
85pub struct SequentialStrategy;
86
87impl SplitterStrategy for SequentialStrategy {
88    fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
89        let chunks = splitter._split_text(text, &splitter.separators);
90
91        if splitter.add_start_index {
92            splitter.annotate_start_index(text, &chunks)
93        } else {
94            chunks.into_iter().map(|s| Chunk::new(s, None)).collect()
95        }
96    }
97
98    fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
99        splitter
100            .split_text(text)
101            .into_iter()
102            .map(|c| c.page_content)
103            .collect()
104    }
105}
106
107/// Parallel splitting strategy using rayon — requires the `rayon` feature.
108#[cfg(feature = "rayon")]
109#[derive(Debug, Clone, Default)]
110pub struct ParallelStrategy;
111
112#[cfg(feature = "rayon")]
113impl SplitterStrategy for ParallelStrategy {
114    fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
115        let chunks = splitter._split_text_par(text, &splitter.separators);
116
117        if splitter.add_start_index {
118            splitter.annotate_start_index(text, &chunks)
119        } else {
120            chunks.into_iter().map(|s| Chunk::new(s, None)).collect()
121        }
122    }
123
124    fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
125        splitter
126            .split_text_par(text)
127            .into_iter()
128            .map(|c| c.page_content)
129            .collect()
130    }
131}
132
133impl Default for RecursiveCharacterTextSplitter {
134    fn default() -> Self {
135        Self::new()
136    }
137}
138
139impl RecursiveCharacterTextSplitter {
140    /// Create a new `RecursiveCharacterTextSplitter` with default separators.
141    ///
142    /// Default separators: `["\n\n", "\n", " ", ""]`
143    /// Default chunk size: 4096 characters
144    /// Default chunk overlap: 0
145    pub fn new() -> Self {
146        Self {
147            separators: vec!["\n\n", "\n", " ", ""],
148            chunk_size: 4096,
149            chunk_overlap: 0,
150            add_start_index: false,
151            strip_whitespace: true,
152            keep_separator: true,
153            length_function: LengthFunction::default(),
154            strategy: std::sync::Arc::new(SequentialStrategy),
155        }
156    }
157
158    /// Set custom separators (tried in priority order)
159    pub fn with_separators(mut self, separators: Vec<&'static str>) -> Self {
160        self.separators = separators;
161        self
162    }
163
164    /// Set the maximum chunk size (in characters by default)
165    pub fn with_chunk_size(mut self, size: usize) -> Self {
166        self.chunk_size = size;
167        self
168    }
169
170    /// Set the chunk overlap (in characters by default)
171    pub fn with_chunk_overlap(mut self, overlap: usize) -> Self {
172        self.chunk_overlap = overlap;
173        self
174    }
175
176    /// Whether to add `start_index` metadata to each chunk
177    pub fn with_add_start_index(mut self, add: bool) -> Self {
178        self.add_start_index = add;
179        self
180    }
181
182    /// Whether to strip whitespace from chunk edges
183    pub fn with_strip_whitespace(mut self, strip: bool) -> Self {
184        self.strip_whitespace = strip;
185        self
186    }
187
188    /// Whether to keep separators attached to each piece (Python default for RecursiveCharacterTextSplitter: True)
189    pub fn with_keep_separator(mut self, keep: bool) -> Self {
190        self.keep_separator = keep;
191        self
192    }
193
194    /// Use byte count instead of character count for chunk size measurement
195    pub fn with_byte_length_function(mut self) -> Self {
196        self.length_function = LengthFunction::ByteCount;
197        self
198    }
199
200    /// Set the splitting strategy (sequential or parallel via rayon).
201    ///
202    /// When the `rayon` feature is enabled, you can select `ParallelStrategy`
203    /// for multi-core processing. When `rayon` is disabled, only
204    /// `SequentialStrategy` is available (which is also the default).
205    ///
206    /// # Example
207    /// ```
208    /// use recursive_text_splitter::{RecursiveCharacterTextSplitter, SequentialStrategy};
209    ///
210    /// let splitter = RecursiveCharacterTextSplitter::new()
211    ///     .with_strategy::<SequentialStrategy>();
212    /// #
213    /// # use recursive_text_splitter::Chunk;
214    /// # #[cfg(feature = "rayon")]
215    /// # use recursive_text_splitter::ParallelStrategy;
216    /// # #[cfg(feature = "rayon")]
217    /// # let splitter = RecursiveCharacterTextSplitter::new()
218    /// #     .with_strategy::<ParallelStrategy>();
219    /// ```
220    pub fn with_strategy<S: SplitterStrategy + Default + Send + Sync + 'static>(mut self) -> Self {
221        self.strategy = std::sync::Arc::new(S::default());
222        self
223    }
224
225    /// Split text into chunks.
226    ///
227    /// Uses the configured strategy (sequential by default).
228    pub fn split_text(&self, text: &str) -> Vec<Chunk> {
229        self.strategy.split(self, text)
230    }
231
232    /// Split text into chunks, returning just the page_content strings (no metadata)
233    pub fn split_text_content(&self, text: &str) -> Vec<String> {
234        self.split_text(text)
235            .into_iter()
236            .map(|c| c.page_content)
237            .collect()
238    }
239
240    /// Parallel version of `split_text` using rayon for multi-core processing.
241    ///
242    /// When the `rayon` feature is enabled, this method uses a thread pool
243    /// to process multiple splits concurrently. This is most effective on
244    /// large texts that produce many oversized splits requiring recursion.
245    ///
246    /// Note: The merge step (adding chunk overlap) remains sequential because
247    /// overlap depends on the order of previous chunks. However, the recursive
248    /// splitting of individual splits is parallelized.
249    ///
250    /// Requires the `rayon` feature: `features = ["rayon"]`
251    #[cfg(feature = "rayon")]
252    pub fn split_text_par(&self, text: &str) -> Vec<Chunk> {
253        let chunks = self._split_text_par(text, &self.separators);
254
255        if self.add_start_index {
256            self.annotate_start_index(text, &chunks)
257        } else {
258            chunks.into_iter().map(|s| Chunk::new(s, None)).collect()
259        }
260    }
261
262    /// Parallel version of `split_text_content`.
263    #[cfg(feature = "rayon")]
264    pub fn split_text_content_par(&self, text: &str) -> Vec<String> {
265        self.split_text_par(text)
266            .into_iter()
267            .map(|c| c.page_content)
268            .collect()
269    }
270
271    /// Check if rayon parallelism is available.
272    #[cfg(feature = "rayon")]
273    pub fn is_parallel(&self) -> bool {
274        true
275    }
276
277    /// Check if rayon parallelism is available.
278    #[cfg(not(feature = "rayon"))]
279    pub fn is_parallel(&self) -> bool {
280        false
281    }
282
283    /// Core recursive splitting logic — mirrors Python's `_split_text`.
284    ///
285    /// 1. Find the best separator for the text (first one in the list found in text)
286    /// 2. Split by that separator (keeping separator attached if keep_separator=True)
287    /// 3. For each split: if it fits, add to good_splits; if not, flush good_splits
288    ///    via merge_splits and recurse on the oversized piece with remaining separators
289    fn _split_text(&self, text: &str, separators: &[&'static str]) -> Vec<String> {
290        let mut final_chunks: Vec<String> = Vec::new();
291
292        // Find appropriate separator to use (first separator found in text)
293        let (separator, new_separators) = self.find_separator_and_rest(text, separators);
294
295        // Split by separator (keeping separator attached)
296        let splits = self.split_by_separator(text, separator);
297
298        // Determine merge separator
299        // When keep_separator=True, merge_sep is "" (we don't re-insert the separator)
300        // When keep_separator=False, merge_sep is the separator itself (re-insert it)
301        let merge_sep = if self.keep_separator { "" } else { separator };
302
303        // Process each split: if it fits, add to good_splits; if not, flush and recurse
304        let mut good_splits: Vec<String> = Vec::new();
305
306        for s in &splits {
307            if self.length_function.len(s) < self.chunk_size {
308                good_splits.push(s.clone());
309            } else {
310                // Flush good_splits before recursing on the oversized piece
311                if !good_splits.is_empty() {
312                    let merged = self.merge_splits_with_sep(&good_splits, merge_sep);
313                    final_chunks.extend(merged);
314                    good_splits.clear();
315                }
316
317                if new_separators.is_empty() {
318                    // No more separators — just add the oversized piece
319                    final_chunks.push(s.clone());
320                } else {
321                    // Recurse with remaining separators
322                    let other = self._split_text(s, &new_separators);
323                    final_chunks.extend(other);
324                }
325            }
326        }
327
328        // Flush remaining good_splits
329        if !good_splits.is_empty() {
330            let merged = self.merge_splits_with_sep(&good_splits, merge_sep);
331            final_chunks.extend(merged);
332        }
333
334        final_chunks
335    }
336
337    /// Parallel recursive splitting logic — mirrors `_split_text` exactly but
338    /// processes oversized splits concurrently using rayon.
339    ///
340    /// The algorithm preserves Python's sequential flush-and-merge ordering:
341    /// 1. Walk through splits sequentially, building "segments"
342    /// 2. Good splits are buffered in good_splits
343    /// 3. When an oversized split is found:
344    ///    a. Flush good_splits → segment::Merged(...)
345    ///    b. If new_separators is empty → segment::Direct(s)
346    ///    c. Else → segment::Deferred(s) (for parallel processing)
347    /// 4. Flush remaining good_splits → segment::Merged(...)
348    /// 5. Process all Deferred segments in PARALLEL via rayon
349    /// 6. Replace Deferred markers with parallel results (in order)
350    /// 7. Flatten all segments
351    ///
352    /// This is correct because: oversized splits are independent recursive
353    /// calls on disjoint subtexts. Their results can be computed in any
354    /// order (or in parallel) and still assembled correctly.
355    #[cfg(feature = "rayon")]
356    fn _split_text_par(&self, text: &str, separators: &[&'static str]) -> Vec<String> {
357        use rayon::prelude::*;
358
359        let (separator, new_separators) = self.find_separator_and_rest(text, separators);
360        let splits = self.split_by_separator(text, separator);
361        let merge_sep = if self.keep_separator { "" } else { separator };
362
363        // Phase 1: Walk splits sequentially, building segments
364        let mut good_splits: Vec<String> = Vec::new();
365        let mut segments: Vec<Segment> = Vec::new();
366        let mut deferred_splits: Vec<String> = Vec::new();
367
368        for s in &splits {
369            if self.length_function.len(s) < self.chunk_size {
370                good_splits.push(s.clone());
371            } else {
372                // Flush good_splits before handling the oversized piece
373                if !good_splits.is_empty() {
374                    segments.push(Segment::Merged(
375                        self.merge_splits_with_sep(&good_splits, merge_sep),
376                    ));
377                    good_splits.clear();
378                }
379
380                if new_separators.is_empty() {
381                    segments.push(Segment::Direct(s.clone()));
382                } else {
383                    deferred_splits.push(s.clone());
384                    segments.push(Segment::Deferred);
385                }
386            }
387        }
388
389        // Flush remaining good_splits
390        if !good_splits.is_empty() {
391            segments.push(Segment::Merged(
392                self.merge_splits_with_sep(&good_splits, merge_sep),
393            ));
394        }
395
396        // Phase 2: Process deferred (oversized) splits in parallel
397        // Note: _split_text_par recursively calls itself, enabling cross-recursion
398        // parallelism — oversized sub-splits at deeper recursion levels are also
399        // parallelized, not just at the top level.
400        if !deferred_splits.is_empty() {
401            let par_results: Vec<Vec<String>> = deferred_splits
402                .par_iter()
403                .map(|s| self._split_text_par(s, &new_separators))
404                .collect();
405
406            // Phase 3: Replace Deferred markers with parallel results (in order)
407            let mut idx = 0;
408            for seg in &mut segments {
409                if matches!(seg, Segment::Deferred) {
410                    *seg = Segment::Merged(par_results[idx].clone());
411                    idx += 1;
412                }
413            }
414        }
415
416        // Phase 4: Flatten all segments
417        segments.into_iter().flat_map(|s| s.into_chunks()).collect()
418    }
419    /// return it along with the remaining separators.
420    fn find_separator_and_rest(
421        &self,
422        text: &str,
423        separators: &[&'static str],
424    ) -> (&'static str, Vec<&'static str>) {
425        for (i, s_) in separators.iter().enumerate() {
426            if s_.is_empty() {
427                // Empty separator is the fallback — use it immediately
428                return (*s_, separators[i + 1..].to_vec());
429            }
430
431            if text.contains(*s_) {
432                return (*s_, separators[i + 1..].to_vec());
433            }
434        }
435
436        // Fallback to the last separator
437        let fallback = *separators.last().unwrap_or(&" ");
438        (fallback, Vec::new())
439    }
440
441    /// Split text by a separator, optionally keeping the separator attached.
442    ///
443    /// When `keep_separator` is True: splits like Python's `re.split(r"(sep)", text)`
444    /// with keep_separator="start" — the separator is prepended to the text
445    /// that follows it.
446    ///
447    /// When `keep_separator` is False: simple split without separators.
448    ///
449    /// When separator is "": splits into individual characters.
450    fn split_by_separator(&self, text: &str, separator: &str) -> Vec<String> {
451        if separator.is_empty() {
452            // Split into individual characters
453            return text.chars().map(|c| c.to_string()).collect();
454        }
455
456        if !self.keep_separator {
457            return text
458                .split(separator)
459                .filter(|s| !s.is_empty())
460                .map(|s| s.to_string())
461                .collect();
462        }
463
464        // keep_separator=True: replicate Python's _split_text_with_regex
465        // Python: re.split(r"(sep)", text) → [before, sep, middle, sep, after]
466        // With keep_separator="start": [sep + middle, sep + after, ...] + [before]
467        // Result: [before, sep + after_1, sep + after_2, ...]
468        //
469        // We replicate by: finding all separator positions, then building
470        // pieces where the separator is prepended to the text that follows it.
471        let mut splits: Vec<String> = Vec::new();
472        let mut start = 0usize;
473        let sep_len = separator.len();
474
475        while let Some(pos) = text[start..].find(separator) {
476            let abs_pos = start + pos;
477            let end = abs_pos + sep_len;
478
479            if abs_pos > start {
480                // Text before the separator
481                splits.push(text[start..abs_pos].to_string());
482            }
483
484            // Find the end of this piece (up to next separator or end of text)
485            if end < text.len() {
486                if let Some(next_pos) = text[end..].find(separator) {
487                    splits.push(text[abs_pos..end + next_pos].to_string());
488                    start = end + next_pos;
489                } else {
490                    // No more separators — rest of text including this separator
491                    splits.push(text[abs_pos..].to_string());
492                    start = text.len();
493                }
494            } else {
495                // Separator at end of text
496                splits.push(separator.to_string());
497                start = end;
498            }
499        }
500
501        // Add remaining text after last separator
502        if start < text.len() {
503            splits.push(text[start..].to_string());
504        }
505
506        if splits.is_empty() {
507            splits.push(text.to_string());
508        }
509
510        splits
511    }
512
513    /// Merge splits with overlap, matching Python's `_merge_splits` behavior.
514    ///
515    /// Algorithm (mirrors Python):
516    /// 1. Greedily add splits to current_doc until adding the next would exceed chunk_size
517    /// 2. When chunk is full, push it and trim from the FRONT until under chunk_overlap
518    /// 3. Then append the new split
519    fn merge_splits_with_sep(&self, splits: &[String], separator: &str) -> Vec<String> {
520        let separator_len = self.length_function.len(separator);
521        let mut docs: Vec<String> = Vec::new();
522        let mut current_doc: Vec<String> = Vec::new();
523        let mut total: usize = 0;
524
525        for d in splits {
526            let len_d = self.length_function.len(d);
527
528            // Check if adding this would exceed chunk_size
529            if !current_doc.is_empty() {
530                let sep_contribution = if current_doc.len() > 1 {
531                    separator_len
532                } else {
533                    0
534                };
535                if total + len_d + sep_contribution > self.chunk_size {
536                    // Flush current doc
537                    if let Some(doc) = self.join_docs(&current_doc, separator) {
538                        docs.push(doc);
539                    }
540
541                    // Pop from front until under chunk_overlap or would still overflow
542                    loop {
543                        if current_doc.is_empty() {
544                            break;
545                        }
546                        let sep_contrib = if current_doc.len() > 1 {
547                            separator_len
548                        } else {
549                            0
550                        };
551                        let combined = total + len_d + sep_contrib;
552                        let should_continue =
553                            total > self.chunk_overlap || (combined > self.chunk_size);
554                        if !should_continue {
555                            break;
556                        }
557                        let first_len = self.length_function.len(&current_doc[0]);
558                        total -= first_len + sep_contrib;
559                        current_doc.remove(0);
560                    }
561                }
562            }
563
564            current_doc.push(d.clone());
565            let sep_contrib = if current_doc.len() > 1 {
566                separator_len
567            } else {
568                0
569            };
570            total += len_d + sep_contrib;
571        }
572
573        // Flush remaining
574        if let Some(doc) = self.join_docs(&current_doc, separator) {
575            docs.push(doc);
576        }
577
578        docs
579    }
580
581    /// Annotate each chunk with its start_index in the original text.
582    /// Mirrors Python's `create_documents` with `add_start_index`.
583    fn annotate_start_index(&self, text: &str, chunks: &[String]) -> Vec<Chunk> {
584        let mut result = Vec::with_capacity(chunks.len());
585        let mut index = 0usize;
586        let mut previous_chunk_len = 0usize;
587
588        for chunk_content in chunks {
589            let search_from = if index == 0 && previous_chunk_len == 0 {
590                0
591            } else {
592                (index + previous_chunk_len).saturating_sub(self.chunk_overlap)
593            };
594
595            let pos = if search_from <= text.len() {
596                text[search_from..].find(chunk_content)
597            } else {
598                None
599            };
600
601            if let Some(p) = pos {
602                result.push(Chunk::new(chunk_content.clone(), Some(search_from + p)));
603                previous_chunk_len = chunk_content.chars().count();
604            } else {
605                result.push(Chunk::new(chunk_content.clone(), Some(0)));
606            }
607
608            index = search_from;
609        }
610
611        result
612    }
613
614    /// Join documents with separator, optionally stripping whitespace.
615    fn join_docs(&self, docs: &[String], separator: &str) -> Option<String> {
616        let text = docs.join(separator);
617        if self.strip_whitespace {
618            let trimmed = text.trim();
619            if trimmed.is_empty() {
620                return None;
621            }
622            Some(trimmed.to_string())
623        } else {
624            if text.is_empty() {
625                return None;
626            }
627            Some(text)
628        }
629    }
630}
631
632/// Internal segment type used by `_split_text_par` for parallel processing.
633#[cfg(feature = "rayon")]
634enum Segment {
635    /// Already-merged chunks from flushing `good_splits`
636    Merged(Vec<String>),
637    /// An oversized split with no remaining separators — added directly
638    Direct(String),
639    /// Placeholder for a split that needs parallel recursive processing
640    Deferred,
641}
642
643#[cfg(feature = "rayon")]
644impl Segment {
645    fn into_chunks(self) -> Vec<String> {
646        match self {
647            Segment::Merged(c) => c,
648            Segment::Direct(s) => vec![s],
649            Segment::Deferred => vec![],
650        }
651    }
652}
653
654#[cfg(test)]
655mod tests {
656    use super::*;
657
658    #[test]
659    fn test_default_splitter() {
660        let splitter = RecursiveCharacterTextSplitter::new();
661        let text = "Hello world.\n\nThis is a test.\nAnother line.";
662        let chunks = splitter.split_text_content(text);
663        assert!(!chunks.is_empty());
664    }
665
666    #[test]
667    fn test_custom_separators() {
668        let splitter = RecursiveCharacterTextSplitter::new()
669            .with_separators(vec!["\nINT.", "\nEXT.", "\n\n", "\n", " ", ""])
670            .with_chunk_size(100)
671            .with_chunk_overlap(20);
672
673        let text = "INT. LIVING ROOM - DAY\nLuke sits on a couch.\n\nEXT. FOREST - NIGHT\nLeia walks through trees.";
674        let chunks = splitter.split_text_content(text);
675
676        assert!(chunks.iter().any(|c| c.contains("INT. LIVING ROOM")));
677        assert!(chunks.iter().any(|c| c.contains("EXT. FOREST")));
678    }
679
680    #[test]
681    fn test_chunk_size_limit() {
682        let splitter = RecursiveCharacterTextSplitter::new()
683            .with_chunk_size(10)
684            .with_chunk_overlap(0);
685
686        let text = "This is a very long sentence that should be split into multiple chunks.";
687        let chunks = splitter.split_text_content(text);
688
689        for chunk in &chunks {
690            assert!(
691                chunk.chars().count() <= 10 || chunk.is_empty(),
692                "Chunk '{}' has {} chars, exceeds chunk size 10",
693                chunk,
694                chunk.chars().count()
695            );
696        }
697    }
698
699    #[test]
700    fn test_chunk_overlap() {
701        let splitter = RecursiveCharacterTextSplitter::new()
702            .with_chunk_size(5)
703            .with_chunk_overlap(3);
704
705        let text = "abcdefghij"; // 10 chars -> should split into 2+ chunks with overlap
706        let chunks = splitter.split_text_content(text);
707
708        assert!(
709            chunks.len() >= 2,
710            "Expected >= 2 chunks, got {}",
711            chunks.len()
712        );
713
714        // With overlap=3, the last chunk should contain characters from the overlap region
715        if chunks.len() >= 2 {
716            let last_chunk = chunks.last().unwrap();
717            let prev_chunk = &chunks[chunks.len() - 2];
718
719            // Check that there's some overlap (at least 1 shared char in the boundary region)
720            let overlap_exists = last_chunk.chars().take(3).any(|c| prev_chunk.contains(c));
721            assert!(
722                overlap_exists,
723                "Expected overlapping characters between chunks"
724            );
725        }
726    }
727
728    #[test]
729    fn test_python_hi_bye() {
730        // Python langchain: chunk_size=10, chunk_overlap=0, text="hi bye"
731        // "hi bye" is 7 chars, fits in chunk_size=10, so one chunk
732        let splitter = RecursiveCharacterTextSplitter::new()
733            .with_chunk_size(10)
734            .with_chunk_overlap(0);
735
736        let text = "hi bye";
737        let chunks = splitter.split_text_content(text);
738        assert_eq!(chunks, vec!["hi bye"]);
739    }
740
741    #[test]
742    fn test_python_hi_bye_small_chunk() {
743        // chunk_size=2, chunk_overlap=0, text = "hi bye"
744        // Each chunk should be <= 2 chars
745        let splitter = RecursiveCharacterTextSplitter::new()
746            .with_chunk_size(2)
747            .with_chunk_overlap(0);
748
749        let text = "hi bye";
750        let chunks = splitter.split_text_content(text);
751
752        for chunk in &chunks {
753            assert!(
754                chunk.chars().count() <= 2,
755                "Chunk '{}' exceeds chunk_size 2",
756                chunk
757            );
758        }
759    }
760
761    #[test]
762    fn test_star_wars_script_style() {
763        let splitter = RecursiveCharacterTextSplitter::new()
764            .with_separators(vec!["\nINT.", "\nEXT.", "\n\n", "\n", " ", ""])
765            .with_chunk_size(200)
766            .with_chunk_overlap(50);
767
768        let text = "INT. COCKPIT - DAY\n\nLUKE: I am a Jedi.\n\nEXT. DESERT - DAY\n\nVADER: Join me.\n\nINT. BRIDGE - NIGHT\n\nLEIA: We need to escape.";
769        let chunks = splitter.split_text_content(text);
770
771        assert!(!chunks.is_empty());
772        for chunk in &chunks {
773            assert!(chunk.chars().count() <= 250);
774        }
775    }
776
777    #[test]
778    fn test_byte_length_function() {
779        let splitter = RecursiveCharacterTextSplitter::new()
780            .with_chunk_size(10)
781            .with_byte_length_function();
782
783        let text = "Hello, 世界";
784        let chunks = splitter.split_text_content(text);
785        assert!(!chunks.is_empty());
786    }
787
788    #[test]
789    fn test_strips_whitespace() {
790        let splitter = RecursiveCharacterTextSplitter::new()
791            .with_chunk_size(5)
792            .with_strip_whitespace(true);
793
794        let text = "  hello  \n\n  world  ";
795        let chunks = splitter.split_text_content(text);
796
797        for chunk in &chunks {
798            assert_eq!(chunk, chunk.trim());
799        }
800    }
801
802    #[test]
803    fn test_no_overlap() {
804        let splitter = RecursiveCharacterTextSplitter::new()
805            .with_chunk_size(10)
806            .with_chunk_overlap(0);
807
808        let text = "abcdefghij";
809        let chunks = splitter.split_text_content(text);
810
811        assert_eq!(chunks.len(), 1);
812    }
813
814    #[test]
815    fn test_large_text_recursive_split() {
816        let splitter = RecursiveCharacterTextSplitter::new()
817            .with_chunk_size(10)
818            .with_chunk_overlap(0);
819
820        let text = "a".repeat(50);
821        let chunks = splitter.split_text_content(&text);
822
823        for chunk in &chunks {
824            assert!(chunk.chars().count() <= 10);
825        }
826    }
827
828    #[test]
829    fn test_keep_separator_false() {
830        let splitter = RecursiveCharacterTextSplitter::new()
831            .with_chunk_size(100)
832            .with_keep_separator(false);
833
834        let text = "line1\nline2\nline3";
835        let chunks = splitter.split_text_content(text);
836
837        assert!(!chunks.is_empty());
838        for chunk in &chunks {
839            assert!(!chunk.ends_with('\n'));
840        }
841    }
842
843    #[test]
844    fn test_empty_string() {
845        let splitter = RecursiveCharacterTextSplitter::new();
846        let chunks = splitter.split_text_content("");
847        assert!(chunks.is_empty() || chunks.iter().all(|c| c.is_empty()));
848    }
849
850    #[test]
851    fn test_single_char_chunk_size() {
852        let splitter = RecursiveCharacterTextSplitter::new()
853            .with_chunk_size(1)
854            .with_chunk_overlap(0);
855
856        let text = "abc";
857        let chunks = splitter.split_text_content(text);
858        assert_eq!(chunks, vec!["a", "b", "c"]);
859    }
860
861    #[test]
862    fn test_multiline_text() {
863        let splitter = RecursiveCharacterTextSplitter::new()
864            .with_chunk_size(10)
865            .with_chunk_overlap(0);
866
867        let text = "first line\nsecond line\nthird line";
868        let chunks = splitter.split_text_content(text);
869
870        for chunk in &chunks {
871            assert!(chunk.chars().count() <= 10);
872        }
873    }
874
875    #[test]
876    fn test_add_start_index_simple() {
877        let splitter = RecursiveCharacterTextSplitter::new()
878            .with_chunk_size(5)
879            .with_add_start_index(true)
880            .with_keep_separator(false);
881
882        let text = "abcdefghij";
883        let chunks = splitter.split_text(text);
884
885        for chunk in &chunks {
886            assert!(chunk.start_index.is_some());
887        }
888    }
889}