1#[derive(Clone)]
14pub struct RecursiveCharacterTextSplitter {
15 separators: Vec<&'static str>,
17 chunk_size: usize,
19 chunk_overlap: usize,
21 add_start_index: bool,
23 strip_whitespace: bool,
25 keep_separator: bool,
27 length_function: LengthFunction,
29 strategy: std::sync::Arc<dyn SplitterStrategy + Send + Sync>,
31}
32
33#[derive(Debug, Clone, PartialEq, Eq)]
35pub struct Chunk {
36 pub page_content: String,
37 pub start_index: Option<usize>,
38}
39
40impl Chunk {
41 pub fn new(content: String, start_index: Option<usize>) -> Self {
42 Self {
43 page_content: content,
44 start_index,
45 }
46 }
47}
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
51pub enum LengthFunction {
52 #[default]
54 CharCount,
55 ByteCount,
57}
58
59impl LengthFunction {
60 #[inline]
61 fn len(&self, text: &str) -> usize {
62 match self {
63 LengthFunction::CharCount => text.chars().count(),
64 LengthFunction::ByteCount => text.len(),
65 }
66 }
67}
68
69pub trait SplitterStrategy {
76 fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk>;
78
79 fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String>;
81}
82
83#[derive(Debug, Clone, Default)]
85pub struct SequentialStrategy;
86
87impl SplitterStrategy for SequentialStrategy {
88 fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
89 let chunks = splitter._split_text(text, &splitter.separators);
90
91 if splitter.add_start_index {
92 splitter.annotate_start_index(text, &chunks)
93 } else {
94 chunks.into_iter().map(|s| Chunk::new(s, None)).collect()
95 }
96 }
97
98 fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
99 splitter
100 .split_text(text)
101 .into_iter()
102 .map(|c| c.page_content)
103 .collect()
104 }
105}
106
107#[cfg(feature = "rayon")]
109#[derive(Debug, Clone, Default)]
110pub struct ParallelStrategy;
111
112#[cfg(feature = "rayon")]
113impl SplitterStrategy for ParallelStrategy {
114 fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
115 let chunks = splitter._split_text_par(text, &splitter.separators);
116
117 if splitter.add_start_index {
118 splitter.annotate_start_index(text, &chunks)
119 } else {
120 chunks.into_iter().map(|s| Chunk::new(s, None)).collect()
121 }
122 }
123
124 fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
125 splitter
126 .split_text_par(text)
127 .into_iter()
128 .map(|c| c.page_content)
129 .collect()
130 }
131}
132
133impl Default for RecursiveCharacterTextSplitter {
134 fn default() -> Self {
135 Self::new()
136 }
137}
138
139impl RecursiveCharacterTextSplitter {
140 pub fn new() -> Self {
146 Self {
147 separators: vec!["\n\n", "\n", " ", ""],
148 chunk_size: 4096,
149 chunk_overlap: 0,
150 add_start_index: false,
151 strip_whitespace: true,
152 keep_separator: true,
153 length_function: LengthFunction::default(),
154 strategy: std::sync::Arc::new(SequentialStrategy),
155 }
156 }
157
158 pub fn with_separators(mut self, separators: Vec<&'static str>) -> Self {
160 self.separators = separators;
161 self
162 }
163
164 pub fn with_chunk_size(mut self, size: usize) -> Self {
166 self.chunk_size = size;
167 self
168 }
169
170 pub fn with_chunk_overlap(mut self, overlap: usize) -> Self {
172 self.chunk_overlap = overlap;
173 self
174 }
175
176 pub fn with_add_start_index(mut self, add: bool) -> Self {
178 self.add_start_index = add;
179 self
180 }
181
182 pub fn with_strip_whitespace(mut self, strip: bool) -> Self {
184 self.strip_whitespace = strip;
185 self
186 }
187
188 pub fn with_keep_separator(mut self, keep: bool) -> Self {
190 self.keep_separator = keep;
191 self
192 }
193
194 pub fn with_byte_length_function(mut self) -> Self {
196 self.length_function = LengthFunction::ByteCount;
197 self
198 }
199
200 pub fn with_strategy<S: SplitterStrategy + Default + Send + Sync + 'static>(mut self) -> Self {
221 self.strategy = std::sync::Arc::new(S::default());
222 self
223 }
224
225 pub fn split_text(&self, text: &str) -> Vec<Chunk> {
229 self.strategy.split(self, text)
230 }
231
232 pub fn split_text_content(&self, text: &str) -> Vec<String> {
234 self.split_text(text)
235 .into_iter()
236 .map(|c| c.page_content)
237 .collect()
238 }
239
240 #[cfg(feature = "rayon")]
252 pub fn split_text_par(&self, text: &str) -> Vec<Chunk> {
253 let chunks = self._split_text_par(text, &self.separators);
254
255 if self.add_start_index {
256 self.annotate_start_index(text, &chunks)
257 } else {
258 chunks.into_iter().map(|s| Chunk::new(s, None)).collect()
259 }
260 }
261
262 #[cfg(feature = "rayon")]
264 pub fn split_text_content_par(&self, text: &str) -> Vec<String> {
265 self.split_text_par(text)
266 .into_iter()
267 .map(|c| c.page_content)
268 .collect()
269 }
270
271 #[cfg(feature = "rayon")]
273 pub fn is_parallel(&self) -> bool {
274 true
275 }
276
277 #[cfg(not(feature = "rayon"))]
279 pub fn is_parallel(&self) -> bool {
280 false
281 }
282
283 fn _split_text(&self, text: &str, separators: &[&'static str]) -> Vec<String> {
290 let mut final_chunks: Vec<String> = Vec::new();
291
292 let (separator, new_separators) = self.find_separator_and_rest(text, separators);
294
295 let splits = self.split_by_separator(text, separator);
297
298 let merge_sep = if self.keep_separator { "" } else { separator };
302
303 let mut good_splits: Vec<String> = Vec::new();
305
306 for s in &splits {
307 if self.length_function.len(s) < self.chunk_size {
308 good_splits.push(s.clone());
309 } else {
310 if !good_splits.is_empty() {
312 let merged = self.merge_splits_with_sep(&good_splits, merge_sep);
313 final_chunks.extend(merged);
314 good_splits.clear();
315 }
316
317 if new_separators.is_empty() {
318 final_chunks.push(s.clone());
320 } else {
321 let other = self._split_text(s, &new_separators);
323 final_chunks.extend(other);
324 }
325 }
326 }
327
328 if !good_splits.is_empty() {
330 let merged = self.merge_splits_with_sep(&good_splits, merge_sep);
331 final_chunks.extend(merged);
332 }
333
334 final_chunks
335 }
336
337 #[cfg(feature = "rayon")]
356 fn _split_text_par(&self, text: &str, separators: &[&'static str]) -> Vec<String> {
357 use rayon::prelude::*;
358
359 let (separator, new_separators) = self.find_separator_and_rest(text, separators);
360 let splits = self.split_by_separator(text, separator);
361 let merge_sep = if self.keep_separator { "" } else { separator };
362
363 let mut good_splits: Vec<String> = Vec::new();
365 let mut segments: Vec<Segment> = Vec::new();
366 let mut deferred_splits: Vec<String> = Vec::new();
367
368 for s in &splits {
369 if self.length_function.len(s) < self.chunk_size {
370 good_splits.push(s.clone());
371 } else {
372 if !good_splits.is_empty() {
374 segments.push(Segment::Merged(
375 self.merge_splits_with_sep(&good_splits, merge_sep),
376 ));
377 good_splits.clear();
378 }
379
380 if new_separators.is_empty() {
381 segments.push(Segment::Direct(s.clone()));
382 } else {
383 deferred_splits.push(s.clone());
384 segments.push(Segment::Deferred);
385 }
386 }
387 }
388
389 if !good_splits.is_empty() {
391 segments.push(Segment::Merged(
392 self.merge_splits_with_sep(&good_splits, merge_sep),
393 ));
394 }
395
396 if !deferred_splits.is_empty() {
401 let par_results: Vec<Vec<String>> = deferred_splits
402 .par_iter()
403 .map(|s| self._split_text_par(s, &new_separators))
404 .collect();
405
406 let mut idx = 0;
408 for seg in &mut segments {
409 if matches!(seg, Segment::Deferred) {
410 *seg = Segment::Merged(par_results[idx].clone());
411 idx += 1;
412 }
413 }
414 }
415
416 segments.into_iter().flat_map(|s| s.into_chunks()).collect()
418 }
419 fn find_separator_and_rest(
421 &self,
422 text: &str,
423 separators: &[&'static str],
424 ) -> (&'static str, Vec<&'static str>) {
425 for (i, s_) in separators.iter().enumerate() {
426 if s_.is_empty() {
427 return (*s_, separators[i + 1..].to_vec());
429 }
430
431 if text.contains(*s_) {
432 return (*s_, separators[i + 1..].to_vec());
433 }
434 }
435
436 let fallback = *separators.last().unwrap_or(&" ");
438 (fallback, Vec::new())
439 }
440
441 fn split_by_separator(&self, text: &str, separator: &str) -> Vec<String> {
451 if separator.is_empty() {
452 return text.chars().map(|c| c.to_string()).collect();
454 }
455
456 if !self.keep_separator {
457 return text
458 .split(separator)
459 .filter(|s| !s.is_empty())
460 .map(|s| s.to_string())
461 .collect();
462 }
463
464 let mut splits: Vec<String> = Vec::new();
472 let mut start = 0usize;
473 let sep_len = separator.len();
474
475 while let Some(pos) = text[start..].find(separator) {
476 let abs_pos = start + pos;
477 let end = abs_pos + sep_len;
478
479 if abs_pos > start {
480 splits.push(text[start..abs_pos].to_string());
482 }
483
484 if end < text.len() {
486 if let Some(next_pos) = text[end..].find(separator) {
487 splits.push(text[abs_pos..end + next_pos].to_string());
488 start = end + next_pos;
489 } else {
490 splits.push(text[abs_pos..].to_string());
492 start = text.len();
493 }
494 } else {
495 splits.push(separator.to_string());
497 start = end;
498 }
499 }
500
501 if start < text.len() {
503 splits.push(text[start..].to_string());
504 }
505
506 if splits.is_empty() {
507 splits.push(text.to_string());
508 }
509
510 splits
511 }
512
513 fn merge_splits_with_sep(&self, splits: &[String], separator: &str) -> Vec<String> {
520 let separator_len = self.length_function.len(separator);
521 let mut docs: Vec<String> = Vec::new();
522 let mut current_doc: Vec<String> = Vec::new();
523 let mut total: usize = 0;
524
525 for d in splits {
526 let len_d = self.length_function.len(d);
527
528 if !current_doc.is_empty() {
530 let sep_contribution = if current_doc.len() > 1 {
531 separator_len
532 } else {
533 0
534 };
535 if total + len_d + sep_contribution > self.chunk_size {
536 if let Some(doc) = self.join_docs(¤t_doc, separator) {
538 docs.push(doc);
539 }
540
541 loop {
543 if current_doc.is_empty() {
544 break;
545 }
546 let sep_contrib = if current_doc.len() > 1 {
547 separator_len
548 } else {
549 0
550 };
551 let combined = total + len_d + sep_contrib;
552 let should_continue =
553 total > self.chunk_overlap || (combined > self.chunk_size);
554 if !should_continue {
555 break;
556 }
557 let first_len = self.length_function.len(¤t_doc[0]);
558 total -= first_len + sep_contrib;
559 current_doc.remove(0);
560 }
561 }
562 }
563
564 current_doc.push(d.clone());
565 let sep_contrib = if current_doc.len() > 1 {
566 separator_len
567 } else {
568 0
569 };
570 total += len_d + sep_contrib;
571 }
572
573 if let Some(doc) = self.join_docs(¤t_doc, separator) {
575 docs.push(doc);
576 }
577
578 docs
579 }
580
581 fn annotate_start_index(&self, text: &str, chunks: &[String]) -> Vec<Chunk> {
584 let mut result = Vec::with_capacity(chunks.len());
585 let mut index = 0usize;
586 let mut previous_chunk_len = 0usize;
587
588 for chunk_content in chunks {
589 let search_from = if index == 0 && previous_chunk_len == 0 {
590 0
591 } else {
592 (index + previous_chunk_len).saturating_sub(self.chunk_overlap)
593 };
594
595 let pos = if search_from <= text.len() {
596 text[search_from..].find(chunk_content)
597 } else {
598 None
599 };
600
601 if let Some(p) = pos {
602 result.push(Chunk::new(chunk_content.clone(), Some(search_from + p)));
603 previous_chunk_len = chunk_content.chars().count();
604 } else {
605 result.push(Chunk::new(chunk_content.clone(), Some(0)));
606 }
607
608 index = search_from;
609 }
610
611 result
612 }
613
614 fn join_docs(&self, docs: &[String], separator: &str) -> Option<String> {
616 let text = docs.join(separator);
617 if self.strip_whitespace {
618 let trimmed = text.trim();
619 if trimmed.is_empty() {
620 return None;
621 }
622 Some(trimmed.to_string())
623 } else {
624 if text.is_empty() {
625 return None;
626 }
627 Some(text)
628 }
629 }
630}
631
632#[cfg(feature = "rayon")]
634enum Segment {
635 Merged(Vec<String>),
637 Direct(String),
639 Deferred,
641}
642
643#[cfg(feature = "rayon")]
644impl Segment {
645 fn into_chunks(self) -> Vec<String> {
646 match self {
647 Segment::Merged(c) => c,
648 Segment::Direct(s) => vec![s],
649 Segment::Deferred => vec![],
650 }
651 }
652}
653
654#[cfg(test)]
655mod tests {
656 use super::*;
657
658 #[test]
659 fn test_default_splitter() {
660 let splitter = RecursiveCharacterTextSplitter::new();
661 let text = "Hello world.\n\nThis is a test.\nAnother line.";
662 let chunks = splitter.split_text_content(text);
663 assert!(!chunks.is_empty());
664 }
665
666 #[test]
667 fn test_custom_separators() {
668 let splitter = RecursiveCharacterTextSplitter::new()
669 .with_separators(vec!["\nINT.", "\nEXT.", "\n\n", "\n", " ", ""])
670 .with_chunk_size(100)
671 .with_chunk_overlap(20);
672
673 let text = "INT. LIVING ROOM - DAY\nLuke sits on a couch.\n\nEXT. FOREST - NIGHT\nLeia walks through trees.";
674 let chunks = splitter.split_text_content(text);
675
676 assert!(chunks.iter().any(|c| c.contains("INT. LIVING ROOM")));
677 assert!(chunks.iter().any(|c| c.contains("EXT. FOREST")));
678 }
679
680 #[test]
681 fn test_chunk_size_limit() {
682 let splitter = RecursiveCharacterTextSplitter::new()
683 .with_chunk_size(10)
684 .with_chunk_overlap(0);
685
686 let text = "This is a very long sentence that should be split into multiple chunks.";
687 let chunks = splitter.split_text_content(text);
688
689 for chunk in &chunks {
690 assert!(
691 chunk.chars().count() <= 10 || chunk.is_empty(),
692 "Chunk '{}' has {} chars, exceeds chunk size 10",
693 chunk,
694 chunk.chars().count()
695 );
696 }
697 }
698
699 #[test]
700 fn test_chunk_overlap() {
701 let splitter = RecursiveCharacterTextSplitter::new()
702 .with_chunk_size(5)
703 .with_chunk_overlap(3);
704
705 let text = "abcdefghij"; let chunks = splitter.split_text_content(text);
707
708 assert!(
709 chunks.len() >= 2,
710 "Expected >= 2 chunks, got {}",
711 chunks.len()
712 );
713
714 if chunks.len() >= 2 {
716 let last_chunk = chunks.last().unwrap();
717 let prev_chunk = &chunks[chunks.len() - 2];
718
719 let overlap_exists = last_chunk.chars().take(3).any(|c| prev_chunk.contains(c));
721 assert!(
722 overlap_exists,
723 "Expected overlapping characters between chunks"
724 );
725 }
726 }
727
728 #[test]
729 fn test_python_hi_bye() {
730 let splitter = RecursiveCharacterTextSplitter::new()
733 .with_chunk_size(10)
734 .with_chunk_overlap(0);
735
736 let text = "hi bye";
737 let chunks = splitter.split_text_content(text);
738 assert_eq!(chunks, vec!["hi bye"]);
739 }
740
741 #[test]
742 fn test_python_hi_bye_small_chunk() {
743 let splitter = RecursiveCharacterTextSplitter::new()
746 .with_chunk_size(2)
747 .with_chunk_overlap(0);
748
749 let text = "hi bye";
750 let chunks = splitter.split_text_content(text);
751
752 for chunk in &chunks {
753 assert!(
754 chunk.chars().count() <= 2,
755 "Chunk '{}' exceeds chunk_size 2",
756 chunk
757 );
758 }
759 }
760
761 #[test]
762 fn test_star_wars_script_style() {
763 let splitter = RecursiveCharacterTextSplitter::new()
764 .with_separators(vec!["\nINT.", "\nEXT.", "\n\n", "\n", " ", ""])
765 .with_chunk_size(200)
766 .with_chunk_overlap(50);
767
768 let text = "INT. COCKPIT - DAY\n\nLUKE: I am a Jedi.\n\nEXT. DESERT - DAY\n\nVADER: Join me.\n\nINT. BRIDGE - NIGHT\n\nLEIA: We need to escape.";
769 let chunks = splitter.split_text_content(text);
770
771 assert!(!chunks.is_empty());
772 for chunk in &chunks {
773 assert!(chunk.chars().count() <= 250);
774 }
775 }
776
777 #[test]
778 fn test_byte_length_function() {
779 let splitter = RecursiveCharacterTextSplitter::new()
780 .with_chunk_size(10)
781 .with_byte_length_function();
782
783 let text = "Hello, 世界";
784 let chunks = splitter.split_text_content(text);
785 assert!(!chunks.is_empty());
786 }
787
788 #[test]
789 fn test_strips_whitespace() {
790 let splitter = RecursiveCharacterTextSplitter::new()
791 .with_chunk_size(5)
792 .with_strip_whitespace(true);
793
794 let text = " hello \n\n world ";
795 let chunks = splitter.split_text_content(text);
796
797 for chunk in &chunks {
798 assert_eq!(chunk, chunk.trim());
799 }
800 }
801
802 #[test]
803 fn test_no_overlap() {
804 let splitter = RecursiveCharacterTextSplitter::new()
805 .with_chunk_size(10)
806 .with_chunk_overlap(0);
807
808 let text = "abcdefghij";
809 let chunks = splitter.split_text_content(text);
810
811 assert_eq!(chunks.len(), 1);
812 }
813
814 #[test]
815 fn test_large_text_recursive_split() {
816 let splitter = RecursiveCharacterTextSplitter::new()
817 .with_chunk_size(10)
818 .with_chunk_overlap(0);
819
820 let text = "a".repeat(50);
821 let chunks = splitter.split_text_content(&text);
822
823 for chunk in &chunks {
824 assert!(chunk.chars().count() <= 10);
825 }
826 }
827
828 #[test]
829 fn test_keep_separator_false() {
830 let splitter = RecursiveCharacterTextSplitter::new()
831 .with_chunk_size(100)
832 .with_keep_separator(false);
833
834 let text = "line1\nline2\nline3";
835 let chunks = splitter.split_text_content(text);
836
837 assert!(!chunks.is_empty());
838 for chunk in &chunks {
839 assert!(!chunk.ends_with('\n'));
840 }
841 }
842
843 #[test]
844 fn test_empty_string() {
845 let splitter = RecursiveCharacterTextSplitter::new();
846 let chunks = splitter.split_text_content("");
847 assert!(chunks.is_empty() || chunks.iter().all(|c| c.is_empty()));
848 }
849
850 #[test]
851 fn test_single_char_chunk_size() {
852 let splitter = RecursiveCharacterTextSplitter::new()
853 .with_chunk_size(1)
854 .with_chunk_overlap(0);
855
856 let text = "abc";
857 let chunks = splitter.split_text_content(text);
858 assert_eq!(chunks, vec!["a", "b", "c"]);
859 }
860
861 #[test]
862 fn test_multiline_text() {
863 let splitter = RecursiveCharacterTextSplitter::new()
864 .with_chunk_size(10)
865 .with_chunk_overlap(0);
866
867 let text = "first line\nsecond line\nthird line";
868 let chunks = splitter.split_text_content(text);
869
870 for chunk in &chunks {
871 assert!(chunk.chars().count() <= 10);
872 }
873 }
874
875 #[test]
876 fn test_add_start_index_simple() {
877 let splitter = RecursiveCharacterTextSplitter::new()
878 .with_chunk_size(5)
879 .with_add_start_index(true)
880 .with_keep_separator(false);
881
882 let text = "abcdefghij";
883 let chunks = splitter.split_text(text);
884
885 for chunk in &chunks {
886 assert!(chunk.start_index.is_some());
887 }
888 }
889}