#[derive(Clone)]
pub struct RecursiveCharacterTextSplitter {
separators: Vec<&'static str>,
chunk_size: usize,
chunk_overlap: usize,
add_start_index: bool,
strip_whitespace: bool,
keep_separator: bool,
length_function: LengthFunction,
strategy: std::sync::Arc<dyn SplitterStrategy + Send + Sync>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Chunk {
pub page_content: String,
pub start_index: Option<usize>,
}
impl Chunk {
pub fn new(content: String, start_index: Option<usize>) -> Self {
Self {
page_content: content,
start_index,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum LengthFunction {
#[default]
CharCount,
ByteCount,
}
impl LengthFunction {
#[inline]
fn len(&self, text: &str) -> usize {
match self {
LengthFunction::CharCount => text.chars().count(),
LengthFunction::ByteCount => text.len(),
}
}
}
pub trait SplitterStrategy {
fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk>;
fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String>;
}
#[derive(Debug, Clone, Default)]
pub struct SequentialStrategy;
impl SplitterStrategy for SequentialStrategy {
fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
let chunks = splitter._split_text(text, &splitter.separators);
if splitter.add_start_index {
splitter.annotate_start_index(text, &chunks)
} else {
chunks.into_iter().map(|s| Chunk::new(s, None)).collect()
}
}
fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
splitter
.split_text(text)
.into_iter()
.map(|c| c.page_content)
.collect()
}
}
#[cfg(feature = "rayon")]
#[derive(Debug, Clone, Default)]
pub struct ParallelStrategy;
#[cfg(feature = "rayon")]
impl SplitterStrategy for ParallelStrategy {
fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
let chunks = splitter._split_text_par(text, &splitter.separators);
if splitter.add_start_index {
splitter.annotate_start_index(text, &chunks)
} else {
chunks.into_iter().map(|s| Chunk::new(s, None)).collect()
}
}
fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
splitter
.split_text_par(text)
.into_iter()
.map(|c| c.page_content)
.collect()
}
}
impl Default for RecursiveCharacterTextSplitter {
fn default() -> Self {
Self::new()
}
}
impl RecursiveCharacterTextSplitter {
pub fn new() -> Self {
Self {
separators: vec!["\n\n", "\n", " ", ""],
chunk_size: 4096,
chunk_overlap: 0,
add_start_index: false,
strip_whitespace: true,
keep_separator: true,
length_function: LengthFunction::default(),
strategy: std::sync::Arc::new(SequentialStrategy),
}
}
pub fn with_separators(mut self, separators: Vec<&'static str>) -> Self {
self.separators = separators;
self
}
pub fn with_chunk_size(mut self, size: usize) -> Self {
self.chunk_size = size;
self
}
pub fn with_chunk_overlap(mut self, overlap: usize) -> Self {
self.chunk_overlap = overlap;
self
}
pub fn with_add_start_index(mut self, add: bool) -> Self {
self.add_start_index = add;
self
}
pub fn with_strip_whitespace(mut self, strip: bool) -> Self {
self.strip_whitespace = strip;
self
}
pub fn with_keep_separator(mut self, keep: bool) -> Self {
self.keep_separator = keep;
self
}
pub fn with_byte_length_function(mut self) -> Self {
self.length_function = LengthFunction::ByteCount;
self
}
pub fn with_strategy<S: SplitterStrategy + Default + Send + Sync + 'static>(mut self) -> Self {
self.strategy = std::sync::Arc::new(S::default());
self
}
pub fn split_text(&self, text: &str) -> Vec<Chunk> {
self.strategy.split(self, text)
}
pub fn split_text_content(&self, text: &str) -> Vec<String> {
self.split_text(text)
.into_iter()
.map(|c| c.page_content)
.collect()
}
#[cfg(feature = "rayon")]
pub fn split_text_par(&self, text: &str) -> Vec<Chunk> {
let chunks = self._split_text_par(text, &self.separators);
if self.add_start_index {
self.annotate_start_index(text, &chunks)
} else {
chunks.into_iter().map(|s| Chunk::new(s, None)).collect()
}
}
#[cfg(feature = "rayon")]
pub fn split_text_content_par(&self, text: &str) -> Vec<String> {
self.split_text_par(text)
.into_iter()
.map(|c| c.page_content)
.collect()
}
#[cfg(feature = "rayon")]
pub fn is_parallel(&self) -> bool {
true
}
#[cfg(not(feature = "rayon"))]
pub fn is_parallel(&self) -> bool {
false
}
fn _split_text(&self, text: &str, separators: &[&'static str]) -> Vec<String> {
let mut final_chunks: Vec<String> = Vec::new();
let (separator, new_separators) = self.find_separator_and_rest(text, separators);
let splits = self.split_by_separator(text, separator);
let merge_sep = if self.keep_separator { "" } else { separator };
let mut good_splits: Vec<String> = Vec::new();
for s in &splits {
if self.length_function.len(s) < self.chunk_size {
good_splits.push(s.clone());
} else {
if !good_splits.is_empty() {
let merged = self.merge_splits_with_sep(&good_splits, merge_sep);
final_chunks.extend(merged);
good_splits.clear();
}
if new_separators.is_empty() {
final_chunks.push(s.clone());
} else {
let other = self._split_text(s, &new_separators);
final_chunks.extend(other);
}
}
}
if !good_splits.is_empty() {
let merged = self.merge_splits_with_sep(&good_splits, merge_sep);
final_chunks.extend(merged);
}
final_chunks
}
#[cfg(feature = "rayon")]
fn _split_text_par(&self, text: &str, separators: &[&'static str]) -> Vec<String> {
use rayon::prelude::*;
let (separator, new_separators) = self.find_separator_and_rest(text, separators);
let splits = self.split_by_separator(text, separator);
let merge_sep = if self.keep_separator { "" } else { separator };
let mut good_splits: Vec<String> = Vec::new();
let mut segments: Vec<Segment> = Vec::new();
let mut deferred_splits: Vec<String> = Vec::new();
for s in &splits {
if self.length_function.len(s) < self.chunk_size {
good_splits.push(s.clone());
} else {
if !good_splits.is_empty() {
segments.push(Segment::Merged(
self.merge_splits_with_sep(&good_splits, merge_sep),
));
good_splits.clear();
}
if new_separators.is_empty() {
segments.push(Segment::Direct(s.clone()));
} else {
deferred_splits.push(s.clone());
segments.push(Segment::Deferred);
}
}
}
if !good_splits.is_empty() {
segments.push(Segment::Merged(
self.merge_splits_with_sep(&good_splits, merge_sep),
));
}
if !deferred_splits.is_empty() {
let par_results: Vec<Vec<String>> = deferred_splits
.par_iter()
.map(|s| self._split_text_par(s, &new_separators))
.collect();
let mut idx = 0;
for seg in &mut segments {
if matches!(seg, Segment::Deferred) {
*seg = Segment::Merged(par_results[idx].clone());
idx += 1;
}
}
}
segments.into_iter().flat_map(|s| s.into_chunks()).collect()
}
fn find_separator_and_rest(
&self,
text: &str,
separators: &[&'static str],
) -> (&'static str, Vec<&'static str>) {
for (i, s_) in separators.iter().enumerate() {
if s_.is_empty() {
return (*s_, separators[i + 1..].to_vec());
}
if text.contains(*s_) {
return (*s_, separators[i + 1..].to_vec());
}
}
let fallback = *separators.last().unwrap_or(&" ");
(fallback, Vec::new())
}
fn split_by_separator(&self, text: &str, separator: &str) -> Vec<String> {
if separator.is_empty() {
return text.chars().map(|c| c.to_string()).collect();
}
if !self.keep_separator {
return text
.split(separator)
.filter(|s| !s.is_empty())
.map(|s| s.to_string())
.collect();
}
let mut splits: Vec<String> = Vec::new();
let mut start = 0usize;
let sep_len = separator.len();
while let Some(pos) = text[start..].find(separator) {
let abs_pos = start + pos;
let end = abs_pos + sep_len;
if abs_pos > start {
splits.push(text[start..abs_pos].to_string());
}
if end < text.len() {
if let Some(next_pos) = text[end..].find(separator) {
splits.push(text[abs_pos..end + next_pos].to_string());
start = end + next_pos;
} else {
splits.push(text[abs_pos..].to_string());
start = text.len();
}
} else {
splits.push(separator.to_string());
start = end;
}
}
if start < text.len() {
splits.push(text[start..].to_string());
}
if splits.is_empty() {
splits.push(text.to_string());
}
splits
}
fn merge_splits_with_sep(&self, splits: &[String], separator: &str) -> Vec<String> {
let separator_len = self.length_function.len(separator);
let mut docs: Vec<String> = Vec::new();
let mut current_doc: Vec<String> = Vec::new();
let mut total: usize = 0;
for d in splits {
let len_d = self.length_function.len(d);
if !current_doc.is_empty() {
let sep_contribution = if current_doc.len() > 1 {
separator_len
} else {
0
};
if total + len_d + sep_contribution > self.chunk_size {
if let Some(doc) = self.join_docs(¤t_doc, separator) {
docs.push(doc);
}
loop {
if current_doc.is_empty() {
break;
}
let sep_contrib = if current_doc.len() > 1 {
separator_len
} else {
0
};
let combined = total + len_d + sep_contrib;
let should_continue =
total > self.chunk_overlap || (combined > self.chunk_size);
if !should_continue {
break;
}
let first_len = self.length_function.len(¤t_doc[0]);
total -= first_len + sep_contrib;
current_doc.remove(0);
}
}
}
current_doc.push(d.clone());
let sep_contrib = if current_doc.len() > 1 {
separator_len
} else {
0
};
total += len_d + sep_contrib;
}
if let Some(doc) = self.join_docs(¤t_doc, separator) {
docs.push(doc);
}
docs
}
fn annotate_start_index(&self, text: &str, chunks: &[String]) -> Vec<Chunk> {
let mut result = Vec::with_capacity(chunks.len());
let mut index = 0usize;
let mut previous_chunk_len = 0usize;
for chunk_content in chunks {
let search_from = if index == 0 && previous_chunk_len == 0 {
0
} else {
(index + previous_chunk_len).saturating_sub(self.chunk_overlap)
};
let pos = if search_from <= text.len() {
text[search_from..].find(chunk_content)
} else {
None
};
if let Some(p) = pos {
result.push(Chunk::new(chunk_content.clone(), Some(search_from + p)));
previous_chunk_len = chunk_content.chars().count();
} else {
result.push(Chunk::new(chunk_content.clone(), Some(0)));
}
index = search_from;
}
result
}
fn join_docs(&self, docs: &[String], separator: &str) -> Option<String> {
let text = docs.join(separator);
if self.strip_whitespace {
let trimmed = text.trim();
if trimmed.is_empty() {
return None;
}
Some(trimmed.to_string())
} else {
if text.is_empty() {
return None;
}
Some(text)
}
}
}
#[cfg(feature = "rayon")]
enum Segment {
Merged(Vec<String>),
Direct(String),
Deferred,
}
#[cfg(feature = "rayon")]
impl Segment {
fn into_chunks(self) -> Vec<String> {
match self {
Segment::Merged(c) => c,
Segment::Direct(s) => vec![s],
Segment::Deferred => vec![],
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_default_splitter() {
let splitter = RecursiveCharacterTextSplitter::new();
let text = "Hello world.\n\nThis is a test.\nAnother line.";
let chunks = splitter.split_text_content(text);
assert!(!chunks.is_empty());
}
#[test]
fn test_custom_separators() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_separators(vec!["\nINT.", "\nEXT.", "\n\n", "\n", " ", ""])
.with_chunk_size(100)
.with_chunk_overlap(20);
let text = "INT. LIVING ROOM - DAY\nLuke sits on a couch.\n\nEXT. FOREST - NIGHT\nLeia walks through trees.";
let chunks = splitter.split_text_content(text);
assert!(chunks.iter().any(|c| c.contains("INT. LIVING ROOM")));
assert!(chunks.iter().any(|c| c.contains("EXT. FOREST")));
}
#[test]
fn test_chunk_size_limit() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(10)
.with_chunk_overlap(0);
let text = "This is a very long sentence that should be split into multiple chunks.";
let chunks = splitter.split_text_content(text);
for chunk in &chunks {
assert!(
chunk.chars().count() <= 10 || chunk.is_empty(),
"Chunk '{}' has {} chars, exceeds chunk size 10",
chunk,
chunk.chars().count()
);
}
}
#[test]
fn test_chunk_overlap() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(5)
.with_chunk_overlap(3);
let text = "abcdefghij"; let chunks = splitter.split_text_content(text);
assert!(
chunks.len() >= 2,
"Expected >= 2 chunks, got {}",
chunks.len()
);
if chunks.len() >= 2 {
let last_chunk = chunks.last().unwrap();
let prev_chunk = &chunks[chunks.len() - 2];
let overlap_exists = last_chunk.chars().take(3).any(|c| prev_chunk.contains(c));
assert!(
overlap_exists,
"Expected overlapping characters between chunks"
);
}
}
#[test]
fn test_python_hi_bye() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(10)
.with_chunk_overlap(0);
let text = "hi bye";
let chunks = splitter.split_text_content(text);
assert_eq!(chunks, vec!["hi bye"]);
}
#[test]
fn test_python_hi_bye_small_chunk() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(2)
.with_chunk_overlap(0);
let text = "hi bye";
let chunks = splitter.split_text_content(text);
for chunk in &chunks {
assert!(
chunk.chars().count() <= 2,
"Chunk '{}' exceeds chunk_size 2",
chunk
);
}
}
#[test]
fn test_star_wars_script_style() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_separators(vec!["\nINT.", "\nEXT.", "\n\n", "\n", " ", ""])
.with_chunk_size(200)
.with_chunk_overlap(50);
let text = "INT. COCKPIT - DAY\n\nLUKE: I am a Jedi.\n\nEXT. DESERT - DAY\n\nVADER: Join me.\n\nINT. BRIDGE - NIGHT\n\nLEIA: We need to escape.";
let chunks = splitter.split_text_content(text);
assert!(!chunks.is_empty());
for chunk in &chunks {
assert!(chunk.chars().count() <= 250);
}
}
#[test]
fn test_byte_length_function() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(10)
.with_byte_length_function();
let text = "Hello, 世界";
let chunks = splitter.split_text_content(text);
assert!(!chunks.is_empty());
}
#[test]
fn test_strips_whitespace() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(5)
.with_strip_whitespace(true);
let text = " hello \n\n world ";
let chunks = splitter.split_text_content(text);
for chunk in &chunks {
assert_eq!(chunk, chunk.trim());
}
}
#[test]
fn test_no_overlap() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(10)
.with_chunk_overlap(0);
let text = "abcdefghij";
let chunks = splitter.split_text_content(text);
assert_eq!(chunks.len(), 1);
}
#[test]
fn test_large_text_recursive_split() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(10)
.with_chunk_overlap(0);
let text = "a".repeat(50);
let chunks = splitter.split_text_content(&text);
for chunk in &chunks {
assert!(chunk.chars().count() <= 10);
}
}
#[test]
fn test_keep_separator_false() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(100)
.with_keep_separator(false);
let text = "line1\nline2\nline3";
let chunks = splitter.split_text_content(text);
assert!(!chunks.is_empty());
for chunk in &chunks {
assert!(!chunk.ends_with('\n'));
}
}
#[test]
fn test_empty_string() {
let splitter = RecursiveCharacterTextSplitter::new();
let chunks = splitter.split_text_content("");
assert!(chunks.is_empty() || chunks.iter().all(|c| c.is_empty()));
}
#[test]
fn test_single_char_chunk_size() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(1)
.with_chunk_overlap(0);
let text = "abc";
let chunks = splitter.split_text_content(text);
assert_eq!(chunks, vec!["a", "b", "c"]);
}
#[test]
fn test_multiline_text() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(10)
.with_chunk_overlap(0);
let text = "first line\nsecond line\nthird line";
let chunks = splitter.split_text_content(text);
for chunk in &chunks {
assert!(chunk.chars().count() <= 10);
}
}
#[test]
fn test_add_start_index_simple() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(5)
.with_add_start_index(true)
.with_keep_separator(false);
let text = "abcdefghij";
let chunks = splitter.split_text(text);
for chunk in &chunks {
assert!(chunk.start_index.is_some());
}
}
}