use recursive_text_splitter::RecursiveCharacterTextSplitter;
#[test]
fn test_python_comparison_hi_bye_cs10() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(10)
.with_chunk_overlap(0);
let chunks = splitter.split_text_content("hi bye");
assert_eq!(chunks, vec!["hi bye"]);
}
#[test]
fn test_python_comparison_hi_bye_cs2() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(2)
.with_chunk_overlap(0);
let chunks = splitter.split_text_content("hi bye");
assert_eq!(chunks, vec!["hi", "b", "ye"]);
}
#[test]
fn test_python_comparison_cs1_abc() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(1)
.with_chunk_overlap(0);
let chunks = splitter.split_text_content("abc");
assert_eq!(chunks, vec!["a", "b", "c"]);
}
#[test]
fn test_python_comparison_50a_cs10() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(10)
.with_chunk_overlap(0);
let text = "a".repeat(50);
let chunks = splitter.split_text_content(&text);
let expected = vec!["aaaaaaaaaa"; 5];
assert_eq!(chunks, expected);
}
#[test]
fn test_python_comparison_10chars_cs10() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(10)
.with_chunk_overlap(0);
let chunks = splitter.split_text_content("abcdefghij");
assert_eq!(chunks, vec!["abcdefghij"]);
}
#[test]
fn test_python_comparison_overlap() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(5)
.with_chunk_overlap(3);
let chunks = splitter.split_text_content("abcdefghij");
assert_eq!(chunks, vec!["abcde", "cdefg", "efghi", "ghij"]);
}
#[test]
fn test_python_comparison_empty() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(5)
.with_chunk_overlap(3);
let chunks = splitter.split_text_content("");
assert!(chunks.is_empty());
}
#[test]
fn test_python_comparison_strip_whitespace() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(5)
.with_chunk_overlap(0);
let chunks = splitter.split_text_content(" hello \n\n world ");
assert_eq!(chunks, vec!["hell", "o", "worl", "d"]);
}
#[test]
fn test_python_comparison_keep_separator_false() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(1000)
.with_chunk_overlap(0)
.with_keep_separator(false);
let chunks = splitter.split_text_content("line1\nline2\nline3");
assert_eq!(chunks, vec!["line1\nline2\nline3"]);
}
#[test]
fn test_python_comparison_keep_separator_end() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_separators(["\n"])
.with_chunk_size(3)
.with_chunk_overlap(0)
.with_keep_separator_mode(recursive_text_splitter::KeepSeparator::End);
let chunks = splitter.split_text_content("a\nb\nc");
assert_eq!(chunks, vec!["a", "b\nc"]);
}
#[test]
fn test_python_comparison_unicode() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_chunk_size(3)
.with_chunk_overlap(0);
let chunks = splitter.split_text_content("🙂aé\n\nβγ🙂 z");
assert_eq!(chunks, vec!["🙂aé", "βγ", "🙂", "z"]);
}
#[test]
fn test_python_comparison_only_separators() {
let splitter = RecursiveCharacterTextSplitter::new()
.with_separators(["abc", ""])
.with_chunk_size(4)
.with_chunk_overlap(2)
.with_keep_separator(false);
let chunks = splitter.split_text_content("abcabcabc");
assert!(chunks.is_empty());
}