synaptic-splitters 0.4.0

Text splitters: Character, Recursive, Markdown, HTML, Language, Token
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

use synaptic_splitters::{Language, RecursiveCharacterTextSplitter, TextSplitter};

#[test]
fn python_splits_on_class_and_def() {
    let code = r#"
class MyClass:
    def __init__(self):
        self.x = 1

    def method(self):
        return self.x

def standalone_function():
    return 42

class AnotherClass:
    def another_method(self):
        pass
"#;

    let splitter = RecursiveCharacterTextSplitter::from_language(Language::Python, 80, 0);
    let chunks = splitter.split_text(code);

    assert!(
        chunks.len() >= 2,
        "expected at least 2 chunks, got {}",
        chunks.len()
    );
    // Each chunk should be within size limit
    for chunk in &chunks {
        assert!(
            chunk.len() <= 80,
            "chunk too long: {} chars: {:?}",
            chunk.len(),
            chunk
        );
    }
}

#[test]
fn rust_splits_on_fn_and_struct() {
    let code = r#"
struct Point {
    x: f64,
    y: f64,
}

impl Point {
    fn new(x: f64, y: f64) -> Self {
        Self { x, y }
    }

    fn distance(&self, other: &Point) -> f64 {
        ((self.x - other.x).powi(2) + (self.y - other.y).powi(2)).sqrt()
    }
}

fn main() {
    let p1 = Point::new(0.0, 0.0);
    let p2 = Point::new(3.0, 4.0);
    println!("{}", p1.distance(&p2));
}
"#;

    let splitter = RecursiveCharacterTextSplitter::from_language(Language::Rust, 100, 0);
    let chunks = splitter.split_text(code);

    assert!(
        chunks.len() >= 2,
        "expected at least 2 chunks, got {}",
        chunks.len()
    );
    for chunk in &chunks {
        assert!(
            chunk.len() <= 100,
            "chunk too long: {} chars: {:?}",
            chunk.len(),
            chunk
        );
    }
}

#[test]
fn language_separators_all_end_with_empty() {
    let languages = [
        Language::Python,
        Language::JavaScript,
        Language::TypeScript,
        Language::Rust,
        Language::Go,
        Language::Java,
        Language::Cpp,
        Language::Ruby,
        Language::Markdown,
        Language::Latex,
        Language::Html,
    ];

    for lang in &languages {
        let seps = lang.separators();
        assert!(!seps.is_empty(), "language {:?} has no separators", lang);
        assert_eq!(
            seps.last().unwrap(),
            "",
            "language {:?} should end with empty separator",
            lang
        );
    }
}

#[test]
fn from_language_with_overlap() {
    let splitter = RecursiveCharacterTextSplitter::from_language(Language::Python, 50, 10);
    let code = "def foo():\n    return 1\n\ndef bar():\n    return 2\n\ndef baz():\n    return 3\n";
    let chunks = splitter.split_text(code);

    assert!(chunks.len() >= 2);
}