julienne 0.1.0

Range-preserving Rust text chunkers for retrieval and embedding pipelines
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

use julienne::{
    character::CharacterTextSplitterBuilder, recursive::RecursiveCharacterTextSplitterBuilder,
    semantic::SemanticChunkerBuilder, semchunk::SemchunkSplitterBuilder,
    sentence::SentenceChunkerBuilder, CharacterTextSplitter, RecursiveCharacterTextSplitter,
    SemanticChunker, SemchunkSplitter, SentenceChunker,
};

fn assert_send_sync<T: Send + Sync>() {}
fn assert_clone<T: Clone>() {}

#[test]
fn public_splitters_and_builders_are_send_sync_clone() {
    assert_send_sync::<CharacterTextSplitter>();
    assert_send_sync::<RecursiveCharacterTextSplitter>();
    assert_send_sync::<SentenceChunker>();
    assert_send_sync::<SemchunkSplitter>();
    assert_send_sync::<SemanticChunker>();

    assert_send_sync::<CharacterTextSplitterBuilder>();
    assert_send_sync::<RecursiveCharacterTextSplitterBuilder>();
    assert_send_sync::<SentenceChunkerBuilder>();
    assert_send_sync::<SemchunkSplitterBuilder>();
    assert_send_sync::<SemanticChunkerBuilder>();

    assert_clone::<CharacterTextSplitter>();
    assert_clone::<RecursiveCharacterTextSplitter>();
    assert_clone::<SentenceChunker>();
    assert_clone::<SemchunkSplitter>();
    assert_clone::<SemanticChunker>();
}