embedcache 0.1.1

High-performance text embedding service with caching capabilities
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

//! Word-based chunking implementation

use async_trait::async_trait;
use super::ContentChunker;

/// Word-based chunking implementation
///
/// This chunker splits text into chunks based on word boundaries.
pub struct WordChunker;

#[async_trait]
impl ContentChunker for WordChunker {
    async fn chunk(&self, content: &str, size: usize) -> Vec<String> {
        content
            .split_whitespace()
            .collect::<Vec<&str>>()
            .chunks(size)
            .map(|chunk| chunk.join(" "))
            .collect()
    }

    fn name(&self) -> &str {
        "words"
    }
}