rag_toolchain/chunkers/
traits.rs

1use crate::common::{Chunk, Chunks};
2use futures::Stream;
3use std::error::Error;
4
5pub trait Chunker {
6    type ErrorType: Error;
7    fn generate_chunks(&self, raw_text: &str) -> Result<Chunks, Self::ErrorType>;
8}
9
10#[allow(unused)]
11pub trait StreamedChunker {
12    type ErrorType: Error;
13    type CharacterStream: Stream<Item = std::io::Result<char>>;
14    type ChunkStream: Stream<Item = Result<Chunk, Self::ErrorType>>;
15    fn generate_chunks(&self, data_stream: Self::CharacterStream) -> Self::ChunkStream;
16}