Skip to main content

wesichain_retrieval/
lib.rs

1mod base_retriever;
2mod ensemble;
3mod error;
4pub mod reranker;
5mod hash_embedder;
6mod in_memory;
7mod indexer;
8mod loader;
9mod multi_query;
10mod retriever;
11mod splitter;
12
13use std::path::PathBuf;
14
15use wesichain_core::Document;
16
17pub use base_retriever::BaseRetriever;
18pub use ensemble::EnsembleRetriever;
19pub use error::{IngestionError, RetrievalError};
20pub use hash_embedder::HashEmbedder;
21pub use in_memory::InMemoryVectorStore;
22pub use indexer::Indexer;
23pub use loader::{load_file_async, load_files_async, PdfLoader, TextLoader};
24pub use multi_query::MultiQueryRetriever;
25pub use reranker::{CrossEncoderRetriever, KeywordReranker, Reranker};
26pub use retriever::Retriever;
27pub use splitter::{RecursiveCharacterTextSplitter, SplitterConfigError, TextSplitter};
28
29pub async fn load_and_split_recursive(
30    paths: Vec<PathBuf>,
31    splitter: &RecursiveCharacterTextSplitter,
32) -> Result<Vec<Document>, IngestionError> {
33    let documents = load_files_async(paths).await?;
34    Ok(splitter.split_documents(&documents))
35}