Skip to main content

synaptic_loaders/
lib.rs

1mod csv_loader;
2mod directory_loader;
3mod file_loader;
4mod json_loader;
5mod markdown_loader;
6mod text_loader;
7mod web_loader;
8
9pub use csv_loader::CsvLoader;
10pub use directory_loader::DirectoryLoader;
11pub use file_loader::FileLoader;
12pub use json_loader::JsonLoader;
13pub use markdown_loader::MarkdownLoader;
14pub use text_loader::TextLoader;
15pub use web_loader::WebBaseLoader;
16
17use std::pin::Pin;
18
19use async_trait::async_trait;
20use futures::Stream;
21use synaptic_core::SynapseError;
22use synaptic_retrieval::Document;
23
24/// Trait for loading documents from various sources.
25#[async_trait]
26pub trait Loader: Send + Sync {
27    /// Load all documents from this source.
28    async fn load(&self) -> Result<Vec<Document>, SynapseError>;
29
30    /// Stream documents lazily. Default implementation wraps load().
31    fn lazy_load(&self) -> Pin<Box<dyn Stream<Item = Result<Document, SynapseError>> + Send + '_>> {
32        Box::pin(async_stream::stream! {
33            match self.load().await {
34                Ok(docs) => {
35                    for doc in docs {
36                        yield Ok(doc);
37                    }
38                }
39                Err(e) => yield Err(e),
40            }
41        })
42    }
43}