Skip to main content

synaptic_loaders/
lib.rs

1mod csv_loader;
2mod directory_loader;
3mod file_loader;
4mod json_loader;
5mod markdown_loader;
6mod text_loader;
7mod web_loader;
8
9pub use csv_loader::CsvLoader;
10pub use directory_loader::DirectoryLoader;
11pub use file_loader::FileLoader;
12pub use json_loader::JsonLoader;
13pub use markdown_loader::MarkdownLoader;
14pub use text_loader::TextLoader;
15pub use web_loader::WebBaseLoader;
16
17use std::pin::Pin;
18
19use async_trait::async_trait;
20use futures::Stream;
21use synaptic_core::SynapticError;
22use synaptic_retrieval::Document;
23
24/// Trait for loading documents from various sources.
25#[async_trait]
26pub trait Loader: Send + Sync {
27    /// Load all documents from this source.
28    async fn load(&self) -> Result<Vec<Document>, SynapticError>;
29
30    /// Stream documents lazily. Default implementation wraps load().
31    fn lazy_load(
32        &self,
33    ) -> Pin<Box<dyn Stream<Item = Result<Document, SynapticError>> + Send + '_>> {
34        Box::pin(async_stream::stream! {
35            match self.load().await {
36                Ok(docs) => {
37                    for doc in docs {
38                        yield Ok(doc);
39                    }
40                }
41                Err(e) => yield Err(e),
42            }
43        })
44    }
45}