lc_rag/loaders/
mod.rs

1// src/retrieval/loaders/mod.rs
2//! Document loader implementations
3//!
4//! Provides document loading from files in various formats, including PDF, CSV, Text, JSON,
5//! Markdown, HTML, etc. v0.4.1 added the WebScraper, Sitemap, and Docx loaders.
6
7mod csv;
8mod docx;
9mod html;
10mod json;
11mod markdown;
12mod pdf;
13mod sitemap;
14mod text;
15mod web_scraper;
16
17pub use csv::CSVLoader;
18pub use docx::DocxLoader;
19pub use html::HTMLLoader;
20pub use json::JSONLoader;
21pub use markdown::MarkdownLoader;
22pub use pdf::PDFLoader;
23pub use sitemap::SitemapLoader;
24pub use text::TextLoader;
25pub use web_scraper::WebScraperLoader;
26
27use async_trait::async_trait;
28use lc_vector_stores::Document;
29
30/// Document loader error type
31#[derive(Debug, thiserror::Error)]
32#[non_exhaustive]
33pub enum LoaderError {
34    /// IO error
35    #[error("IO error: {0}")]
36    IoError(#[from] std::io::Error),
37
38    /// CSV parse error
39    #[error("CSV parse error: {0}")]
40    CsvError(String),
41
42    /// PDF parse error
43    #[error("PDF parse error: {0}")]
44    PdfError(String),
45
46    /// JSON parse error
47    #[error("JSON parse error: {0}")]
48    JsonError(String),
49
50    /// Unknown error
51    #[error("unknown error: {0}")]
52    Other(String),
53}
54
55impl From<pdf_extract::Error> for LoaderError {
56    fn from(err: pdf_extract::Error) -> Self {
57        LoaderError::PdfError(err.to_string())
58    }
59}
60
61/// Document loader trait
62///
63/// Defines the common interface for loading documents from a source.
64#[async_trait]
65pub trait DocumentLoader: Send + Sync {
66    /// Loads documents from the source
67    ///
68    /// # Returns
69    /// The loaded documents
70    async fn load(&self) -> Result<Vec<Document>, LoaderError>;
71}
lc_rag/loaders/mod.rs

lc_rag/loaders/
mod.rs
