cognis-rag 0.3.1

RAG primitives for Cognis: embeddings, vector stores (in-memory, FAISS, Chroma, Qdrant, Pinecone, Weaviate), retrievers, text splitters, document loaders, and incremental indexing pipelines.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

//! Document loaders — read sources into [`Document`]s.
//!
//! Each loader returns either:
//! - `Result<Vec<Document>>` — for small, bounded sources (one file).
//! - `RunnableStream<Document>` — for unbounded / large sources (a directory walk).
//!
//! The unifying trait is [`DocumentLoader`]: `load() -> Stream<Item=Result<Document>>`,
//! so callers can compose with `futures` combinators (e.g. `.take(100)`,
//! `.filter`).

use async_trait::async_trait;
use futures::Stream;

use cognis_core::Result;

use crate::document::Document;

#[cfg(feature = "csv-loader")]
pub mod csv_loader;
pub mod directory;
#[cfg(feature = "html-loader")]
pub mod html;
pub mod json;
pub mod markdown;
#[cfg(feature = "pdf-loader")]
pub mod pdf;
pub mod text;
#[cfg(feature = "toml-loader")]
pub mod toml_loader;
#[cfg(feature = "web-loader")]
pub mod web;
#[cfg(feature = "yaml-loader")]
pub mod yaml;

#[cfg(feature = "csv-loader")]
pub use csv_loader::CsvLoader;
pub use directory::DirectoryLoader;
#[cfg(feature = "html-loader")]
pub use html::HtmlLoader;
pub use json::JsonLoader;
pub use markdown::MarkdownLoader;
#[cfg(feature = "pdf-loader")]
pub use pdf::PdfLoader;
pub use text::TextLoader;
#[cfg(feature = "toml-loader")]
pub use toml_loader::TomlLoader;
#[cfg(feature = "web-loader")]
pub use web::WebLoader;
#[cfg(feature = "yaml-loader")]
pub use yaml::YamlLoader;

/// A document loader.
///
/// Implementations stream documents — preferred over collecting into a
/// `Vec` so callers can early-terminate large sources.
#[async_trait]
pub trait DocumentLoader: Send + Sync {
    /// Stream of documents. Errors are per-item so a partial-failure source
    /// can still yield successful items.
    async fn load(&self) -> Result<DocumentStream>;

    /// Convenience: collect every yielded document. Stops at the first error.
    async fn load_all(&self) -> Result<Vec<Document>> {
        use futures::StreamExt;
        let mut s = self.load().await?;
        let mut out = Vec::new();
        while let Some(doc) = s.next().await {
            out.push(doc?);
        }
        Ok(out)
    }
}

/// Boxed stream returned by [`DocumentLoader::load`].
pub type DocumentStream = std::pin::Pin<Box<dyn Stream<Item = Result<Document>> + Send>>;