rig_core/loaders/
file.rs

1//! Lazy UTF-8 loading from filesystem paths or in-memory bytes.
2//!
3//! ```
4//! use rig_core::loaders::FileLoader;
5//!
6//! let documents = FileLoader::from_bytes(b"hello".to_vec())
7//!     .read().into_iter().collect::<Result<Vec<_>, _>>()?;
8//! assert_eq!(documents, vec!["hello"]);
9//! # Ok::<(), rig_core::loaders::file::FileLoaderError>(())
10//! ```
11
12use std::{fs, path::PathBuf, string::FromUtf8Error};
13
14use thiserror::Error;
15
16#[derive(Error, Debug)]
17pub enum FileLoaderError {
18    #[error("Invalid glob pattern: {0}")]
19    InvalidGlobPattern(String),
20
21    #[error("IO error: {0}")]
22    IoError(#[from] std::io::Error),
23
24    #[error("Pattern error: {0}")]
25    PatternError(#[from] glob::PatternError),
26
27    #[error("Glob error: {0}")]
28    GlobError(#[from] glob::GlobError),
29
30    #[error("String conversion error: {0}")]
31    StringUtf8Error(#[from] FromUtf8Error),
32}
33
34loadable_trait!(Readable, FileLoaderError, String, read, read_with_path);
35
36impl Readable for PathBuf {
37    fn read(self) -> Result<String, FileLoaderError> {
38        fs::read_to_string(self).map_err(FileLoaderError::IoError)
39    }
40    fn read_with_path(self) -> Result<(PathBuf, String), FileLoaderError> {
41        let contents = fs::read_to_string(&self);
42        Ok((self, contents?))
43    }
44}
45
46impl Readable for Vec<u8> {
47    fn read(self) -> Result<String, FileLoaderError> {
48        Ok(String::from_utf8(self)?)
49    }
50
51    fn read_with_path(self) -> Result<(PathBuf, String), FileLoaderError> {
52        let res = String::from_utf8(self)?;
53
54        Ok((PathBuf::from("<memory>"), res))
55    }
56}
57
58/// Iterator pipeline for loading UTF-8 documents. Reads happen synchronously
59/// during iteration; per-item I/O and decoding errors are yielded unless filtered.
60pub struct FileLoader<'a, T> {
61    iterator: Box<dyn Iterator<Item = T> + 'a>,
62}
63
64#[allow(private_bounds)] // `Readable` deliberately seals which states expose these methods
65impl<'a, T: Readable + 'a> FileLoader<'a, T> {
66    /// Decodes each input as UTF-8 during iteration, yielding I/O or decoding errors.
67    ///
68    /// ```no_run
69    /// # use rig_core::loaders::FileLoader;
70    /// # fn run() -> Result<(), Box<dyn std::error::Error>> {
71    /// let content = FileLoader::with_glob("files/*.txt")?.read();
72    /// for result in content {
73    ///     match result {
74    ///         Ok(content) => println!("{content}"),
75    ///         Err(e) => eprintln!("Error reading file: {e}"),
76    ///     }
77    /// }
78    /// # Ok(())
79    /// # }
80    /// ```
81    pub fn read(self) -> FileLoader<'a, Result<String, FileLoaderError>> {
82        FileLoader {
83            iterator: Box::new(self.iterator.map(Readable::read)),
84        }
85    }
86    /// Decodes each input as UTF-8 and pairs it with its source path, yielding
87    /// I/O or decoding errors. In-memory inputs use the path `<memory>`.
88    ///
89    /// ```no_run
90    /// # use rig_core::loaders::FileLoader;
91    /// # fn run() -> Result<(), Box<dyn std::error::Error>> {
92    /// let content = FileLoader::with_glob("files/*.txt")?.read_with_path();
93    /// for result in content {
94    ///     match result {
95    ///         Ok((path, content)) => println!("{path:?} {content}"),
96    ///         Err(e) => eprintln!("Error reading file: {e}"),
97    ///     }
98    /// }
99    /// # Ok(())
100    /// # }
101    /// ```
102    pub fn read_with_path(self) -> FileLoader<'a, Result<(PathBuf, String), FileLoaderError>> {
103        FileLoader {
104            iterator: Box::new(self.iterator.map(Readable::read_with_path)),
105        }
106    }
107}
108
109loader_scaffold!(FileLoader, FileLoaderError, dir: files_only);
110loader_from_bytes!(FileLoader);
111
112#[cfg(test)]
113mod tests;
rig_core/loaders/file.rs

rig_core/loaders/
file.rs
