1use std::fs;
2use text_splitter::TextSplitter;
3use std::path::PathBuf;
4use anyhow::{Context, Result};
5use log::{info, debug};
6
7pub struct FileInputAdapter {
8 input_folder: PathBuf,
9 chunk_size: usize,
10}
11
12impl FileInputAdapter {
13 pub fn new(input_folder: PathBuf, chunk_size: usize) -> Self {
14 info!("Creating new FileInputAdapter with input folder: {:?} and chunk size: {}", input_folder, chunk_size);
15 Self {
16 input_folder: input_folder,
17 chunk_size,
18 }
19 }
20
21 fn read_files_in_folder(&self) -> Result<Vec<String>> {
22 info!("Reading files from folder: {:?}", self.input_folder);
23 let mut contents = Vec::new();
24 for entry in fs::read_dir(&self.input_folder)
25 .with_context(|| format!("Failed to read directory: {:?}", self.input_folder))?
26 {
27 let entry = entry.context("Failed to read directory entry")?;
28 let path = entry.path();
29 if path.is_file() {
30 if let Some(extension) = path.extension() {
31 if extension == "txt" || extension == "md" {
32 debug!("Reading file: {:?}", path.to_str().unwrap());
33 let content = fs::read_to_string(&path)
34 .with_context(|| format!("Failed to read file: {:?}", path))?;
35 contents.push(content);
36 }
37 }
38 }
39 }
40 info!("Read {} files from folder", contents.len());
41 Ok(contents)
42 }
43
44 pub fn fetch_chunks(&self) -> Result<Vec<String>> {
45 info!("Fetching chunks with size: {}", self.chunk_size);
46 let file_contents = self.read_files_in_folder()?;
47 let mut all_chunks = Vec::new();
48 let splitter = TextSplitter::new(self.chunk_size);
49 for (index, content) in file_contents.iter().enumerate() {
50 let chunks: Vec<String> = splitter.chunks(content).map(|s| s.to_string()).collect();
51 debug!("Split content {} into {} chunks", index + 1, chunks.len());
52 all_chunks.extend(chunks);
53 }
54 info!("Total chunks fetched: {}", all_chunks.len());
55 Ok(all_chunks)
56 }
57}