use crate::{config::YekConfig, priority::get_file_priority, Result};
use content_inspector::{inspect, ContentType};
use glob::glob;
use ignore::gitignore::GitignoreBuilder;
use path_slash::PathBufExt;
use rayon::prelude::*;
use std::{
collections::HashMap,
fs,
path::Path,
sync::{mpsc, Arc},
};
use tracing::debug;
#[derive(Debug, Clone)]
pub struct ProcessedFile {
pub priority: i32,
pub file_index: usize,
pub rel_path: String,
pub content: String,
}
fn process_single_file(
file_path: &Path,
base_dir: &Path,
config: &YekConfig,
boost_map: &HashMap<String, i32>,
) -> Result<Vec<ProcessedFile>> {
let rel_path = normalize_path(file_path, base_dir);
let file_parent = file_path.parent().unwrap_or(Path::new(""));
let mut gitignore_builder = GitignoreBuilder::new(file_parent);
for pattern in &config.ignore_patterns {
gitignore_builder.add_line(None, pattern)?;
}
let gitignore_file = file_parent.join(".gitignore");
if gitignore_file.exists() {
gitignore_builder.add(&gitignore_file);
}
let gitignore = gitignore_builder.build()?;
if gitignore.matched(file_path, false).is_ignore() {
debug!("Skipping ignored file: {rel_path}");
return Ok(Vec::new());
}
let mut processed_files = Vec::new();
match fs::read(file_path) {
Ok(content) => {
if inspect(&content) == ContentType::BINARY {
debug!("Skipping binary file: {rel_path}");
} else {
let rule_priority = get_file_priority(&rel_path, &config.priority_rules);
let boost = boost_map.get(&rel_path).copied().unwrap_or(0);
let combined_priority = rule_priority + boost;
processed_files.push(ProcessedFile {
priority: combined_priority,
file_index: 0, rel_path,
content: String::from_utf8_lossy(&content).to_string(),
});
}
}
Err(e) => {
debug!("Failed to read {rel_path}: {e}");
}
}
Ok(processed_files)
}
pub fn process_files_parallel(
base_path: &Path,
config: &YekConfig,
boost_map: &HashMap<String, i32>,
) -> Result<Vec<ProcessedFile>> {
let mut expanded_paths = Vec::new();
let path_str = base_path.to_string_lossy();
for entry in glob(&path_str)? {
match entry {
Ok(path) => expanded_paths.push(path),
Err(e) => debug!("Glob entry error: {:?}", e),
}
}
let base_dir = if path_str.contains('*') || path_str.contains('?') {
std::env::current_dir().unwrap_or_else(|_| Path::new(".").to_path_buf())
} else if base_path.is_file() {
base_path.parent().unwrap_or(Path::new(".")).to_path_buf()
} else {
base_path.to_path_buf()
};
if expanded_paths.len() == 1 && expanded_paths[0].is_file() {
return process_single_file(&expanded_paths[0], &base_dir, config, boost_map);
}
let mut all_processed_files = Vec::new();
for path in expanded_paths {
if path.is_file() {
all_processed_files.extend(process_single_file(&path, &base_dir, config, boost_map)?);
} else if path.is_dir() {
all_processed_files.extend(process_files_parallel_internal(&path, config, boost_map)?);
}
}
Ok(all_processed_files)
}
fn process_files_parallel_internal(
base_path: &Path,
config: &YekConfig,
boost_map: &HashMap<String, i32>,
) -> Result<Vec<ProcessedFile>> {
let mut walk_builder = ignore::WalkBuilder::new(base_path);
walk_builder
.follow_links(false)
.standard_filters(true)
.require_git(false);
let mut gitignore_builder = GitignoreBuilder::new(base_path);
for pattern in &config.ignore_patterns {
gitignore_builder.add_line(None, pattern)?;
}
let gitignore_file = base_path.join(".gitignore");
if gitignore_file.exists() {
gitignore_builder.add(&gitignore_file);
}
let gitignore = Arc::new(gitignore_builder.build()?);
let (processed_files_tx, processed_files_rx) = mpsc::channel::<(std::path::PathBuf, String)>();
let process_thread = std::thread::spawn({
let priority_rules = config.priority_rules.clone();
let boost_map = boost_map.clone();
move || {
let mut processed = Vec::new();
for (path, rel_path) in processed_files_rx {
match fs::read(&path) {
Ok(content) => {
if inspect(&content) == ContentType::BINARY {
debug!("Skipping binary file: {rel_path}");
continue;
}
let rule_priority = get_file_priority(&rel_path, &priority_rules);
let boost = boost_map.get(&rel_path).copied().unwrap_or(0);
let combined = rule_priority + boost;
processed.push(ProcessedFile {
priority: combined,
file_index: 0, rel_path,
content: String::from_utf8_lossy(&content).to_string(),
});
}
Err(e) => {
debug!("Failed to read {rel_path}: {e}");
}
}
}
processed
}
});
let base_cloned = base_path.to_owned();
let walker_tx = processed_files_tx.clone();
walk_builder.build_parallel().run(move || {
let base_dir = base_cloned.clone();
let processed_files_tx = walker_tx.clone();
let gitignore = Arc::clone(&gitignore);
Box::new(move |entry| {
let entry = match entry {
Ok(e) => e,
Err(_) => return ignore::WalkState::Continue,
};
if !entry.file_type().is_some_and(|ft| ft.is_file()) {
return ignore::WalkState::Continue;
}
let path = entry.path().to_path_buf();
let rel_path = normalize_path(&path, &base_dir);
if gitignore.matched(&path, false).is_ignore() {
debug!("Skipping ignored file: {rel_path}");
return ignore::WalkState::Continue;
}
processed_files_tx.send((path, rel_path)).ok();
ignore::WalkState::Continue
})
});
drop(processed_files_tx);
let mut processed_files = process_thread.join().unwrap();
let mut counters = HashMap::new();
for f in &mut processed_files {
let ctr = counters.entry(f.priority).or_insert(0);
f.file_index = *ctr;
*ctr += 1;
}
if config.debug {
debug!(
"Processed {} files in parallel for base_path: {}",
processed_files.len(),
base_path.display()
);
}
processed_files.par_sort_by(|a, b| {
a.priority
.cmp(&b.priority)
.reverse()
.then_with(|| a.file_index.cmp(&b.file_index))
});
Ok(processed_files)
}
pub fn normalize_path(path: &Path, base: &Path) -> String {
path.strip_prefix(base)
.unwrap_or(path)
.to_path_buf()
.to_slash()
.unwrap_or_default()
.to_string()
}