use std::fs::{File, Metadata};
use std::io::{self, Read};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering::Relaxed};
use std::sync::Mutex;
use std::time::SystemTime;
use ignore::{WalkBuilder, WalkState};
use crate::error::{Error, Result};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileEntry {
pub path: PathBuf,
pub inode: u64,
pub mtime: SystemTime,
pub size: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FileKind {
Text,
Binary,
TooLarge,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CrawledFile {
pub entry: FileEntry,
pub kind: FileKind,
}
#[derive(Debug, Clone)]
pub struct CrawlConfig {
pub max_size: u64,
pub sniff_bytes: usize,
pub include_hidden: bool,
pub respect_global_gitignore: bool,
pub threads: usize,
}
impl Default for CrawlConfig {
fn default() -> Self {
Self {
max_size: 10 * 1024 * 1024,
sniff_bytes: 8 * 1024,
include_hidden: true,
respect_global_gitignore: true,
threads: 0,
}
}
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct CrawlStats {
pub files_seen: u64,
pub skipped_too_large: u64,
pub skipped_binary: u64,
pub errors: u64,
pub indexable: u64,
pub indexable_bytes: u64,
}
#[derive(Default)]
struct Counters {
files_seen: AtomicU64,
too_large: AtomicU64,
binary: AtomicU64,
errors: AtomicU64,
}
pub fn crawl(root: &Path, config: &CrawlConfig) -> Result<(Vec<FileEntry>, CrawlStats)> {
crawl_with_progress(root, config, &|_| {})
}
pub fn crawl_with_progress(
root: &Path,
config: &CrawlConfig,
progress: &(dyn Fn(u64) + Sync),
) -> Result<(Vec<FileEntry>, CrawlStats)> {
let (all, stats) = crawl_all_with_progress(root, config, progress)?;
let text = all.into_iter().filter(|f| f.kind == FileKind::Text).map(|f| f.entry).collect();
Ok((text, stats))
}
pub fn crawl_all(root: &Path, config: &CrawlConfig) -> Result<(Vec<CrawledFile>, CrawlStats)> {
crawl_all_with_progress(root, config, &|_| {})
}
pub fn crawl_all_with_progress(
root: &Path,
config: &CrawlConfig,
progress: &(dyn Fn(u64) + Sync),
) -> Result<(Vec<CrawledFile>, CrawlStats)> {
if !root.is_dir() {
return Err(Error::NotADirectory(root.to_path_buf()));
}
let threads = if config.threads == 0 {
std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1)
} else {
config.threads
};
let mut builder = WalkBuilder::new(root);
builder
.hidden(!config.include_hidden)
.require_git(false)
.git_global(config.respect_global_gitignore)
.follow_links(false)
.threads(threads)
.filter_entry(|e| e.file_name() != ".git" && e.file_name() != crate::store::DIR_NAME);
let entries: Mutex<Vec<CrawledFile>> = Mutex::new(Vec::new());
let counters = Counters::default();
builder.build_parallel().run(|| {
Box::new(|result| {
match result {
Err(err) => {
log::warn!("crawl: {err}");
counters.errors.fetch_add(1, Relaxed);
}
Ok(entry) => {
if entry.file_type().is_some_and(|t| t.is_file()) {
let seen = counters.files_seen.fetch_add(1, Relaxed) + 1;
if seen % 256 == 0 {
progress(seen);
}
let path = entry.path();
match inspect(path, config) {
Ok(cf) => {
match cf.kind {
FileKind::TooLarge => {
log::debug!("{}: larger than {} bytes", path.display(), config.max_size);
counters.too_large.fetch_add(1, Relaxed);
}
FileKind::Binary => {
log::debug!("{}: binary", path.display());
counters.binary.fetch_add(1, Relaxed);
}
FileKind::Text => {}
}
entries.lock().unwrap().push(cf);
}
Err(err) => {
log::warn!("skipping {}: {err}", path.display());
counters.errors.fetch_add(1, Relaxed);
}
}
}
}
}
WalkState::Continue
})
});
let mut files = entries.into_inner().unwrap();
files.sort_by(|a, b| a.entry.path.cmp(&b.entry.path));
let stats = CrawlStats {
files_seen: counters.files_seen.load(Relaxed),
skipped_too_large: counters.too_large.load(Relaxed),
skipped_binary: counters.binary.load(Relaxed),
errors: counters.errors.load(Relaxed),
indexable: files.iter().filter(|f| f.kind == FileKind::Text).count() as u64,
indexable_bytes: files.iter().filter(|f| f.kind == FileKind::Text).map(|f| f.entry.size).sum(),
};
Ok((files, stats))
}
fn inspect(path: &Path, config: &CrawlConfig) -> io::Result<CrawledFile> {
let mut file = File::open(path)?;
let meta = file.metadata()?;
let entry = |file: &File, meta: &Metadata| FileEntry {
path: path.to_path_buf(),
inode: file_id(file, meta),
mtime: meta.modified().unwrap_or(SystemTime::UNIX_EPOCH),
size: meta.len(),
};
if meta.len() > config.max_size {
return Ok(CrawledFile { entry: entry(&file, &meta), kind: FileKind::TooLarge });
}
let want = config.sniff_bytes.min(meta.len() as usize);
let mut buf = vec![0u8; want];
let mut filled = 0;
while filled < want {
match file.read(&mut buf[filled..])? {
0 => break,
n => filled += n,
}
}
let kind = if looks_binary(&buf[..filled]) { FileKind::Binary } else { FileKind::Text };
Ok(CrawledFile { entry: entry(&file, &meta), kind })
}
pub fn looks_binary(prefix: &[u8]) -> bool {
prefix.contains(&0)
}
#[cfg(unix)]
fn file_id(_file: &File, meta: &Metadata) -> u64 {
use std::os::unix::fs::MetadataExt;
meta.ino()
}
#[cfg(windows)]
fn file_id(file: &File, _meta: &Metadata) -> u64 {
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Storage::FileSystem::{GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION};
let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
let ok = unsafe { GetFileInformationByHandle(file.as_raw_handle() as _, &mut info) };
if ok == 0 {
return 0;
}
((info.nFileIndexHigh as u64) << 32) | info.nFileIndexLow as u64
}
#[cfg(not(any(unix, windows)))]
fn file_id(_file: &File, _meta: &Metadata) -> u64 {
0
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn binary_sniff() {
assert!(!looks_binary(b""));
assert!(!looks_binary(b"plain text\n"));
assert!(!looks_binary("ünïcödé 日本語".as_bytes()));
assert!(looks_binary(b"\x00"));
assert!(looks_binary(b"MZ\x90\x00\x03"));
assert!(looks_binary(b"text then \x00 nul"));
}
#[test]
fn not_a_directory_is_fatal() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("f.txt");
std::fs::write(&file, "x").unwrap();
assert!(matches!(crawl(&file, &CrawlConfig::default()), Err(Error::NotADirectory(_))));
assert!(matches!(
crawl(&dir.path().join("missing"), &CrawlConfig::default()),
Err(Error::NotADirectory(_))
));
}
#[test]
fn file_ids_distinguish_files_and_are_stable() {
let dir = tempfile::tempdir().unwrap();
let a = dir.path().join("a.txt");
let b = dir.path().join("b.txt");
std::fs::write(&a, "a").unwrap();
std::fs::write(&b, "b").unwrap();
let id = |p: &Path| {
let f = File::open(p).unwrap();
let m = f.metadata().unwrap();
file_id(&f, &m)
};
assert_eq!(id(&a), id(&a));
if id(&a) != 0 {
assert_ne!(id(&a), id(&b));
}
}
#[test]
fn crawl_all_reports_binary_and_too_large_with_identity() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("a.txt"), "hello").unwrap();
std::fs::write(dir.path().join("b.bin"), b"\x00\x01\x02").unwrap();
std::fs::write(dir.path().join("c.big"), vec![b'x'; 100]).unwrap();
let cfg = CrawlConfig { max_size: 50, respect_global_gitignore: false, ..CrawlConfig::default() };
let (all, stats) = crawl_all(dir.path(), &cfg).unwrap();
assert_eq!(all.len(), 3);
let kinds: Vec<(String, FileKind)> =
all.iter().map(|f| (f.entry.path.file_name().unwrap().to_string_lossy().into_owned(), f.kind)).collect();
assert_eq!(kinds, vec![("a.txt".into(), FileKind::Text), ("b.bin".into(), FileKind::Binary), ("c.big".into(), FileKind::TooLarge)]);
assert_eq!(stats.indexable, 1);
assert_eq!(stats.skipped_binary, 1);
assert_eq!(stats.skipped_too_large, 1);
assert!(all[1].entry.size == 3);
assert!(all[2].entry.size == 100);
let (text, stats2) = crawl(dir.path(), &cfg).unwrap();
assert_eq!(text.len(), 1);
assert_eq!(text[0].path.file_name().unwrap(), "a.txt");
assert_eq!(stats2, stats);
}
}