use std::io::Read;
use std::path::Path;
use std::time::{Duration, Instant};
const NEVER_WALK: &[&str] = &[
".git",
".hg",
".svn",
".leindex",
"node_modules",
"__pycache__",
".mypy_cache",
".pytest_cache",
".tox",
];
const POLL_MIN: Duration = Duration::from_millis(1);
const POLL_MAX: Duration = Duration::from_millis(20);
const DEADLINE_CHECK_STRIDE: usize = 64;
fn run_captured(mut command: std::process::Command, deadline: Option<Instant>) -> Option<Vec<u8>> {
let mut child = command
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::null())
.spawn()
.ok()?;
let mut pipe = child.stdout.take()?;
let reader = std::thread::spawn(move || {
let mut raw = Vec::new();
let _ = pipe.read_to_end(&mut raw);
raw
});
let mut poll = POLL_MIN;
let status = loop {
match child.try_wait() {
Ok(Some(status)) => break Some(status),
Ok(None) => {}
Err(_) => break None,
}
if deadline.is_some_and(|d| Instant::now() >= d) {
let _ = child.kill();
let _ = child.wait();
break None;
}
let now = Instant::now();
let sleep = match deadline {
Some(d) => poll.min(d.saturating_duration_since(now)),
None => poll,
};
std::thread::sleep(sleep);
poll = (poll * 2).min(POLL_MAX);
};
if !status.is_some_and(|status| status.success()) {
return None;
}
reader.join().ok()
}
fn sorted_unique(mut files: Vec<String>) -> Vec<String> {
files.sort_unstable();
files.dedup();
files
}
fn git_inventory(
root: &Path,
deadline: Option<Instant>,
admit: &impl Fn(&str) -> bool,
) -> GitInventory {
if deadline.is_some_and(|d| Instant::now() >= d) {
return GitInventory::Absent;
}
let mut command = std::process::Command::new("git");
command
.args([
"ls-files",
"-z",
"--cached",
"--others",
"--exclude-standard",
])
.current_dir(root);
let Some(raw) = run_captured(command, deadline) else {
return GitInventory::Absent;
};
let canon_root = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf());
let mut files: Vec<String> = Vec::new();
let mut truncation = InventoryTruncation::default();
for (index, entry) in raw
.split(|b| *b == 0)
.filter(|entry| !entry.is_empty())
.enumerate()
{
if index % DEADLINE_CHECK_STRIDE == 0 && deadline.is_some_and(|d| Instant::now() >= d) {
truncation.deadline = true;
return GitInventory::Present(sorted_unique(files), truncation);
}
let Ok(rel) = std::str::from_utf8(entry) else {
continue;
};
if rel.split('/').any(|seg| seg == ".leindex" || seg == ".git") {
continue;
}
let path = root.join(rel);
if !std::fs::canonicalize(&path)
.is_ok_and(|resolved| resolved.starts_with(&canon_root) && resolved.is_file())
{
continue;
}
if admit(rel) {
files.push(rel.to_string());
}
}
GitInventory::Present(sorted_unique(files), truncation)
}
#[derive(Debug)]
enum GitInventory {
Absent,
Present(Vec<String>, InventoryTruncation),
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct InventoryTruncation {
pub limit: bool,
pub deadline: bool,
pub skip: bool,
}
impl InventoryTruncation {
pub fn truncated(self) -> bool {
self.limit || self.deadline || self.skip
}
}
const WORK_BUDGET_PER_ADMITTED_FILE: usize = 64;
fn walk_inventory(
root: &Path,
limit: usize,
deadline: Option<Instant>,
admit: impl Fn(&str) -> bool,
) -> (Vec<String>, InventoryTruncation) {
let work_budget = limit.saturating_mul(WORK_BUDGET_PER_ADMITTED_FILE);
let mut seen = 0usize;
let mut out = Vec::new();
let mut truncation = InventoryTruncation::default();
fn finish(
mut out: Vec<String>,
truncation: InventoryTruncation,
) -> (Vec<String>, InventoryTruncation) {
out.sort_unstable();
(out, truncation)
}
let mut stack = vec![(root.to_path_buf(), String::new())];
while let Some((dir, prefix)) = stack.pop() {
if deadline.is_some_and(|d| Instant::now() >= d) {
truncation.deadline = true;
return finish(out, truncation);
}
let Ok(entries) = std::fs::read_dir(&dir) else {
truncation.skip = true;
continue;
};
let mut since_check = 0usize;
for entry in entries {
let Ok(entry) = entry else {
truncation.skip = true;
continue;
};
since_check += 1;
if since_check >= DEADLINE_CHECK_STRIDE {
since_check = 0;
if deadline.is_some_and(|d| Instant::now() >= d) {
truncation.deadline = true;
return finish(out, truncation);
}
}
let Ok(kind) = entry.file_type() else {
truncation.skip = true;
continue;
};
let name = entry.file_name();
let Some(name) = name.to_str() else {
continue;
};
let rel = if prefix.is_empty() {
name.to_string()
} else {
format!("{prefix}/{name}")
};
if kind.is_dir() {
if !NEVER_WALK.contains(&name) {
stack.push((entry.path(), rel));
}
continue;
}
if !kind.is_file() {
continue;
}
if seen >= work_budget {
truncation.limit = true;
return finish(out, truncation);
}
seen += 1;
if admit(&rel) {
if out.len() >= limit {
truncation.limit = true;
return finish(out, truncation);
}
out.push(rel);
}
}
}
finish(out, truncation)
}
pub fn list_files(dir: &Path, limit: usize, deadline: Option<Instant>) -> Vec<String> {
list_files_reporting(dir, limit, deadline).0
}
pub fn list_files_reporting(
dir: &Path,
limit: usize,
deadline: Option<Instant>,
) -> (Vec<String>, InventoryTruncation) {
list_files_matching(dir, limit, deadline, |_| true)
}
pub fn list_files_matching(
dir: &Path,
limit: usize,
deadline: Option<Instant>,
admit: impl Fn(&str) -> bool,
) -> (Vec<String>, InventoryTruncation) {
let (mut files, mut truncation) = match git_inventory(dir, deadline, &admit) {
GitInventory::Present(files, trunc) => (files, trunc),
GitInventory::Absent => walk_inventory(dir, limit, deadline, admit),
};
if files.len() > limit {
truncation.limit = true;
}
files.truncate(limit);
(files, truncation)
}
#[cfg(test)]
mod test {
use super::*;
#[cfg(unix)]
fn exits_now() -> std::process::Command {
let mut command = std::process::Command::new("sh");
command.args(["-c", "printf 'a\\0b\\0'"]);
command
}
#[cfg(unix)]
#[test]
fn test_run_captured_reads_a_command_that_exits() {
let out = run_captured(exits_now(), None).expect("a quick command must be captured");
assert_eq!(out, b"a\0b\0");
}
#[cfg(unix)]
#[test]
fn test_run_captured_kills_a_child_that_overruns_the_deadline() {
let started = Instant::now();
let mut command = std::process::Command::new("sleep");
command.arg("30");
let out = run_captured(command, Some(Instant::now()));
assert!(out.is_none(), "an overrunning child must report no output");
assert!(
started.elapsed() < Duration::from_secs(5),
"the child must be KILLED, not waited out; took {:?}",
started.elapsed()
);
}
#[cfg(unix)]
#[test]
fn test_run_captured_reports_a_failing_command_as_none() {
let mut command = std::process::Command::new("sh");
command.args(["-c", "exit 3"]);
assert!(
run_captured(command, None).is_none(),
"a non-zero exit is not a listing"
);
}
#[cfg(unix)]
#[test]
fn test_walk_reports_an_unreadable_subtree_as_truncated() {
let dir = tempfile::tempdir().unwrap();
let locked = dir.path().join("locked");
std::fs::create_dir(&locked).unwrap();
std::fs::write(locked.join("inner.rs"), "fn inner() {}").unwrap();
let mut perms = std::fs::metadata(&locked).unwrap().permissions();
std::os::unix::fs::PermissionsExt::set_mode(&mut perms, 0o000);
std::fs::set_permissions(&locked, perms).unwrap();
let unreadable = std::fs::read_dir(&locked).is_err();
let (out, truncation) = walk_inventory(dir.path(), 100, None, |rel| rel.ends_with(".rs"));
let mut restore = std::fs::metadata(&locked).unwrap().permissions();
std::os::unix::fs::PermissionsExt::set_mode(&mut restore, 0o755);
std::fs::set_permissions(&locked, restore).unwrap();
if !unreadable {
return; }
assert!(out.is_empty(), "the only file is inside the locked subtree");
assert!(
truncation.skip,
"a dropped subtree must be reported, not silently swallowed"
);
assert!(truncation.truncated());
assert!(
!truncation.deadline,
"the OS refused the directory; the clock did not stop it"
);
}
#[test]
fn test_list_files_matching_bounds_the_matching_files() {
let dir = tempfile::tempdir().unwrap();
for name in ["aaa.txt", "aab.txt", "zzz.rs", "zzz2.rs"] {
std::fs::write(dir.path().join(name), "x").unwrap();
}
let (matched, truncation) =
list_files_matching(dir.path(), 2, None, |rel| rel.ends_with(".rs"));
assert_eq!(
matched,
vec!["zzz.rs".to_string(), "zzz2.rs".to_string()],
"the limit must be spent on admitted files, not consumed by rejects"
);
assert!(
!truncation.truncated(),
"both matching files fit, so nothing was cut short"
);
let (one, truncation) =
list_files_matching(dir.path(), 1, None, |rel| rel.ends_with(".rs"));
assert_eq!(one.len(), 1, "the limit bounds the files returned");
assert!(
one[0].ends_with(".rs"),
"the one file returned must be an admitted match, got {:?}",
one
);
assert!(
truncation.limit,
"a second matching file exists, so the listing is incomplete"
);
}
#[test]
fn test_list_files_reporting_admits_everything() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("a.rs"), "x").unwrap();
std::fs::write(dir.path().join("b.txt"), "x").unwrap();
let (files, truncation) = walk_inventory(dir.path(), 10, None, |_| true);
assert_eq!(files, vec!["a.rs".to_string(), "b.txt".to_string()]);
assert!(!truncation.truncated());
}
#[test]
fn test_walk_stops_on_a_work_budget_a_selective_predicate_never_spends() {
let dir = tempfile::tempdir().unwrap();
for i in 0..WORK_BUDGET_PER_ADMITTED_FILE + 1 {
std::fs::write(dir.path().join(format!("f{i}.txt")), "x").unwrap();
}
let examined = std::cell::Cell::new(0usize);
let (matched, truncation) = walk_inventory(dir.path(), 1, None, |_| {
examined.set(examined.get() + 1);
false
});
assert!(
matched.is_empty(),
"nothing is admitted, so nothing may be returned"
);
assert_eq!(
examined.get(),
WORK_BUDGET_PER_ADMITTED_FILE,
"the walk must examine exactly the work budget, not one directory's worth: \
without it the walk reads the whole tree"
);
assert!(
truncation.limit,
"the work budget cut the traversal, so files remain unlisted"
);
assert!(
!truncation.deadline,
"no deadline was given, so this is a file budget, not time pressure"
);
}
#[test]
fn test_walk_stops_once_the_result_budget_is_admitted() {
let dir = tempfile::tempdir().unwrap();
for name in ["a.rs", "b.rs", "c.rs"] {
std::fs::write(dir.path().join(name), "x").unwrap();
}
let (matched, truncation) = walk_inventory(dir.path(), 2, None, |_| true);
assert_eq!(matched.len(), 2, "the limit bounds the files returned");
assert!(
truncation.limit,
"a third file exists, so the listing is incomplete"
);
assert!(!truncation.deadline);
}
#[test]
fn test_walk_of_exactly_the_admitted_budget_is_complete() {
let dir = tempfile::tempdir().unwrap();
for name in ["a.rs", "b.rs"] {
std::fs::write(dir.path().join(name), "x").unwrap();
}
let (matched, truncation) = walk_inventory(dir.path(), 2, None, |_| true);
assert_eq!(matched, vec!["a.rs".to_string(), "b.rs".to_string()]);
assert!(
!truncation.truncated(),
"every file in the tree was admitted, so nothing was cut short"
);
}
#[test]
fn test_walk_with_a_zero_limit_reports_truncation() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("only.rs"), "x").unwrap();
let (matched, truncation) = walk_inventory(dir.path(), 0, None, |_| true);
assert!(matched.is_empty(), "a zero limit admits nothing");
assert!(truncation.limit, "the tree was never listed");
}
#[test]
fn test_filling_the_result_budget_on_rejects_alone_is_not_truncation() {
let dir = tempfile::tempdir().unwrap();
for name in ["a.rs", "b.rs", "c.rs", "d.rs"] {
std::fs::write(dir.path().join(name), "x").unwrap();
}
let admitted = std::cell::Cell::new(0usize);
let (matched, truncation) = walk_inventory(dir.path(), 2, None, |_| {
let n = admitted.get();
admitted.set(n + 1);
n < 2
});
assert_eq!(matched.len(), 2, "the first two dirents are admitted");
assert!(
!truncation.truncated(),
"every match in the tree was admitted and the rejects withheld \
nothing — reporting truncation here is the CI flake"
);
}
#[test]
fn test_sorted_unique_deduplicates_and_sorts() {
let input = vec![
"b.rs".to_string(),
"a.rs".to_string(),
"b.rs".to_string(),
"a.rs".to_string(),
];
assert_eq!(
sorted_unique(input),
vec!["a.rs".to_string(), "b.rs".to_string()]
);
}
#[test]
#[cfg(unix)]
fn test_git_inventory_non_utf8_name_is_never_truncation() {
use std::os::unix::ffi::OsStrExt;
let dir = tempfile::tempdir().unwrap();
let ok = std::process::Command::new("git")
.args(["init", "-q"])
.current_dir(dir.path())
.status()
.is_ok_and(|s| s.success());
if !ok {
return;
}
let bad_name = std::ffi::OsStr::from_bytes(b"bad\xffname.txt");
std::fs::write(dir.path().join(bad_name), "x").unwrap();
std::fs::write(dir.path().join("good.txt"), "x").unwrap();
let (files, truncation) = list_files_matching(dir.path(), 100, None, |_| true);
assert_eq!(files, vec!["good.txt".to_string()]);
assert!(
!truncation.truncated(),
"a non-UTF-8 name is never matchable, so the listing of matchable \
files is complete"
);
assert!(!truncation.deadline);
}
#[test]
fn test_git_inventory_deterministic_repo() {
let dir = tempfile::tempdir().unwrap();
let ok = std::process::Command::new("git")
.args(["init", "-q"])
.current_dir(dir.path())
.status()
.is_ok_and(|s| s.success());
if !ok {
return;
}
std::fs::write(dir.path().join("a.rs"), "x").unwrap();
std::fs::write(dir.path().join("b.txt"), "x").unwrap();
std::fs::write(dir.path().join("c.rs"), "x").unwrap();
let (matched, truncation) =
list_files_matching(dir.path(), 10, None, |p| p.ends_with(".rs"));
assert_eq!(matched, vec!["a.rs".to_string(), "c.rs".to_string()]);
assert!(!truncation.truncated());
let (limited, truncation) =
list_files_matching(dir.path(), 1, None, |p| p.ends_with(".rs"));
assert_eq!(limited.len(), 1);
assert!(limited[0].ends_with(".rs"));
assert!(truncation.limit);
}
}