use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use ahash::AHashMap;
use crate::config::Config;
use crate::config::root_guard::{self, RootRefusal};
use crate::scanner::{ScanCancel, ScanError};
use crate::scanner_filter::{Filters, ignore_walk_builder};
const LARGE_SCAN_CANDIDATE_WARN: usize = 50_000;
const VISIT_BUDGET_MULTIPLIER: usize = 20;
const MIN_VISIT_BUDGET: usize = 100_000;
const SURVEY_VISIT_GRACE: usize = 200_000;
const MAX_TALLY_KEYS: usize = 4096;
const TOP_CONTRIBUTORS: usize = 5;
const ROOT_FILES_LABEL: &str = "(files in the root)";
pub const ALLOW_EXTRA_ROOTS_ENV: &str = "BASEMIND_ALLOW_EXTRA_ROOTS";
static EXTRA_ROOTS_GRANT: AtomicBool = AtomicBool::new(false);
pub fn allow_extra_roots(allow: bool) {
EXTRA_ROOTS_GRANT.store(allow, Ordering::Relaxed);
}
fn extra_roots_granted(repo_root: &Path) -> bool {
EXTRA_ROOTS_GRANT.load(Ordering::Relaxed)
|| std::env::var(ALLOW_EXTRA_ROOTS_ENV).is_ok_and(|v| extra_roots_grant_covers(&v, repo_root))
}
fn extra_roots_grant_covers(value: &str, repo_root: &Path) -> bool {
if is_truthy(value) {
return true;
}
let repo_root = repo_root.canonicalize().unwrap_or_else(|_| repo_root.to_path_buf());
std::env::split_paths(value.trim())
.filter(|entry| {
let absolute = entry.is_absolute();
if !absolute && !entry.as_os_str().is_empty() {
crate::config::trust::warn_once(format!(
"{ALLOW_EXTRA_ROOTS_ENV} entry {entry:?} ignored: use 1/true/yes or a list of absolute workspace paths"
));
}
absolute
})
.any(|entry| repo_root.starts_with(entry.canonicalize().unwrap_or(entry)))
}
fn is_sensitive_root(path: &Path) -> bool {
const SENSITIVE_DIRS: &[&str] = &[".ssh", ".aws", ".gnupg"];
path.starts_with("/etc")
|| path.starts_with("/private/etc")
|| path
.components()
.any(|c| SENSITIVE_DIRS.iter().any(|d| c.as_os_str() == std::ffi::OsStr::new(d)))
}
fn is_truthy(value: &str) -> bool {
let value = value.trim();
value.eq_ignore_ascii_case("1") || value.eq_ignore_ascii_case("true") || value.eq_ignore_ascii_case("yes")
}
#[derive(Default)]
struct Tally(AHashMap<String, usize>);
impl Tally {
fn bump(&mut self, label: &str) {
if let Some(count) = self.0.get_mut(label) {
*count += 1;
} else if self.0.len() < MAX_TALLY_KEYS {
self.0.insert(label.to_string(), 1);
}
}
fn top(self) -> Vec<(String, usize)> {
let mut top: Vec<(String, usize)> = self.0.into_iter().collect();
top.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
top.truncate(TOP_CONTRIBUTORS);
top
}
}
#[derive(PartialEq, Eq, Debug)]
enum Flow {
Continue,
Stop,
}
struct Budget {
cap: usize,
visit_budget: usize,
keys: Vec<String>,
candidates: Tally,
visits: Tally,
accepted: usize,
visited: usize,
over_cap: bool,
over_walk: bool,
survey_until: usize,
}
impl Budget {
fn new(cap: usize) -> Self {
let visit_budget = if cap == 0 {
0
} else {
cap.saturating_mul(VISIT_BUDGET_MULTIPLIER).max(MIN_VISIT_BUDGET)
};
Self {
cap,
visit_budget,
keys: Vec::new(),
candidates: Tally::default(),
visits: Tally::default(),
accepted: 0,
visited: 0,
over_cap: false,
over_walk: false,
survey_until: 0,
}
}
fn visit(&mut self, label: &str) -> Flow {
self.visited += 1;
self.visits.bump(label);
if self.over_cap {
return if self.visited > self.survey_until {
Flow::Stop
} else {
Flow::Continue
};
}
if self.visit_budget != 0 && self.visited > self.visit_budget {
self.over_walk = true;
return Flow::Stop;
}
Flow::Continue
}
fn accept(&mut self, label: &str, key: &str) -> Flow {
self.accepted += 1;
self.candidates.bump(label);
if self.cap == 0 || self.accepted <= self.cap {
self.keys.push(key.to_string());
return Flow::Continue;
}
if !self.over_cap {
self.over_cap = true;
self.survey_until = self.visited.saturating_add(SURVEY_VISIT_GRACE);
}
if self.visited > self.survey_until {
Flow::Stop
} else {
Flow::Continue
}
}
fn into_keys(self) -> Vec<String> {
self.keys
}
fn finish(self, root: &Path) -> Result<Vec<String>, ScanError> {
if self.over_cap {
return Err(ScanError::TooManyCandidates {
candidates: self.accepted,
cap: self.cap,
root: root.display().to_string(),
top_dirs: self.candidates.top(),
});
}
if self.over_walk {
return Err(ScanError::WalkTooLarge {
visited: self.visited,
candidates: self.accepted,
cap: self.cap,
root: root.display().to_string(),
top_dirs: self.visits.top(),
});
}
Ok(self.keys)
}
}
fn label_for(rel: &str) -> &str {
let Some(first) = rel.find('/') else {
return ROOT_FILES_LABEL;
};
match rel[first + 1..].find('/') {
Some(second) => &rel[..first + 1 + second],
None => &rel[..first],
}
}
fn rel_str<'a>(base: &Path, path: &'a Path, buf: &'a mut String) -> Option<&'a str> {
let rel = path.strip_prefix(base).ok()?.to_str()?;
#[cfg(windows)]
{
buf.clear();
buf.push_str(&rel.replace('\\', "/"));
Some(buf.as_str())
}
#[cfg(not(windows))]
{
let _ = buf;
Some(rel)
}
}
pub(crate) fn walk_candidates(
root: &Path,
config: &Config,
filters: &Filters,
cancel: &ScanCancel,
) -> Result<Vec<String>, ScanError> {
let mut budget = Budget::new(config.scan.max_candidates);
let pruner = filters.dir_pruner(Some(root));
let walker = ignore_walk_builder(root, config.scan.respect_gitignore, config.scan.follow_symlinks)
.filter_entry(move |dent| pruner.keep(dent))
.build();
let mut buf = String::new();
for dent in walker.flatten() {
if cancel.is_cancelled() {
return Ok(budget.into_keys());
}
let path = dent.path();
let Some(rel) = rel_str(root, path, &mut buf) else {
continue;
};
if rel.is_empty() {
continue;
}
let label = label_for(rel);
if budget.visit(label) == Flow::Stop {
break;
}
if !dent.file_type().map(|t| t.is_file()).unwrap_or(false) {
continue;
}
if !filters.allows(rel) {
continue;
}
if budget.accept(label, rel) == Flow::Stop {
break;
}
}
if !budget.over_cap && !budget.over_walk && !cancel.is_cancelled() {
walk_extra_roots(root, config, filters, cancel, &mut budget);
}
if cancel.is_cancelled() {
return Ok(budget.into_keys());
}
let out = budget.finish(root)?;
if out.len() > LARGE_SCAN_CANDIDATE_WARN {
tracing::warn!(
candidates = out.len(),
"scan candidate set is very large; check .gitignore / [scan] exclude globs for generated or vendored trees"
);
}
Ok(out)
}
fn walk_extra_roots(root: &Path, config: &Config, filters: &Filters, cancel: &ScanCancel, budget: &mut Budget) {
if config.scan.extra_roots.is_empty() {
return;
}
if !extra_roots_granted(root) {
tracing::warn!(
roots = config.scan.extra_roots.len(),
env = ALLOW_EXTRA_ROOTS_ENV,
"scan.extra_roots ignored: it names directories outside the repository but was read from \
the repository's own basemind.toml; set {ALLOW_EXTRA_ROOTS_ENV}=1 (or a list of workspace \
roots) in the environment to opt in",
);
return;
}
let repo_root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
for raw_root in &config.scan.extra_roots {
if cancel.is_cancelled() {
return;
}
let Some(extra) = vet_extra_root(raw_root, &repo_root) else {
continue;
};
let Some(root_display) = extra.to_str().map(display_key) else {
tracing::warn!(root = %extra.display(), "extra_root skipped: path is not valid UTF-8");
continue;
};
let pruner = filters.dir_pruner_for_extra_root(&extra);
let walker = ignore_walk_builder(&extra, config.scan.respect_gitignore, config.scan.follow_symlinks)
.filter_entry(move |dent| pruner.keep(dent))
.build();
let mut rel_buf = String::new();
let mut key_buf = String::new();
let mut label_buf = String::new();
for dent in walker.flatten() {
if cancel.is_cancelled() {
return;
}
let path = dent.path();
let Some(rel) = rel_str(&extra, path, &mut rel_buf) else {
continue;
};
let label = extra_label(&mut label_buf, &root_display, rel);
if budget.visit(label) == Flow::Stop {
return;
}
if !dent.file_type().map(|t| t.is_file()).unwrap_or(false) {
continue;
}
let Some(key) = key_str(path, &mut key_buf) else {
continue;
};
if !filters.allows_in_extra_root(rel) {
continue;
}
if budget.accept(label, key) == Flow::Stop {
return;
}
}
}
}
fn display_key(raw: &str) -> String {
#[cfg(windows)]
{
raw.replace('\\', "/")
}
#[cfg(not(windows))]
{
raw.to_string()
}
}
fn key_str<'a>(path: &'a Path, buf: &'a mut String) -> Option<&'a str> {
let raw = path.to_str()?;
#[cfg(windows)]
{
buf.clear();
buf.push_str(&raw.replace('\\', "/"));
Some(buf.as_str())
}
#[cfg(not(windows))]
{
let _ = buf;
Some(raw)
}
}
fn extra_label<'a>(buf: &'a mut String, root_display: &str, rel: &str) -> &'a str {
buf.clear();
buf.push_str(root_display);
let sub = label_for(rel);
if sub != ROOT_FILES_LABEL {
buf.push('/');
buf.push_str(sub);
}
buf.as_str()
}
fn vet_extra_root(raw_root: &Path, repo_root: &Path) -> Option<PathBuf> {
let extra = match raw_root.canonicalize() {
Ok(p) => p,
Err(e) => {
tracing::warn!(root = %raw_root.display(), error = %e, "extra_root skipped: cannot access");
return None;
}
};
if let Err(RootRefusal::FilesystemRoot) = root_guard::workspace_root_verdict(&extra) {
tracing::warn!(
root = %extra.display(),
"extra_root refused: that is a filesystem or volume root; indexing it would walk the entire machine"
);
return None;
}
if is_sensitive_root(&extra) {
tracing::warn!(root = %extra.display(), "extra_root refused: credential or system configuration directory");
return None;
}
if !extra.is_dir() {
tracing::warn!(root = %extra.display(), "extra_root skipped: not a directory");
return None;
}
if extra.starts_with(repo_root) {
tracing::warn!(root = %extra.display(), "extra_root skipped: inside the repository root (already indexed)");
return None;
}
Some(extra)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn label_takes_two_segments_and_buckets_root_files() {
assert_eq!(label_for("packages/app/src/x.rs"), "packages/app");
assert_eq!(label_for("src/a.rs"), "src");
assert_eq!(label_for("src"), ROOT_FILES_LABEL);
assert_eq!(label_for("basemind.toml"), ROOT_FILES_LABEL);
}
#[test]
fn exactly_cap_candidates_is_not_a_breach() {
let mut budget = Budget::new(3);
for i in 0..3 {
assert_eq!(budget.visit("src"), Flow::Continue);
assert_eq!(budget.accept("src", &format!("src/f{i}.rs")), Flow::Continue);
}
for _ in 0..50 {
assert_eq!(
budget.visit("src"),
Flow::Continue,
"non-candidate entries never breach"
);
}
assert_eq!(budget.finish(Path::new("/repo")).expect("no breach").len(), 3);
}
#[test]
fn breach_survives_to_name_the_real_offender() {
let mut budget = Budget::new(20);
for i in 0..30 {
budget.visit("aaa");
budget.accept("aaa", &format!("aaa/f{i}.rs"));
}
for i in 0..5000 {
budget.visit("zzz");
budget.accept("zzz", &format!("zzz/f{i}.rs"));
}
let err = budget.finish(Path::new("/repo")).expect_err("cap breached");
match err {
ScanError::TooManyCandidates {
candidates,
cap,
top_dirs,
..
} => {
assert_eq!(cap, 20);
assert_eq!(candidates, 5030, "the honest count, not the cap");
assert_eq!(top_dirs.first().map(|(n, c)| (n.as_str(), *c)), Some(("zzz", 5000)));
}
other => panic!("unexpected error: {other}"),
}
}
#[test]
fn survey_window_stops_the_walk() {
let mut budget = Budget::new(1);
budget.visit("a");
assert_eq!(budget.accept("a", "a/one.rs"), Flow::Continue);
budget.visit("a");
assert_eq!(budget.accept("a", "a/two.rs"), Flow::Continue, "survey opens");
for _ in 0..SURVEY_VISIT_GRACE {
budget.visit("a");
}
assert_eq!(budget.visit("a"), Flow::Stop, "survey window closed");
assert_eq!(budget.keys.len(), 1, "nothing past the cap is stored");
}
#[test]
fn walk_budget_bounds_a_walk_that_accepts_almost_nothing() {
let mut budget = Budget::new(1_000_000);
let limit = budget.visit_budget;
assert_eq!(limit, 20_000_000);
budget.visited = limit - 1;
assert_eq!(budget.visit("noise/tree"), Flow::Continue);
assert_eq!(budget.visit("noise/tree"), Flow::Stop);
match budget.finish(Path::new("/repo")).expect_err("walk budget breached") {
ScanError::WalkTooLarge {
visited,
candidates,
top_dirs,
..
} => {
assert_eq!(visited, limit + 1);
assert_eq!(candidates, 0);
assert_eq!(top_dirs.first().map(|(n, _)| n.as_str()), Some("noise/tree"));
}
other => panic!("unexpected error: {other}"),
}
}
#[test]
fn zero_cap_disables_both_bounds() {
let mut budget = Budget::new(0);
assert_eq!(budget.visit_budget, 0);
for i in 0..1000 {
assert_eq!(budget.visit("a"), Flow::Continue);
assert_eq!(budget.accept("a", &format!("a/{i}.rs")), Flow::Continue);
}
assert_eq!(budget.finish(Path::new("/repo")).expect("unlimited").len(), 1000);
}
#[test]
fn tally_stops_growing_but_keeps_counting_known_labels() {
let mut tally = Tally::default();
for i in 0..(MAX_TALLY_KEYS + 100) {
tally.bump(&format!("d{i}"));
}
tally.bump("d0");
let top = tally.top();
assert_eq!(top.first().map(|(n, c)| (n.as_str(), *c)), Some(("d0", 2)));
}
#[test]
#[cfg(unix)]
fn filesystem_root_is_refused_even_via_a_symlink() {
let tmp = tempfile::tempdir().expect("tempdir");
let link = tmp.path().join("to-root");
std::os::unix::fs::symlink("/", &link).expect("symlink");
let repo = tmp.path().canonicalize().expect("canonicalize");
assert!(vet_extra_root(Path::new("/"), &repo).is_none());
assert!(
vet_extra_root(&link, &repo).is_none(),
"a symlink to / resolves to / and is refused"
);
}
#[test]
fn truthy_grant_values() {
for yes in ["1", "true", " TRUE ", "yes"] {
assert!(is_truthy(yes), "{yes:?}");
}
for no in ["0", "false", "no", ""] {
assert!(!is_truthy(no), "{no:?}");
}
}
#[test]
fn extra_roots_grant_can_be_scoped_to_a_list_of_workspaces() {
let dir = tempfile::tempdir().expect("tempdir");
let allowed = dir.path().join("allowed");
let other = dir.path().join("other");
std::fs::create_dir_all(allowed.join("sub")).expect("mkdir");
std::fs::create_dir_all(&other).expect("mkdir");
assert!(
extra_roots_grant_covers("1", &other),
"the bare word grants every workspace"
);
let list = std::env::join_paths([&allowed, Path::new("/nonexistent/elsewhere")])
.expect("join")
.into_string()
.expect("utf8");
assert!(extra_roots_grant_covers(&list, &allowed));
assert!(
extra_roots_grant_covers(&list, &allowed.join("sub")),
"descendants are covered"
);
assert!(!extra_roots_grant_covers(&list, &other), "an unlisted workspace is not");
assert!(!extra_roots_grant_covers("0", &allowed));
assert!(
!extra_roots_grant_covers(".", &allowed),
"a relative entry never grants"
);
assert!(!extra_roots_grant_covers("on", &allowed));
assert!(is_sensitive_root(Path::new("/home/u/.ssh")));
assert!(is_sensitive_root(Path::new("/etc/ssl")));
assert!(!is_sensitive_root(Path::new("/home/u/code/external")));
}
}