use std::collections::HashSet;
use std::path::{Path, PathBuf};
use crate::model::DetectConfidence;
use crate::{Error, Registry};
#[derive(Debug, Clone, Copy, Default)]
#[non_exhaustive]
pub struct ExpandOptions {
pub recursive: bool,
pub glob: bool,
pub dataset_dir: Option<fn(&Registry, &Path) -> bool>,
}
#[derive(Debug)]
pub struct BatchInput {
pub path: PathBuf,
pub relative: PathBuf,
pub error: Option<Error>,
}
#[derive(Debug, Default)]
pub struct Expansion {
pub items: Vec<BatchInput>,
pub expanded: bool,
}
pub fn has_glob_meta(p: &Path) -> bool {
p.to_string_lossy().contains(['*', '?', '['])
}
pub fn is_dataset_dir(reg: &Registry, dir: &Path) -> bool {
reg.detect(dir)
.is_ok_and(|(_, d)| d.confidence != DetectConfidence::ExtensionOnly)
}
pub fn is_dataset_dir_in_walk(reg: &Registry, dir: &Path) -> bool {
let Ok((_, det)) = reg.detect(dir) else {
return false;
};
match det.confidence {
DetectConfidence::Definite => true,
DetectConfidence::ExtensionOnly => false,
DetectConfidence::Likely => {
let Ok(rd) = std::fs::read_dir(dir) else {
return true;
};
!rd.filter_map(std::result::Result::ok).any(|e| {
!e.file_name().to_string_lossy().starts_with('.')
&& e.file_type().is_ok_and(|t| t.is_dir())
&& reg
.detect(&e.path())
.is_ok_and(|(_, d)| match d.confidence {
DetectConfidence::Definite => true,
DetectConfidence::Likely => {
d.format_id != det.format_id
|| !is_dataset_dir_in_walk(reg, &e.path())
}
DetectConfidence::ExtensionOnly => false,
})
})
}
}
}
pub const SIDECAR_SUFFIX: &str = ".openreadout.json";
pub fn expand(reg: &Registry, args: &[PathBuf], opts: ExpandOptions) -> Expansion {
let mut out = Expansion::default();
let mut visited = HashSet::new();
for arg in args {
if arg.as_os_str() == "-" {
out.items.push(BatchInput {
path: arg.clone(),
relative: arg.clone(),
error: None,
});
continue;
}
if opts.glob && !arg.exists() && has_glob_meta(arg) {
out.expanded = true;
let pattern = arg.to_string_lossy();
match glob::glob(&pattern) {
Err(e) => out.items.push(BatchInput {
path: arg.clone(),
relative: arg.clone(),
error: Some(Error::Usage(format!("bad glob pattern '{pattern}': {e}"))),
}),
Ok(paths) => {
let mut matched: Vec<PathBuf> =
paths.filter_map(std::result::Result::ok).collect();
matched.sort();
if matched.is_empty() {
out.items.push(BatchInput {
path: arg.clone(),
relative: arg.clone(),
error: Some(Error::io(
arg,
std::io::Error::new(
std::io::ErrorKind::NotFound,
"no file matches this pattern",
),
)),
});
}
for m in matched {
add_path(reg, &m, &m, opts, true, &mut visited, &mut out);
}
}
}
continue;
}
add_path(reg, arg, arg, opts, true, &mut visited, &mut out);
}
out
}
fn relative_to(path: &Path, root: &Path) -> PathBuf {
match path.strip_prefix(root) {
Ok(r) if !r.as_os_str().is_empty() => r.to_path_buf(),
_ => path
.file_name()
.map_or_else(|| path.to_path_buf(), PathBuf::from),
}
}
fn add_path(
reg: &Registry,
path: &Path,
root: &Path,
opts: ExpandOptions,
top: bool,
visited: &mut HashSet<PathBuf>,
out: &mut Expansion,
) {
let meta = match std::fs::metadata(path) {
Ok(m) => m,
Err(e) => {
out.items.push(BatchInput {
path: path.to_path_buf(),
relative: relative_to(path, root),
error: Some(Error::io(path, e)),
});
return;
}
};
let dataset_dir = opts.dataset_dir.unwrap_or(if opts.recursive {
is_dataset_dir_in_walk
} else {
is_dataset_dir
});
if !meta.is_dir() || dataset_dir(reg, path) {
out.items.push(BatchInput {
path: path.to_path_buf(),
relative: relative_to(path, root),
error: None,
});
return;
}
if !top && !opts.recursive {
return;
}
let canon = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
if !visited.insert(canon) {
return;
}
out.expanded = true;
let rd = match std::fs::read_dir(path) {
Ok(rd) => rd,
Err(e) => {
out.items.push(BatchInput {
path: path.to_path_buf(),
relative: relative_to(path, root),
error: Some(Error::io(path, e)),
});
return;
}
};
let mut entries: Vec<PathBuf> = rd
.filter_map(std::result::Result::ok)
.filter(|e| {
let n = e.file_name();
let n = n.to_string_lossy();
!n.starts_with('.') && !n.ends_with(SIDECAR_SUFFIX)
})
.map(|e| e.path())
.collect();
entries.sort();
for e in entries {
add_path(reg, &e, root, opts, false, visited, out);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn walks_sorted_skips_hidden_and_respects_recursion() {
let base = std::env::temp_dir().join(format!("openreadout-batch-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&base);
std::fs::create_dir_all(base.join("sub/deeper")).unwrap();
for f in [
"b.txt",
"a.txt",
".hidden",
"a.txt.openreadout.json",
"sub/c.txt",
"sub/deeper/d.txt",
] {
std::fs::write(base.join(f), b"x").unwrap();
}
let reg = Registry::new();
let flat = expand(®, std::slice::from_ref(&base), ExpandOptions::default());
let names: Vec<String> = flat
.items
.iter()
.map(|i| i.relative.to_string_lossy().replace('\\', "/"))
.collect();
assert_eq!(names, ["a.txt", "b.txt"]);
assert!(flat.expanded);
let deep = expand(
®,
std::slice::from_ref(&base),
ExpandOptions {
recursive: true,
glob: true,
dataset_dir: None,
},
);
let names: Vec<String> = deep
.items
.iter()
.map(|i| i.relative.to_string_lossy().replace('\\', "/"))
.collect();
assert_eq!(names, ["a.txt", "b.txt", "sub/c.txt", "sub/deeper/d.txt"]);
let pat = base.join("*.txt");
let g = expand(
®,
&[pat],
ExpandOptions {
recursive: false,
glob: true,
dataset_dir: None,
},
);
assert_eq!(g.items.len(), 2);
let none = expand(
®,
&[base.join("*.nothing")],
ExpandOptions {
recursive: false,
glob: true,
dataset_dir: None,
},
);
assert_eq!(none.items.len(), 1);
assert_eq!(none.items[0].error.as_ref().unwrap().exit_code(), 5);
let missing = expand(®, &[base.join("nope")], ExpandOptions::default());
assert!(!missing.expanded);
assert!(missing.items[0].error.is_some());
let _ = std::fs::remove_dir_all(&base);
}
#[cfg(unix)]
#[test]
fn links_are_followed_without_loops() {
let base = tempfile::tempdir().unwrap();
let data = base.path().join("data");
std::fs::create_dir_all(data.join("more")).unwrap();
std::fs::write(data.join("a.txt"), b"x").unwrap();
std::fs::write(data.join("more/b.txt"), b"x").unwrap();
let lab = base.path().join("lab");
std::fs::create_dir(&lab).unwrap();
std::os::unix::fs::symlink(data.join("a.txt"), lab.join("a.txt")).unwrap();
std::os::unix::fs::symlink(data.join("more"), lab.join("more")).unwrap();
std::os::unix::fs::symlink(&lab, lab.join("up")).unwrap();
let reg = Registry::new();
let opts = ExpandOptions {
recursive: true,
glob: false,
dataset_dir: None,
};
let names: Vec<String> = expand(®, std::slice::from_ref(&lab), opts)
.items
.iter()
.map(|i| i.relative.to_string_lossy().replace('\\', "/"))
.collect();
assert_eq!(names, ["a.txt", "more/b.txt"]);
}
#[derive(Debug)]
struct Folders;
impl crate::FormatReader for Folders {
fn descriptor(&self) -> crate::model::FormatDescriptor {
crate::model::FormatDescriptor {
id: "folders".into(),
name: "Folder test format".into(),
vendor: String::new(),
extensions: Vec::new(),
family: "test".into(),
can_read: true,
can_write: false,
confidence: crate::provenance::Confidence::Low,
known_gaps: Vec::new(),
}
}
fn sniff(&self, _head: &[u8], path: &Path) -> Option<crate::reader::Detection> {
let ext = path.extension()?.to_str()?;
let (format_id, confidence) = match ext {
"proj" | "tp" => ("folders", DetectConfidence::Likely),
"exp" => ("experiment", DetectConfidence::Definite),
_ => return None,
};
Some(crate::reader::Detection {
format_id,
confidence,
note: None,
})
}
fn open(&self, path: &Path) -> crate::Result<Box<dyn crate::Dataset>> {
Err(Error::Other(format!("opened {}", path.display())))
}
}
#[test]
fn recursive_walks_enter_likely_folders_holding_data_sets() {
let dir = tempfile::tempdir().unwrap();
let base = dir.path();
for d in ["mixed.proj/1.exp", "mixed.proj/2.exp", "plate.proj/t1.tp"] {
std::fs::create_dir_all(base.join(d)).unwrap();
}
std::fs::write(base.join("mixed.proj/notes.txt"), b"x").unwrap();
let reg = Registry::new().with(Box::new(Folders));
let names = |opts: ExpandOptions, arg: &Path| -> Vec<String> {
expand(®, &[arg.to_path_buf()], opts)
.items
.iter()
.map(|i| i.relative.to_string_lossy().replace('\\', "/"))
.collect()
};
let rec = ExpandOptions {
recursive: true,
..ExpandOptions::default()
};
assert_eq!(
names(rec, base),
[
"mixed.proj/1.exp",
"mixed.proj/2.exp",
"mixed.proj/notes.txt",
"plate.proj"
]
);
assert_eq!(
names(ExpandOptions::default(), &base.join("mixed.proj")),
["mixed.proj"]
);
assert!(is_dataset_dir(®, &base.join("mixed.proj")));
assert!(!is_dataset_dir_in_walk(®, &base.join("mixed.proj")));
assert!(is_dataset_dir_in_walk(®, &base.join("plate.proj")));
}
}