use crate::dataset::Dataset;
use std::path::Path;
pub(super) fn portability_findings(ds: &Dataset) -> Vec<(&'static str, String)> {
let templates = [
("data_path", Some(ds.info.data_path.as_str())),
("video_path", ds.info.video_path.as_deref()),
];
let mut out: Vec<(&'static str, String)> = templates
.into_iter()
.filter_map(|(name, t)| Some((name, t?)))
.filter(|(_, t)| is_absolute_template(t))
.map(|(name, t)| {
(
"path_absolute",
format!("{name} is the absolute path `{t}`"),
)
})
.collect();
let scan = scan_files(&ds.root, LARGE_FILE);
if !scan.stray.is_empty() {
out.push((
"data_stray_file",
format!(
"data/ holds files that are not parquet: {}",
list_some(&scan.stray)
),
));
}
if !scan.symlinks.is_empty() && !ds.root.components().any(|c| c.as_os_str() == "snapshots") {
out.push((
"symlink",
format!(
"symbolic links break when the dataset is copied or uploaded: {}",
list_some(&scan.symlinks)
),
));
}
if !scan.large.is_empty() {
out.push((
"file_oversized",
format!("files over 5 GB: {}", list_some(&scan.large)),
));
}
out
}
pub(super) fn is_absolute_template(t: &str) -> bool {
let b = t.as_bytes();
t.starts_with('/')
|| t.starts_with('\\')
|| (b.len() > 2 && b[0].is_ascii_alphabetic() && b[1] == b':')
}
pub(super) const LARGE_FILE: u64 = 5 << 30;
pub(super) fn list_some(items: &[String]) -> String {
let shown: Vec<&str> = items.iter().take(5).map(String::as_str).collect();
match items.len().saturating_sub(shown.len()) {
0 => shown.join(", "),
more => format!("{} and {more} more", shown.join(", ")),
}
}
#[derive(Default)]
pub(super) struct Scan {
pub(super) stray: Vec<String>,
pub(super) symlinks: Vec<String>,
pub(super) large: Vec<String>,
}
pub(super) fn scan_files(root: &Path, limit: u64) -> Scan {
let mut out = Scan::default();
let mut stack = vec![root.to_path_buf()];
while let Some(d) = stack.pop() {
let Ok(rd) = std::fs::read_dir(&d) else {
continue;
};
for e in rd.flatten() {
let (p, Ok(kind)) = (e.path(), e.file_type()) else {
continue;
};
let name = || p.strip_prefix(root).unwrap_or(&p).display().to_string();
if kind.is_symlink() {
out.symlinks.push(name());
} else if kind.is_dir() {
stack.push(p);
} else {
if p.extension().is_none_or(|x| x != "parquet") && p.starts_with(root.join("data"))
{
out.stray.push(name());
}
if let Ok(m) = e.metadata()
&& m.len() >= limit
{
out.large.push(format!(
"{} ({:.1} GB)",
name(),
m.len() as f64 / (1u64 << 30) as f64
));
}
}
}
}
out.stray.sort();
out.symlinks.sort();
out.large.sort();
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn scanning_a_directory_that_cannot_be_read_finds_nothing() {
let scan = scan_files(std::path::Path::new("no/such/dir"), 1);
assert!(scan.stray.is_empty() && scan.symlinks.is_empty() && scan.large.is_empty());
}
#[test]
fn absolute_templates_are_recognised() {
assert!(is_absolute_template("/data/ep_{episode_index}.parquet"));
assert!(is_absolute_template("C:\\data\\ep.parquet"));
assert!(!is_absolute_template(
"data/chunk-{episode_chunk:03d}/ep.parquet"
));
}
}