use std::fs;
use std::path::{Path, PathBuf};
use serde_yaml::Value;
use std::collections::BTreeMap;
use crate::hash::raw_hash_bytes as compute_hash_hex;
fn is_port_open(port: u16) -> bool {
use std::net::{TcpStream, SocketAddr};
use std::time::Duration;
let addr = SocketAddr::from(([127, 0, 0, 1], port));
if !TcpStream::connect_timeout(&addr, Duration::from_millis(500)).is_ok() {
return false;
}
if port == 9000 || port == 5001 {
let container = if port == 9000 { "minio_server" } else { "ipfs_host" };
let docker_check = std::process::Command::new("docker")
.args(["inspect", "-f", "{{.State.Running}}", container])
.output()
.map(|o| String::from_utf8_lossy(&o.stdout).trim() == "true")
.unwrap_or(false);
if !docker_check {
return false;
}
let daemon_check = if port == 9000 {
std::process::Command::new("docker")
.args(["exec", "minio_server", "mc", "alias", "list", "local"])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
} else {
std::process::Command::new("docker")
.args(["exec", "ipfs_host", "ipfs", "id"])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
};
if !daemon_check {
return false;
}
}
true
}
fn list_ipfs_recursive_live(current_cid: &str, current_path: &str, out: &mut Vec<(String, String)>) {
let output = std::process::Command::new("docker")
.args(["exec", "ipfs_host", "ipfs", "ls", current_cid])
.output();
if let Ok(o) = output {
let stdout_str = String::from_utf8_lossy(&o.stdout);
for line in stdout_str.lines() {
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.len() >= 3 {
let hash = parts[0];
let size_str = parts[1];
let name = parts[2].trim_end_matches('/');
let is_dir = size_str == "-" || parts[2].ends_with('/');
let next_path = if current_path.is_empty() {
name.to_string()
} else {
format!("{}/{}", current_path, name)
};
if is_dir {
list_ipfs_recursive_live(&format!("/ipfs/{}", hash), &next_path, out);
} else {
let cat_out = std::process::Command::new("docker")
.args(["exec", "ipfs_host", "ipfs", "cat", &format!("/ipfs/{}", hash)])
.output();
if let Ok(co) = cat_out {
let file_hash = compute_hash_hex(&co.stdout);
out.push((next_path, file_hash));
}
}
}
}
}
}
fn scan_local_dir(
base_dir: &Path,
current_dir: &Path,
storage_type: &str,
top_dir_name: &str,
mapping: &mut BTreeMap<String, String>,
) -> Result<(), String> {
if !current_dir.exists() {
return Ok(());
}
let mut entries = Vec::new();
if let Ok(read_dir) = fs::read_dir(current_dir) {
for entry in read_dir.flatten() {
entries.push(entry);
}
}
entries.sort_by_key(|e| e.path());
for entry in entries {
let path = entry.path();
if path.is_file() {
let rel_path = path.strip_prefix(base_dir)
.map_err(|e| format!("Failed to compute relative path: {}", e))?
.to_string_lossy()
.replace('\\', "/");
let content_bytes = fs::read(&path)
.map_err(|e| format!("Failed to read file {}: {}", path.display(), e))?;
let hash = compute_hash_hex(&content_bytes);
let handle = format!("{}://{}/{}", storage_type, top_dir_name, rel_path);
mapping.insert(handle, hash);
} else if path.is_dir() {
scan_local_dir(base_dir, &path, storage_type, top_dir_name, mapping)?;
}
}
Ok(())
}
fn collect_files_recursive(dir: &Path, out: &mut Vec<PathBuf>) -> Result<(), String> {
if dir.is_file() {
out.push(dir.to_path_buf());
return Ok(());
}
let mut entries = Vec::new();
if let Ok(read_dir) = fs::read_dir(dir) {
for entry in read_dir.flatten() {
entries.push(entry);
}
}
entries.sort_by_key(|e| e.path());
for entry in entries {
let path = entry.path();
if path.is_file() {
out.push(path);
} else if path.is_dir() {
collect_files_recursive(&path, out)?;
}
}
Ok(())
}
fn uuid_snippet() -> String {
use std::sync::atomic::Ordering;
let count = crate::TEMP_COUNTER.fetch_add(1, Ordering::SeqCst);
format!("{}_{}", std::process::id(), count)
}
fn copy_dir_all(src: &Path, dst: &Path) -> std::io::Result<()> {
fs::create_dir_all(dst)?;
for entry in fs::read_dir(src)? {
let entry = entry?;
let ty = entry.file_type()?;
if ty.is_dir() {
copy_dir_all(&entry.path(), &dst.join(entry.file_name()))?;
} else {
fs::copy(entry.path(), dst.join(entry.file_name()))?;
}
}
Ok(())
}
pub fn directory_fs_scan(clm_root: &str, clm_file: &str) -> Result<(i32, String), String> {
let clm_payload = crate::cross_cutting::context::CLMContext::get_scoped_or_env("CLM_PAYLOAD").unwrap_or_default();
let (dataset_dir, mut storage_type) = if !clm_payload.is_empty() && clm_payload != "null" {
(clm_payload, "fs".to_string())
} else {
let content = fs::read_to_string(clm_file)
.map_err(|e| format!("Failed to read CLM spec file: {}", e))?;
let parsed: Value = serde_yaml::from_str(&content)
.map_err(|e| format!("Failed to parse spec YAML: {}", e))?;
let clm_val = parsed.get("clm").unwrap_or(&parsed);
let process_val = clm_val.get("concrete_impl")
.and_then(|v| v.get("process"))
.or_else(|| clm_val.get("process"));
let dir = process_val
.and_then(|v| v.get("dataset_dir"))
.and_then(|v| v.as_str())
.unwrap_or("tests/test_fixtures/small_dataset")
.to_string();
let st = process_val
.and_then(|v| v.get("storage_type"))
.and_then(|v| v.as_str())
.unwrap_or("fs")
.to_string();
(dir, st)
};
if dataset_dir.starts_with("s3://") {
storage_type = "s3".to_string();
} else if dataset_dir.starts_with("ipfs://") {
storage_type = "ipfs".to_string();
}
let top_dir_name = if storage_type == "s3" {
let uri_path = dataset_dir.trim_start_matches("s3://");
let bucket = uri_path.split('/').next().unwrap_or(uri_path);
bucket.to_string()
} else if storage_type == "ipfs" {
let uri_path = dataset_dir.trim_start_matches("ipfs://");
let cid = uri_path.split('/').next().unwrap_or(uri_path);
cid.to_string()
} else {
let path = Path::new(&dataset_dir);
path.file_name()
.unwrap_or_else(|| std::ffi::OsStr::new("small_dataset"))
.to_string_lossy()
.into_owned()
};
let mut mapping = BTreeMap::new();
if storage_type == "s3" || storage_type == "ipfs" {
let port = if storage_type == "s3" { 9000 } else { 5001 };
let live = is_port_open(port);
if live {
if storage_type == "s3" {
let s3_path = format!("local/{}", dataset_dir.trim_start_matches("s3://"));
let output = std::process::Command::new("docker")
.args(["exec", "minio_server", "mc", "ls", "--json", "--recursive", &s3_path])
.output();
if let Ok(out) = output {
let stdout_str = String::from_utf8_lossy(&out.stdout);
for line in stdout_str.lines() {
if let Ok(val) = serde_json::from_str::<serde_json::Value>(line)
&& val.get("type").and_then(|t| t.as_str()) == Some("file")
&& let Some(key) = val.get("key").and_then(|k| k.as_str()) {
let cat_out = std::process::Command::new("docker")
.args(["exec", "minio_server", "mc", "cat", &format!("{}/{}", s3_path, key)])
.output();
if let Ok(co) = cat_out {
let file_hash = compute_hash_hex(&co.stdout);
let handle = format!("s3://{}/{}", top_dir_name, key);
mapping.insert(handle, file_hash);
}
}
}
}
} else {
let ipfs_path = format!("/ipfs/{}", dataset_dir.trim_start_matches("ipfs://"));
let mut listed = Vec::new();
list_ipfs_recursive_live(&ipfs_path, "", &mut listed);
for (rel_path, file_hash) in listed {
let handle = format!("ipfs://{}/{}", top_dir_name, rel_path);
mapping.insert(handle, file_hash);
}
}
} else {
let fixture_dir = Path::new(clm_root).join("tests/test_fixtures/small_dataset");
scan_local_dir(&fixture_dir, &fixture_dir, &storage_type, &top_dir_name, &mut mapping)?;
}
} else {
let target_path = if Path::new(&dataset_dir).is_absolute() {
PathBuf::from(&dataset_dir)
} else {
Path::new(clm_root).join(&dataset_dir)
};
scan_local_dir(&target_path, &target_path, &storage_type, &top_dir_name, &mut mapping)?;
}
let stdout = serde_json::to_string_pretty(&mapping)
.map_err(|e| format!("Failed to serialize mapping to JSON: {}", e))?;
Ok((0, format!("{}\n", stdout)))
}
#[path = "directory_ingest_batch.rs"]
mod directory_ingest_batch;
#[allow(unused_imports)]
pub use directory_ingest_batch::*;