use std::fs;
use std::path::{Path, PathBuf};
use serde_yaml::Value;
use std::collections::BTreeMap;
use crate::hash::raw_hash_bytes as compute_hash_hex;
fn is_port_open(port: u16) -> bool {
use std::net::{TcpStream, SocketAddr};
use std::time::Duration;
let addr = SocketAddr::from(([127, 0, 0, 1], port));
if !TcpStream::connect_timeout(&addr, Duration::from_millis(500)).is_ok() {
return false;
}
if port == 9000 || port == 5001 {
let container = if port == 9000 { "minio_server" } else { "ipfs_host" };
let docker_check = std::process::Command::new("docker")
.args(["inspect", "-f", "{{.State.Running}}", container])
.output()
.map(|o| String::from_utf8_lossy(&o.stdout).trim() == "true")
.unwrap_or(false);
if !docker_check {
return false;
}
let daemon_check = if port == 9000 {
std::process::Command::new("docker")
.args(["exec", "minio_server", "mc", "alias", "list", "local"])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
} else {
std::process::Command::new("docker")
.args(["exec", "ipfs_host", "ipfs", "id"])
.output()
.map(|o| o.status.success())
.unwrap_or(false)
};
if !daemon_check {
return false;
}
}
true
}
fn list_ipfs_recursive_live(current_cid: &str, current_path: &str, out: &mut Vec<(String, String)>) {
let output = std::process::Command::new("docker")
.args(["exec", "ipfs_host", "ipfs", "ls", current_cid])
.output();
if let Ok(o) = output {
let stdout_str = String::from_utf8_lossy(&o.stdout);
for line in stdout_str.lines() {
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.len() >= 3 {
let hash = parts[0];
let size_str = parts[1];
let name = parts[2].trim_end_matches('/');
let is_dir = size_str == "-" || parts[2].ends_with('/');
let next_path = if current_path.is_empty() {
name.to_string()
} else {
format!("{}/{}", current_path, name)
};
if is_dir {
list_ipfs_recursive_live(&format!("/ipfs/{}", hash), &next_path, out);
} else {
let cat_out = std::process::Command::new("docker")
.args(["exec", "ipfs_host", "ipfs", "cat", &format!("/ipfs/{}", hash)])
.output();
if let Ok(co) = cat_out {
let file_hash = compute_hash_hex(&co.stdout);
out.push((next_path, file_hash));
}
}
}
}
}
}
fn scan_local_dir(
base_dir: &Path,
current_dir: &Path,
storage_type: &str,
top_dir_name: &str,
mapping: &mut BTreeMap<String, String>,
) -> Result<(), String> {
if !current_dir.exists() {
return Ok(());
}
let mut entries = Vec::new();
if let Ok(read_dir) = fs::read_dir(current_dir) {
for entry in read_dir.flatten() {
entries.push(entry);
}
}
entries.sort_by_key(|e| e.path());
for entry in entries {
let path = entry.path();
if path.is_file() {
let rel_path = path.strip_prefix(base_dir)
.map_err(|e| format!("Failed to compute relative path: {}", e))?
.to_string_lossy()
.into_owned();
let content_bytes = fs::read(&path)
.map_err(|e| format!("Failed to read file {}: {}", path.display(), e))?;
let hash = compute_hash_hex(&content_bytes);
let handle = format!("{}://{}/{}", storage_type, top_dir_name, rel_path);
mapping.insert(handle, hash);
} else if path.is_dir() {
scan_local_dir(base_dir, &path, storage_type, top_dir_name, mapping)?;
}
}
Ok(())
}
fn collect_files_recursive(dir: &Path, out: &mut Vec<PathBuf>) -> Result<(), String> {
if dir.is_file() {
out.push(dir.to_path_buf());
return Ok(());
}
let mut entries = Vec::new();
if let Ok(read_dir) = fs::read_dir(dir) {
for entry in read_dir.flatten() {
entries.push(entry);
}
}
entries.sort_by_key(|e| e.path());
for entry in entries {
let path = entry.path();
if path.is_file() {
out.push(path);
} else if path.is_dir() {
collect_files_recursive(&path, out)?;
}
}
Ok(())
}
fn uuid_snippet() -> String {
use std::sync::atomic::Ordering;
let count = crate::TEMP_COUNTER.fetch_add(1, Ordering::SeqCst);
format!("{}_{}", std::process::id(), count)
}
fn copy_dir_all(src: &Path, dst: &Path) -> std::io::Result<()> {
fs::create_dir_all(dst)?;
for entry in fs::read_dir(src)? {
let entry = entry?;
let ty = entry.file_type()?;
if ty.is_dir() {
copy_dir_all(&entry.path(), &dst.join(entry.file_name()))?;
} else {
fs::copy(entry.path(), dst.join(entry.file_name()))?;
}
}
Ok(())
}
pub fn directory_fs_scan(clm_root: &str, clm_file: &str) -> Result<(i32, String), String> {
let clm_payload = std::env::var("CLM_PAYLOAD").unwrap_or_default();
let (dataset_dir, mut storage_type) = if !clm_payload.is_empty() && clm_payload != "null" {
(clm_payload, "fs".to_string())
} else {
let content = fs::read_to_string(clm_file)
.map_err(|e| format!("Failed to read CLM spec file: {}", e))?;
let parsed: Value = serde_yaml::from_str(&content)
.map_err(|e| format!("Failed to parse spec YAML: {}", e))?;
let clm_val = parsed.get("clm").unwrap_or(&parsed);
let process_val = clm_val.get("concrete_impl")
.and_then(|v| v.get("process"))
.or_else(|| clm_val.get("process"));
let dir = process_val
.and_then(|v| v.get("dataset_dir"))
.and_then(|v| v.as_str())
.unwrap_or("tests/test_fixtures/small_dataset")
.to_string();
let st = process_val
.and_then(|v| v.get("storage_type"))
.and_then(|v| v.as_str())
.unwrap_or("fs")
.to_string();
(dir, st)
};
if dataset_dir.starts_with("s3://") {
storage_type = "s3".to_string();
} else if dataset_dir.starts_with("ipfs://") {
storage_type = "ipfs".to_string();
}
let top_dir_name = if storage_type == "s3" {
let uri_path = dataset_dir.trim_start_matches("s3://");
let bucket = uri_path.split('/').next().unwrap_or(uri_path);
bucket.to_string()
} else if storage_type == "ipfs" {
let uri_path = dataset_dir.trim_start_matches("ipfs://");
let cid = uri_path.split('/').next().unwrap_or(uri_path);
cid.to_string()
} else {
let path = Path::new(&dataset_dir);
path.file_name()
.unwrap_or_else(|| std::ffi::OsStr::new("small_dataset"))
.to_string_lossy()
.into_owned()
};
let mut mapping = BTreeMap::new();
if storage_type == "s3" || storage_type == "ipfs" {
let port = if storage_type == "s3" { 9000 } else { 5001 };
let live = is_port_open(port);
if live {
if storage_type == "s3" {
let s3_path = format!("local/{}", dataset_dir.trim_start_matches("s3://"));
let output = std::process::Command::new("docker")
.args(["exec", "minio_server", "mc", "ls", "--json", "--recursive", &s3_path])
.output();
if let Ok(out) = output {
let stdout_str = String::from_utf8_lossy(&out.stdout);
for line in stdout_str.lines() {
if let Ok(val) = serde_json::from_str::<serde_json::Value>(line)
&& val.get("type").and_then(|t| t.as_str()) == Some("file")
&& let Some(key) = val.get("key").and_then(|k| k.as_str()) {
let cat_out = std::process::Command::new("docker")
.args(["exec", "minio_server", "mc", "cat", &format!("{}/{}", s3_path, key)])
.output();
if let Ok(co) = cat_out {
let file_hash = compute_hash_hex(&co.stdout);
let handle = format!("s3://{}/{}", top_dir_name, key);
mapping.insert(handle, file_hash);
}
}
}
}
} else {
let ipfs_path = format!("/ipfs/{}", dataset_dir.trim_start_matches("ipfs://"));
let mut listed = Vec::new();
list_ipfs_recursive_live(&ipfs_path, "", &mut listed);
for (rel_path, file_hash) in listed {
let handle = format!("ipfs://{}/{}", top_dir_name, rel_path);
mapping.insert(handle, file_hash);
}
}
} else {
let fixture_dir = Path::new(clm_root).join("tests/test_fixtures/small_dataset");
scan_local_dir(&fixture_dir, &fixture_dir, &storage_type, &top_dir_name, &mut mapping)?;
}
} else {
let target_path = if Path::new(&dataset_dir).is_absolute() {
PathBuf::from(&dataset_dir)
} else {
Path::new(clm_root).join(&dataset_dir)
};
scan_local_dir(&target_path, &target_path, &storage_type, &top_dir_name, &mut mapping)?;
}
let stdout = serde_json::to_string_pretty(&mapping)
.map_err(|e| format!("Failed to serialize mapping to JSON: {}", e))?;
Ok((0, format!("{}\n", stdout)))
}
pub fn directory_to_db(clm_root: &str, clm_file: &str) -> Result<(i32, String), String> {
let clm_payload = std::env::var("CLM_PAYLOAD").unwrap_or_default();
let (dataset_dir, mut storage_type) = if !clm_payload.is_empty() && clm_payload != "null" {
(clm_payload, "fs".to_string())
} else {
let content = fs::read_to_string(clm_file)
.map_err(|e| format!("Failed to read CLM spec file: {}", e))?;
let parsed: Value = serde_yaml::from_str(&content)
.map_err(|e| format!("Failed to parse spec YAML: {}", e))?;
let clm_val = parsed.get("clm").unwrap_or(&parsed);
let process_val = clm_val.get("concrete_impl")
.and_then(|v| v.get("process"))
.or_else(|| clm_val.get("process"));
let dir = process_val
.and_then(|v| v.get("dataset_dir"))
.and_then(|v| v.as_str())
.unwrap_or("tests/test_fixtures/small_dataset")
.to_string();
let st = process_val
.and_then(|v| v.get("storage_type"))
.and_then(|v| v.as_str())
.unwrap_or("fs")
.to_string();
(dir, st)
};
if dataset_dir.starts_with("s3://") {
storage_type = "s3".to_string();
} else if dataset_dir.starts_with("ipfs://") {
storage_type = "ipfs".to_string();
}
let top_dir_name = if storage_type == "s3" {
let uri_path = dataset_dir.trim_start_matches("s3://");
let bucket = uri_path.split('/').next().unwrap_or(uri_path);
bucket.to_string()
} else if storage_type == "ipfs" {
let uri_path = dataset_dir.trim_start_matches("ipfs://");
let cid = uri_path.split('/').next().unwrap_or(uri_path);
cid.to_string()
} else {
let path = Path::new(&dataset_dir);
path.file_name()
.unwrap_or_else(|| std::ffi::OsStr::new("small_dataset"))
.to_string_lossy()
.into_owned()
};
let mut is_remote = false;
let mut temp_dir_opt = None;
let target_dir;
if storage_type == "s3" {
is_remote = true;
let t_dir = clm_root.to_string() + "/data/execution_logs/.tmp/s3_fetch_" + &uuid_snippet();
let _ = fs::create_dir_all(&t_dir);
let live = is_port_open(9000);
if live {
let s3_path = format!("local/{}", dataset_dir.trim_start_matches("s3://"));
let _ = std::process::Command::new("docker")
.args(["exec", "minio_server", "rm", "-rf", "/tmp/s3_fetch_out"])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
let _ = std::process::Command::new("docker")
.args(["exec", "minio_server", "mkdir", "-p", "/tmp/s3_fetch_out"])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
let _ = std::process::Command::new("docker")
.args(["exec", "minio_server", "mc", "cp", "--recursive", &format!("{}/", s3_path), "/tmp/s3_fetch_out/"])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
let _ = std::process::Command::new("docker")
.args(["cp", "minio_server:/tmp/s3_fetch_out/.", &t_dir])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
} else {
let fixture_dir = Path::new(clm_root).join("tests/test_fixtures/small_dataset");
if !fixture_dir.exists() {
return Err(format!("Local fallback fixture directory not found: {} (clm_root={})", fixture_dir.display(), clm_root));
}
copy_dir_all(&fixture_dir, Path::new(&t_dir))
.map_err(|e| format!("Failed to copy fixture dir {} to {}: {}", fixture_dir.display(), t_dir, e))?;
}
target_dir = PathBuf::from(&t_dir);
temp_dir_opt = Some(t_dir);
} else if storage_type == "ipfs" {
is_remote = true;
let t_dir = clm_root.to_string() + "/data/execution_logs/.tmp/ipfs_fetch_" + &uuid_snippet();
let _ = fs::create_dir_all(&t_dir);
let live = is_port_open(5001);
if live {
let ipfs_path = format!("/ipfs/{}", dataset_dir.trim_start_matches("ipfs://"));
let _ = std::process::Command::new("docker")
.args(["exec", "ipfs_host", "rm", "-rf", "/tmp/fetch_out"])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
let _ = std::process::Command::new("docker")
.args(["exec", "ipfs_host", "ipfs", "get", "-o", "/tmp/fetch_out", &ipfs_path])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
let _ = std::process::Command::new("docker")
.args(["cp", "ipfs_host:/tmp/fetch_out/.", &t_dir])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
} else {
let fixture_dir = Path::new(clm_root).join("tests/test_fixtures/small_dataset");
if !fixture_dir.exists() {
return Err(format!("Local fallback fixture directory not found: {} (clm_root={})", fixture_dir.display(), clm_root));
}
copy_dir_all(&fixture_dir, Path::new(&t_dir))
.map_err(|e| format!("Failed to copy fixture dir {} to {}: {}", fixture_dir.display(), t_dir, e))?;
}
target_dir = PathBuf::from(&t_dir);
temp_dir_opt = Some(t_dir);
} else {
target_dir = if Path::new(&dataset_dir).is_absolute() {
PathBuf::from(&dataset_dir)
} else {
Path::new(clm_root).join(&dataset_dir)
};
}
let db_file = if std::env::var("CLM_ZERO_FS").unwrap_or_default() == "true" {
let snap_dir = std::env::var("CLM_MEMBROKER_SNAP_DIR")
.unwrap_or_else(|_| "/tmp".to_string());
Path::new(&snap_dir).join("sprint38_cas.db")
} else {
Path::new(clm_root).join("db").join("sprint38_cas.db")
};
if let Some(parent) = db_file.parent() {
let _ = fs::create_dir_all(parent);
}
let mut conn = rusqlite::Connection::open(&db_file)
.map_err(|e| format!("Failed to open db file {}: {}", db_file.display(), e))?;
conn.busy_timeout(std::time::Duration::from_millis(10000))
.map_err(|e| format!("Failed to set busy timeout: {}", e))?;
let has_tables: Result<bool, _> = conn.query_row(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='card';",
[],
|_| Ok(true),
);
if has_tables.is_err() {
let project_path = Path::new(clm_root);
let mut schema_file = project_path.join("schema").join("mcard_schema.sql");
if !schema_file.exists() {
schema_file = project_path.join("..").join("schema").join("mcard_schema.sql");
}
let schema_path = if schema_file.exists() { Some(schema_file) } else { None };
if let Some(schema_path) = schema_path {
let schema_sql = fs::read_to_string(schema_path)
.map_err(|e| format!("Failed to read schema file: {}", e))?;
conn.execute_batch(&schema_sql)
.map_err(|e| format!("Failed to apply schema SQL: {}", e))?;
} else {
conn.execute_batch(crate::db::CANONICAL_SCHEMA)
.map_err(|e| format!("Failed to create tables: {}", e))?;
}
}
let mut files_list = Vec::new();
collect_files_recursive(&target_dir, &mut files_list)?;
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
.map_err(|e| format!("Failed to start transaction: {}", e))?;
let now = crate::utils::now_iso8601();
for filepath in files_list {
let rel_path = filepath.strip_prefix(&target_dir)
.map_err(|e| format!("Failed to compute relative path: {}", e))?
.to_string_lossy()
.into_owned();
let content_bytes = fs::read(&filepath)
.map_err(|e| format!("Failed to read file {}: {}", filepath.display(), e))?;
let hash = compute_hash_hex(&content_bytes);
let handle = format!("{}://{}/{}", storage_type, top_dir_name, rel_path);
tx.execute(
"INSERT OR IGNORE INTO card (hash, content, g_time) VALUES (?, ?, ?);",
rusqlite::params![hash, content_bytes, now],
).map_err(|e| format!("Failed to insert card: {}", e))?;
tx.execute(
"INSERT OR IGNORE INTO handle_registry (handle, current_hash, created_at, updated_at) VALUES (?, ?, ?, ?);",
rusqlite::params![handle, hash, now, now],
).map_err(|e| format!("Failed to insert handle registry: {}", e))?;
}
tx.commit()
.map_err(|e| format!("Failed to commit database transaction: {}", e))?;
let mut stmt = conn.prepare("SELECT handle, current_hash FROM handle_registry ORDER BY handle ASC;")
.map_err(|e| format!("Failed to prepare select query: {}", e))?;
let rows = stmt.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
}).map_err(|e| format!("Failed to run select query: {}", e))?;
let filter_prefix = format!("{}://{}/", storage_type, top_dir_name);
let mut results = BTreeMap::new();
for row_res in rows {
if let Ok((h, ch)) = row_res
&& h.starts_with(&filter_prefix) {
results.insert(h, ch);
}
}
if is_remote
&& let Some(t_dir) = temp_dir_opt {
let _ = fs::remove_dir_all(&t_dir);
}
let stdout = serde_json::to_string_pretty(&results)
.map_err(|e| format!("Failed to serialize results to JSON: {}", e))?;
Ok((0, format!("{}\n", stdout)))
}