#![allow(unused_imports)]
use super::*;
pub fn directory_to_db(clm_root: &str, clm_file: &str) -> Result<(i32, String), String> {
let clm_payload = crate::cross_cutting::context::CLMContext::get_scoped_or_env(crate::layer0::vocabulary_env::ENV_CLM_PAYLOAD).unwrap_or_default();
let (dataset_dir, mut storage_type) = if !clm_payload.is_empty() && clm_payload != "null" {
(clm_payload, "fs".to_string())
} else {
let content = fs::read_to_string(clm_file)
.map_err(|e| format!("Failed to read CLM spec file: {}", e))?;
let parsed: Value = serde_yaml::from_str(&content)
.map_err(|e| format!("Failed to parse spec YAML: {}", e))?;
let clm_val = parsed.get("clm").unwrap_or(&parsed);
let process_val = clm_val.get("concrete_impl")
.and_then(|v| v.get("process"))
.or_else(|| clm_val.get("process"));
let dir = process_val
.and_then(|v| v.get("dataset_dir"))
.and_then(|v| v.as_str())
.unwrap_or("tests/test_fixtures/small_dataset")
.to_string();
let st = process_val
.and_then(|v| v.get("storage_type"))
.and_then(|v| v.as_str())
.unwrap_or("fs")
.to_string();
(dir, st)
};
if dataset_dir.starts_with(crate::layer0::vocabulary::URI_SCHEME_S3) {
storage_type = "s3".to_string();
} else if dataset_dir.starts_with(crate::layer0::vocabulary::URI_SCHEME_IPFS) {
storage_type = "ipfs".to_string();
}
let top_dir_name = if storage_type == "s3" {
let uri_path = dataset_dir.trim_start_matches(crate::layer0::vocabulary::URI_SCHEME_S3);
let bucket = uri_path.split('/').next().unwrap_or(uri_path);
bucket.to_string()
} else if storage_type == "ipfs" {
let uri_path = dataset_dir.trim_start_matches(crate::layer0::vocabulary::URI_SCHEME_IPFS);
let cid = uri_path.split('/').next().unwrap_or(uri_path);
cid.to_string()
} else {
let path = Path::new(&dataset_dir);
path.file_name()
.unwrap_or_else(|| std::ffi::OsStr::new("small_dataset"))
.to_string_lossy()
.into_owned()
};
let mut is_remote = false;
let mut temp_dir_opt = None;
let target_dir;
if storage_type == "s3" {
is_remote = true;
let t_dir = clm_root.to_string() + "/data/execution_logs/.tmp/s3_fetch_" + &uuid_snippet();
let _ = fs::create_dir_all(&t_dir);
let live = is_port_open(crate::layer0::provider_profiles::provider_port(crate::layer0::vocabulary_types::PROVIDER_STORAGE, crate::layer0::vocabulary_types::PROVIDER_FIELD_S3_PORT));
if live {
let s3_path = format!("local/{}", dataset_dir.trim_start_matches(crate::layer0::vocabulary::URI_SCHEME_S3));
let _ = std::process::Command::new("docker")
.args(["exec", "minio_server", "rm", "-rf", "/tmp/s3_fetch_out"])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
let _ = std::process::Command::new("docker")
.args(["exec", "minio_server", "mkdir", "-p", "/tmp/s3_fetch_out"])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
let _ = std::process::Command::new("docker")
.args(["exec", "minio_server", "mc", "cp", "--recursive", &format!("{}/", s3_path), "/tmp/s3_fetch_out/"])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
let _ = std::process::Command::new("docker")
.args(["cp", "minio_server:/tmp/s3_fetch_out/.", &t_dir])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
} else {
let fixture_dir = Path::new(clm_root).join("tests/test_fixtures/small_dataset");
if !fixture_dir.exists() {
return Err(format!("Local fallback fixture directory not found: {} (clm_root={})", fixture_dir.display(), clm_root));
}
copy_dir_all(&fixture_dir, Path::new(&t_dir))
.map_err(|e| format!("Failed to copy fixture dir {} to {}: {}", fixture_dir.display(), t_dir, e))?;
}
target_dir = PathBuf::from(&t_dir);
temp_dir_opt = Some(t_dir);
} else if storage_type == "ipfs" {
is_remote = true;
let t_dir = clm_root.to_string() + "/data/execution_logs/.tmp/ipfs_fetch_" + &uuid_snippet();
let _ = fs::create_dir_all(&t_dir);
let live = is_port_open(crate::layer0::provider_profiles::provider_port(crate::layer0::vocabulary_types::PROVIDER_STORAGE, crate::layer0::vocabulary_types::PROVIDER_FIELD_IPFS_PORT));
if live {
let ipfs_path = format!("/ipfs/{}", dataset_dir.trim_start_matches(crate::layer0::vocabulary::URI_SCHEME_IPFS));
let _ = std::process::Command::new("docker")
.args(["exec", "ipfs_host", "rm", "-rf", "/tmp/fetch_out"])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
let _ = std::process::Command::new("docker")
.args(["exec", "ipfs_host", "ipfs", "get", "-o", "/tmp/fetch_out", &ipfs_path])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
let _ = std::process::Command::new("docker")
.args(["cp", "ipfs_host:/tmp/fetch_out/.", &t_dir])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.status();
} else {
let fixture_dir = Path::new(clm_root).join("tests/test_fixtures/small_dataset");
if !fixture_dir.exists() {
return Err(format!("Local fallback fixture directory not found: {} (clm_root={})", fixture_dir.display(), clm_root));
}
copy_dir_all(&fixture_dir, Path::new(&t_dir))
.map_err(|e| format!("Failed to copy fixture dir {} to {}: {}", fixture_dir.display(), t_dir, e))?;
}
target_dir = PathBuf::from(&t_dir);
temp_dir_opt = Some(t_dir);
} else {
target_dir = if Path::new(&dataset_dir).is_absolute() {
PathBuf::from(&dataset_dir)
} else {
Path::new(clm_root).join(&dataset_dir)
};
}
let db_file = if crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_ZERO_FS).unwrap_or_default() == "true" {
let snap_dir = crate::layer0::environment::var(crate::layer0::vocabulary_env::ENV_CLM_MEMBROKER_SNAP_DIR)
.unwrap_or_else(|_| "/tmp".to_string());
let snap_path = if cfg!(windows) && (snap_dir == "/tmp" || snap_dir == "\\tmp") {
std::env::temp_dir()
} else {
PathBuf::from(&snap_dir)
};
snap_path.join("sprint38_cas.db")
} else {
Path::new(clm_root).join("db").join("sprint38_cas.db")
};
if let Some(parent) = db_file.parent() {
let _ = fs::create_dir_all(parent);
}
let mut conn = rusqlite::Connection::open(&db_file)
.map_err(|e| format!("Failed to open db file {}: {}", db_file.display(), e))?;
conn.busy_timeout(std::time::Duration::from_millis(u64::from(crate::layer0::runtime_contract::SQLITE_BUSY_TIMEOUT_MS)))
.map_err(|e| format!("Failed to set busy timeout: {}", e))?;
let has_tables: Result<bool, _> = conn.query_row(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='card';",
[],
|_| Ok(true),
);
if has_tables.is_err() {
let project_path = Path::new(clm_root);
let mut schema_file = project_path.join("schema").join("mcard_schema.sql");
if !schema_file.exists() {
schema_file = project_path.join("..").join("schema").join("mcard_schema.sql");
}
let schema_path = if schema_file.exists() { Some(schema_file) } else { None };
if let Some(schema_path) = schema_path {
let schema_sql = fs::read_to_string(schema_path)
.map_err(|e| format!("Failed to read schema file: {}", e))?;
let sql_to_use = if schema_sql.contains("CREATE TABLE") {
&schema_sql
} else {
crate::db::CANONICAL_SCHEMA
};
conn.execute_batch(sql_to_use)
.map_err(|e| format!("Failed to apply schema SQL: {}", e))?;
} else {
conn.execute_batch(crate::db::CANONICAL_SCHEMA)
.map_err(|e| format!("Failed to create tables: {}", e))?;
}
}
let mut files_list = Vec::new();
collect_files_recursive(&target_dir, &mut files_list)?;
let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
.map_err(|e| format!("Failed to start transaction: {}", e))?;
let now = crate::utils::now_iso8601();
let g_time = crate::fs_utils::compute_g_time();
for filepath in files_list {
let rel_path = filepath.strip_prefix(&target_dir)
.map_err(|e| format!("Failed to compute relative path: {}", e))?
.to_string_lossy()
.replace('\\', "/");
let content_bytes = fs::read(&filepath)
.map_err(|e| format!("Failed to read file {}: {}", filepath.display(), e))?;
let hash = compute_hash_hex(&content_bytes);
let handle = format!("{}://{}/{}", storage_type, top_dir_name, rel_path);
crate::layer2::record_store::insert_record(&tx, &crate::layer0::mcard_record::MCardRecord {
hash: hash.clone(), content: content_bytes, g_time: g_time.clone(),
}).map_err(|e| format!("Failed to insert card: {}", e))?;
tx.execute(
"INSERT OR IGNORE INTO handle_registry (handle, current_hash, updated_at) VALUES (?, ?, ?);",
rusqlite::params![handle, hash, now],
).map_err(|e| format!("Failed to insert handle registry: {}", e))?;
}
tx.commit()
.map_err(|e| format!("Failed to commit database transaction: {}", e))?;
let mut stmt = conn.prepare("SELECT handle, current_hash FROM handle_registry ORDER BY handle ASC;")
.map_err(|e| format!("Failed to prepare select query: {}", e))?;
let rows = stmt.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
}).map_err(|e| format!("Failed to run select query: {}", e))?;
let filter_prefix = format!("{}://{}/", storage_type, top_dir_name);
let mut results = BTreeMap::new();
for row_res in rows {
if let Ok((h, ch)) = row_res
&& h.starts_with(&filter_prefix) {
results.insert(h, ch);
}
}
if is_remote
&& let Some(t_dir) = temp_dir_opt {
let _ = fs::remove_dir_all(&t_dir);
}
let stdout = serde_json::to_string_pretty(&results)
.map_err(|e| format!("Failed to serialize results to JSON: {}", e))?;
Ok((0, format!("{}\n", stdout)))
}