clm-kernel 0.0.21

Cubical Logic Model (CLM) sovereign execution engine, CPN-VM, and Cordis spatiotemporal substrate
Documentation
use std::fs;
use std::path::{Path, PathBuf};
use serde_yaml::Value;
use std::collections::BTreeMap;
use crate::hash::raw_hash_bytes as compute_hash_hex;

fn is_port_open(port: u16) -> bool {
    use std::net::{TcpStream, SocketAddr};
    use std::time::Duration;
    let addr = SocketAddr::from(([127, 0, 0, 1], port));
    if TcpStream::connect_timeout(&addr, Duration::from_millis(500)).is_err() {
        return false;
    }
    if port == 9000 || port == 5001 {
        let container = if port == 9000 { "minio_server" } else { "ipfs_host" };
        let docker_check = std::process::Command::new("docker")
            .args(["inspect", "-f", "{{.State.Running}}", container])
            .output()
            .map(|o| String::from_utf8_lossy(&o.stdout).trim() == "true")
            .unwrap_or(false);
        if !docker_check {
            return false;
        }
        let daemon_check = if port == 9000 {
            std::process::Command::new("docker")
                .args(["exec", "minio_server", "mc", "alias", "list", "local"])
                .output()
                .map(|o| o.status.success())
                .unwrap_or(false)
        } else {
            std::process::Command::new("docker")
                .args(["exec", "ipfs_host", "ipfs", "id"])
                .output()
                .map(|o| o.status.success())
                .unwrap_or(false)
        };
        if !daemon_check {
            return false;
        }
    }
    true
}

fn list_ipfs_recursive_live(current_cid: &str, current_path: &str, out: &mut Vec<(String, String)>) {
    let output = std::process::Command::new("docker")
        .args(["exec", "ipfs_host", "ipfs", "ls", current_cid])
        .output();
    if let Ok(o) = output {
        let stdout_str = String::from_utf8_lossy(&o.stdout);
        for line in stdout_str.lines() {
            let parts: Vec<&str> = line.split_whitespace().collect();
            if parts.len() >= 3 {
                let hash = parts[0];
                let size_str = parts[1];
                let name = parts[2].trim_end_matches('/');
                
                let is_dir = size_str == "-" || parts[2].ends_with('/');
                let next_path = if current_path.is_empty() {
                    name.to_string()
                } else {
                    format!("{}/{}", current_path, name)
                };
                
                if is_dir {
                    list_ipfs_recursive_live(&format!("/ipfs/{}", hash), &next_path, out);
                } else {
                    let cat_out = std::process::Command::new("docker")
                        .args(["exec", "ipfs_host", "ipfs", "cat", &format!("/ipfs/{}", hash)])
                        .output();
                    if let Ok(co) = cat_out {
                        let file_hash = compute_hash_hex(&co.stdout);
                        out.push((next_path, file_hash));
                    }
                }
            }
        }
    }
}

fn scan_local_dir(
    base_dir: &Path,
    current_dir: &Path,
    storage_type: &str,
    top_dir_name: &str,
    mapping: &mut BTreeMap<String, String>,
) -> Result<(), String> {
    if !current_dir.exists() {
        return Ok(());
    }
    
    let mut entries = Vec::new();
    if let Ok(read_dir) = fs::read_dir(current_dir) {
        for entry in read_dir.flatten() {
            entries.push(entry);
        }
    }
    // Sort entries by path to ensure deterministic order (crucial for parity!)
    entries.sort_by_key(|e| e.path());

    for entry in entries {
        let path = entry.path();
        if path.is_file() {
            let rel_path = path.strip_prefix(base_dir)
                .map_err(|e| format!("Failed to compute relative path: {}", e))?
                .to_string_lossy()
                .replace('\\', "/");
            
            let content_bytes = fs::read(&path)
                .map_err(|e| format!("Failed to read file {}: {}", path.display(), e))?;
            
            let hash = compute_hash_hex(&content_bytes);
            let handle = format!("{}://{}/{}", storage_type, top_dir_name, rel_path);
            mapping.insert(handle, hash);
        } else if path.is_dir() {
            scan_local_dir(base_dir, &path, storage_type, top_dir_name, mapping)?;
        }
    }
    Ok(())
}

fn collect_files_recursive(dir: &Path, out: &mut Vec<PathBuf>) -> Result<(), String> {
    if dir.is_file() {
        out.push(dir.to_path_buf());
        return Ok(());
    }
    let mut entries = Vec::new();
    if let Ok(read_dir) = fs::read_dir(dir) {
        for entry in read_dir.flatten() {
            entries.push(entry);
        }
    }
    entries.sort_by_key(|e| e.path());
    for entry in entries {
        let path = entry.path();
        if path.is_file() {
            out.push(path);
        } else if path.is_dir() {
            collect_files_recursive(&path, out)?;
        }
    }
    Ok(())
}

fn uuid_snippet() -> String {
    use std::sync::atomic::Ordering;
    let count = crate::TEMP_COUNTER.fetch_add(1, Ordering::SeqCst);
    format!("{}_{}", std::process::id(), count)
}

fn copy_dir_all(src: &Path, dst: &Path) -> std::io::Result<()> {
    fs::create_dir_all(dst)?;
    for entry in fs::read_dir(src)? {
        let entry = entry?;
        let ty = entry.file_type()?;
        if ty.is_dir() {
            copy_dir_all(&entry.path(), &dst.join(entry.file_name()))?;
        } else {
            fs::copy(entry.path(), dst.join(entry.file_name()))?;
        }
    }
    Ok(())
}

pub fn directory_fs_scan(clm_root: &str, clm_file: &str) -> Result<(i32, String), String> {
    let clm_payload = crate::cross_cutting::context::CLMContext::get_scoped_or_env(crate::layer0::vocabulary_env::ENV_CLM_PAYLOAD).unwrap_or_default();
    
    let (dataset_dir, mut storage_type) = if !clm_payload.is_empty() && clm_payload != "null" {
        (clm_payload, "fs".to_string())
    } else {
        let content = fs::read_to_string(clm_file)
            .map_err(|e| format!("Failed to read CLM spec file: {}", e))?;
        let parsed: Value = serde_yaml::from_str(&content)
            .map_err(|e| format!("Failed to parse spec YAML: {}", e))?;
        
        let clm_val = parsed.get("clm").unwrap_or(&parsed);
        let process_val = clm_val.get("concrete_impl")
            .and_then(|v| v.get("process"))
            .or_else(|| clm_val.get("process"));
        
        let dir = process_val
            .and_then(|v| v.get("dataset_dir"))
            .and_then(|v| v.as_str())
            .unwrap_or("tests/test_fixtures/small_dataset")
            .to_string();
        
        let st = process_val
            .and_then(|v| v.get("storage_type"))
            .and_then(|v| v.as_str())
            .unwrap_or("fs")
            .to_string();
            
        (dir, st)
    };

    if dataset_dir.starts_with(crate::layer0::vocabulary::URI_SCHEME_S3) {
        storage_type = "s3".to_string();
    } else if dataset_dir.starts_with(crate::layer0::vocabulary::URI_SCHEME_IPFS) {
        storage_type = "ipfs".to_string();
    }

    let top_dir_name = if storage_type == "s3" {
        let uri_path = dataset_dir.trim_start_matches(crate::layer0::vocabulary::URI_SCHEME_S3);
        let bucket = uri_path.split('/').next().unwrap_or(uri_path);
        bucket.to_string()
    } else if storage_type == "ipfs" {
        let uri_path = dataset_dir.trim_start_matches(crate::layer0::vocabulary::URI_SCHEME_IPFS);
        let cid = uri_path.split('/').next().unwrap_or(uri_path);
        cid.to_string()
    } else {
        let path = Path::new(&dataset_dir);
        path.file_name()
            .unwrap_or_else(|| std::ffi::OsStr::new("small_dataset"))
            .to_string_lossy()
            .into_owned()
    };

    let mut mapping = BTreeMap::new();

    if storage_type == "s3" || storage_type == "ipfs" {
        let port = crate::layer0::provider_profiles::provider_port(crate::layer0::vocabulary_types::PROVIDER_STORAGE, if storage_type == "s3" { crate::layer0::vocabulary_types::PROVIDER_FIELD_S3_PORT } else { crate::layer0::vocabulary_types::PROVIDER_FIELD_IPFS_PORT });
        let live = is_port_open(port);
        if live {
            if storage_type == "s3" {
                let s3_path = format!("local/{}", dataset_dir.trim_start_matches(crate::layer0::vocabulary::URI_SCHEME_S3));
                let output = std::process::Command::new("docker")
                    .args(["exec", "minio_server", "mc", "ls", "--json", "--recursive", &s3_path])
                    .output();
                if let Ok(out) = output {
                    let stdout_str = String::from_utf8_lossy(&out.stdout);
                    for line in stdout_str.lines() {
                        if let Ok(val) = serde_json::from_str::<serde_json::Value>(line)
                            && val.get("type").and_then(|t| t.as_str()) == Some("file")
                                && let Some(key) = val.get("key").and_then(|k| k.as_str()) {
                                    let cat_out = std::process::Command::new("docker")
                                        .args(["exec", "minio_server", "mc", "cat", &format!("{}/{}", s3_path, key)])
                                        .output();
                                    if let Ok(co) = cat_out {
                                        let file_hash = compute_hash_hex(&co.stdout);
                                        let handle = format!("s3://{}/{}", top_dir_name, key);
                                        mapping.insert(handle, file_hash);
                                    }
                                }
                    }
                }
            } else {
                let ipfs_path = format!("/ipfs/{}", dataset_dir.trim_start_matches(crate::layer0::vocabulary::URI_SCHEME_IPFS));
                let mut listed = Vec::new();
                list_ipfs_recursive_live(&ipfs_path, "", &mut listed);
                for (rel_path, file_hash) in listed {
                    let handle = format!("ipfs://{}/{}", top_dir_name, rel_path);
                    mapping.insert(handle, file_hash);
                }
            }
        } else {
            let fixture_dir = Path::new(clm_root).join("tests/test_fixtures/small_dataset");
            scan_local_dir(&fixture_dir, &fixture_dir, &storage_type, &top_dir_name, &mut mapping)?;
        }
    } else {
        let target_path = if Path::new(&dataset_dir).is_absolute() {
            PathBuf::from(&dataset_dir)
        } else {
            Path::new(clm_root).join(&dataset_dir)
        };
        scan_local_dir(&target_path, &target_path, &storage_type, &top_dir_name, &mut mapping)?;
    }

    let stdout = serde_json::to_string_pretty(&mapping)
        .map_err(|e| format!("Failed to serialize mapping to JSON: {}", e))?;
    
    Ok((0, format!("{}\n", stdout)))
}


#[path = "directory_ingest_batch.rs"]
mod directory_ingest_batch;
#[allow(unused_imports)]
pub use directory_ingest_batch::*;