clm-kernel 0.0.7

Cubical Logic Model (CLM) sovereign execution engine, CPN-VM, and Cordis spatiotemporal substrate
Documentation
#![allow(unused_imports)]
use super::*;

pub fn directory_to_db(clm_root: &str, clm_file: &str) -> Result<(i32, String), String> {
    let clm_payload = crate::cross_cutting::context::CLMContext::get_scoped_or_env("CLM_PAYLOAD").unwrap_or_default();
    
    let (dataset_dir, mut storage_type) = if !clm_payload.is_empty() && clm_payload != "null" {
        (clm_payload, "fs".to_string())
    } else {
        let content = fs::read_to_string(clm_file)
            .map_err(|e| format!("Failed to read CLM spec file: {}", e))?;
        let parsed: Value = serde_yaml::from_str(&content)
            .map_err(|e| format!("Failed to parse spec YAML: {}", e))?;
        
        let clm_val = parsed.get("clm").unwrap_or(&parsed);
        let process_val = clm_val.get("concrete_impl")
            .and_then(|v| v.get("process"))
            .or_else(|| clm_val.get("process"));
        
        let dir = process_val
            .and_then(|v| v.get("dataset_dir"))
            .and_then(|v| v.as_str())
            .unwrap_or("tests/test_fixtures/small_dataset")
            .to_string();
        
        let st = process_val
            .and_then(|v| v.get("storage_type"))
            .and_then(|v| v.as_str())
            .unwrap_or("fs")
            .to_string();
            
        (dir, st)
    };

    if dataset_dir.starts_with("s3://") {
        storage_type = "s3".to_string();
    } else if dataset_dir.starts_with("ipfs://") {
        storage_type = "ipfs".to_string();
    }

    let top_dir_name = if storage_type == "s3" {
        let uri_path = dataset_dir.trim_start_matches("s3://");
        let bucket = uri_path.split('/').next().unwrap_or(uri_path);
        bucket.to_string()
    } else if storage_type == "ipfs" {
        let uri_path = dataset_dir.trim_start_matches("ipfs://");
        let cid = uri_path.split('/').next().unwrap_or(uri_path);
        cid.to_string()
    } else {
        let path = Path::new(&dataset_dir);
        path.file_name()
            .unwrap_or_else(|| std::ffi::OsStr::new("small_dataset"))
            .to_string_lossy()
            .into_owned()
    };

    let mut is_remote = false;
    let mut temp_dir_opt = None;
    let target_dir;

    if storage_type == "s3" {
        is_remote = true;
        let t_dir = clm_root.to_string() + "/data/execution_logs/.tmp/s3_fetch_" + &uuid_snippet();
        let _ = fs::create_dir_all(&t_dir);
        let live = is_port_open(9000);
        if live {
            let s3_path = format!("local/{}", dataset_dir.trim_start_matches("s3://"));
            let _ = std::process::Command::new("docker")
                .args(["exec", "minio_server", "rm", "-rf", "/tmp/s3_fetch_out"])
                .stdout(std::process::Stdio::null())
                .stderr(std::process::Stdio::null())
                .status();
            let _ = std::process::Command::new("docker")
                .args(["exec", "minio_server", "mkdir", "-p", "/tmp/s3_fetch_out"])
                .stdout(std::process::Stdio::null())
                .stderr(std::process::Stdio::null())
                .status();
            let _ = std::process::Command::new("docker")
                .args(["exec", "minio_server", "mc", "cp", "--recursive", &format!("{}/", s3_path), "/tmp/s3_fetch_out/"])
                .stdout(std::process::Stdio::null())
                .stderr(std::process::Stdio::null())
                .status();
            let _ = std::process::Command::new("docker")
                .args(["cp", "minio_server:/tmp/s3_fetch_out/.", &t_dir])
                .stdout(std::process::Stdio::null())
                .stderr(std::process::Stdio::null())
                .status();
        } else {
            let fixture_dir = Path::new(clm_root).join("tests/test_fixtures/small_dataset");
            if !fixture_dir.exists() {
                return Err(format!("Local fallback fixture directory not found: {} (clm_root={})", fixture_dir.display(), clm_root));
            }
            copy_dir_all(&fixture_dir, Path::new(&t_dir))
                .map_err(|e| format!("Failed to copy fixture dir {} to {}: {}", fixture_dir.display(), t_dir, e))?;
        }
        target_dir = PathBuf::from(&t_dir);
        temp_dir_opt = Some(t_dir);
    } else if storage_type == "ipfs" {
        is_remote = true;
        let t_dir = clm_root.to_string() + "/data/execution_logs/.tmp/ipfs_fetch_" + &uuid_snippet();
        let _ = fs::create_dir_all(&t_dir);
        let live = is_port_open(5001);
        if live {
            let ipfs_path = format!("/ipfs/{}", dataset_dir.trim_start_matches("ipfs://"));
            let _ = std::process::Command::new("docker")
                .args(["exec", "ipfs_host", "rm", "-rf", "/tmp/fetch_out"])
                .stdout(std::process::Stdio::null())
                .stderr(std::process::Stdio::null())
                .status();
            let _ = std::process::Command::new("docker")
                .args(["exec", "ipfs_host", "ipfs", "get", "-o", "/tmp/fetch_out", &ipfs_path])
                .stdout(std::process::Stdio::null())
                .stderr(std::process::Stdio::null())
                .status();
            let _ = std::process::Command::new("docker")
                .args(["cp", "ipfs_host:/tmp/fetch_out/.", &t_dir])
                .stdout(std::process::Stdio::null())
                .stderr(std::process::Stdio::null())
                .status();
        } else {
            let fixture_dir = Path::new(clm_root).join("tests/test_fixtures/small_dataset");
            if !fixture_dir.exists() {
                return Err(format!("Local fallback fixture directory not found: {} (clm_root={})", fixture_dir.display(), clm_root));
            }
            copy_dir_all(&fixture_dir, Path::new(&t_dir))
                .map_err(|e| format!("Failed to copy fixture dir {} to {}: {}", fixture_dir.display(), t_dir, e))?;
        }
        target_dir = PathBuf::from(&t_dir);
        temp_dir_opt = Some(t_dir);
    } else {
        target_dir = if Path::new(&dataset_dir).is_absolute() {
            PathBuf::from(&dataset_dir)
        } else {
            Path::new(clm_root).join(&dataset_dir)
        };
    }

    let db_file = if std::env::var("CLM_ZERO_FS").unwrap_or_default() == "true" {
        let snap_dir = std::env::var("CLM_MEMBROKER_SNAP_DIR")
            .unwrap_or_else(|_| "/tmp".to_string());
        let snap_path = if cfg!(windows) && (snap_dir == "/tmp" || snap_dir == "\\tmp") {
            std::env::temp_dir()
        } else {
            PathBuf::from(&snap_dir)
        };
        snap_path.join("sprint38_cas.db")
    } else {
        Path::new(clm_root).join("db").join("sprint38_cas.db")
    };

    if let Some(parent) = db_file.parent() {
        let _ = fs::create_dir_all(parent);
    }

    let mut conn = rusqlite::Connection::open(&db_file)
        .map_err(|e| format!("Failed to open db file {}: {}", db_file.display(), e))?;

    conn.busy_timeout(std::time::Duration::from_millis(10000))
        .map_err(|e| format!("Failed to set busy timeout: {}", e))?;

    let has_tables: Result<bool, _> = conn.query_row(
        "SELECT 1 FROM sqlite_master WHERE type='table' AND name='card';",
        [],
        |_| Ok(true),
    );
    if has_tables.is_err() {
        let project_path = Path::new(clm_root);
        let mut schema_file = project_path.join("schema").join("mcard_schema.sql");
        if !schema_file.exists() {
            schema_file = project_path.join("..").join("schema").join("mcard_schema.sql");
        }
        let schema_path = if schema_file.exists() { Some(schema_file) } else { None };
        if let Some(schema_path) = schema_path {
            let schema_sql = fs::read_to_string(schema_path)
                .map_err(|e| format!("Failed to read schema file: {}", e))?;
            let sql_to_use = if schema_sql.contains("CREATE TABLE") {
                &schema_sql
            } else {
                crate::db::CANONICAL_SCHEMA
            };
            conn.execute_batch(sql_to_use)
                .map_err(|e| format!("Failed to apply schema SQL: {}", e))?;
        } else {
            conn.execute_batch(crate::db::CANONICAL_SCHEMA)
                .map_err(|e| format!("Failed to create tables: {}", e))?;
        }
    }

    let mut files_list = Vec::new();
    collect_files_recursive(&target_dir, &mut files_list)?;

    let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)
        .map_err(|e| format!("Failed to start transaction: {}", e))?;

    let now = crate::utils::now_iso8601();
    // The card row's coordinate must name the algorithm that produced its digest (charter L2).
    let g_time = crate::fs_utils::compute_g_time();

    for filepath in files_list {
        let rel_path = filepath.strip_prefix(&target_dir)
            .map_err(|e| format!("Failed to compute relative path: {}", e))?
            .to_string_lossy()
            .replace('\\', "/");
        
        let content_bytes = fs::read(&filepath)
            .map_err(|e| format!("Failed to read file {}: {}", filepath.display(), e))?;
        
        let hash = compute_hash_hex(&content_bytes);
        let handle = format!("{}://{}/{}", storage_type, top_dir_name, rel_path);

        tx.execute(
            "INSERT OR IGNORE INTO card (hash, content, g_time) VALUES (?, ?, ?);",
            rusqlite::params![hash, content_bytes, g_time],
        ).map_err(|e| format!("Failed to insert card: {}", e))?;

        tx.execute(
            "INSERT OR IGNORE INTO handle_registry (handle, current_hash, updated_at) VALUES (?, ?, ?);",
            rusqlite::params![handle, hash, now],
        ).map_err(|e| format!("Failed to insert handle registry: {}", e))?;
    }

    tx.commit()
        .map_err(|e| format!("Failed to commit database transaction: {}", e))?;

    let mut stmt = conn.prepare("SELECT handle, current_hash FROM handle_registry ORDER BY handle ASC;")
        .map_err(|e| format!("Failed to prepare select query: {}", e))?;
    
    let rows = stmt.query_map([], |row| {
        Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
    }).map_err(|e| format!("Failed to run select query: {}", e))?;

    let filter_prefix = format!("{}://{}/", storage_type, top_dir_name);
    let mut results = BTreeMap::new();
    for row_res in rows {
        if let Ok((h, ch)) = row_res
            && h.starts_with(&filter_prefix) {
                results.insert(h, ch);
            }
    }

    if is_remote
        && let Some(t_dir) = temp_dir_opt {
            let _ = fs::remove_dir_all(&t_dir);
        }

    let stdout = serde_json::to_string_pretty(&results)
        .map_err(|e| format!("Failed to serialize results to JSON: {}", e))?;

    Ok((0, format!("{}\n", stdout)))
}