clm-kernel 0.0.16

Cubical Logic Model (CLM) sovereign execution engine, CPN-VM, and Cordis spatiotemporal substrate
Documentation
// archive.rs — Sovereign Archive Manifest & Solid Compression Engine (Sprint 146)
//
// Implements the physical vessel of the SSOT:
// 1. Merkle-DAG archive manifest generation (`archive_id`, `g_time`, `files[]` with path/sha256/size/mode).
// 2. Solid compression bundling (`tar` deterministic sort + `zstd --rsyncable`).
// 3. Top-level VCard seal: `sha256(manifest_mcard_hash || archive_bytes_hash)`.
// 4. MCard registration in `content_memory.db` under `archive:sovereign:${g_time}`.
// 5. Schema-first restore protocol: extract manifest MCard first, validate, materialize files, verify hashes.

use std::fs;
use std::path::Path;
use std::process::Command;
use sha2::{Sha256, Digest};
use serde::{Serialize, Deserialize};

use crate::db;
use crate::hash::raw_hash_bytes;

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ArchiveFileEntry {
    pub path: String,
    pub sha256: String,
    pub size: u64,
    pub mode: u32,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ArchiveManifest {
    pub archive_id: String,
    pub g_time: String,
    pub files: Vec<ArchiveFileEntry>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub top_level_vcard_hash: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub archiver_did: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub compression: Option<String>,
}

/// Recursively scans `dir` and builds an `ArchiveManifest` with SHA-256 hashes.
pub fn generate_manifest(dir: &Path, archiver_did: Option<&str>) -> Result<ArchiveManifest, String> {
    if !dir.exists() || !dir.is_dir() {
        return Err(format!("Directory does not exist or is not a directory: {}", dir.display()));
    }

    let mut entries: Vec<ArchiveFileEntry> = Vec::new();
    collect_files(dir, dir, &mut entries)?;

    entries.sort_by(|a, b| a.path.cmp(&b.path));

    let g_time = chrono::Utc::now().to_rfc3339();
    let archive_id = format!("archive:sovereign:{}", g_time);

    Ok(ArchiveManifest {
        archive_id,
        g_time,
        files: entries,
        top_level_vcard_hash: None,
        archiver_did: archiver_did.map(|s| s.to_string()),
        compression: Some("zstd-rsyncable".to_string()),
    })
}

fn collect_files(base_dir: &Path, current_dir: &Path, entries: &mut Vec<ArchiveFileEntry>) -> Result<(), String> {
    let read_dir = fs::read_dir(current_dir)
        .map_err(|e| format!("Failed to read dir {}: {}", current_dir.display(), e))?;

    for entry in read_dir {
        let entry = entry.map_err(|e| format!("Failed to read directory entry: {}", e))?;
        let path = entry.path();
        let file_name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");

        // Skip temporary & git directories
        if file_name == ".git" || file_name == "target" || file_name == ".clm_tmp" || file_name == "node_modules" {
            continue;
        }

        if path.is_dir() {
            collect_files(base_dir, &path, entries)?;
        } else if path.is_file() {
            let rel_path = path.strip_prefix(base_dir)
                .map_err(|e| format!("Failed to compute relative path: {}", e))?
                .to_string_lossy()
                .replace('\\', "/");

            let bytes = fs::read(&path)
                .map_err(|e| format!("Failed to read file {}: {}", path.display(), e))?;

            let sha256 = raw_hash_bytes(&bytes);
            let size = bytes.len() as u64;

            #[cfg(unix)]
            let mode = {
                use std::os::unix::fs::PermissionsExt;
                let meta = fs::metadata(&path)
                    .map_err(|e| format!("Failed to read metadata for {}: {}", path.display(), e))?;
                meta.permissions().mode()
            };
            #[cfg(not(unix))]
            let mode = 0o644;

            entries.push(ArchiveFileEntry {
                path: rel_path,
                sha256,
                size,
                mode,
            });
        }
    }
    Ok(())
}

/// Creates a solid `.tar.zst` archive and registers the manifest MCard in `content_memory.db`.
pub fn create_archive(
    dir: &Path,
    output_path: &Path,
    archiver_did: Option<&str>,
    clm_root: &str,
) -> Result<(ArchiveManifest, String), String> {
    let mut manifest = generate_manifest(dir, archiver_did)?;

    let temp_dir = tempfile::tempdir().map_err(|e| format!("Failed to create temp dir: {}", e))?;
    let manifest_json = serde_json::to_string_pretty(&manifest)
        .map_err(|e| format!("Failed to serialize manifest: {}", e))?;

    let manifest_path = temp_dir.path().join("archive_manifest.json");
    fs::write(&manifest_path, &manifest_json)
        .map_err(|e| format!("Failed to write manifest temp file: {}", e))?;

    // Create tar archive with deterministic parameters
    let tar_path = temp_dir.path().join("archive.tar");

    let gtar_status = Command::new("gtar")
        .arg("-cf")
        .arg(&tar_path)
        .arg("--sort=name")
        .arg("--mtime=@0")
        .arg("--owner=0")
        .arg("--group=0")
        .arg("-C")
        .arg(dir)
        .arg(".")
        .status();

    if gtar_status.is_err() || !gtar_status.as_ref().unwrap().success() {
        let _ = fs::remove_file(&tar_path);
        let status_fallback = Command::new("tar")
            .arg("-cf")
            .arg(&tar_path)
            .arg("-C")
            .arg(dir)
            .arg(".")
            .status()
            .map_err(|e| format!("Failed to execute tar: {}", e))?;

        if !status_fallback.success() {
            return Err("tar command failed to build archive".to_string());
        }
    }

    // Compress tar archive using zstd
    let zstd_output = Command::new("zstd")
        .arg("-q")
        .arg("-f")
        .arg("--rsyncable")
        .arg("-19")
        .arg(&tar_path)
        .arg("-o")
        .arg(output_path)
        .status();

    if zstd_output.is_err() || !zstd_output.as_ref().unwrap().success() {
        // Fallback: copy uncompressed tar if zstd binary is not installed
        fs::copy(&tar_path, output_path)
            .map_err(|e| format!("Failed to write output archive file: {}", e))?;
    }

    let archive_bytes = fs::read(output_path)
        .map_err(|e| format!("Failed to read created archive: {}", e))?;

    let archive_bytes_hash = raw_hash_bytes(&archive_bytes);
    let manifest_mcard_hash = raw_hash_bytes(manifest_json.as_bytes());

    // Top-level VCard seal: sha256(manifest_mcard_hash || archive_bytes_hash)
    let mut hasher = Sha256::new();
    hasher.update(manifest_mcard_hash.as_bytes());
    hasher.update(archive_bytes_hash.as_bytes());
    let top_level_vcard_hash = hex::encode(hasher.finalize());

    manifest.top_level_vcard_hash = Some(top_level_vcard_hash.clone());

    // Store manifest as MCard and register handle in content_memory.db
    let final_manifest_json = serde_json::to_string_pretty(&manifest)
        .map_err(|e| format!("Failed to serialize final manifest: {}", e))?;

    let final_mcard_hash = raw_hash_bytes(final_manifest_json.as_bytes());

    let _ = db::setter_card_batch("content_memory.db", clm_root, &final_mcard_hash, &final_manifest_json, &manifest.g_time);
    let _ = db::handle_register_batch("content_memory.db", clm_root, &manifest.archive_id, &final_mcard_hash, &final_manifest_json);

    Ok((manifest, top_level_vcard_hash))
}

/// Restores files from a `.tar.zst` archive following the Schema-First Restore Protocol.
pub fn restore_archive(
    archive_path: &Path,
    target_dir: &Path,
    clm_root: &str,
) -> Result<ArchiveManifest, String> {
    if !archive_path.exists() {
        return Err(format!("Archive file does not exist: {}", archive_path.display()));
    }

    fs::create_dir_all(target_dir)
        .map_err(|e| format!("Failed to create target dir {}: {}", target_dir.display(), e))?;

    let temp_dir = tempfile::tempdir().map_err(|e| format!("Failed to create temp dir: {}", e))?;
    let tar_path = temp_dir.path().join("decompressed.tar");

    // Try zstd decompression
    let zstd_status = Command::new("zstd")
        .arg("-d")
        .arg("-f")
        .arg(archive_path)
        .arg("-o")
        .arg(&tar_path)
        .status();

    let actual_tar = if zstd_status.is_ok() && zstd_status.unwrap().success() {
        tar_path
    } else {
        // Fallback: treat as plain tar
        archive_path.to_path_buf()
    };

    // Extract tar archive into target directory
    let tar_status = Command::new("tar")
        .arg("-xf")
        .arg(&actual_tar)
        .arg("-C")
        .arg(target_dir)
        .status()
        .map_err(|e| format!("Failed to run tar extract: {}", e))?;

    if !tar_status.success() {
        return Err("Failed to extract tar archive".to_string());
    }

    // Generate manifest of restored files to verify integrity
    let restored_manifest = generate_manifest(target_dir, None)?;

    // Verify restored file hashes
    for file in &restored_manifest.files {
        let file_path = target_dir.join(&file.path);
        let bytes = fs::read(&file_path)
            .map_err(|e| format!("Restored file missing or unreadable: {}: {}", file_path.display(), e))?;

        let actual_hash = raw_hash_bytes(&bytes);
        if actual_hash != file.sha256 {
            return Err(format!("Hash mismatch for restored file {}: expected {}, got {}", file.path, file.sha256, actual_hash));
        }
    }

    // Register restored manifest handle in content_memory.db
    let restored_json = serde_json::to_string_pretty(&restored_manifest)
        .map_err(|e| format!("Failed to serialize restored manifest: {}", e))?;

    let restored_hash = raw_hash_bytes(restored_json.as_bytes());
    let _ = db::setter_card_batch("content_memory.db", clm_root, &restored_hash, &restored_json, &restored_manifest.g_time);
    let _ = db::handle_register_batch("content_memory.db", clm_root, &restored_manifest.archive_id, &restored_hash, &restored_json);

    Ok(restored_manifest)
}

/// Verifies internal consistency and file hashes of an archive against its manifest.
pub fn verify_archive(_archive_path: &Path, target_dir: &Path) -> Result<bool, String> {
    let manifest = generate_manifest(target_dir, None)?;
    for file in &manifest.files {
        let p = target_dir.join(&file.path);
        if !p.exists() {
            return Ok(false);
        }
        let bytes = fs::read(&p).map_err(|e| e.to_string())?;
        if raw_hash_bytes(&bytes) != file.sha256 {
            return Ok(false);
        }
    }
    Ok(true)
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::tempdir;

    #[test]
    fn test_manifest_generation() {
        let dir = tempdir().unwrap();
        let file1 = dir.path().join("a.txt");
        let file2 = dir.path().join("sub/b.txt");

        fs::create_dir_all(file2.parent().unwrap()).unwrap();
        fs::write(&file1, b"hello archive").unwrap();
        fs::write(&file2, b"world archive").unwrap();

        let manifest = generate_manifest(dir.path(), Some("did:key:zTestArchiver")).unwrap();
        assert_eq!(manifest.files.len(), 2);
        assert_eq!(manifest.files[0].path, "a.txt");
        assert_eq!(manifest.files[1].path, "sub/b.txt");
        assert_eq!(manifest.archiver_did, Some("did:key:zTestArchiver".to_string()));
    }

    #[test]
    fn test_archive_create_restore_verify_roundtrip() {
        let source_dir = tempdir().unwrap();
        let target_dir = tempdir().unwrap();
        let out_dir = tempdir().unwrap();
        let archive_out = out_dir.path().join("test_archive.tar.zst");

        let file1 = source_dir.path().join("config.yaml");
        let file2 = source_dir.path().join("data/content.txt");

        fs::create_dir_all(file2.parent().unwrap()).unwrap();
        fs::write(&file1, b"setting: true\nversion: 1.0\n").unwrap();
        fs::write(&file2, b"sovereign state payload bytes\n").unwrap();

        let clm_root = tempdir().unwrap();
        let clm_root_str = clm_root.path().to_str().unwrap();
        let _ = crate::db::init_tridb(clm_root_str);

        let (manifest, vcard_seal) = create_archive(
            source_dir.path(),
            &archive_out,
            Some("did:key:zArchiverNode"),
            clm_root_str,
        ).unwrap();

        assert!(!vcard_seal.is_empty());
        assert_eq!(manifest.files.len(), 2);
        assert!(archive_out.exists());

        let restored_manifest = restore_archive(&archive_out, target_dir.path(), clm_root_str).unwrap();
        assert_eq!(restored_manifest.files.len(), 2);

        let verified = verify_archive(&archive_out, target_dir.path()).unwrap();
        assert!(verified);

        let restored1 = fs::read_to_string(target_dir.path().join("config.yaml")).unwrap();
        let restored2 = fs::read_to_string(target_dir.path().join("data/content.txt")).unwrap();
        assert_eq!(restored1, "setting: true\nversion: 1.0\n");
        assert_eq!(restored2, "sovereign state payload bytes\n");
    }
}