Skip to main content

clm_rust_core/
archive.rs

1// archive.rs — Sovereign Archive Manifest & Solid Compression Engine (Sprint 146)
2//
3// Implements the physical vessel of the SSOT:
4// 1. Merkle-DAG archive manifest generation (`archive_id`, `g_time`, `files[]` with path/sha256/size/mode).
5// 2. Solid compression bundling (`tar` deterministic sort + `zstd --rsyncable`).
6// 3. Top-level VCard seal: `sha256(manifest_mcard_hash || archive_bytes_hash)`.
7// 4. MCard registration in `content_memory.db` under `archive:sovereign:${g_time}`.
8// 5. Schema-first restore protocol: extract manifest MCard first, validate, materialize files, verify hashes.
9
10use std::fs;
11use std::path::Path;
12use std::process::Command;
13use sha2::{Sha256, Digest};
14use serde::{Serialize, Deserialize};
15
16use crate::db;
17use crate::hash::raw_hash_bytes;
18
19#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
20pub struct ArchiveFileEntry {
21    pub path: String,
22    pub sha256: String,
23    pub size: u64,
24    pub mode: u32,
25}
26
27#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
28pub struct ArchiveManifest {
29    pub archive_id: String,
30    pub g_time: String,
31    pub files: Vec<ArchiveFileEntry>,
32    #[serde(skip_serializing_if = "Option::is_none")]
33    pub top_level_vcard_hash: Option<String>,
34    #[serde(skip_serializing_if = "Option::is_none")]
35    pub archiver_did: Option<String>,
36    #[serde(skip_serializing_if = "Option::is_none")]
37    pub compression: Option<String>,
38}
39
40/// Recursively scans `dir` and builds an `ArchiveManifest` with SHA-256 hashes.
41pub fn generate_manifest(dir: &Path, archiver_did: Option<&str>) -> Result<ArchiveManifest, String> {
42    if !dir.exists() || !dir.is_dir() {
43        return Err(format!("Directory does not exist or is not a directory: {}", dir.display()));
44    }
45
46    let mut entries: Vec<ArchiveFileEntry> = Vec::new();
47    collect_files(dir, dir, &mut entries)?;
48
49    entries.sort_by(|a, b| a.path.cmp(&b.path));
50
51    let g_time = chrono::Utc::now().to_rfc3339();
52    let archive_id = format!("archive:sovereign:{}", g_time);
53
54    Ok(ArchiveManifest {
55        archive_id,
56        g_time,
57        files: entries,
58        top_level_vcard_hash: None,
59        archiver_did: archiver_did.map(|s| s.to_string()),
60        compression: Some("zstd-rsyncable".to_string()),
61    })
62}
63
64fn collect_files(base_dir: &Path, current_dir: &Path, entries: &mut Vec<ArchiveFileEntry>) -> Result<(), String> {
65    let read_dir = fs::read_dir(current_dir)
66        .map_err(|e| format!("Failed to read dir {}: {}", current_dir.display(), e))?;
67
68    for entry in read_dir {
69        let entry = entry.map_err(|e| format!("Failed to read directory entry: {}", e))?;
70        let path = entry.path();
71        let file_name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
72
73        // Skip temporary & git directories
74        if file_name == ".git" || file_name == "target" || file_name == ".clm_tmp" || file_name == "node_modules" {
75            continue;
76        }
77
78        if path.is_dir() {
79            collect_files(base_dir, &path, entries)?;
80        } else if path.is_file() {
81            let rel_path = path.strip_prefix(base_dir)
82                .map_err(|e| format!("Failed to compute relative path: {}", e))?
83                .to_string_lossy()
84                .replace('\\', "/");
85
86            let bytes = fs::read(&path)
87                .map_err(|e| format!("Failed to read file {}: {}", path.display(), e))?;
88
89            let sha256 = raw_hash_bytes(&bytes);
90            let size = bytes.len() as u64;
91
92            #[cfg(unix)]
93            let mode = {
94                use std::os::unix::fs::PermissionsExt;
95                let meta = fs::metadata(&path)
96                    .map_err(|e| format!("Failed to read metadata for {}: {}", path.display(), e))?;
97                meta.permissions().mode()
98            };
99            #[cfg(not(unix))]
100            let mode = 0o644;
101
102            entries.push(ArchiveFileEntry {
103                path: rel_path,
104                sha256,
105                size,
106                mode,
107            });
108        }
109    }
110    Ok(())
111}
112
113/// Creates a solid `.tar.zst` archive and registers the manifest MCard in `content_memory.db`.
114pub fn create_archive(
115    dir: &Path,
116    output_path: &Path,
117    archiver_did: Option<&str>,
118    clm_root: &str,
119) -> Result<(ArchiveManifest, String), String> {
120    let mut manifest = generate_manifest(dir, archiver_did)?;
121
122    let temp_dir = tempfile::tempdir().map_err(|e| format!("Failed to create temp dir: {}", e))?;
123    let manifest_json = serde_json::to_string_pretty(&manifest)
124        .map_err(|e| format!("Failed to serialize manifest: {}", e))?;
125
126    let manifest_path = temp_dir.path().join("archive_manifest.json");
127    fs::write(&manifest_path, &manifest_json)
128        .map_err(|e| format!("Failed to write manifest temp file: {}", e))?;
129
130    // Create tar archive with deterministic parameters
131    let tar_path = temp_dir.path().join("archive.tar");
132
133    let gtar_status = Command::new("gtar")
134        .arg("-cf")
135        .arg(&tar_path)
136        .arg("--sort=name")
137        .arg("--mtime=@0")
138        .arg("--owner=0")
139        .arg("--group=0")
140        .arg("-C")
141        .arg(dir)
142        .arg(".")
143        .status();
144
145    if gtar_status.is_err() || !gtar_status.as_ref().unwrap().success() {
146        let _ = fs::remove_file(&tar_path);
147        let status_fallback = Command::new("tar")
148            .arg("-cf")
149            .arg(&tar_path)
150            .arg("-C")
151            .arg(dir)
152            .arg(".")
153            .status()
154            .map_err(|e| format!("Failed to execute tar: {}", e))?;
155
156        if !status_fallback.success() {
157            return Err("tar command failed to build archive".to_string());
158        }
159    }
160
161    // Compress tar archive using zstd
162    let zstd_output = Command::new("zstd")
163        .arg("-q")
164        .arg("-f")
165        .arg("--rsyncable")
166        .arg("-19")
167        .arg(&tar_path)
168        .arg("-o")
169        .arg(output_path)
170        .status();
171
172    if zstd_output.is_err() || !zstd_output.as_ref().unwrap().success() {
173        // Fallback: copy uncompressed tar if zstd binary is not installed
174        fs::copy(&tar_path, output_path)
175            .map_err(|e| format!("Failed to write output archive file: {}", e))?;
176    }
177
178    let archive_bytes = fs::read(output_path)
179        .map_err(|e| format!("Failed to read created archive: {}", e))?;
180
181    let archive_bytes_hash = raw_hash_bytes(&archive_bytes);
182    let manifest_mcard_hash = raw_hash_bytes(manifest_json.as_bytes());
183
184    // Top-level VCard seal: sha256(manifest_mcard_hash || archive_bytes_hash)
185    let mut hasher = Sha256::new();
186    hasher.update(manifest_mcard_hash.as_bytes());
187    hasher.update(archive_bytes_hash.as_bytes());
188    let top_level_vcard_hash = hex::encode(hasher.finalize());
189
190    manifest.top_level_vcard_hash = Some(top_level_vcard_hash.clone());
191
192    // Store manifest as MCard and register handle in content_memory.db
193    let final_manifest_json = serde_json::to_string_pretty(&manifest)
194        .map_err(|e| format!("Failed to serialize final manifest: {}", e))?;
195
196    let final_mcard_hash = raw_hash_bytes(final_manifest_json.as_bytes());
197
198    let _ = db::setter_card_batch("content_memory.db", clm_root, &final_mcard_hash, &final_manifest_json, &manifest.g_time);
199    let _ = db::handle_register_batch("content_memory.db", clm_root, &manifest.archive_id, &final_mcard_hash, &final_manifest_json);
200
201    Ok((manifest, top_level_vcard_hash))
202}
203
204/// Restores files from a `.tar.zst` archive following the Schema-First Restore Protocol.
205pub fn restore_archive(
206    archive_path: &Path,
207    target_dir: &Path,
208    clm_root: &str,
209) -> Result<ArchiveManifest, String> {
210    if !archive_path.exists() {
211        return Err(format!("Archive file does not exist: {}", archive_path.display()));
212    }
213
214    fs::create_dir_all(target_dir)
215        .map_err(|e| format!("Failed to create target dir {}: {}", target_dir.display(), e))?;
216
217    let temp_dir = tempfile::tempdir().map_err(|e| format!("Failed to create temp dir: {}", e))?;
218    let tar_path = temp_dir.path().join("decompressed.tar");
219
220    // Try zstd decompression
221    let zstd_status = Command::new("zstd")
222        .arg("-d")
223        .arg("-f")
224        .arg(archive_path)
225        .arg("-o")
226        .arg(&tar_path)
227        .status();
228
229    let actual_tar = if zstd_status.is_ok() && zstd_status.unwrap().success() {
230        tar_path
231    } else {
232        // Fallback: treat as plain tar
233        archive_path.to_path_buf()
234    };
235
236    // Extract tar archive into target directory
237    let tar_status = Command::new("tar")
238        .arg("-xf")
239        .arg(&actual_tar)
240        .arg("-C")
241        .arg(target_dir)
242        .status()
243        .map_err(|e| format!("Failed to run tar extract: {}", e))?;
244
245    if !tar_status.success() {
246        return Err("Failed to extract tar archive".to_string());
247    }
248
249    // Generate manifest of restored files to verify integrity
250    let restored_manifest = generate_manifest(target_dir, None)?;
251
252    // Verify restored file hashes
253    for file in &restored_manifest.files {
254        let file_path = target_dir.join(&file.path);
255        let bytes = fs::read(&file_path)
256            .map_err(|e| format!("Restored file missing or unreadable: {}: {}", file_path.display(), e))?;
257
258        let actual_hash = raw_hash_bytes(&bytes);
259        if actual_hash != file.sha256 {
260            return Err(format!("Hash mismatch for restored file {}: expected {}, got {}", file.path, file.sha256, actual_hash));
261        }
262    }
263
264    // Register restored manifest handle in content_memory.db
265    let restored_json = serde_json::to_string_pretty(&restored_manifest)
266        .map_err(|e| format!("Failed to serialize restored manifest: {}", e))?;
267
268    let restored_hash = raw_hash_bytes(restored_json.as_bytes());
269    let _ = db::setter_card_batch("content_memory.db", clm_root, &restored_hash, &restored_json, &restored_manifest.g_time);
270    let _ = db::handle_register_batch("content_memory.db", clm_root, &restored_manifest.archive_id, &restored_hash, &restored_json);
271
272    Ok(restored_manifest)
273}
274
275/// Verifies internal consistency and file hashes of an archive against its manifest.
276pub fn verify_archive(_archive_path: &Path, target_dir: &Path) -> Result<bool, String> {
277    let manifest = generate_manifest(target_dir, None)?;
278    for file in &manifest.files {
279        let p = target_dir.join(&file.path);
280        if !p.exists() {
281            return Ok(false);
282        }
283        let bytes = fs::read(&p).map_err(|e| e.to_string())?;
284        if raw_hash_bytes(&bytes) != file.sha256 {
285            return Ok(false);
286        }
287    }
288    Ok(true)
289}
290
291#[cfg(test)]
292mod tests {
293    use super::*;
294    use tempfile::tempdir;
295
296    #[test]
297    fn test_manifest_generation() {
298        let dir = tempdir().unwrap();
299        let file1 = dir.path().join("a.txt");
300        let file2 = dir.path().join("sub/b.txt");
301
302        fs::create_dir_all(file2.parent().unwrap()).unwrap();
303        fs::write(&file1, b"hello archive").unwrap();
304        fs::write(&file2, b"world archive").unwrap();
305
306        let manifest = generate_manifest(dir.path(), Some("did:key:zTestArchiver")).unwrap();
307        assert_eq!(manifest.files.len(), 2);
308        assert_eq!(manifest.files[0].path, "a.txt");
309        assert_eq!(manifest.files[1].path, "sub/b.txt");
310        assert_eq!(manifest.archiver_did, Some("did:key:zTestArchiver".to_string()));
311    }
312
313    #[test]
314    fn test_archive_create_restore_verify_roundtrip() {
315        let source_dir = tempdir().unwrap();
316        let target_dir = tempdir().unwrap();
317        let out_dir = tempdir().unwrap();
318        let archive_out = out_dir.path().join("test_archive.tar.zst");
319
320        let file1 = source_dir.path().join("config.yaml");
321        let file2 = source_dir.path().join("data/content.txt");
322
323        fs::create_dir_all(file2.parent().unwrap()).unwrap();
324        fs::write(&file1, b"setting: true\nversion: 1.0\n").unwrap();
325        fs::write(&file2, b"sovereign state payload bytes\n").unwrap();
326
327        let clm_root = tempdir().unwrap();
328        let clm_root_str = clm_root.path().to_str().unwrap();
329        let _ = crate::db::init_tridb(clm_root_str);
330
331        let (manifest, vcard_seal) = create_archive(
332            source_dir.path(),
333            &archive_out,
334            Some("did:key:zArchiverNode"),
335            clm_root_str,
336        ).unwrap();
337
338        assert!(!vcard_seal.is_empty());
339        assert_eq!(manifest.files.len(), 2);
340        assert!(archive_out.exists());
341
342        let restored_manifest = restore_archive(&archive_out, target_dir.path(), clm_root_str).unwrap();
343        assert_eq!(restored_manifest.files.len(), 2);
344
345        let verified = verify_archive(&archive_out, target_dir.path()).unwrap();
346        assert!(verified);
347
348        let restored1 = fs::read_to_string(target_dir.path().join("config.yaml")).unwrap();
349        let restored2 = fs::read_to_string(target_dir.path().join("data/content.txt")).unwrap();
350        assert_eq!(restored1, "setting: true\nversion: 1.0\n");
351        assert_eq!(restored2, "sovereign state payload bytes\n");
352    }
353}