sema_core/
archive.rs

1//! VFS archive binary format for bundling files into a standalone executable.
2//!
3//! The archive is appended to the sema binary to create a self-contained
4//! executable. It stores metadata key-value pairs and file contents, with
5//! a CRC32 checksum for integrity validation.
6//!
7//! ## Binary Layout
8//!
9//! ```text
10//! Header:
11//!   format_version: u16 LE (= 1)
12//!   flags:          u16 LE (= 0, reserved)
13//!   checksum:       u32 LE (CRC32 of everything after this field)
14//!   metadata_count: u32 LE
15//! Metadata entries (repeated metadata_count times):
16//!   key_len: u16 LE
17//!   key:     [u8; key_len] (UTF-8)
18//!   val_len: u32 LE
19//!   val:     [u8; val_len]
20//! TOC:
21//!   entry_count: u32 LE
22//!   entries (repeated entry_count times):
23//!     path_len: u32 LE
24//!     path:     [u8; path_len] (UTF-8)
25//!     offset:   u64 LE (relative to file data start)
26//!     size:     u64 LE
27//! File data:
28//!   raw bytes for all files
29//!
30//! Trailer (appended after archive, for ELF detection):
31//!   archive_size: u64 LE
32//!   magic:        "SEMAEXEC" (8 bytes)
33//! ```
34
35use std::collections::HashMap;
36use std::io::{self, Read, Seek, SeekFrom, Write};
37use std::path::Path;
38
39/// Magic bytes written at the end of a bundled executable.
40pub const MAGIC: &[u8; 8] = b"SEMAEXEC";
41
42/// Size of the trailer in bytes (u64 archive_size + 8-byte magic).
43pub const TRAILER_SIZE: usize = 16;
44
45/// Current archive format version.
46pub const FORMAT_VERSION: u16 = 1;
47
48// ---------------------------------------------------------------------------
49// Archive struct
50// ---------------------------------------------------------------------------
51
52/// An in-memory representation of a VFS archive.
53#[derive(Debug, Clone)]
54#[allow(dead_code)]
55pub struct Archive {
56    pub format_version: u16,
57    pub flags: u16,
58    pub metadata: HashMap<String, Vec<u8>>,
59    pub files: HashMap<String, Vec<u8>>,
60}
61
62impl Default for Archive {
63    fn default() -> Self {
64        Self::new()
65    }
66}
67
68impl Archive {
69    /// Create a new empty archive with the current format version.
70    #[allow(dead_code)]
71    pub fn new() -> Self {
72        Self {
73            format_version: FORMAT_VERSION,
74            flags: 0,
75            metadata: HashMap::new(),
76            files: HashMap::new(),
77        }
78    }
79}
80
81// ---------------------------------------------------------------------------
82// Detection
83// ---------------------------------------------------------------------------
84
85/// Check whether `path` contains an embedded SEMAEXEC archive by reading the
86/// last 16 bytes and checking for the magic trailer.
87pub fn has_embedded_archive(path: &Path) -> io::Result<bool> {
88    let mut file = std::fs::File::open(path)?;
89    let file_len = file.metadata()?.len();
90    if file_len < TRAILER_SIZE as u64 {
91        return Ok(false);
92    }
93    file.seek(SeekFrom::End(-(TRAILER_SIZE as i64)))?;
94    let mut trailer = [0u8; TRAILER_SIZE];
95    file.read_exact(&mut trailer)?;
96    let magic = &trailer[8..16];
97    Ok(magic == MAGIC)
98}
99
100// ---------------------------------------------------------------------------
101// Extraction
102// ---------------------------------------------------------------------------
103
104/// Read a bundled executable at `path`, locate the embedded archive using the
105/// trailer, and deserialize it.
106#[allow(dead_code)]
107pub fn extract_archive(path: &Path) -> io::Result<Archive> {
108    let data = std::fs::read(path)?;
109    let len = data.len();
110
111    if len < TRAILER_SIZE {
112        return Err(io::Error::new(
113            io::ErrorKind::InvalidData,
114            "file too small to contain archive trailer",
115        ));
116    }
117
118    // Read trailer
119    let trailer = &data[len - TRAILER_SIZE..];
120    let magic = &trailer[8..16];
121    if magic != MAGIC {
122        return Err(io::Error::new(
123            io::ErrorKind::InvalidData,
124            "SEMAEXEC magic not found in trailer",
125        ));
126    }
127
128    let archive_size = u64::from_le_bytes(trailer[0..8].try_into().unwrap()) as usize;
129    let archive_start = len - TRAILER_SIZE - archive_size;
130
131    if archive_start > len - TRAILER_SIZE {
132        return Err(io::Error::new(
133            io::ErrorKind::InvalidData,
134            "archive size exceeds file size",
135        ));
136    }
137
138    let archive_bytes = &data[archive_start..archive_start + archive_size];
139    deserialize_archive(archive_bytes)
140}
141
142// ---------------------------------------------------------------------------
143// Deserialization (private)
144// ---------------------------------------------------------------------------
145
146/// Helper to read a `u16` LE from a cursor position, advancing it.
147fn read_u16(data: &[u8], pos: &mut usize) -> io::Result<u16> {
148    if *pos + 2 > data.len() {
149        return Err(io::Error::new(
150            io::ErrorKind::UnexpectedEof,
151            "unexpected end of archive (u16)",
152        ));
153    }
154    let val = u16::from_le_bytes(data[*pos..*pos + 2].try_into().unwrap());
155    *pos += 2;
156    Ok(val)
157}
158
159/// Helper to read a `u32` LE from a cursor position, advancing it.
160fn read_u32(data: &[u8], pos: &mut usize) -> io::Result<u32> {
161    if *pos + 4 > data.len() {
162        return Err(io::Error::new(
163            io::ErrorKind::UnexpectedEof,
164            "unexpected end of archive (u32)",
165        ));
166    }
167    let val = u32::from_le_bytes(data[*pos..*pos + 4].try_into().unwrap());
168    *pos += 4;
169    Ok(val)
170}
171
172/// Helper to read a `u64` LE from a cursor position, advancing it.
173fn read_u64(data: &[u8], pos: &mut usize) -> io::Result<u64> {
174    if *pos + 8 > data.len() {
175        return Err(io::Error::new(
176            io::ErrorKind::UnexpectedEof,
177            "unexpected end of archive (u64)",
178        ));
179    }
180    let val = u64::from_le_bytes(data[*pos..*pos + 8].try_into().unwrap());
181    *pos += 8;
182    Ok(val)
183}
184
185/// Helper to read `n` bytes from a cursor position, advancing it.
186fn read_bytes<'a>(data: &'a [u8], pos: &mut usize, n: usize) -> io::Result<&'a [u8]> {
187    if *pos + n > data.len() {
188        return Err(io::Error::new(
189            io::ErrorKind::UnexpectedEof,
190            "unexpected end of archive (bytes)",
191        ));
192    }
193    let slice = &data[*pos..*pos + n];
194    *pos += n;
195    Ok(slice)
196}
197
198/// Parse raw archive bytes into an `Archive`. Validates the CRC32 checksum.
199fn deserialize_archive(data: &[u8]) -> io::Result<Archive> {
200    let mut pos = 0;
201
202    // Header
203    let format_version = read_u16(data, &mut pos)?;
204    if format_version != FORMAT_VERSION {
205        return Err(io::Error::new(
206            io::ErrorKind::InvalidData,
207            format!(
208                "unsupported archive format version {format_version}, expected {FORMAT_VERSION}"
209            ),
210        ));
211    }
212
213    let flags = read_u16(data, &mut pos)?;
214    let stored_checksum = read_u32(data, &mut pos)?;
215    // pos is now 8 -- everything from here on is checksummed
216    let checksum_start = pos;
217
218    // Validate CRC32
219    let computed_checksum = crc32fast::hash(&data[checksum_start..]);
220    if stored_checksum != computed_checksum {
221        return Err(io::Error::new(
222            io::ErrorKind::InvalidData,
223            format!(
224                "archive checksum mismatch: stored {stored_checksum:#010x}, computed {computed_checksum:#010x}"
225            ),
226        ));
227    }
228
229    // Metadata
230    let metadata_count = read_u32(data, &mut pos)? as usize;
231    // Clamp capacity to avoid OOM from malicious archives — each metadata entry
232    // is at least 8 bytes (u16 key_len + u32 val_len + 2 bytes min), so the
233    // remaining data bounds how many entries can actually exist.
234    let remaining = data.len().saturating_sub(pos);
235    let mut metadata = HashMap::with_capacity(metadata_count.min(remaining / 8));
236    for _ in 0..metadata_count {
237        let key_len = read_u16(data, &mut pos)? as usize;
238        let key_bytes = read_bytes(data, &mut pos, key_len)?;
239        let key = String::from_utf8(key_bytes.to_vec()).map_err(|e| {
240            io::Error::new(
241                io::ErrorKind::InvalidData,
242                format!("metadata key is not valid UTF-8: {e}"),
243            )
244        })?;
245        let val_len = read_u32(data, &mut pos)? as usize;
246        let val = read_bytes(data, &mut pos, val_len)?.to_vec();
247        metadata.insert(key, val);
248    }
249
250    // TOC
251    let entry_count = read_u32(data, &mut pos)? as usize;
252
253    struct TocEntry {
254        path: String,
255        offset: u64,
256        size: u64,
257    }
258
259    // Clamp capacity — each TOC entry is at least 20 bytes (u32 path_len + u64 offset + u64 size).
260    let remaining = data.len().saturating_sub(pos);
261    let mut toc = Vec::with_capacity(entry_count.min(remaining / 20));
262    for _ in 0..entry_count {
263        let path_len = read_u32(data, &mut pos)? as usize;
264        let path_bytes = read_bytes(data, &mut pos, path_len)?;
265        let path = String::from_utf8(path_bytes.to_vec()).map_err(|e| {
266            io::Error::new(
267                io::ErrorKind::InvalidData,
268                format!("file path is not valid UTF-8: {e}"),
269            )
270        })?;
271        let offset = read_u64(data, &mut pos)?;
272        let size = read_u64(data, &mut pos)?;
273        toc.push(TocEntry { path, offset, size });
274    }
275
276    // File data starts at current pos
277    let file_data_start = pos;
278    let mut files = HashMap::with_capacity(toc.len());
279    for entry in &toc {
280        let start = file_data_start + entry.offset as usize;
281        let end = start + entry.size as usize;
282        if end > data.len() {
283            return Err(io::Error::new(
284                io::ErrorKind::InvalidData,
285                format!(
286                    "file entry '{}' extends beyond archive (offset={}, size={}, data_len={})",
287                    entry.path,
288                    entry.offset,
289                    entry.size,
290                    data.len()
291                ),
292            ));
293        }
294        files.insert(entry.path.clone(), data[start..end].to_vec());
295    }
296
297    Ok(Archive {
298        format_version,
299        flags,
300        metadata,
301        files,
302    })
303}
304
305/// Public entry point for deserializing archive bytes (used by the libsui
306/// path where we already have raw bytes extracted from the binary).
307pub fn deserialize_archive_from_bytes(data: &[u8]) -> io::Result<Archive> {
308    deserialize_archive(data)
309}
310
311// ---------------------------------------------------------------------------
312// Serialization
313// ---------------------------------------------------------------------------
314
315/// Build archive bytes from metadata and file maps.
316///
317/// Keys are sorted before serialization for deterministic output. The CRC32
318/// checksum is computed over everything after the checksum field and then
319/// backfilled into position.
320pub fn serialize_archive(
321    metadata: &HashMap<String, Vec<u8>>,
322    files: &HashMap<String, Vec<u8>>,
323) -> Vec<u8> {
324    let mut buf: Vec<u8> = Vec::new();
325
326    // -- Header --
327    buf.extend_from_slice(&FORMAT_VERSION.to_le_bytes()); // format_version: u16
328    buf.extend_from_slice(&0u16.to_le_bytes()); // flags: u16
329    buf.extend_from_slice(&0u32.to_le_bytes()); // checksum placeholder: u32
330                                                // checksum field ends at byte 8
331
332    // -- Metadata --
333    let mut meta_keys: Vec<&String> = metadata.keys().collect();
334    meta_keys.sort();
335
336    buf.extend_from_slice(&(meta_keys.len() as u32).to_le_bytes()); // metadata_count
337    for key in &meta_keys {
338        let key_bytes = key.as_bytes();
339        buf.extend_from_slice(&(key_bytes.len() as u16).to_le_bytes());
340        buf.extend_from_slice(key_bytes);
341        let val = &metadata[*key];
342        buf.extend_from_slice(&(val.len() as u32).to_le_bytes());
343        buf.extend_from_slice(val);
344    }
345
346    // -- TOC + File data --
347    // We need to build the TOC and file data together so offsets are correct.
348    let mut file_keys: Vec<&String> = files.keys().collect();
349    file_keys.sort();
350
351    // First pass: compute offsets
352    struct FileEntry<'a> {
353        path: &'a str,
354        data: &'a [u8],
355        offset: u64,
356    }
357
358    let mut entries = Vec::with_capacity(file_keys.len());
359    let mut current_offset: u64 = 0;
360    for key in &file_keys {
361        let data = &files[*key];
362        entries.push(FileEntry {
363            path: key.as_str(),
364            data,
365            offset: current_offset,
366        });
367        current_offset += data.len() as u64;
368    }
369
370    // Write entry_count
371    buf.extend_from_slice(&(entries.len() as u32).to_le_bytes());
372
373    // Write TOC entries
374    for entry in &entries {
375        let path_bytes = entry.path.as_bytes();
376        buf.extend_from_slice(&(path_bytes.len() as u32).to_le_bytes());
377        buf.extend_from_slice(path_bytes);
378        buf.extend_from_slice(&entry.offset.to_le_bytes());
379        buf.extend_from_slice(&(entry.data.len() as u64).to_le_bytes());
380    }
381
382    // Write file data
383    for entry in &entries {
384        buf.extend_from_slice(entry.data);
385    }
386
387    // -- Backfill CRC32 --
388    let checksum = crc32fast::hash(&buf[8..]); // everything after the checksum field
389    buf[4..8].copy_from_slice(&checksum.to_le_bytes());
390
391    buf
392}
393
394// ---------------------------------------------------------------------------
395// Bundle writer
396// ---------------------------------------------------------------------------
397
398/// Copy the runtime binary at `runtime_path` to `output_path`, then append the
399/// archive bytes and a 16-byte trailer. On Unix, the output is made executable.
400#[allow(dead_code)]
401pub fn write_bundled_executable(
402    runtime_path: &Path,
403    output_path: &Path,
404    archive_bytes: &[u8],
405) -> io::Result<()> {
406    // Read the runtime binary
407    let runtime = std::fs::read(runtime_path)?;
408
409    // Write: runtime + archive + trailer
410    let mut out = std::fs::File::create(output_path)?;
411    out.write_all(&runtime)?;
412    out.write_all(archive_bytes)?;
413
414    // Trailer
415    let archive_size = archive_bytes.len() as u64;
416    out.write_all(&archive_size.to_le_bytes())?;
417    out.write_all(MAGIC)?;
418
419    out.flush()?;
420    drop(out);
421
422    // Make executable on Unix
423    #[cfg(unix)]
424    {
425        use std::os::unix::fs::PermissionsExt;
426        let perms = std::fs::Permissions::from_mode(0o755);
427        std::fs::set_permissions(output_path, perms)?;
428    }
429
430    Ok(())
431}
432
433/// Like `write_bundled_executable`, but takes the runtime binary as a byte
434/// slice instead of reading from disk. Used for cross-compilation where the
435/// runtime bytes were downloaded/cached.
436pub fn write_bundled_executable_from_bytes(
437    runtime: &[u8],
438    output_path: &Path,
439    archive_bytes: &[u8],
440) -> io::Result<()> {
441    let mut out = std::fs::File::create(output_path)?;
442    out.write_all(runtime)?;
443    out.write_all(archive_bytes)?;
444
445    // Trailer
446    let archive_size = archive_bytes.len() as u64;
447    out.write_all(&archive_size.to_le_bytes())?;
448    out.write_all(MAGIC)?;
449
450    out.flush()?;
451    drop(out);
452
453    #[cfg(unix)]
454    {
455        use std::os::unix::fs::PermissionsExt;
456        let perms = std::fs::Permissions::from_mode(0o755);
457        std::fs::set_permissions(output_path, perms)?;
458    }
459
460    Ok(())
461}
462
463// ---------------------------------------------------------------------------
464// Tests
465// ---------------------------------------------------------------------------
466
467#[cfg(test)]
468mod tests {
469    use super::*;
470
471    #[test]
472    fn test_archive_roundtrip() {
473        let mut metadata = HashMap::new();
474        metadata.insert("entry".to_string(), b"main.semac".to_vec());
475        metadata.insert("version".to_string(), b"1".to_vec());
476
477        let mut files = HashMap::new();
478        files.insert("main.semac".to_string(), vec![0xDE, 0xAD, 0xBE, 0xEF]);
479        files.insert("lib/utils.sema".to_string(), b"(define x 42)".to_vec());
480
481        let bytes = serialize_archive(&metadata, &files);
482        let archive = deserialize_archive(&bytes).expect("deserialize should succeed");
483
484        assert_eq!(archive.format_version, FORMAT_VERSION);
485        assert_eq!(archive.flags, 0);
486        assert_eq!(archive.metadata.len(), 2);
487        assert_eq!(archive.metadata.get("entry").unwrap(), b"main.semac");
488        assert_eq!(archive.metadata.get("version").unwrap(), b"1");
489        assert_eq!(archive.files.len(), 2);
490        assert_eq!(
491            archive.files.get("main.semac").unwrap(),
492            &vec![0xDE, 0xAD, 0xBE, 0xEF]
493        );
494        assert_eq!(
495            archive.files.get("lib/utils.sema").unwrap(),
496            b"(define x 42)"
497        );
498    }
499
500    #[test]
501    fn test_archive_empty() {
502        let metadata = HashMap::new();
503        let files = HashMap::new();
504
505        let bytes = serialize_archive(&metadata, &files);
506        let archive = deserialize_archive(&bytes).expect("deserialize should succeed");
507
508        assert_eq!(archive.format_version, FORMAT_VERSION);
509        assert_eq!(archive.flags, 0);
510        assert!(archive.metadata.is_empty());
511        assert!(archive.files.is_empty());
512    }
513
514    #[test]
515    fn test_crc32_known_value() {
516        // CRC32 of empty data should be 0x00000000
517        // Actually, CRC32 of empty is 0x00000000 for our implementation
518        let empty = crc32fast::hash(b"");
519        assert_eq!(empty, 0x0000_0000);
520
521        // CRC32 of "123456789" is 0xCBF43926 (well-known test vector)
522        let check = crc32fast::hash(b"123456789");
523        assert_eq!(check, 0xCBF4_3926);
524    }
525
526    #[test]
527    fn test_checksum_validation() {
528        let metadata = HashMap::new();
529        let files = HashMap::new();
530        let mut bytes = serialize_archive(&metadata, &files);
531
532        // Corrupt the checksum
533        bytes[4] ^= 0xFF;
534
535        let result = deserialize_archive(&bytes);
536        assert!(result.is_err());
537        let err = result.unwrap_err();
538        assert!(
539            err.to_string().contains("checksum mismatch"),
540            "error should mention checksum: {err}"
541        );
542    }
543
544    #[test]
545    fn test_deserialize_archive_from_bytes_public() {
546        let metadata = HashMap::new();
547        let mut files = HashMap::new();
548        files.insert("test.txt".to_string(), b"hello".to_vec());
549
550        let bytes = serialize_archive(&metadata, &files);
551        let archive =
552            deserialize_archive_from_bytes(&bytes).expect("public deserialize should succeed");
553
554        assert_eq!(archive.files.len(), 1);
555        assert_eq!(archive.files.get("test.txt").unwrap(), b"hello");
556    }
557
558    /// Build a minimal archive with a tampered metadata_count or entry_count.
559    /// Recomputes the CRC32 so the checksum passes.
560    fn craft_archive_with_counts(metadata_count: u32, entry_count: u32) -> Vec<u8> {
561        let mut buf: Vec<u8> = Vec::new();
562        // Header
563        buf.extend_from_slice(&FORMAT_VERSION.to_le_bytes());
564        buf.extend_from_slice(&0u16.to_le_bytes()); // flags
565        buf.extend_from_slice(&0u32.to_le_bytes()); // checksum placeholder
566
567        // metadata_count (potentially huge)
568        buf.extend_from_slice(&metadata_count.to_le_bytes());
569        // No actual metadata entries — the loop will hit EOF immediately
570
571        // entry_count (potentially huge)
572        buf.extend_from_slice(&entry_count.to_le_bytes());
573        // No actual TOC entries
574
575        // Backfill CRC32
576        let checksum = crc32fast::hash(&buf[8..]);
577        buf[4..8].copy_from_slice(&checksum.to_le_bytes());
578        buf
579    }
580
581    #[test]
582    fn test_huge_metadata_count_does_not_oom() {
583        // A crafted archive claiming u32::MAX metadata entries but containing none.
584        // Should fail gracefully with an error, not panic/OOM.
585        let data = craft_archive_with_counts(u32::MAX, 0);
586        let result = deserialize_archive(&data);
587        assert!(result.is_err(), "should fail, not OOM");
588    }
589
590    #[test]
591    fn test_huge_entry_count_does_not_oom() {
592        // A crafted archive claiming u32::MAX file entries but containing none.
593        let data = craft_archive_with_counts(0, u32::MAX);
594        let result = deserialize_archive(&data);
595        assert!(result.is_err(), "should fail, not OOM");
596    }
597
598    #[test]
599    fn test_write_and_detect_bundled() {
600        use std::io::Write;
601
602        let dir = std::env::temp_dir().join("sema_archive_test");
603        let _ = std::fs::create_dir_all(&dir);
604
605        let runtime_path = dir.join("fake_runtime");
606        let output_path = dir.join("bundled_output");
607
608        // Create a fake "runtime" binary
609        {
610            let mut f = std::fs::File::create(&runtime_path).unwrap();
611            f.write_all(b"FAKE_RUNTIME_BINARY").unwrap();
612        }
613
614        // Build archive
615        let mut metadata = HashMap::new();
616        metadata.insert("entry".to_string(), b"main.semac".to_vec());
617        let mut files = HashMap::new();
618        files.insert("main.semac".to_string(), vec![1, 2, 3, 4]);
619
620        let archive_bytes = serialize_archive(&metadata, &files);
621
622        // Write bundled executable
623        write_bundled_executable(&runtime_path, &output_path, &archive_bytes).unwrap();
624
625        // Should detect the archive
626        assert!(has_embedded_archive(&output_path).unwrap());
627
628        // Should NOT detect archive in the plain runtime
629        assert!(!has_embedded_archive(&runtime_path).unwrap());
630
631        // Extract and verify
632        let extracted = extract_archive(&output_path).unwrap();
633        assert_eq!(extracted.metadata.get("entry").unwrap(), b"main.semac");
634        assert_eq!(
635            extracted.files.get("main.semac").unwrap(),
636            &vec![1, 2, 3, 4]
637        );
638
639        // Cleanup
640        let _ = std::fs::remove_dir_all(&dir);
641    }
642
643    #[test]
644    fn test_write_bundled_from_bytes_roundtrip() {
645        let dir = std::env::temp_dir().join("sema_archive_from_bytes_test");
646        let _ = std::fs::create_dir_all(&dir);
647        let output_path = dir.join("bundled_from_bytes");
648
649        let runtime = b"FAKE_RUNTIME_BINARY";
650        let mut metadata = HashMap::new();
651        metadata.insert("entry".to_string(), b"main.semac".to_vec());
652        let mut files = HashMap::new();
653        files.insert("main.semac".to_string(), vec![1, 2, 3, 4]);
654        let archive_bytes = serialize_archive(&metadata, &files);
655
656        write_bundled_executable_from_bytes(runtime, &output_path, &archive_bytes).unwrap();
657
658        assert!(has_embedded_archive(&output_path).unwrap());
659        let extracted = extract_archive(&output_path).unwrap();
660        assert_eq!(extracted.metadata.get("entry").unwrap(), b"main.semac");
661        assert_eq!(
662            extracted.files.get("main.semac").unwrap(),
663            &vec![1, 2, 3, 4]
664        );
665
666        let _ = std::fs::remove_dir_all(&dir);
667    }
668}
sema_core/archive.rs

sema_core/
archive.rs
