void_core/index/

io.rs

//! Read/write operations for the workspace index with sharded encryption.
//!
//! On-disk format (sharded):
//!   .void/index/
//!     index.bin      — header: version, commit_cid, shard_count, per-shard metadata
//!     shard_00.bin   — ~4096 entries each, independently zstd + AES-GCM encrypted
//!     shard_01.bin
//!     ...
//!
//! Each shard contains a CBOR-serialized Vec<IndexEntry>, compressed with zstd level 3,
//! then encrypted with AES-256-GCM. Shards are read/written in parallel with rayon.

use std::fs;
use std::path::{Path, PathBuf};

use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use void_crypto::EncryptedIndex;
use void_crypto::CommitCid;

use crate::crypto::SecretKey;
use crate::{Result, VoidError};

use super::types::IndexEntry;
use super::workspace::WorkspaceIndex;

const INDEX_FILENAME: &str = "index.bin";
const ZSTD_LEVEL: i32 = 3;
const SHARD_TARGET_ENTRIES: usize = 4096;

fn index_dir(void_dir: &Path) -> PathBuf {
    void_dir.join("index")
}

fn index_path(void_dir: &Path) -> PathBuf {
    index_dir(void_dir).join(INDEX_FILENAME)
}

fn shard_path(void_dir: &Path, shard_index: usize) -> PathBuf {
    index_dir(void_dir).join(format!("shard_{:04}.bin", shard_index))
}

/// Compress bytes with zstd.
fn compress(data: &[u8]) -> Result<Vec<u8>> {
    zstd::encode_all(data, ZSTD_LEVEL).map_err(VoidError::Io)
}

/// Decompress zstd bytes.
fn decompress(data: &[u8]) -> Result<Vec<u8>> {
    zstd::decode_all(data).map_err(VoidError::Io)
}

// ============================================================================
// Sharded index header
// ============================================================================

#[derive(Serialize, Deserialize)]
struct ShardedIndexHeader {
    version: u32,
    commit_cid: Option<CommitCid>,
    shard_count: u32,
    entries_per_shard: u32,
    total_entries: u64,
    shard_meta: Vec<ShardMeta>,
}

#[derive(Serialize, Deserialize)]
struct ShardMeta {
    first_path: String,
    last_path: String,
    entry_count: u32,
}

const SHARDED_INDEX_VERSION: u32 = 2;

// ============================================================================
// Write
// ============================================================================

/// Writes the encrypted workspace index to disk (sharded).
pub fn write_index(
    void_dir: impl AsRef<Path>,
    key: &SecretKey,
    commit_cid: Option<CommitCid>,
    entries: Vec<IndexEntry>,
) -> Result<()> {
    let index = WorkspaceIndex::new(commit_cid, entries);
    write_workspace_index(void_dir, key, &index)
}

/// Writes a WorkspaceIndex directly to disk (sharded).
pub fn write_workspace_index(
    void_dir: impl AsRef<Path>,
    key: &SecretKey,
    index: &WorkspaceIndex,
) -> Result<()> {
    let void_dir = void_dir.as_ref();
    let dir = index_dir(void_dir);
    fs::create_dir_all(&dir)?;

    let entries = &index.entries;
    let chunks: Vec<&[IndexEntry]> = if entries.is_empty() {
        Vec::new()
    } else {
        entries.chunks(SHARD_TARGET_ENTRIES).collect()
    };
    let shard_count = chunks.len();

    // Write shard files in parallel
    let shard_meta: Vec<ShardMeta> = chunks
        .par_iter()
        .enumerate()
        .map(|(i, chunk)| {
            let cbor = crate::support::cbor_to_vec(chunk)?;
            let compressed = compress(&cbor)?;
            let blob = EncryptedIndex::encrypt(key.as_bytes(), &compressed)?;

            let path = shard_path(void_dir, i);
            let temp = path.with_extension("tmp");
            fs::write(&temp, blob.as_bytes())?;
            fs::rename(&temp, &path)?;

            Ok(ShardMeta {
                first_path: chunk.first().map(|e| e.path.clone()).unwrap_or_default(),
                last_path: chunk.last().map(|e| e.path.clone()).unwrap_or_default(),
                entry_count: chunk.len() as u32,
            })
        })
        .collect::<Result<Vec<_>>>()?;

    // Write header
    let header = ShardedIndexHeader {
        version: SHARDED_INDEX_VERSION,
        commit_cid: index.commit_cid.clone(),
        shard_count: shard_count as u32,
        entries_per_shard: SHARD_TARGET_ENTRIES as u32,
        total_entries: entries.len() as u64,
        shard_meta,
    };
    let cbor = crate::support::cbor_to_vec(&header)?;
    let compressed = compress(&cbor)?;
    let blob = EncryptedIndex::encrypt(key.as_bytes(), &compressed)?;

    let header_path = index_path(void_dir);
    let temp = header_path.with_extension("tmp");
    fs::write(&temp, blob.as_bytes())?;
    fs::rename(&temp, &header_path)?;

    // Clean up stale shard files
    for i in shard_count.. {
        let stale = shard_path(void_dir, i);
        if stale.exists() {
            let _ = fs::remove_file(&stale);
        } else {
            break;
        }
    }

    Ok(())
}

// ============================================================================
// Read
// ============================================================================

/// Reads the encrypted workspace index from disk (sharded).
pub fn read_index(void_dir: impl AsRef<Path>, key: &SecretKey) -> Result<WorkspaceIndex> {
    let void_dir = void_dir.as_ref();
    let header_path = index_path(void_dir);

    let raw = fs::read(&header_path).map_err(|e| {
        if e.kind() == std::io::ErrorKind::NotFound {
            VoidError::NotFound("index".into())
        } else {
            VoidError::Io(e)
        }
    })?;

    let blob = EncryptedIndex::from_bytes(raw);
    let decrypted = blob.decrypt(key.as_bytes())?;
    let decompressed = decompress(&decrypted).map_err(|_| {
        VoidError::Serialization(
            "index format not recognized — delete .void/index/ and re-run void add".into(),
        )
    })?;

    let header: ShardedIndexHeader = ciborium::from_reader(&decompressed[..]).map_err(|_| {
        VoidError::Serialization(
            "index format not recognized — delete .void/index/ and re-run void add".into(),
        )
    })?;

    if header.shard_count == 0 {
        return Ok(WorkspaceIndex::new(header.commit_cid, Vec::new()));
    }

    // Read shard files in parallel
    let shard_entries: Vec<Vec<IndexEntry>> = (0..header.shard_count as usize)
        .into_par_iter()
        .map(|i| {
            let path = shard_path(void_dir, i);
            let raw = fs::read(&path).map_err(|e| {
                if e.kind() == std::io::ErrorKind::NotFound {
                    VoidError::NotFound(format!("index shard {}", i))
                } else {
                    VoidError::Io(e)
                }
            })?;
            let blob = EncryptedIndex::from_bytes(raw);
            let decrypted = blob.decrypt(key.as_bytes())?;
            let decompressed = decompress(&decrypted)?;
            let entries: Vec<IndexEntry> = ciborium::from_reader(&decompressed[..])
                .map_err(|e| VoidError::Serialization(e.to_string()))?;
            Ok(entries)
        })
        .collect::<Result<Vec<_>>>()?;

    // Concatenate in order — shards are contiguous sorted path ranges
    let total = header.total_entries as usize;
    let mut entries = Vec::with_capacity(total);
    for shard in shard_entries {
        entries.extend(shard);
    }

    Ok(WorkspaceIndex::new(header.commit_cid, entries))
}