clm-kernel 0.0.23

Cubical Logic Model (CLM) sovereign execution engine, CPN-VM, and Cordis spatiotemporal substrate
Documentation
//! Large-Object Content-Defined Chunking, Streaming Range Reads & Anchored Pruning (Sprint 513)
//! Canonical Path: kernel/clm_rust_core/src/layer2/chunk_pruning.rs

use crate::cpn::mcard_token::TokenMCard;
use crate::cpn::token_namespace::MCardMarking;
use crate::layer0::mcard_record::{verify_record, MCardRecord, RecordVerification};
use std::collections::{BTreeMap, BTreeSet};

pub use super::chunk_manifest::*;

fn hash_bytes_fast(algo: &str, content: &[u8]) -> Result<String, String> {
    if algo == crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3 {
        Ok(blake3::hash(content).to_hex().to_string())
    } else {
        crate::hash::raw_hash_bytes_for(algo, content)
    }
}

pub struct ChunkerProcessPCard {
    pub min_size: usize,
    pub avg_size: usize,
    pub max_size: usize,
    pub algorithm: String,
}

impl Default for ChunkerProcessPCard {
    fn default() -> Self {
        Self {
            min_size: 64,
            avg_size: 256,
            max_size: 1024,
            algorithm: crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3.to_string(),
        }
    }
}

impl ChunkerProcessPCard {
    pub fn chunk_bytes(
        &self,
        data: &[u8],
    ) -> Result<(ChunkManifest, BTreeMap<String, MCardRecord>), String> {
        let mut store = BTreeMap::new();
        let boundaries = cdc_split_boundaries(data, self.min_size, self.avg_size, self.max_size);
        let mut chunks = Vec::with_capacity(boundaries.len());
        let mut shared_meta: Option<(String, String)> = None;

        for (idx, (offset, length)) in boundaries.into_iter().enumerate() {
            let slice = &data[offset..offset + length];
            let qref = if let Some((ref algo, ref g_time)) = shared_meta {
                let hash = hash_bytes_fast(algo, slice)?;
                let qref = format!("{}:{hash}", self.algorithm);
                store.entry(qref.clone()).or_insert_with(|| MCardRecord {
                    hash,
                    content: slice.to_vec(),
                    g_time: g_time.clone(),
                });
                qref
            } else {
                let first = MCardRecord::create(slice, Some(&self.algorithm))?;
                let algo = first.g_time.split('|').next().unwrap_or(crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3).to_string();
                let qref = format!("{}:{}", self.algorithm, first.hash);
                shared_meta = Some((algo, first.g_time.clone()));
                store.insert(qref.clone(), first);
                qref
            };
            chunks.push(ChunkDescriptor {
                index: idx,
                offset,
                length,
                qualified_ref: qref,
            });
        }

        let manifest = ChunkManifest {
            chunker_id: CHUNKER_ID.to_string(),
            chunker_version: CHUNKER_VERSION,
            min_size: self.min_size,
            avg_size: self.avg_size,
            max_size: self.max_size,
            total_bytes: data.len(),
            chunks,
            algorithm: self.algorithm.clone(),
            cached_qualified_ref: MemoizedRef::default(),
        };
        let m_bytes = manifest.to_canonical_bytes();
        let m_rec = if let Some((algo, g_time)) = shared_meta {
            let hash = hash_bytes_fast(&algo, &m_bytes)?;
            MCardRecord {
                hash,
                content: m_bytes,
                g_time,
            }
        } else {
            MCardRecord::create(&m_bytes, Some(&self.algorithm))?
        };
        let m_qref = format!("{}:{}", self.algorithm, m_rec.hash);
        manifest.set_cached_qualified_ref(m_qref.clone());
        store.insert(m_qref, m_rec);
        Ok((manifest, store))
    }
}

#[derive(Debug, Clone)]
pub struct StreamRangeResult {
    pub committed: bool,
    pub trapped_to_exception: bool,
    pub data: Vec<u8>,
    pub peak_buffer_bytes: usize,
    pub offending_ref: Option<String>,
    pub exception_token: Option<TokenMCard>,
}

fn chunk_exception_result(kind: &str, offending_ref: &str, peak_buf: usize) -> StreamRangeResult {
    let exc_val = serde_json::json!({
        "kind": kind,
        "offending_ref": offending_ref,
    });
    let exc_tok = TokenMCard::from_json(&exc_val, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_CHUNK_EXCEPTION, None).ok();
    StreamRangeResult {
        committed: false,
        trapped_to_exception: true,
        data: Vec::new(),
        peak_buffer_bytes: peak_buf,
        offending_ref: Some(offending_ref.to_string()),
        exception_token: exc_tok,
    }
}

pub struct StreamRangeProcessPCard {
    pub max_buffer_bytes: usize,
}

impl Default for StreamRangeProcessPCard {
    fn default() -> Self {
        Self {
            max_buffer_bytes: 262_144,
        }
    }
}

impl StreamRangeProcessPCard {
    pub fn read_range(
        &self,
        manifest: &ChunkManifest,
        store: &BTreeMap<String, MCardRecord>,
        start: usize,
        end: Option<usize>,
    ) -> StreamRangeResult {
        let target_end = end.unwrap_or(manifest.total_bytes).min(manifest.total_bytes);
        if start >= target_end || start >= manifest.total_bytes {
            return StreamRangeResult {
                committed: true,
                trapped_to_exception: false,
                data: Vec::new(),
                peak_buffer_bytes: 0,
                offending_ref: None,
                exception_token: None,
            };
        }

        let mut out = Vec::with_capacity(target_end - start);
        let mut peak_buf = 0usize;

        for desc in &manifest.chunks {
            let c_start = desc.offset;
            let c_end = desc.offset + desc.length;
            if c_end <= start || c_start >= target_end {
                continue;
            }
            let Some(rec) = store.get(&desc.qualified_ref) else {
                return chunk_exception_result("chunk_missing_exception", &desc.qualified_ref, peak_buf);
            };

            let (algo, expected_hex) = desc
                .qualified_ref
                .split_once(':')
                .unwrap_or((crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3, &desc.qualified_ref));
            let verified = match verify_record(rec) {
                RecordVerification::DeclaredOk { algorithm: rec_algo } => {
                    let rec_raw = rec.hash.split_once(':').map(|(_, h)| h).unwrap_or(&rec.hash);
                    rec_algo.eq_ignore_ascii_case(algo) && rec_raw.eq_ignore_ascii_case(expected_hex)
                }
                _ => false,
            };
            if !verified {
                return chunk_exception_result("chunk_corrupt_exception", &desc.qualified_ref, peak_buf);
            }

            peak_buf = peak_buf.max(rec.content.len());
            let rel_start = start.saturating_sub(c_start);
            let rel_end = (target_end - c_start).min(desc.length);
            out.extend_from_slice(&rec.content[rel_start..rel_end]);
        }

        StreamRangeResult {
            committed: true,
            trapped_to_exception: false,
            data: out,
            peak_buffer_bytes: peak_buf,
            offending_ref: None,
            exception_token: None,
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PruningReachabilityReport {
    pub reachable_refs: Vec<String>,
    pub candidate_refs: Vec<String>,
}

pub fn compute_reachability(
    all_object_refs: &BTreeSet<String>,
    handle_heads: &BTreeSet<String>,
    pinned_refs: &BTreeSet<String>,
    active_markings: &[MCardMarking],
    manifest_children: &BTreeMap<String, Vec<String>>,
) -> PruningReachabilityReport {
    let mut roots: BTreeSet<String> = handle_heads.union(pinned_refs).cloned().collect();
    for marking in active_markings {
        for refs in marking.by_place.values() {
            for qref in refs {
                roots.insert(qref.clone());
            }
        }
    }

    let mut reachable = BTreeSet::new();
    let mut stack: Vec<String> = roots.into_iter().collect();
    while let Some(cur) = stack.pop() {
        if reachable.contains(&cur) || !all_object_refs.contains(&cur) {
            continue;
        }
        reachable.insert(cur.clone());
        if let Some(children) = manifest_children.get(&cur) {
            for child in children {
                if !reachable.contains(child) {
                    stack.push(child.clone());
                }
            }
        }
    }

    let candidates: Vec<String> = all_object_refs.difference(&reachable).cloned().collect();
    PruningReachabilityReport {
        reachable_refs: reachable.into_iter().collect(),
        candidate_refs: candidates,
    }
}

#[derive(Debug, Clone)]
pub struct PruningAnchorResult {
    pub committed: bool,
    pub trapped_to_exception: bool,
    pub anchor_mcard: TokenMCard,
    pub archived_refs: Vec<String>,
    pub remaining_working_refs: Vec<String>,
}

fn pruning_exception_result(
    error: &str,
    working_store: &BTreeMap<String, MCardRecord>,
) -> PruningAnchorResult {
    let exc_val = serde_json::json!({
        "error": error,
        "kind": "pruning_exception"
    });
    let exc = TokenMCard::from_json(&exc_val, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_PRUNING_EXCEPTION, None)
        .expect("valid exception token");
    PruningAnchorResult {
        committed: false,
        trapped_to_exception: true,
        anchor_mcard: exc,
        archived_refs: Vec::new(),
        remaining_working_refs: working_store.keys().cloned().collect(),
    }
}

#[derive(Debug, Clone, Default)]
pub struct AnchoredPruningProcessPCard;

impl AnchoredPruningProcessPCard {
    pub fn new() -> Self {
        Self
    }

    pub fn compute_reachability(
        &self,
        all_object_refs: &BTreeSet<String>,
        handle_heads: &BTreeSet<String>,
        pinned_refs: &BTreeSet<String>,
        active_markings: &[MCardMarking],
        manifest_children: &BTreeMap<String, Vec<String>>,
    ) -> PruningReachabilityReport {
        compute_reachability(
            all_object_refs,
            handle_heads,
            pinned_refs,
            active_markings,
            manifest_children,
        )
    }

    pub fn execute_anchored_prune(
        &self,
        working_store: &mut BTreeMap<String, MCardRecord>,
        report: &PruningReachabilityReport,
        sponsor_did: Option<&str>,
        archive_uri: Option<&str>,
        archive_store: Option<&mut BTreeMap<String, MCardRecord>>,
    ) -> PruningAnchorResult {
        let Some(did) = sponsor_did else {
            return pruning_exception_result("missing_operator_authorization", working_store);
        };
        if !did.starts_with(crate::layer0::vocabulary_identities::DID_KEY) && !did.starts_with(crate::layer0::vocabulary::URI_SCHEME_DID_CLM) {
            return pruning_exception_result("missing_operator_authorization", working_store);
        }
        let Some(uri) = archive_uri else {
            return pruning_exception_result("missing_or_invalid_archive_uri", working_store);
        };
        if !uri.starts_with(crate::layer0::vocabulary::HANDLE_MCARD_ARCHIVE) {
            return pruning_exception_result("missing_or_invalid_archive_uri", working_store);
        }

        let mut archived = Vec::new();
        let mut target_archive = archive_store;
        for cref in &report.candidate_refs {
            if let Some(rec) = working_store.get(cref) {
                if let Some(ref mut arc) = target_archive {
                    arc.insert(cref.clone(), rec.clone());
                }
                archived.push(cref.clone());
            }
        }

        let anchor_payload = serde_json::json!({
            "archive_uri": uri,
            "archived_refs": archived,
            "reachable_refs": report.reachable_refs,
            "schema": crate::layer0::vocabulary::HANDLE_CLM_LAYER2_PRUNING_ANCHOR_V1,
            "sponsor_did": did,
        });
        let anchor_tok =
            TokenMCard::from_json(&anchor_payload, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_PRUNING_ANCHOR, None)
                .expect("valid pruning anchor token");

        for cref in &archived {
            working_store.remove(cref);
        }
        working_store.insert(anchor_tok.qualified_hash(), anchor_tok.card.clone());

        PruningAnchorResult {
            committed: true,
            trapped_to_exception: false,
            anchor_mcard: anchor_tok,
            archived_refs: archived,
            remaining_working_refs: working_store.keys().cloned().collect(),
        }
    }
}