use crate::cpn::mcard_token::TokenMCard;
use crate::cpn::token_namespace::MCardMarking;
use crate::layer0::mcard_record::{verify_record, MCardRecord, RecordVerification};
use std::collections::{BTreeMap, BTreeSet};
pub use super::chunk_manifest::*;
fn hash_bytes_fast(algo: &str, content: &[u8]) -> Result<String, String> {
if algo == crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3 {
Ok(blake3::hash(content).to_hex().to_string())
} else {
crate::hash::raw_hash_bytes_for(algo, content)
}
}
pub struct ChunkerProcessPCard {
pub min_size: usize,
pub avg_size: usize,
pub max_size: usize,
pub algorithm: String,
}
impl Default for ChunkerProcessPCard {
fn default() -> Self {
Self {
min_size: 64,
avg_size: 256,
max_size: 1024,
algorithm: crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3.to_string(),
}
}
}
impl ChunkerProcessPCard {
pub fn chunk_bytes(
&self,
data: &[u8],
) -> Result<(ChunkManifest, BTreeMap<String, MCardRecord>), String> {
let mut store = BTreeMap::new();
let boundaries = cdc_split_boundaries(data, self.min_size, self.avg_size, self.max_size);
let mut chunks = Vec::with_capacity(boundaries.len());
let mut shared_meta: Option<(String, String)> = None;
for (idx, (offset, length)) in boundaries.into_iter().enumerate() {
let slice = &data[offset..offset + length];
let qref = if let Some((ref algo, ref g_time)) = shared_meta {
let hash = hash_bytes_fast(algo, slice)?;
let qref = format!("{}:{hash}", self.algorithm);
store.entry(qref.clone()).or_insert_with(|| MCardRecord {
hash,
content: slice.to_vec(),
g_time: g_time.clone(),
});
qref
} else {
let first = MCardRecord::create(slice, Some(&self.algorithm))?;
let algo = first.g_time.split('|').next().unwrap_or(crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3).to_string();
let qref = format!("{}:{}", self.algorithm, first.hash);
shared_meta = Some((algo, first.g_time.clone()));
store.insert(qref.clone(), first);
qref
};
chunks.push(ChunkDescriptor {
index: idx,
offset,
length,
qualified_ref: qref,
});
}
let manifest = ChunkManifest {
chunker_id: CHUNKER_ID.to_string(),
chunker_version: CHUNKER_VERSION,
min_size: self.min_size,
avg_size: self.avg_size,
max_size: self.max_size,
total_bytes: data.len(),
chunks,
algorithm: self.algorithm.clone(),
cached_qualified_ref: MemoizedRef::default(),
};
let m_bytes = manifest.to_canonical_bytes();
let m_rec = if let Some((algo, g_time)) = shared_meta {
let hash = hash_bytes_fast(&algo, &m_bytes)?;
MCardRecord {
hash,
content: m_bytes,
g_time,
}
} else {
MCardRecord::create(&m_bytes, Some(&self.algorithm))?
};
let m_qref = format!("{}:{}", self.algorithm, m_rec.hash);
manifest.set_cached_qualified_ref(m_qref.clone());
store.insert(m_qref, m_rec);
Ok((manifest, store))
}
}
#[derive(Debug, Clone)]
pub struct StreamRangeResult {
pub committed: bool,
pub trapped_to_exception: bool,
pub data: Vec<u8>,
pub peak_buffer_bytes: usize,
pub offending_ref: Option<String>,
pub exception_token: Option<TokenMCard>,
}
fn chunk_exception_result(kind: &str, offending_ref: &str, peak_buf: usize) -> StreamRangeResult {
let exc_val = serde_json::json!({
"kind": kind,
"offending_ref": offending_ref,
});
let exc_tok = TokenMCard::from_json(&exc_val, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_CHUNK_EXCEPTION, None).ok();
StreamRangeResult {
committed: false,
trapped_to_exception: true,
data: Vec::new(),
peak_buffer_bytes: peak_buf,
offending_ref: Some(offending_ref.to_string()),
exception_token: exc_tok,
}
}
pub struct StreamRangeProcessPCard {
pub max_buffer_bytes: usize,
}
impl Default for StreamRangeProcessPCard {
fn default() -> Self {
Self {
max_buffer_bytes: 262_144,
}
}
}
impl StreamRangeProcessPCard {
pub fn read_range(
&self,
manifest: &ChunkManifest,
store: &BTreeMap<String, MCardRecord>,
start: usize,
end: Option<usize>,
) -> StreamRangeResult {
let target_end = end.unwrap_or(manifest.total_bytes).min(manifest.total_bytes);
if start >= target_end || start >= manifest.total_bytes {
return StreamRangeResult {
committed: true,
trapped_to_exception: false,
data: Vec::new(),
peak_buffer_bytes: 0,
offending_ref: None,
exception_token: None,
};
}
let mut out = Vec::with_capacity(target_end - start);
let mut peak_buf = 0usize;
for desc in &manifest.chunks {
let c_start = desc.offset;
let c_end = desc.offset + desc.length;
if c_end <= start || c_start >= target_end {
continue;
}
let Some(rec) = store.get(&desc.qualified_ref) else {
return chunk_exception_result("chunk_missing_exception", &desc.qualified_ref, peak_buf);
};
let (algo, expected_hex) = desc
.qualified_ref
.split_once(':')
.unwrap_or((crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3, &desc.qualified_ref));
let verified = match verify_record(rec) {
RecordVerification::DeclaredOk { algorithm: rec_algo } => {
let rec_raw = rec.hash.split_once(':').map(|(_, h)| h).unwrap_or(&rec.hash);
rec_algo.eq_ignore_ascii_case(algo) && rec_raw.eq_ignore_ascii_case(expected_hex)
}
_ => false,
};
if !verified {
return chunk_exception_result("chunk_corrupt_exception", &desc.qualified_ref, peak_buf);
}
peak_buf = peak_buf.max(rec.content.len());
let rel_start = start.saturating_sub(c_start);
let rel_end = (target_end - c_start).min(desc.length);
out.extend_from_slice(&rec.content[rel_start..rel_end]);
}
StreamRangeResult {
committed: true,
trapped_to_exception: false,
data: out,
peak_buffer_bytes: peak_buf,
offending_ref: None,
exception_token: None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PruningReachabilityReport {
pub reachable_refs: Vec<String>,
pub candidate_refs: Vec<String>,
}
pub fn compute_reachability(
all_object_refs: &BTreeSet<String>,
handle_heads: &BTreeSet<String>,
pinned_refs: &BTreeSet<String>,
active_markings: &[MCardMarking],
manifest_children: &BTreeMap<String, Vec<String>>,
) -> PruningReachabilityReport {
let mut roots: BTreeSet<String> = handle_heads.union(pinned_refs).cloned().collect();
for marking in active_markings {
for refs in marking.by_place.values() {
for qref in refs {
roots.insert(qref.clone());
}
}
}
let mut reachable = BTreeSet::new();
let mut stack: Vec<String> = roots.into_iter().collect();
while let Some(cur) = stack.pop() {
if reachable.contains(&cur) || !all_object_refs.contains(&cur) {
continue;
}
reachable.insert(cur.clone());
if let Some(children) = manifest_children.get(&cur) {
for child in children {
if !reachable.contains(child) {
stack.push(child.clone());
}
}
}
}
let candidates: Vec<String> = all_object_refs.difference(&reachable).cloned().collect();
PruningReachabilityReport {
reachable_refs: reachable.into_iter().collect(),
candidate_refs: candidates,
}
}
#[derive(Debug, Clone)]
pub struct PruningAnchorResult {
pub committed: bool,
pub trapped_to_exception: bool,
pub anchor_mcard: TokenMCard,
pub archived_refs: Vec<String>,
pub remaining_working_refs: Vec<String>,
}
fn pruning_exception_result(
error: &str,
working_store: &BTreeMap<String, MCardRecord>,
) -> PruningAnchorResult {
let exc_val = serde_json::json!({
"error": error,
"kind": "pruning_exception"
});
let exc = TokenMCard::from_json(&exc_val, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_PRUNING_EXCEPTION, None)
.expect("valid exception token");
PruningAnchorResult {
committed: false,
trapped_to_exception: true,
anchor_mcard: exc,
archived_refs: Vec::new(),
remaining_working_refs: working_store.keys().cloned().collect(),
}
}
#[derive(Debug, Clone, Default)]
pub struct AnchoredPruningProcessPCard;
impl AnchoredPruningProcessPCard {
pub fn new() -> Self {
Self
}
pub fn compute_reachability(
&self,
all_object_refs: &BTreeSet<String>,
handle_heads: &BTreeSet<String>,
pinned_refs: &BTreeSet<String>,
active_markings: &[MCardMarking],
manifest_children: &BTreeMap<String, Vec<String>>,
) -> PruningReachabilityReport {
compute_reachability(
all_object_refs,
handle_heads,
pinned_refs,
active_markings,
manifest_children,
)
}
pub fn execute_anchored_prune(
&self,
working_store: &mut BTreeMap<String, MCardRecord>,
report: &PruningReachabilityReport,
sponsor_did: Option<&str>,
archive_uri: Option<&str>,
archive_store: Option<&mut BTreeMap<String, MCardRecord>>,
) -> PruningAnchorResult {
let Some(did) = sponsor_did else {
return pruning_exception_result("missing_operator_authorization", working_store);
};
if !did.starts_with(crate::layer0::vocabulary_identities::DID_KEY) && !did.starts_with(crate::layer0::vocabulary::URI_SCHEME_DID_CLM) {
return pruning_exception_result("missing_operator_authorization", working_store);
}
let Some(uri) = archive_uri else {
return pruning_exception_result("missing_or_invalid_archive_uri", working_store);
};
if !uri.starts_with(crate::layer0::vocabulary::HANDLE_MCARD_ARCHIVE) {
return pruning_exception_result("missing_or_invalid_archive_uri", working_store);
}
let mut archived = Vec::new();
let mut target_archive = archive_store;
for cref in &report.candidate_refs {
if let Some(rec) = working_store.get(cref) {
if let Some(ref mut arc) = target_archive {
arc.insert(cref.clone(), rec.clone());
}
archived.push(cref.clone());
}
}
let anchor_payload = serde_json::json!({
"archive_uri": uri,
"archived_refs": archived,
"reachable_refs": report.reachable_refs,
"schema": crate::layer0::vocabulary::HANDLE_CLM_LAYER2_PRUNING_ANCHOR_V1,
"sponsor_did": did,
});
let anchor_tok =
TokenMCard::from_json(&anchor_payload, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_PRUNING_ANCHOR, None)
.expect("valid pruning anchor token");
for cref in &archived {
working_store.remove(cref);
}
working_store.insert(anchor_tok.qualified_hash(), anchor_tok.card.clone());
PruningAnchorResult {
committed: true,
trapped_to_exception: false,
anchor_mcard: anchor_tok,
archived_refs: archived,
remaining_working_refs: working_store.keys().cloned().collect(),
}
}
}