Skip to main content

clm_rust_core/layer2/
chunk_pruning.rs

1//! Large-Object Content-Defined Chunking, Streaming Range Reads & Anchored Pruning (Sprint 513)
2//! Canonical Path: kernel/clm_rust_core/src/layer2/chunk_pruning.rs
3
4use crate::cpn::mcard_token::TokenMCard;
5use crate::cpn::token_namespace::MCardMarking;
6use crate::layer0::mcard_record::{verify_record, MCardRecord, RecordVerification};
7use std::collections::{BTreeMap, BTreeSet};
8
9pub use super::chunk_manifest::*;
10
11fn hash_bytes_fast(algo: &str, content: &[u8]) -> Result<String, String> {
12    if algo == crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3 {
13        Ok(blake3::hash(content).to_hex().to_string())
14    } else {
15        crate::hash::raw_hash_bytes_for(algo, content)
16    }
17}
18
19pub struct ChunkerProcessPCard {
20    pub min_size: usize,
21    pub avg_size: usize,
22    pub max_size: usize,
23    pub algorithm: String,
24}
25
26impl Default for ChunkerProcessPCard {
27    fn default() -> Self {
28        Self {
29            min_size: 64,
30            avg_size: 256,
31            max_size: 1024,
32            algorithm: crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3.to_string(),
33        }
34    }
35}
36
37impl ChunkerProcessPCard {
38    pub fn chunk_bytes(
39        &self,
40        data: &[u8],
41    ) -> Result<(ChunkManifest, BTreeMap<String, MCardRecord>), String> {
42        let mut store = BTreeMap::new();
43        let boundaries = cdc_split_boundaries(data, self.min_size, self.avg_size, self.max_size);
44        let mut chunks = Vec::with_capacity(boundaries.len());
45        let mut shared_meta: Option<(String, String)> = None;
46
47        for (idx, (offset, length)) in boundaries.into_iter().enumerate() {
48            let slice = &data[offset..offset + length];
49            let qref = if let Some((ref algo, ref g_time)) = shared_meta {
50                let hash = hash_bytes_fast(algo, slice)?;
51                let qref = format!("{}:{hash}", self.algorithm);
52                store.entry(qref.clone()).or_insert_with(|| MCardRecord {
53                    hash,
54                    content: slice.to_vec(),
55                    g_time: g_time.clone(),
56                });
57                qref
58            } else {
59                let first = MCardRecord::create(slice, Some(&self.algorithm))?;
60                let algo = first.g_time.split('|').next().unwrap_or(crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3).to_string();
61                let qref = format!("{}:{}", self.algorithm, first.hash);
62                shared_meta = Some((algo, first.g_time.clone()));
63                store.insert(qref.clone(), first);
64                qref
65            };
66            chunks.push(ChunkDescriptor {
67                index: idx,
68                offset,
69                length,
70                qualified_ref: qref,
71            });
72        }
73
74        let manifest = ChunkManifest {
75            chunker_id: CHUNKER_ID.to_string(),
76            chunker_version: CHUNKER_VERSION,
77            min_size: self.min_size,
78            avg_size: self.avg_size,
79            max_size: self.max_size,
80            total_bytes: data.len(),
81            chunks,
82            algorithm: self.algorithm.clone(),
83            cached_qualified_ref: MemoizedRef::default(),
84        };
85        let m_bytes = manifest.to_canonical_bytes();
86        let m_rec = if let Some((algo, g_time)) = shared_meta {
87            let hash = hash_bytes_fast(&algo, &m_bytes)?;
88            MCardRecord {
89                hash,
90                content: m_bytes,
91                g_time,
92            }
93        } else {
94            MCardRecord::create(&m_bytes, Some(&self.algorithm))?
95        };
96        let m_qref = format!("{}:{}", self.algorithm, m_rec.hash);
97        manifest.set_cached_qualified_ref(m_qref.clone());
98        store.insert(m_qref, m_rec);
99        Ok((manifest, store))
100    }
101}
102
103#[derive(Debug, Clone)]
104pub struct StreamRangeResult {
105    pub committed: bool,
106    pub trapped_to_exception: bool,
107    pub data: Vec<u8>,
108    pub peak_buffer_bytes: usize,
109    pub offending_ref: Option<String>,
110    pub exception_token: Option<TokenMCard>,
111}
112
113fn chunk_exception_result(kind: &str, offending_ref: &str, peak_buf: usize) -> StreamRangeResult {
114    let exc_val = serde_json::json!({
115        "kind": kind,
116        "offending_ref": offending_ref,
117    });
118    let exc_tok = TokenMCard::from_json(&exc_val, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_CHUNK_EXCEPTION, None).ok();
119    StreamRangeResult {
120        committed: false,
121        trapped_to_exception: true,
122        data: Vec::new(),
123        peak_buffer_bytes: peak_buf,
124        offending_ref: Some(offending_ref.to_string()),
125        exception_token: exc_tok,
126    }
127}
128
129pub struct StreamRangeProcessPCard {
130    pub max_buffer_bytes: usize,
131}
132
133impl Default for StreamRangeProcessPCard {
134    fn default() -> Self {
135        Self {
136            max_buffer_bytes: 262_144,
137        }
138    }
139}
140
141impl StreamRangeProcessPCard {
142    pub fn read_range(
143        &self,
144        manifest: &ChunkManifest,
145        store: &BTreeMap<String, MCardRecord>,
146        start: usize,
147        end: Option<usize>,
148    ) -> StreamRangeResult {
149        let target_end = end.unwrap_or(manifest.total_bytes).min(manifest.total_bytes);
150        if start >= target_end || start >= manifest.total_bytes {
151            return StreamRangeResult {
152                committed: true,
153                trapped_to_exception: false,
154                data: Vec::new(),
155                peak_buffer_bytes: 0,
156                offending_ref: None,
157                exception_token: None,
158            };
159        }
160
161        let mut out = Vec::with_capacity(target_end - start);
162        let mut peak_buf = 0usize;
163
164        for desc in &manifest.chunks {
165            let c_start = desc.offset;
166            let c_end = desc.offset + desc.length;
167            if c_end <= start || c_start >= target_end {
168                continue;
169            }
170            let Some(rec) = store.get(&desc.qualified_ref) else {
171                return chunk_exception_result("chunk_missing_exception", &desc.qualified_ref, peak_buf);
172            };
173
174            let (algo, expected_hex) = desc
175                .qualified_ref
176                .split_once(':')
177                .unwrap_or((crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3, &desc.qualified_ref));
178            let verified = match verify_record(rec) {
179                RecordVerification::DeclaredOk { algorithm: rec_algo } => {
180                    let rec_raw = rec.hash.split_once(':').map(|(_, h)| h).unwrap_or(&rec.hash);
181                    rec_algo.eq_ignore_ascii_case(algo) && rec_raw.eq_ignore_ascii_case(expected_hex)
182                }
183                _ => false,
184            };
185            if !verified {
186                return chunk_exception_result("chunk_corrupt_exception", &desc.qualified_ref, peak_buf);
187            }
188
189            peak_buf = peak_buf.max(rec.content.len());
190            let rel_start = start.saturating_sub(c_start);
191            let rel_end = (target_end - c_start).min(desc.length);
192            out.extend_from_slice(&rec.content[rel_start..rel_end]);
193        }
194
195        StreamRangeResult {
196            committed: true,
197            trapped_to_exception: false,
198            data: out,
199            peak_buffer_bytes: peak_buf,
200            offending_ref: None,
201            exception_token: None,
202        }
203    }
204}
205
206#[derive(Debug, Clone, PartialEq, Eq)]
207pub struct PruningReachabilityReport {
208    pub reachable_refs: Vec<String>,
209    pub candidate_refs: Vec<String>,
210}
211
212pub fn compute_reachability(
213    all_object_refs: &BTreeSet<String>,
214    handle_heads: &BTreeSet<String>,
215    pinned_refs: &BTreeSet<String>,
216    active_markings: &[MCardMarking],
217    manifest_children: &BTreeMap<String, Vec<String>>,
218) -> PruningReachabilityReport {
219    let mut roots: BTreeSet<String> = handle_heads.union(pinned_refs).cloned().collect();
220    for marking in active_markings {
221        for refs in marking.by_place.values() {
222            for qref in refs {
223                roots.insert(qref.clone());
224            }
225        }
226    }
227
228    let mut reachable = BTreeSet::new();
229    let mut stack: Vec<String> = roots.into_iter().collect();
230    while let Some(cur) = stack.pop() {
231        if reachable.contains(&cur) || !all_object_refs.contains(&cur) {
232            continue;
233        }
234        reachable.insert(cur.clone());
235        if let Some(children) = manifest_children.get(&cur) {
236            for child in children {
237                if !reachable.contains(child) {
238                    stack.push(child.clone());
239                }
240            }
241        }
242    }
243
244    let candidates: Vec<String> = all_object_refs.difference(&reachable).cloned().collect();
245    PruningReachabilityReport {
246        reachable_refs: reachable.into_iter().collect(),
247        candidate_refs: candidates,
248    }
249}
250
251#[derive(Debug, Clone)]
252pub struct PruningAnchorResult {
253    pub committed: bool,
254    pub trapped_to_exception: bool,
255    pub anchor_mcard: TokenMCard,
256    pub archived_refs: Vec<String>,
257    pub remaining_working_refs: Vec<String>,
258}
259
260fn pruning_exception_result(
261    error: &str,
262    working_store: &BTreeMap<String, MCardRecord>,
263) -> PruningAnchorResult {
264    let exc_val = serde_json::json!({
265        "error": error,
266        "kind": "pruning_exception"
267    });
268    let exc = TokenMCard::from_json(&exc_val, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_PRUNING_EXCEPTION, None)
269        .expect("valid exception token");
270    PruningAnchorResult {
271        committed: false,
272        trapped_to_exception: true,
273        anchor_mcard: exc,
274        archived_refs: Vec::new(),
275        remaining_working_refs: working_store.keys().cloned().collect(),
276    }
277}
278
279#[derive(Debug, Clone, Default)]
280pub struct AnchoredPruningProcessPCard;
281
282impl AnchoredPruningProcessPCard {
283    pub fn new() -> Self {
284        Self
285    }
286
287    pub fn compute_reachability(
288        &self,
289        all_object_refs: &BTreeSet<String>,
290        handle_heads: &BTreeSet<String>,
291        pinned_refs: &BTreeSet<String>,
292        active_markings: &[MCardMarking],
293        manifest_children: &BTreeMap<String, Vec<String>>,
294    ) -> PruningReachabilityReport {
295        compute_reachability(
296            all_object_refs,
297            handle_heads,
298            pinned_refs,
299            active_markings,
300            manifest_children,
301        )
302    }
303
304    pub fn execute_anchored_prune(
305        &self,
306        working_store: &mut BTreeMap<String, MCardRecord>,
307        report: &PruningReachabilityReport,
308        sponsor_did: Option<&str>,
309        archive_uri: Option<&str>,
310        archive_store: Option<&mut BTreeMap<String, MCardRecord>>,
311    ) -> PruningAnchorResult {
312        let Some(did) = sponsor_did else {
313            return pruning_exception_result("missing_operator_authorization", working_store);
314        };
315        if !did.starts_with(crate::layer0::vocabulary_identities::DID_KEY) && !did.starts_with(crate::layer0::vocabulary::URI_SCHEME_DID_CLM) {
316            return pruning_exception_result("missing_operator_authorization", working_store);
317        }
318        let Some(uri) = archive_uri else {
319            return pruning_exception_result("missing_or_invalid_archive_uri", working_store);
320        };
321        if !uri.starts_with(crate::layer0::vocabulary::HANDLE_MCARD_ARCHIVE) {
322            return pruning_exception_result("missing_or_invalid_archive_uri", working_store);
323        }
324
325        let mut archived = Vec::new();
326        let mut target_archive = archive_store;
327        for cref in &report.candidate_refs {
328            if let Some(rec) = working_store.get(cref) {
329                if let Some(ref mut arc) = target_archive {
330                    arc.insert(cref.clone(), rec.clone());
331                }
332                archived.push(cref.clone());
333            }
334        }
335
336        let anchor_payload = serde_json::json!({
337            "archive_uri": uri,
338            "archived_refs": archived,
339            "reachable_refs": report.reachable_refs,
340            "schema": crate::layer0::vocabulary::HANDLE_CLM_LAYER2_PRUNING_ANCHOR_V1,
341            "sponsor_did": did,
342        });
343        let anchor_tok =
344            TokenMCard::from_json(&anchor_payload, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_PRUNING_ANCHOR, None)
345                .expect("valid pruning anchor token");
346
347        for cref in &archived {
348            working_store.remove(cref);
349        }
350        working_store.insert(anchor_tok.qualified_hash(), anchor_tok.card.clone());
351
352        PruningAnchorResult {
353            committed: true,
354            trapped_to_exception: false,
355            anchor_mcard: anchor_tok,
356            archived_refs: archived,
357            remaining_working_refs: working_store.keys().cloned().collect(),
358        }
359    }
360}