Skip to main content

clm_rust_core/layer2/
chunk_pruning.rs

1//! Large-Object Content-Defined Chunking, Streaming Range Reads & Anchored Pruning (Sprint 513)
2//! Canonical Path: kernel/clm_rust_core/src/layer2/chunk_pruning.rs
3
4use crate::cpn::mcard_token::TokenMCard;
5use crate::cpn::token_namespace::MCardMarking;
6use crate::layer0::mcard_record::{verify_record, MCardRecord, RecordVerification};
7use std::collections::{BTreeMap, BTreeSet};
8
9pub use super::chunk_manifest::*;
10
11fn hash_bytes_fast(algo: &str, content: &[u8]) -> Result<String, String> {
12    if algo == crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3 {
13        Ok(blake3::hash(content).to_hex().to_string())
14    } else {
15        crate::hash::raw_hash_bytes_for(algo, content)
16    }
17}
18
19pub struct ChunkerProcessPCard {
20    pub min_size: usize,
21    pub avg_size: usize,
22    pub max_size: usize,
23    pub algorithm: String,
24}
25
26impl Default for ChunkerProcessPCard {
27    fn default() -> Self {
28        Self {
29            min_size: 64,
30            avg_size: 256,
31            max_size: 1024,
32            algorithm: crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3.to_string(),
33        }
34    }
35}
36
37impl ChunkerProcessPCard {
38    pub fn chunk_bytes(
39        &self,
40        data: &[u8],
41    ) -> Result<(ChunkManifest, BTreeMap<String, MCardRecord>), String> {
42        let mut store = BTreeMap::new();
43        let boundaries = cdc_split_boundaries(data, self.min_size, self.avg_size, self.max_size);
44        let mut chunks = Vec::with_capacity(boundaries.len());
45        let mut shared_meta: Option<(String, String)> = None;
46
47        for (idx, (offset, length)) in boundaries.into_iter().enumerate() {
48            let slice = &data[offset..offset + length];
49            let qref = if let Some((ref algo, ref g_time)) = shared_meta {
50                let hash = hash_bytes_fast(algo, slice)?;
51                let qref = format!("{}:{hash}", self.algorithm);
52                store.entry(qref.clone()).or_insert_with(|| MCardRecord {
53                    hash,
54                    content: slice.to_vec(),
55                    g_time: g_time.clone(),
56                });
57                qref
58            } else {
59                let first = MCardRecord::create(slice, Some(&self.algorithm))?;
60                let algo = first.g_time.split('|').next().unwrap_or(crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3).to_string();
61                let qref = format!("{}:{}", self.algorithm, first.hash);
62                shared_meta = Some((algo, first.g_time.clone()));
63                store.insert(qref.clone(), first);
64                qref
65            };
66            chunks.push(ChunkDescriptor {
67                index: idx,
68                offset,
69                length,
70                qualified_ref: qref,
71            });
72        }
73
74        let manifest = ChunkManifest {
75            chunker_id: CHUNKER_ID.to_string(),
76            chunker_version: CHUNKER_VERSION,
77            min_size: self.min_size,
78            avg_size: self.avg_size,
79            max_size: self.max_size,
80            total_bytes: data.len(),
81            chunks,
82            algorithm: self.algorithm.clone(),
83            cached_qualified_ref: MemoizedRef::default(),
84        };
85        let m_bytes = manifest.to_canonical_bytes();
86        let m_rec = if let Some((algo, g_time)) = shared_meta {
87            let hash = hash_bytes_fast(&algo, &m_bytes)?;
88            MCardRecord {
89                hash,
90                content: m_bytes,
91                g_time,
92            }
93        } else {
94            MCardRecord::create(&m_bytes, Some(&self.algorithm))?
95        };
96        let m_qref = format!("{}:{}", self.algorithm, m_rec.hash);
97        manifest.set_cached_qualified_ref(m_qref.clone());
98        store.insert(m_qref, m_rec);
99        Ok((manifest, store))
100    }
101}
102
103#[derive(Debug, Clone)]
104pub struct StreamRangeResult {
105    pub committed: bool,
106    pub trapped_to_exception: bool,
107    pub data: Vec<u8>,
108    pub peak_buffer_bytes: usize,
109    pub offending_ref: Option<String>,
110    pub exception_token: Option<TokenMCard>,
111}
112
113fn chunk_exception_result(kind: &str, offending_ref: &str, peak_buf: usize) -> StreamRangeResult {
114    let exc_val = serde_json::json!({
115        "kind": kind,
116        "offending_ref": offending_ref,
117    });
118    let exc_tok = TokenMCard::from_json(&exc_val, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_CHUNK_EXCEPTION, None).ok();
119    StreamRangeResult {
120        committed: false,
121        trapped_to_exception: true,
122        data: Vec::new(),
123        peak_buffer_bytes: peak_buf,
124        offending_ref: Some(offending_ref.to_string()),
125        exception_token: exc_tok,
126    }
127}
128
129pub struct StreamRangeProcessPCard {
130    pub max_buffer_bytes: usize,
131}
132
133impl Default for StreamRangeProcessPCard {
134    fn default() -> Self {
135        Self {
136            max_buffer_bytes: 262_144,
137        }
138    }
139}
140
141impl StreamRangeProcessPCard {
142    pub fn read_range(
143        &self,
144        manifest: &ChunkManifest,
145        store: &BTreeMap<String, MCardRecord>,
146        start: usize,
147        end: Option<usize>,
148    ) -> StreamRangeResult {
149        let target_end = end.unwrap_or(manifest.total_bytes).min(manifest.total_bytes);
150        if start >= target_end || start >= manifest.total_bytes {
151            return StreamRangeResult {
152                committed: true,
153                trapped_to_exception: false,
154                data: Vec::new(),
155                peak_buffer_bytes: 0,
156                offending_ref: None,
157                exception_token: None,
158            };
159        }
160
161        let mut out = Vec::with_capacity(target_end - start);
162        let mut peak_buf = 0usize;
163
164        for desc in &manifest.chunks {
165            let c_start = desc.offset;
166            let c_end = desc.offset + desc.length;
167            if c_end <= start || c_start >= target_end {
168                continue;
169            }
170            let Some(rec) = store.get(&desc.qualified_ref) else {
171                return chunk_exception_result("chunk_missing_exception", &desc.qualified_ref, peak_buf);
172            };
173
174            let (algo, expected_hex) = desc
175                .qualified_ref
176                .split_once(':')
177                .unwrap_or((crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3, &desc.qualified_ref));
178            let verified = match verify_record(rec) {
179                RecordVerification::DeclaredOk { algorithm: rec_algo } => {
180                    let rec_raw = rec.hash.split_once(':').map(|(_, h)| h).unwrap_or(&rec.hash);
181                    rec_algo.eq_ignore_ascii_case(algo) && rec_raw.eq_ignore_ascii_case(expected_hex)
182                }
183                _ => false,
184            };
185            if !verified {
186                return chunk_exception_result("chunk_corrupt_exception", &desc.qualified_ref, peak_buf);
187            }
188
189            peak_buf = peak_buf.max(rec.content.len());
190            let rel_start = start.saturating_sub(c_start);
191            let rel_end = (target_end - c_start).min(desc.length);
192            out.extend_from_slice(&rec.content[rel_start..rel_end]);
193        }
194
195        StreamRangeResult {
196            committed: true,
197            trapped_to_exception: false,
198            data: out,
199            peak_buffer_bytes: peak_buf,
200            offending_ref: None,
201            exception_token: None,
202        }
203    }
204}
205
206#[derive(Debug, Clone, PartialEq, Eq)]
207pub struct PruningReachabilityReport {
208    pub reachable_refs: Vec<String>,
209    pub candidate_refs: Vec<String>,
210}
211
212pub fn compute_reachability(
213    all_object_refs: &BTreeSet<String>,
214    handle_heads: &BTreeSet<String>,
215    pinned_refs: &BTreeSet<String>,
216    active_markings: &[MCardMarking],
217    manifest_children: &BTreeMap<String, Vec<String>>,
218) -> PruningReachabilityReport {
219    let mut roots: BTreeSet<String> = handle_heads.union(pinned_refs).cloned().collect();
220    for marking in active_markings {
221        for refs in marking.by_place.values() {
222            for qref in refs {
223                roots.insert(qref.clone());
224            }
225        }
226    }
227
228    let reachable: BTreeSet<String> = crate::layer2::traversal::reachable_from(
229        roots.into_iter().filter(|r| all_object_refs.contains(r)),
230        |cur| {
231            manifest_children
232                .get(cur)
233                .into_iter()
234                .flatten()
235                .filter(|child| all_object_refs.contains(*child))
236                .cloned()
237                .collect::<Vec<String>>()
238        },
239    )
240    .into_iter()
241    .collect();
242
243    let candidates: Vec<String> = all_object_refs.difference(&reachable).cloned().collect();
244    PruningReachabilityReport {
245        reachable_refs: reachable.into_iter().collect(),
246        candidate_refs: candidates,
247    }
248}
249
250#[derive(Debug, Clone)]
251pub struct PruningAnchorResult {
252    pub committed: bool,
253    pub trapped_to_exception: bool,
254    pub anchor_mcard: TokenMCard,
255    pub archived_refs: Vec<String>,
256    pub remaining_working_refs: Vec<String>,
257}
258
259fn pruning_exception_result(
260    error: &str,
261    working_store: &BTreeMap<String, MCardRecord>,
262) -> PruningAnchorResult {
263    let exc_val = serde_json::json!({
264        "error": error,
265        "kind": "pruning_exception"
266    });
267    let exc = TokenMCard::from_json(&exc_val, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_PRUNING_EXCEPTION, None)
268        .expect("valid exception token");
269    PruningAnchorResult {
270        committed: false,
271        trapped_to_exception: true,
272        anchor_mcard: exc,
273        archived_refs: Vec::new(),
274        remaining_working_refs: working_store.keys().cloned().collect(),
275    }
276}
277
278#[derive(Debug, Clone, Default)]
279pub struct AnchoredPruningProcessPCard;
280
281impl AnchoredPruningProcessPCard {
282    pub fn new() -> Self {
283        Self
284    }
285
286    pub fn compute_reachability(
287        &self,
288        all_object_refs: &BTreeSet<String>,
289        handle_heads: &BTreeSet<String>,
290        pinned_refs: &BTreeSet<String>,
291        active_markings: &[MCardMarking],
292        manifest_children: &BTreeMap<String, Vec<String>>,
293    ) -> PruningReachabilityReport {
294        compute_reachability(
295            all_object_refs,
296            handle_heads,
297            pinned_refs,
298            active_markings,
299            manifest_children,
300        )
301    }
302
303    pub fn execute_anchored_prune(
304        &self,
305        working_store: &mut BTreeMap<String, MCardRecord>,
306        report: &PruningReachabilityReport,
307        sponsor_did: Option<&str>,
308        archive_uri: Option<&str>,
309        archive_store: Option<&mut BTreeMap<String, MCardRecord>>,
310    ) -> PruningAnchorResult {
311        let Some(did) = sponsor_did else {
312            return pruning_exception_result("missing_operator_authorization", working_store);
313        };
314        if !did.starts_with(crate::layer0::vocabulary_identities::DID_KEY) && !did.starts_with(crate::layer0::vocabulary::URI_SCHEME_DID_CLM) {
315            return pruning_exception_result("missing_operator_authorization", working_store);
316        }
317        let Some(uri) = archive_uri else {
318            return pruning_exception_result("missing_or_invalid_archive_uri", working_store);
319        };
320        if !uri.starts_with(crate::layer0::vocabulary::HANDLE_MCARD_ARCHIVE) {
321            return pruning_exception_result("missing_or_invalid_archive_uri", working_store);
322        }
323
324        let mut archived = Vec::new();
325        let mut target_archive = archive_store;
326        for cref in &report.candidate_refs {
327            if let Some(rec) = working_store.get(cref) {
328                if let Some(ref mut arc) = target_archive {
329                    arc.insert(cref.clone(), rec.clone());
330                }
331                archived.push(cref.clone());
332            }
333        }
334
335        let anchor_payload = serde_json::json!({
336            "archive_uri": uri,
337            "archived_refs": archived,
338            "reachable_refs": report.reachable_refs,
339            "schema": crate::layer0::vocabulary::HANDLE_CLM_LAYER2_PRUNING_ANCHOR_V1,
340            "sponsor_did": did,
341        });
342        let anchor_tok =
343            TokenMCard::from_json(&anchor_payload, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_PRUNING_ANCHOR, None)
344                .expect("valid pruning anchor token");
345
346        for cref in &archived {
347            working_store.remove(cref);
348        }
349        working_store.insert(anchor_tok.qualified_hash(), anchor_tok.card.clone());
350
351        PruningAnchorResult {
352            committed: true,
353            trapped_to_exception: false,
354            anchor_mcard: anchor_tok,
355            archived_refs: archived,
356            remaining_working_refs: working_store.keys().cloned().collect(),
357        }
358    }
359}