1use crate::cpn::mcard_token::TokenMCard;
5use crate::cpn::token_namespace::MCardMarking;
6use crate::layer0::mcard_record::{verify_record, MCardRecord, RecordVerification};
7use std::collections::{BTreeMap, BTreeSet};
8
9pub use super::chunk_manifest::*;
10
11fn hash_bytes_fast(algo: &str, content: &[u8]) -> Result<String, String> {
12 if algo == crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3 {
13 Ok(blake3::hash(content).to_hex().to_string())
14 } else {
15 crate::hash::raw_hash_bytes_for(algo, content)
16 }
17}
18
19pub struct ChunkerProcessPCard {
20 pub min_size: usize,
21 pub avg_size: usize,
22 pub max_size: usize,
23 pub algorithm: String,
24}
25
26impl Default for ChunkerProcessPCard {
27 fn default() -> Self {
28 Self {
29 min_size: 64,
30 avg_size: 256,
31 max_size: 1024,
32 algorithm: crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3.to_string(),
33 }
34 }
35}
36
37impl ChunkerProcessPCard {
38 pub fn chunk_bytes(
39 &self,
40 data: &[u8],
41 ) -> Result<(ChunkManifest, BTreeMap<String, MCardRecord>), String> {
42 let mut store = BTreeMap::new();
43 let boundaries = cdc_split_boundaries(data, self.min_size, self.avg_size, self.max_size);
44 let mut chunks = Vec::with_capacity(boundaries.len());
45 let mut shared_meta: Option<(String, String)> = None;
46
47 for (idx, (offset, length)) in boundaries.into_iter().enumerate() {
48 let slice = &data[offset..offset + length];
49 let qref = if let Some((ref algo, ref g_time)) = shared_meta {
50 let hash = hash_bytes_fast(algo, slice)?;
51 let qref = format!("{}:{hash}", self.algorithm);
52 store.entry(qref.clone()).or_insert_with(|| MCardRecord {
53 hash,
54 content: slice.to_vec(),
55 g_time: g_time.clone(),
56 });
57 qref
58 } else {
59 let first = MCardRecord::create(slice, Some(&self.algorithm))?;
60 let algo = first.g_time.split('|').next().unwrap_or(crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3).to_string();
61 let qref = format!("{}:{}", self.algorithm, first.hash);
62 shared_meta = Some((algo, first.g_time.clone()));
63 store.insert(qref.clone(), first);
64 qref
65 };
66 chunks.push(ChunkDescriptor {
67 index: idx,
68 offset,
69 length,
70 qualified_ref: qref,
71 });
72 }
73
74 let manifest = ChunkManifest {
75 chunker_id: CHUNKER_ID.to_string(),
76 chunker_version: CHUNKER_VERSION,
77 min_size: self.min_size,
78 avg_size: self.avg_size,
79 max_size: self.max_size,
80 total_bytes: data.len(),
81 chunks,
82 algorithm: self.algorithm.clone(),
83 cached_qualified_ref: MemoizedRef::default(),
84 };
85 let m_bytes = manifest.to_canonical_bytes();
86 let m_rec = if let Some((algo, g_time)) = shared_meta {
87 let hash = hash_bytes_fast(&algo, &m_bytes)?;
88 MCardRecord {
89 hash,
90 content: m_bytes,
91 g_time,
92 }
93 } else {
94 MCardRecord::create(&m_bytes, Some(&self.algorithm))?
95 };
96 let m_qref = format!("{}:{}", self.algorithm, m_rec.hash);
97 manifest.set_cached_qualified_ref(m_qref.clone());
98 store.insert(m_qref, m_rec);
99 Ok((manifest, store))
100 }
101}
102
103#[derive(Debug, Clone)]
104pub struct StreamRangeResult {
105 pub committed: bool,
106 pub trapped_to_exception: bool,
107 pub data: Vec<u8>,
108 pub peak_buffer_bytes: usize,
109 pub offending_ref: Option<String>,
110 pub exception_token: Option<TokenMCard>,
111}
112
113fn chunk_exception_result(kind: &str, offending_ref: &str, peak_buf: usize) -> StreamRangeResult {
114 let exc_val = serde_json::json!({
115 "kind": kind,
116 "offending_ref": offending_ref,
117 });
118 let exc_tok = TokenMCard::from_json(&exc_val, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_CHUNK_EXCEPTION, None).ok();
119 StreamRangeResult {
120 committed: false,
121 trapped_to_exception: true,
122 data: Vec::new(),
123 peak_buffer_bytes: peak_buf,
124 offending_ref: Some(offending_ref.to_string()),
125 exception_token: exc_tok,
126 }
127}
128
129pub struct StreamRangeProcessPCard {
130 pub max_buffer_bytes: usize,
131}
132
133impl Default for StreamRangeProcessPCard {
134 fn default() -> Self {
135 Self {
136 max_buffer_bytes: 262_144,
137 }
138 }
139}
140
141impl StreamRangeProcessPCard {
142 pub fn read_range(
143 &self,
144 manifest: &ChunkManifest,
145 store: &BTreeMap<String, MCardRecord>,
146 start: usize,
147 end: Option<usize>,
148 ) -> StreamRangeResult {
149 let target_end = end.unwrap_or(manifest.total_bytes).min(manifest.total_bytes);
150 if start >= target_end || start >= manifest.total_bytes {
151 return StreamRangeResult {
152 committed: true,
153 trapped_to_exception: false,
154 data: Vec::new(),
155 peak_buffer_bytes: 0,
156 offending_ref: None,
157 exception_token: None,
158 };
159 }
160
161 let mut out = Vec::with_capacity(target_end - start);
162 let mut peak_buf = 0usize;
163
164 for desc in &manifest.chunks {
165 let c_start = desc.offset;
166 let c_end = desc.offset + desc.length;
167 if c_end <= start || c_start >= target_end {
168 continue;
169 }
170 let Some(rec) = store.get(&desc.qualified_ref) else {
171 return chunk_exception_result("chunk_missing_exception", &desc.qualified_ref, peak_buf);
172 };
173
174 let (algo, expected_hex) = desc
175 .qualified_ref
176 .split_once(':')
177 .unwrap_or((crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3, &desc.qualified_ref));
178 let verified = match verify_record(rec) {
179 RecordVerification::DeclaredOk { algorithm: rec_algo } => {
180 let rec_raw = rec.hash.split_once(':').map(|(_, h)| h).unwrap_or(&rec.hash);
181 rec_algo.eq_ignore_ascii_case(algo) && rec_raw.eq_ignore_ascii_case(expected_hex)
182 }
183 _ => false,
184 };
185 if !verified {
186 return chunk_exception_result("chunk_corrupt_exception", &desc.qualified_ref, peak_buf);
187 }
188
189 peak_buf = peak_buf.max(rec.content.len());
190 let rel_start = start.saturating_sub(c_start);
191 let rel_end = (target_end - c_start).min(desc.length);
192 out.extend_from_slice(&rec.content[rel_start..rel_end]);
193 }
194
195 StreamRangeResult {
196 committed: true,
197 trapped_to_exception: false,
198 data: out,
199 peak_buffer_bytes: peak_buf,
200 offending_ref: None,
201 exception_token: None,
202 }
203 }
204}
205
206#[derive(Debug, Clone, PartialEq, Eq)]
207pub struct PruningReachabilityReport {
208 pub reachable_refs: Vec<String>,
209 pub candidate_refs: Vec<String>,
210}
211
212pub fn compute_reachability(
213 all_object_refs: &BTreeSet<String>,
214 handle_heads: &BTreeSet<String>,
215 pinned_refs: &BTreeSet<String>,
216 active_markings: &[MCardMarking],
217 manifest_children: &BTreeMap<String, Vec<String>>,
218) -> PruningReachabilityReport {
219 let mut roots: BTreeSet<String> = handle_heads.union(pinned_refs).cloned().collect();
220 for marking in active_markings {
221 for refs in marking.by_place.values() {
222 for qref in refs {
223 roots.insert(qref.clone());
224 }
225 }
226 }
227
228 let reachable: BTreeSet<String> = crate::layer2::traversal::reachable_from(
229 roots.into_iter().filter(|r| all_object_refs.contains(r)),
230 |cur| {
231 manifest_children
232 .get(cur)
233 .into_iter()
234 .flatten()
235 .filter(|child| all_object_refs.contains(*child))
236 .cloned()
237 .collect::<Vec<String>>()
238 },
239 )
240 .into_iter()
241 .collect();
242
243 let candidates: Vec<String> = all_object_refs.difference(&reachable).cloned().collect();
244 PruningReachabilityReport {
245 reachable_refs: reachable.into_iter().collect(),
246 candidate_refs: candidates,
247 }
248}
249
250#[derive(Debug, Clone)]
251pub struct PruningAnchorResult {
252 pub committed: bool,
253 pub trapped_to_exception: bool,
254 pub anchor_mcard: TokenMCard,
255 pub archived_refs: Vec<String>,
256 pub remaining_working_refs: Vec<String>,
257}
258
259fn pruning_exception_result(
260 error: &str,
261 working_store: &BTreeMap<String, MCardRecord>,
262) -> PruningAnchorResult {
263 let exc_val = serde_json::json!({
264 "error": error,
265 "kind": "pruning_exception"
266 });
267 let exc = TokenMCard::from_json(&exc_val, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_PRUNING_EXCEPTION, None)
268 .expect("valid exception token");
269 PruningAnchorResult {
270 committed: false,
271 trapped_to_exception: true,
272 anchor_mcard: exc,
273 archived_refs: Vec::new(),
274 remaining_working_refs: working_store.keys().cloned().collect(),
275 }
276}
277
278#[derive(Debug, Clone, Default)]
279pub struct AnchoredPruningProcessPCard;
280
281impl AnchoredPruningProcessPCard {
282 pub fn new() -> Self {
283 Self
284 }
285
286 pub fn compute_reachability(
287 &self,
288 all_object_refs: &BTreeSet<String>,
289 handle_heads: &BTreeSet<String>,
290 pinned_refs: &BTreeSet<String>,
291 active_markings: &[MCardMarking],
292 manifest_children: &BTreeMap<String, Vec<String>>,
293 ) -> PruningReachabilityReport {
294 compute_reachability(
295 all_object_refs,
296 handle_heads,
297 pinned_refs,
298 active_markings,
299 manifest_children,
300 )
301 }
302
303 pub fn execute_anchored_prune(
304 &self,
305 working_store: &mut BTreeMap<String, MCardRecord>,
306 report: &PruningReachabilityReport,
307 sponsor_did: Option<&str>,
308 archive_uri: Option<&str>,
309 archive_store: Option<&mut BTreeMap<String, MCardRecord>>,
310 ) -> PruningAnchorResult {
311 let Some(did) = sponsor_did else {
312 return pruning_exception_result("missing_operator_authorization", working_store);
313 };
314 if !did.starts_with(crate::layer0::vocabulary_identities::DID_KEY) && !did.starts_with(crate::layer0::vocabulary::URI_SCHEME_DID_CLM) {
315 return pruning_exception_result("missing_operator_authorization", working_store);
316 }
317 let Some(uri) = archive_uri else {
318 return pruning_exception_result("missing_or_invalid_archive_uri", working_store);
319 };
320 if !uri.starts_with(crate::layer0::vocabulary::HANDLE_MCARD_ARCHIVE) {
321 return pruning_exception_result("missing_or_invalid_archive_uri", working_store);
322 }
323
324 let mut archived = Vec::new();
325 let mut target_archive = archive_store;
326 for cref in &report.candidate_refs {
327 if let Some(rec) = working_store.get(cref) {
328 if let Some(ref mut arc) = target_archive {
329 arc.insert(cref.clone(), rec.clone());
330 }
331 archived.push(cref.clone());
332 }
333 }
334
335 let anchor_payload = serde_json::json!({
336 "archive_uri": uri,
337 "archived_refs": archived,
338 "reachable_refs": report.reachable_refs,
339 "schema": crate::layer0::vocabulary::HANDLE_CLM_LAYER2_PRUNING_ANCHOR_V1,
340 "sponsor_did": did,
341 });
342 let anchor_tok =
343 TokenMCard::from_json(&anchor_payload, crate::layer0::vocabulary_types::TYPE_URI_CLM_TYPE_STORAGE_PRUNING_ANCHOR, None)
344 .expect("valid pruning anchor token");
345
346 for cref in &archived {
347 working_store.remove(cref);
348 }
349 working_store.insert(anchor_tok.qualified_hash(), anchor_tok.card.clone());
350
351 PruningAnchorResult {
352 committed: true,
353 trapped_to_exception: false,
354 anchor_mcard: anchor_tok,
355 archived_refs: archived,
356 remaining_working_refs: working_store.keys().cloned().collect(),
357 }
358 }
359}