use std::sync::OnceLock;
use serde::{Deserialize, Serialize};
use crate::layer0::mcard_record::MCardRecord;
pub const CHUNKER_ID: &str = "clm-cdc-v1";
pub const CHUNKER_VERSION: u32 = 1;
pub fn cdc_split_boundaries(
data: &[u8],
min_size: usize,
avg_size: usize,
max_size: usize,
) -> Vec<(usize, usize)> {
let n = data.len();
if n == 0 {
return Vec::new();
}
let modulus = (avg_size.saturating_sub(min_size)).max(1) as u32;
let low_mask = (1u32 << modulus.trailing_zeros()).wrapping_sub(1);
let mut slices = Vec::with_capacity(n / avg_size.max(1) + 2);
let mut chunk_start = 0usize;
while chunk_start < n {
let remaining = n - chunk_start;
if remaining <= min_size {
slices.push((chunk_start, remaining));
break;
}
let limit = (chunk_start + max_size).min(n);
let skip_until = (chunk_start + min_size.saturating_sub(1)).min(limit);
let mut h: u32 = 0;
for &b in &data[chunk_start..skip_until] {
h = h.wrapping_mul(31).wrapping_add(b as u32);
}
let mut pos = skip_until;
while pos < limit {
h = h.wrapping_mul(31).wrapping_add(data[pos] as u32);
let span = pos - chunk_start + 1;
let cut = (h & low_mask) == 0 && h.is_multiple_of(modulus);
if span >= min_size && (cut || (pos + 1) == limit) {
slices.push((chunk_start, span));
chunk_start = pos + 1;
break;
}
pos += 1;
}
}
slices
}
#[derive(Default)]
pub struct MemoizedRef(OnceLock<String>);
impl std::fmt::Debug for MemoizedRef {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("MemoizedRef").field(&self.0.get()).finish()
}
}
impl Clone for MemoizedRef {
fn clone(&self) -> Self {
let lock = OnceLock::new();
if let Some(val) = self.0.get() {
let _ = lock.set(val.clone());
}
Self(lock)
}
}
impl PartialEq for MemoizedRef {
fn eq(&self, _other: &Self) -> bool {
true
}
}
impl Eq for MemoizedRef {}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ChunkDescriptor {
pub index: usize,
pub offset: usize,
pub length: usize,
pub qualified_ref: String,
}
#[derive(Serialize)]
struct CanonicalChunk<'a> {
index: usize,
length: usize,
offset: usize,
qualified_ref: &'a str,
}
struct CanonicalChunks<'a>(&'a [ChunkDescriptor]);
impl Serialize for CanonicalChunks<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
use serde::ser::SerializeSeq;
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
for c in self.0 {
seq.serialize_element(&CanonicalChunk {
index: c.index,
length: c.length,
offset: c.offset,
qualified_ref: &c.qualified_ref,
})?;
}
seq.end()
}
}
#[derive(Serialize)]
struct CanonicalManifest<'a> {
algorithm: &'a str,
avg_size: usize,
chunker_id: &'a str,
chunker_version: u32,
chunks: CanonicalChunks<'a>,
max_size: usize,
min_size: usize,
schema: &'static str,
total_bytes: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ChunkManifest {
pub chunker_id: String,
pub chunker_version: u32,
pub min_size: usize,
pub avg_size: usize,
pub max_size: usize,
pub total_bytes: usize,
pub chunks: Vec<ChunkDescriptor>,
pub algorithm: String,
#[serde(skip, default)]
pub cached_qualified_ref: MemoizedRef,
}
impl ChunkManifest {
pub fn to_canonical_bytes(&self) -> Vec<u8> {
let doc = CanonicalManifest {
algorithm: &self.algorithm,
avg_size: self.avg_size,
chunker_id: &self.chunker_id,
chunker_version: self.chunker_version,
chunks: CanonicalChunks(&self.chunks),
max_size: self.max_size,
min_size: self.min_size,
schema: crate::layer0::vocabulary::HANDLE_CLM_LAYER2_CHUNK_MANIFEST_V1,
total_bytes: self.total_bytes,
};
serde_json::to_vec(&doc).unwrap_or_default()
}
pub fn to_mcard_record(&self) -> Result<MCardRecord, String> {
MCardRecord::create(&self.to_canonical_bytes(), Some(&self.algorithm))
}
pub fn qualified_ref(&self) -> Result<String, String> {
if let Some(qref) = self.cached_qualified_ref.0.get() {
return Ok(qref.clone());
}
let selected = if self.algorithm.trim().is_empty() {
crate::hash::config()
.active_algorithm
.trim()
.trim_end_matches(':')
.to_lowercase()
} else {
self.algorithm.trim().trim_end_matches(':').to_lowercase()
};
let bytes = self.to_canonical_bytes();
let hash = if selected == crate::layer0::vocabulary::HASH_ALGORITHM_BLAKE3 {
blake3::hash(&bytes).to_hex().to_string()
} else {
crate::hash::raw_hash_bytes_for(&selected, &bytes)?
};
let prefix = if self.algorithm.is_empty() {
selected.as_str()
} else {
self.algorithm.as_str()
};
let qref = format!("{prefix}:{hash}");
let _ = self.cached_qualified_ref.0.set(qref.clone());
Ok(qref)
}
#[cfg(feature = "native")]
pub(crate) fn set_cached_qualified_ref(&self, qref: String) {
let _ = self.cached_qualified_ref.0.set(qref);
}
}